From: Vincent Vanwaelscappel <vincent@cubedesigners.com>
Date: Fri, 8 Sep 2023 15:37:11 +0000 (+0200)
Subject: wip #6188 @3
X-Git-Url: http://git.cubedesigners.com/?a=commitdiff_plain;h=3b7597d807f8a6a7babf8495f1c40b230d4f5c30;p=cubist_pdf.git

wip #6188 @3
---

diff --git a/resources/tools/fwstk/.idea/workspace.xml b/resources/tools/fwstk/.idea/workspace.xml
index b9d43a9..24a3a2f 100644
--- a/resources/tools/fwstk/.idea/workspace.xml
+++ b/resources/tools/fwstk/.idea/workspace.xml
@@ -95,29 +95,30 @@
     <option name="hideEmptyMiddlePackages" value="true" />
     <option name="showLibraryContents" value="true" />
   </component>
-  <component name="PropertiesComponent">{
-  &quot;keyToString&quot;: {
-    &quot;WebServerToolWindowFactoryState&quot;: &quot;true&quot;,
-    &quot;WebServerToolWindowPanel.toolwindow.highlight.mappings&quot;: &quot;true&quot;,
-    &quot;WebServerToolWindowPanel.toolwindow.highlight.symlinks&quot;: &quot;true&quot;,
-    &quot;WebServerToolWindowPanel.toolwindow.show.date&quot;: &quot;false&quot;,
-    &quot;WebServerToolWindowPanel.toolwindow.show.permissions&quot;: &quot;false&quot;,
-    &quot;WebServerToolWindowPanel.toolwindow.show.size&quot;: &quot;false&quot;,
-    &quot;git-widget-placeholder&quot;: &quot;master&quot;,
-    &quot;ignore.virus.scanning.warn.message&quot;: &quot;true&quot;,
-    &quot;last_opened_file_path&quot;: &quot;D:/Works/cubist_pdf/resources/tools/fwstk&quot;,
-    &quot;node.js.detected.package.eslint&quot;: &quot;true&quot;,
-    &quot;node.js.detected.package.tslint&quot;: &quot;true&quot;,
-    &quot;node.js.selected.package.eslint&quot;: &quot;(autodetect)&quot;,
-    &quot;node.js.selected.package.tslint&quot;: &quot;(autodetect)&quot;,
-    &quot;project.structure.last.edited&quot;: &quot;SDKs&quot;,
-    &quot;project.structure.proportion&quot;: &quot;0.15&quot;,
-    &quot;project.structure.side.proportion&quot;: &quot;0.2&quot;,
-    &quot;ruby.rails.projectView.checked&quot;: &quot;true&quot;,
-    &quot;settings.editor.selected.configurable&quot;: &quot;preferences.lookFeel&quot;,
-    &quot;vue.rearranger.settings.migration&quot;: &quot;true&quot;
+  <component name="PropertiesComponent"><![CDATA[{
+  "keyToString": {
+    "RunOnceActivity.ShowReadmeOnStart": "true",
+    "WebServerToolWindowFactoryState": "true",
+    "WebServerToolWindowPanel.toolwindow.highlight.mappings": "true",
+    "WebServerToolWindowPanel.toolwindow.highlight.symlinks": "true",
+    "WebServerToolWindowPanel.toolwindow.show.date": "false",
+    "WebServerToolWindowPanel.toolwindow.show.permissions": "false",
+    "WebServerToolWindowPanel.toolwindow.show.size": "false",
+    "git-widget-placeholder": "master",
+    "ignore.virus.scanning.warn.message": "true",
+    "last_opened_file_path": "D:/Works/cubist_pdf/resources/tools/fwstk",
+    "node.js.detected.package.eslint": "true",
+    "node.js.detected.package.tslint": "true",
+    "node.js.selected.package.eslint": "(autodetect)",
+    "node.js.selected.package.tslint": "(autodetect)",
+    "project.structure.last.edited": "SDKs",
+    "project.structure.proportion": "0.15",
+    "project.structure.side.proportion": "0.2",
+    "ruby.rails.projectView.checked": "true",
+    "settings.editor.selected.configurable": "preferences.lookFeel",
+    "vue.rearranger.settings.migration": "true"
   }
-}</component>
+}]]></component>
   <component name="RecentsManager">
     <key name="CopyFile.RECENT_KEYS">
       <recent name="H:\Works\cubeExtranet\fluidbook\tools\fwstk\lib" />
@@ -486,6 +487,7 @@
       <workItem from="1692974658841" duration="8000" />
       <workItem from="1692974700537" duration="688000" />
       <workItem from="1694090487471" duration="6051000" />
+      <workItem from="1694157597994" duration="9552000" />
     </task>
     <task id="LOCAL-00001" summary="wip #1111 @0.5">
       <created>1487172253077</created>
@@ -611,17 +613,6 @@
     <MESSAGE value="wip #5410" />
     <option name="LAST_COMMIT_MESSAGE" value="wip #5410" />
   </component>
-  <component name="XDebuggerManager">
-    <breakpoint-manager>
-      <breakpoints>
-        <line-breakpoint enabled="true" type="java-line">
-          <url>file://$PROJECT_DIR$/src/com/fluidbook/fwstk/layout/Word.java</url>
-          <line>168</line>
-          <option name="timeStamp" value="1" />
-        </line-breakpoint>
-      </breakpoints>
-    </breakpoint-manager>
-  </component>
   <component name="antWorkspaceConfiguration">
     <option name="IS_AUTOSCROLL_TO_SOURCE" value="false" />
     <option name="FILTER_TARGETS" value="false" />
diff --git a/resources/tools/fwstk/bin/com/fluidbook/fwstk/TextsThread.class b/resources/tools/fwstk/bin/com/fluidbook/fwstk/TextsThread.class
index 44713c5..62eb436 100644
Binary files a/resources/tools/fwstk/bin/com/fluidbook/fwstk/TextsThread.class and b/resources/tools/fwstk/bin/com/fluidbook/fwstk/TextsThread.class differ
diff --git a/resources/tools/fwstk/bin/cube/util/StringUtil.class b/resources/tools/fwstk/bin/cube/util/StringUtil.class
index 16a9c9e..18ef6cf 100644
Binary files a/resources/tools/fwstk/bin/cube/util/StringUtil.class and b/resources/tools/fwstk/bin/cube/util/StringUtil.class differ
diff --git a/resources/tools/fwstk/out/artifacts/fwstk_jar/fwstk.jar b/resources/tools/fwstk/out/artifacts/fwstk_jar/fwstk.jar
index d8f6b30..e493b0c 100644
Binary files a/resources/tools/fwstk/out/artifacts/fwstk_jar/fwstk.jar and b/resources/tools/fwstk/out/artifacts/fwstk_jar/fwstk.jar differ
diff --git a/resources/tools/fwstk/src/com/fluidbook/fwstk/TextsThread.java b/resources/tools/fwstk/src/com/fluidbook/fwstk/TextsThread.java
index 3eefe21..0c37553 100644
--- a/resources/tools/fwstk/src/com/fluidbook/fwstk/TextsThread.java
+++ b/resources/tools/fwstk/src/com/fluidbook/fwstk/TextsThread.java
@@ -197,18 +197,22 @@ public class TextsThread extends Thread {
                     String fhfile = file.replaceFirst("%s", "fh");
 
                     fbtext = html2text(fbtext);
-
-                    String fbindex = text2index(fbtext);
+                    if (robust) {
+                        fbtext=StringUtil.removeSpaces(fbtext);
+                    }
 
                     try {
                         out = new BufferedWriter(new OutputStreamWriter(
                                 new FileOutputStream(fpfile), "UTF8"));
                         out.write(fbtext);
                         out.close();
-                        out = new BufferedWriter(new OutputStreamWriter(
-                                new FileOutputStream(fifile), "UTF8"));
-                        out.write(fbindex);
-                        out.close();
+                        if (!robust) {
+                            String fbindex = text2index(fbtext);
+                            out = new BufferedWriter(new OutputStreamWriter(
+                                    new FileOutputStream(fifile), "UTF8"));
+                            out.write(fbindex);
+                            out.close();
+                        }
                         out = new BufferedWriter(new OutputStreamWriter(
                                 new FileOutputStream(fhfile), "UTF8"));
                         out.write(fbhtml);
diff --git a/resources/tools/fwstk/src/cube/util/StringUtil.java b/resources/tools/fwstk/src/cube/util/StringUtil.java
index c6d0d49..87bd062 100644
--- a/resources/tools/fwstk/src/cube/util/StringUtil.java
+++ b/resources/tools/fwstk/src/cube/util/StringUtil.java
@@ -6,134 +6,154 @@ import java.util.HashMap;
 
 public class StringUtil {
 
-	public static String removeAccents(String in) {
-		return AsciiUtils.convertNonAscii(in);
-	}
-
-	public static String condenseWhite(String in) {
-		return in.replaceAll("\\p{Space}++", " ");
-	}
-
-	public static String removePoints(String in) {
-		return StringUtil.removePoints(in, "");
-	}
-
-	public static String separateLigatures(String in) {
-		in = in.replaceAll("\u00C6", "AE");
-		in = in.replaceAll("\u00E6", "ae");
-		in = in.replaceAll("\u0152", "OE");
-		in = in.replaceAll("\u0153", "oe");
-		in = in.replaceAll("\u0132", "IJ");
-		in = in.replaceAll("\u0133", "ij");
-		in = in.replaceAll("\ufb00", "ff");
-		in = in.replaceAll("\ufb01", "fi");
-		in = in.replaceAll("\ufb02", "fl");
-		in = in.replaceAll("\ufb03", "ffi");
-		in = in.replaceAll("\ufb04", "ffl");
-
-		return in;
-	}
-
-	public static String removePoints(String in, String ignore) {
-		String res = in.replaceAll(
-				separatorsRegexp(ignore),
-				" ");
-
-		return res;
-	}
-
-	public static String separatorsRegexp(String ignore) {
-		if (ignore != "") {
-			String ig = "";
-			for (int i = 0; i < ignore.length(); i++) {
-				ig += "\\" + ignore.substring(i, i + 1);
-			}
-			ignore = "&&[^" + ig + "]";
-		}
-		return "[\\u2000-\\u20ff\\u2122\\x21-\\x2f\\x3a-\\x3f\\x5b-\\x60\\x7b-\\xa0\\xaa-\\xbf\\ufffd" + ignore + "]";
-	}
-
-	public static String normalizeWhite(String in) {
-		String res = in.replaceAll(
-				"[\\u2000-\\u20ff\\ufffd]",
-				" ");
-
-		return res;
-	}
-
-	public static String removeControl(String in) {
-		return in.replaceAll("\\p{Cntrl}", " ");
-	}
-
-	public static String removeTags(String in) {
-		in = in.replaceAll("<.+>", " ");
-		return condenseWhite(in);
-	}
-
-	public static String trim(String str) {
-		return str.trim();
-	}
-
-	public static String trim(String str, String[] chars) {
-
-		Boolean suite = false;
-		for (String c : chars) {
-			if (str.contains(c)) {
-				suite = true;
-				break;
-			}
-		}
-		if (!suite) {
-			return str;
-		}
-
-		str = ltrim(str, chars);
-		str = rtrim(str, chars);
-
-		return str;
-	}
-
-	public static String rtrim(String str, String[] charsToTrim) {
-		int length = str.length() - 1;
-
-		rightloop:
-		for (; length >= 0; length--) {
-			for (String c : charsToTrim) {
-				if (str.indexOf(c) == length) {
-					continue rightloop;
-				}
-			}
-			break;
-		}
-
-		str = str.substring(0, length + 1);
-		return str;
-	}
-
-	public static String[] splitStr(String str) {
-
-		byte[] chars = str.getBytes();
-		String[] res = new String[chars.length];
-		for (int i = 0; i < chars.length; i++) {
-			res[i] = String.valueOf((char) chars[i]);
-		}
-
-		return res;
-	}
-
-	public static String ltrim(String str, String[] charsToTrim) {
-		int startIndex = 0;
-		leftloop:
-		for (startIndex = 0; startIndex <= str.length(); startIndex++) {
-			for (String c : charsToTrim) {
-				if (str.indexOf(c) == startIndex) {
-					continue leftloop;
-				}
-			}
-			break;
-		}
-		str = str.substring(startIndex);
-		return str;
-
-	}
+    public static String removeAccents(String in) {
+        return AsciiUtils.convertNonAscii(in);
+    }
+
+    public static String condenseWhite(String in) {
+        return in.replaceAll("\\p{Space}++", " ");
+    }
+
+    public static String removePoints(String in) {
+        return StringUtil.removePoints(in, "");
+    }
+
+    public static String separateLigatures(String in) {
+        in = in.replaceAll("\u00C6", "AE");
+        in = in.replaceAll("\u00E6", "ae");
+        in = in.replaceAll("\u0152", "OE");
+        in = in.replaceAll("\u0153", "oe");
+        in = in.replaceAll("\u0132", "IJ");
+        in = in.replaceAll("\u0133", "ij");
+        in = in.replaceAll("\ufb00", "ff");
+        in = in.replaceAll("\ufb01", "fi");
+        in = in.replaceAll("\ufb02", "fl");
+        in = in.replaceAll("\ufb03", "ffi");
+        in = in.replaceAll("\ufb04", "ffl");
+
+        return in;
+    }
+
+    public static String removePoints(String in, String ignore) {
+        String res = in.replaceAll(
+                punctuationRegexp(ignore),
+                " ");
+        return res;
+    }
+
+    public static String punctuationRegexp(String ignore) {
+        if (ignore != "") {
+            String ig = "";
+            for (int i = 0; i < ignore.length(); i++) {
+                ig += "\\" + ignore.substring(i, i + 1);
+            }
+            ignore = "&&[^" + ig + "]";
+        }
+        String common = "-\\/_";
+        String zh = "』『「」！？｡。。」﹂”』’》）］｝〕〗〙〛〉】＂＃＄％＆＇（）＊＋，－／：；＜＝＞＠［＼］＾＿｀｛｜｝～｟｠｢｣､　、〃〈〉《》【】〕〔〗〖〙〘〛〚〜〞〝〟〰〾〿–—‘’“‛”„‟…‧﹏﹑﹔·〇一。";
+        return "[" + common + zh + "\\u2000-\\u20ff\\u2122\\x21-\\x2f\\x3a-\\x3f\\x5b-\\x60\\x7b-\\xa0\\xaa-\\xbf\\ufffd" + ignore + "]";
+    }
+
+    public static String separatorsRegexp(String ignore) {
+        if (ignore != "") {
+            String ig = "";
+            for (int i = 0; i < ignore.length(); i++) {
+                ig += "\\" + ignore.substring(i, i + 1);
+            }
+            ignore = "&&[^" + ig + "]";
+        }
+        return "[\\u2000-\\u20ff\\u2122\\x21-\\x2f\\x3a-\\x3f\\x5b-\\x60\\x7b-\\xa0\\xaa-\\xbf\\ufffd" + ignore + "]";
+    }
+
+    public static String normalizeWhite(String in) {
+        String res = in.replaceAll(
+                "[\\u2000-\\u20ff\\ufffd]",
+                " ");
+
+        return res;
+    }
+
+    public static String removeSpaces(String in) {
+        String res = in.replaceAll(
+                "[\\u2000-\\u20ff\\ufffd]",
+                "");
+
+        return res;
+    }
+
+    public static String removeControl(String in) {
+        return in.replaceAll("\\p{Cntrl}", " ");
+    }
+
+    public static String removeTags(String in) {
+        in = in.replaceAll("<.+>", " ");
+        return condenseWhite(in);
+    }
+
+    public static String trim(String str) {
+        return str.trim();
+    }
+
+    public static String trim(String str, String[] chars) {
+
+        Boolean suite = false;
+        for (String c : chars) {
+            if (str.contains(c)) {
+                suite = true;
+                break;
+            }
+        }
+        if (!suite) {
+            return str;
+        }
+
+        str = ltrim(str, chars);
+        str = rtrim(str, chars);
+
+        return str;
+    }
+
+    public static String rtrim(String str, String[] charsToTrim) {
+        int length = str.length() - 1;
+
+        rightloop:
+        for (; length >= 0; length--) {
+            for (String c : charsToTrim) {
+                if (str.indexOf(c) == length) {
+                    continue rightloop;
+                }
+            }
+            break;
+        }
+
+        str = str.substring(0, length + 1);
+        return str;
+    }
+
+    public static String[] splitStr(String str) {
+
+        byte[] chars = str.getBytes();
+        String[] res = new String[chars.length];
+        for (int i = 0; i < chars.length; i++) {
+            res[i] = String.valueOf((char) chars[i]);
+        }
+
+        return res;
+    }
+
+    public static String ltrim(String str, String[] charsToTrim) {
+        int startIndex = 0;
+        leftloop:
+        for (startIndex = 0; startIndex <= str.length(); startIndex++) {
+            for (String c : charsToTrim) {
+                if (str.indexOf(c) == startIndex) {
+                    continue leftloop;
+                }
+            }
+            break;
+        }
+        str = str.substring(startIndex);
+        return str;
+
+    }
 }