change dictionary format

build dictionary from aspell debian package

diff --git a/dict-dists-extractor/Makefile b/dict-dists-extractor/Makefile
index 67f7385f61df5a5a181299e03582cddf40570060..f13a7ab00c12df208d4ed8b6eba913f5c2372c79 100644 (file)
--- a/dict-dists-extractor/Makefile
+++ b/dict-dists-extractor/Makefile
@@ -2,8 +2,11 @@
 
 ROOT_DICT = ../public/dict
 
-LANGS = bg ca da de en eo es fr it nl no po sv uk
-# pl not working
+LANGS = bg ca da de en eo es fr it nl no pl pt sv uk
+# to add: ar br cs cy el et eu fi? ga he hr hu hy is lv ro ru sk sl
+#LANGS = af ar bg br ca cs cy da de el en eo es et eu fi fr ga he hr hu hy is it la lv ms nl no pl pt ro ru sk sl sv tr uk xk
+
+LOWS = $(addsuffix .low,$(LANGS))
 
 DICTS = $(addsuffix .dict,$(LANGS))
 
@@ -18,53 +21,64 @@ $(ROOT_DICT):
 count:
        wc -l $(ROOT_DICT)/*
 
-dicts: $(DICTS)
+low: $(LOWS)
+
+INSTALL_ASPELL_DICT = apt search aspell-$(1) && \
+    ( dpkg -l aspell-$(1) | grep -q ^ii || sudo apt install aspell-$(1) )
+
+ASPELL_DUMP_MASTER = aspell -d $(1) dump master $(1) | \
+    aspell -l $(1) expand | \
+    tr -s '[:space:]' '\n' | \
+    LC_ALL=C sort -S28G -u > $(1).low
+
+%.low:
+       echo Building list of word $@
+       $(call INSTALL_ASPELL_DICT,$(@:.low=))
+       $(call ASPELL_DUMP_MASTER,$(@:.low=))
+
+no.low:
+       echo Building list of word $@
+       $(call INSTALL_ASPELL_DICT,no)
+       $(call ASPELL_DUMP_MASTER,nb)
+       mv nb.low $@
 
-$(DICTS):%:
-       lang=$(@:.dict=); \
-       apt search aspell-$$lang && \
-       ( dpkg -l aspell-$$lang | grep ^ii || \
-         sudo apt install aspell-$$lang ) && \
-       aspell -d $$lang dump master | aspell -l $$lang expand | \
-           tr -s '[:space:]' '\n' | \
-               LC_ALL=C sort -S28G -u > $$lang.dict
+pt.low:
+       echo Building list of word $@
+       $(call INSTALL_ASPELL_DICT,pt-pt)
+       $(call ASPELL_DUMP_MASTER,pt_PT)
+       mv pt_PT.low $@
 
-$(ROOT_DICT)/bg.json: /usr/share/dict/bulgarian Makefile
+$(ROOT_DICT)/bg.dict: bg.low Makefile
        echo Building $@ from $<
-       iconv -f Windows-1251 -t utf-8 $< | \
+       cat $< | \
         grep -v '^.$$' | \
         grep -v "['-]" | \
+        grep -v "^[a-zA-Z]" | \
                tr '[абвгдежзийклмнопрстуфхцчшщъьюя]' '[АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯ]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
-$(ROOT_DICT)/ca.json: /usr/share/dict/catalan Makefile
+$(ROOT_DICT)/ca.dict: ca.low Makefile
        echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
-        grep -v "['-]" | \
+        grep -v "[.'-]" | \
         sed -e 's/[áàãÁÀ]/a/g' \
             -e 's/[éèÉÈ]/e/g' \
-            -e 's/[ç]/c/g' \
+            -e 's/[çÇ]/c/g' \
                        -e 's/[íïÍ]/i/g' \
             -e 's/[ñ]/n/g' \
             -e 's/[óöòÓÒ]/o/g' \
             -e 's/[úüùÚ]/u/g' | \
         tr '[a-z]' '[A-Z]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
-$(ROOT_DICT)/da.json: /usr/share/dict/danish Makefile
+$(ROOT_DICT)/da.dict: da.low Makefile
        echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
-        grep -v "['-]" | \
+        grep -v "[.'-]" | \
         sed -e 's/[áä]/a/g' \
             -e 's/[ð]/d/g' \
             -e 's/[éëè]/e/g' \
@@ -73,12 +87,9 @@ $(ROOT_DICT)/da.json: /usr/share/dict/danish Makefile
             -e 's/[ü]/u/g' | \
         tr '[a-zæøå]' '[A-ZÆØÅ]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
-$(ROOT_DICT)/de.json: /usr/share/dict/ngerman Makefile
+$(ROOT_DICT)/de.dict: de.low Makefile
        echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
@@ -90,12 +101,9 @@ $(ROOT_DICT)/de.json: /usr/share/dict/ngerman Makefile
             -e 's/[ß]/ss/g' | \
         tr '[a-zäöü]' '[A-ZÄÖÜ]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
-$(ROOT_DICT)/en.json: /usr/share/dict/british-english-huge Makefile
+$(ROOT_DICT)/en.dict: en.low Makefile
        echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
@@ -109,76 +117,62 @@ $(ROOT_DICT)/en.json: /usr/share/dict/british-english-huge Makefile
             -e 's/[úûüùiÜ]/u/g' | \
         tr '[a-z]' '[A-Z]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
-$(ROOT_DICT)/eo.json: /usr/share/dict/esperanto Makefile
+$(ROOT_DICT)/eo.dict: eo.low Makefile
        echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
         grep -v "[.'-]" | \
         tr '[a-zĉĝĥĵŝŭ]' '[A-ZĈĜĤĴŜŬ]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
-$(ROOT_DICT)/es.json: /usr/share/dict/spanish Makefile
+$(ROOT_DICT)/es.dict: es.low Makefile
        echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
-        grep -v "['-]" | \
-        sed -e 's/[á]/a/g' \
+        grep -v "[.'-]" | \
+        sed -e 's/[áÁ]/a/g' \
             -e 's/[é]/e/g' \
                        -e 's/[í]/i/g' \
             -e 's/[ó]/o/g' \
             -e 's/[úü]/u/g' | \
         tr '[a-zñ]' '[A-ZÑ]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
-$(ROOT_DICT)/fr.json: /usr/share/dict/french Makefile
+$(ROOT_DICT)/fr.dict: fr.low Makefile
        echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
-        grep -v "[.'ú-]" | \
-        sed -e 's/[àâ]/a/g' \
+        grep -v "[.'-]" | \
+        sed -e 's/[àâä]/a/g' \
             -e 's/[ç]/c/g' \
-            -e 's/[éèêë]/e/g' \
+            -e 's/[Ã\89éèêë]/e/g' \
             -e 's/[ïî]/i/g' \
             -e 's/[ôö]/o/g' \
-            -e 's/[ùûü]/u/g' | \
+            -e 's/[úùûü]/u/g' | \
         tr '[a-z]' '[A-Z]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
-$(ROOT_DICT)/it.json: /usr/share/dict/italian Makefile
+$(ROOT_DICT)/it.dict: it.low Makefile
        echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
         grep -v "[.'-]" | \
-        sed -e 's/[à]/a/g' \
+        sed -e 's/[àâ]/a/g' \
+            -e 's/[ç]/c/g' \
             -e 's/[éè]/e/g' \
             -e 's/[ì]/i/g' \
-            -e 's/[ò]/o/g' \
+            -e 's/[òiô]/o/g' \
             -e 's/[ù]/u/g' | \
         tr '[a-z]' '[A-Z]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
-$(ROOT_DICT)/nl.json: /usr/share/dict/dutch Makefile
+$(ROOT_DICT)/nl.dict: nl.low Makefile
        echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
@@ -192,14 +186,11 @@ $(ROOT_DICT)/nl.json: /usr/share/dict/dutch Makefile
             -e 's/[úüûù]/u/g' | \
         tr '[a-z]' '[A-Z]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
-$(ROOT_DICT)/no.json: /usr/share/dict/bokmaal Makefile
+$(ROOT_DICT)/no.dict: no.low Makefile
        echo Building $@ from $<
-       iconv -f iso-8859-1 -t utf-8 $< | \
+       cat $< | \
         grep -v '^.$$' | \
         grep -v "['-]" | \
         sed -e 's/[áäÄ]/a/g' \
@@ -211,12 +202,9 @@ $(ROOT_DICT)/no.json: /usr/share/dict/bokmaal Makefile
             -e 's/[ü]/u/g' | \
         tr '[a-zæøå]' '[A-ZÆØÅ]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
-$(ROOT_DICT)/pl.json: /usr/share/dict/polish Makefile
+$(ROOT_DICT)/pl.dict: pl.low Makefile
        echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
@@ -230,14 +218,11 @@ $(ROOT_DICT)/pl.json: /usr/share/dict/polish Makefile
             -e 's/[û]/u/g' | \
         tr '[a-złąęóśżćńź]' '[A-ZŁĄĘÓŚŻĆŃŹ]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
-$(ROOT_DICT)/po.json: /usr/share/dict/portuguese Makefile
+$(ROOT_DICT)/pt.dict: pt.low Makefile
        echo Building $@ from $<
-       iconv -f iso-8859-15 -t utf-8 $< | \
+       cat $< | \
         grep -v '^.$$' | \
         grep -v "['-]" | \
         sed -e 's/[áãâàÁÂ]/a/g' \
@@ -247,14 +232,11 @@ $(ROOT_DICT)/po.json: /usr/share/dict/portuguese Makefile
             -e 's/[úüÚ]/u/g' | \
         tr '[a-zç]' '[A-ZÇ]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
-$(ROOT_DICT)/sv.json: /usr/share/dict/swedish Makefile
+$(ROOT_DICT)/sv.dict: sv.low Makefile
        echo Building $@ from $<
-       iconv -f iso-8859-15 -t utf-8 $< | \
+       cat $< | \
         grep -v '^.$$' | \
         grep -v "['-]" | \
         sed -e 's/[æ]/ae/g' \
@@ -263,60 +245,52 @@ $(ROOT_DICT)/sv.json: /usr/share/dict/swedish Makefile
             -e 's/[ü]/u/g' | \
         tr '[a-zåäö]' '[A-ZÅÄÖ]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
-$(ROOT_DICT)/uk.json: /usr/share/dict/ukrainian Makefile
+$(ROOT_DICT)/uk.dict: uk.low Makefile
        echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
         grep -v "[-]" | \
         tr '[абвгґдеєжзиіїйклмнопрстуфхцчшщьюя]' '[АБВГҐДЕЄЖЗИІЇЙКЛМНОПРСТУФХЦЧШЩЬЮЯ]' | \
         sort | \
-        uniq | \
-        awk 'BEGIN { print "[" } \
-            { print "    \"" $$0 "\"," } \
-            END { print "    \"\"\n]" }' > $@
+        uniq > $@
 
 CHECKER = cat $(1) | \
-    grep -v "\"[$(2)]*\"" | grep -v "\"$3\"" | \
-       grep -v '\[' | grep -v '\]' | grep -v '""' \
-       && exit 1 || echo "$< is correct"
+    grep -v "^[$(2)]*$$" | egrep -v "$(3)" && exit 1 || echo "$< is correct"
 
-check-bg:$(ROOT_DICT)/bg.json
+check-bg:$(ROOT_DICT)/bg.dict
        $(call CHECKER,$<,АОЕИТНПРСВМБДКЛГЪЖЗУЧЯЙХЦШЮФЩЬ,)
 
-check-ca:$(ROOT_DICT)/ca.json
-       $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZ,[A-Z]*L·L[A-Z]*)
+check-ca:$(ROOT_DICT)/ca.dict
+       $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZ,L·L[A-Z]*(L·L)*)
 
-check-da:$(ROOT_DICT)/da.json
+check-da:$(ROOT_DICT)/da.dict
        $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÆØÅ,)
 
-check-de:$(ROOT_DICT)/de.json
+check-de:$(ROOT_DICT)/de.dict
        $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÄÖÜ,)
 
-check-eo:$(ROOT_DICT)/eo.json
+check-eo:$(ROOT_DICT)/eo.dict
        $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZĈĜĤĴŜŬ,)
 
-check-es:$(ROOT_DICT)/es.json
+check-es:$(ROOT_DICT)/es.dict
        $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÑ,)
 
-check-no:$(ROOT_DICT)/no.json
+check-no:$(ROOT_DICT)/no.dict
        $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÆØÅ,)
 
-check-pl:$(ROOT_DICT)/pl.json
+check-pl:$(ROOT_DICT)/pl.dict
        $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZŁĄĘÓŚŻĆŃŹ,)
 
-check-po:$(ROOT_DICT)/po.json
+check-po:$(ROOT_DICT)/po.dict
        $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÇ,)
 
-check-sv:$(ROOT_DICT)/sv.json
+check-sv:$(ROOT_DICT)/sv.dict
        $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÅÄÖ,)
 
-check-uk:$(ROOT_DICT)/uk.json
+check-uk:$(ROOT_DICT)/uk.dict
        $(call CHECKER,$<,ОАИНВЕІТКРСДЛМУПЗЯЬБГЧХЄЇЙЖЦШЮҐФЩ',)
 
-check-%:$(ROOT_DICT)/%.json
+check-%:$(ROOT_DICT)/%.dict
        $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZ,)

diff --git a/public/trivabble.js b/public/trivabble.js
index 43d571049eff3e702091ec476bb758d82d245872..fdd2d19d471c177b7ec0919daeb6be025ec4f894 100644 (file)
--- a/public/trivabble.js
+++ b/public/trivabble.js
@@ -159,12 +159,12 @@
         downloadDictionary.code = code;
 
         const file = new XMLHttpRequest();
-        file.overrideMimeType("application/json");
-        file.open("GET", "dict/" + code + ".json", true);
+        file.overrideMimeType("application/dictionary");
+        file.open("GET", "dict/" + code + ".dict", true);
         file.onreadystatechange = function () {
             if ((file.readyState === 4) && (file.status == "200")) {
                 try {
-                    dictionary = JSON.parse(file.responseText);
+                    dictionary = file.responseText.split("\n");
                     document.getElementById("check-spelling").disabled = false;
                 } catch (e) {
                     console.error(e);