update dictionnary building rules

diff --git a/dict-dists-extractor/Makefile b/dict-dists-extractor/Makefile
index 6f2bb584c4a3ea98e5d8d7ddf2a96d545ef500e7..3abbaff4a32f8911eee62c5e87bc559aee7e2cbe 100644 (file)
--- a/dict-dists-extractor/Makefile
+++ b/dict-dists-extractor/Makefile
@@ -2,7 +2,10 @@
 
 ROOT_DICT = ../public/dict
 
-LANGS = bg ca da de en eo fr it nl no
+LANGS = bg ca da de en eo fr it nl no po sp sv uk
+# pl not working
+
+MAKEFLAGS = -s
 
 all:
        make $(addprefix check-,$(LANGS))
@@ -13,8 +16,8 @@ $(ROOT_DICT):
 count:
        wc -l $(ROOT_DICT)/*
 
-#sed -e 's/Ѝ/И/g' 
 $(ROOT_DICT)/bg.json: /usr/share/dict/bulgarian Makefile
+       echo Building $@ from $<
        iconv -f Windows-1251 -t utf-8 $< | \
         grep -v '^.$$' | \
         grep -v "['-]" | \
@@ -26,17 +29,17 @@ $(ROOT_DICT)/bg.json: /usr/share/dict/bulgarian Makefile
             END { print "    \"\"\n]" }' > $@
 
 $(ROOT_DICT)/ca.json: /usr/share/dict/catalan Makefile
+       echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
         grep -v "['-]" | \
-        grep -v "^[A-Z]" | \
-        sed -e 's/[áàã]/a/g' \
-            -e 's/[éè]/e/g' \
+        sed -e 's/[áàãÁÀ]/a/g' \
+            -e 's/[éèÉÈ]/e/g' \
             -e 's/[ç]/c/g' \
-            -e 's/[íï]/i/g' \
+                       -e 's/[íïÍ]/i/g' \
             -e 's/[ñ]/n/g' \
-            -e 's/[óöò]/o/g' \
-            -e 's/[úüù]/u/g' | \
+            -e 's/[óöòÓÒ]/o/g' \
+            -e 's/[úüùÚ]/u/g' | \
         tr '[a-z]' '[A-Z]' | \
         sort | \
         uniq | \
@@ -45,10 +48,10 @@ $(ROOT_DICT)/ca.json: /usr/share/dict/catalan Makefile
             END { print "    \"\"\n]" }' > $@
 
 $(ROOT_DICT)/da.json: /usr/share/dict/danish Makefile
+       echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
         grep -v "['-]" | \
-        grep -v "^[A-Z]" | \
         sed -e 's/[áä]/a/g' \
             -e 's/[ð]/d/g' \
             -e 's/[éëè]/e/g' \
@@ -63,6 +66,7 @@ $(ROOT_DICT)/da.json: /usr/share/dict/danish Makefile
             END { print "    \"\"\n]" }' > $@
 
 $(ROOT_DICT)/de.json: /usr/share/dict/ngerman Makefile
+       echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
         grep -v "[.'-]" | \
@@ -79,17 +83,17 @@ $(ROOT_DICT)/de.json: /usr/share/dict/ngerman Makefile
             END { print "    \"\"\n]" }' > $@
 
 $(ROOT_DICT)/en.json: /usr/share/dict/british-english-huge Makefile
+       echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
         grep -v "[.'-]" | \
-        grep -v "^[A-Z]" | \
-        sed -e 's/[áâàäå]/a/g' \
+        sed -e 's/[áâàäåÅ]/a/g' \
             -e 's/[ç]/c/g' \
             -e 's/[éêèë]/e/g' \
             -e 's/[íîï]/i/g' \
             -e 's/[ñ]/n/g' \
-            -e 's/[óôöø]/o/g' \
-            -e 's/[úûüù]/u/g' | \
+            -e 's/[óôöøiÖ]/o/g' \
+            -e 's/[úûüùiÜ]/u/g' | \
         tr '[a-z]' '[A-Z]' | \
         sort | \
         uniq | \
@@ -98,6 +102,7 @@ $(ROOT_DICT)/en.json: /usr/share/dict/british-english-huge Makefile
             END { print "    \"\"\n]" }' > $@
 
 $(ROOT_DICT)/eo.json: /usr/share/dict/esperanto Makefile
+       echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
         grep -v "[.'-]" | \
@@ -109,10 +114,10 @@ $(ROOT_DICT)/eo.json: /usr/share/dict/esperanto Makefile
             END { print "    \"\"\n]" }' > $@
 
 $(ROOT_DICT)/fr.json: /usr/share/dict/french Makefile
+       echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
         grep -v "[.'ú-]" | \
-        grep -v "^[A-Z]" | \
         sed -e 's/[àâ]/a/g' \
             -e 's/[ç]/c/g' \
             -e 's/[éèêë]/e/g' \
@@ -127,10 +132,10 @@ $(ROOT_DICT)/fr.json: /usr/share/dict/french Makefile
             END { print "    \"\"\n]" }' > $@
 
 $(ROOT_DICT)/it.json: /usr/share/dict/italian Makefile
+       echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
         grep -v "[.'-]" | \
-        grep -v "^[A-Z]" | \
         sed -e 's/[à]/a/g' \
             -e 's/[éè]/e/g' \
             -e 's/[ì]/i/g' \
@@ -144,10 +149,11 @@ $(ROOT_DICT)/it.json: /usr/share/dict/italian Makefile
             END { print "    \"\"\n]" }' > $@
 
 $(ROOT_DICT)/nl.json: /usr/share/dict/dutch Makefile
+       echo Building $@ from $<
        cat $< | \
         grep -v '^.$$' | \
         grep -v "[1-9+/' .-]" | \
-        sed -e 's/[áäâà]/a/g' \
+        sed -e 's/[áäâàÅ]/a/g' \
             -e 's/[ç]/c/g' \
             -e 's/[éëèê]/e/g' \
             -e 's/[íïî]/i/g' \
@@ -162,15 +168,16 @@ $(ROOT_DICT)/nl.json: /usr/share/dict/dutch Makefile
             END { print "    \"\"\n]" }' > $@
 
 $(ROOT_DICT)/no.json: /usr/share/dict/bokmaal Makefile
-       cat $< | \
+       echo Building $@ from $<
+       iconv -f iso-8859-1 -t utf-8 $< | \
         grep -v '^.$$' | \
         grep -v "['-]" | \
-        grep -v "^[A-Z]" | \
-        sed -e 's/[áä]/a/g' \
+        sed -e 's/[áäÄ]/a/g' \
+            -e 's/[ç]/c/g' \
             -e 's/[ð]/d/g' \
-            -e 's/[éëè]/e/g' \
+            -e 's/[éëèê]/e/g' \
             -e 's/[íì]/i/g' \
-            -e 's/[óöô]/o/g' \
+            -e 's/[óöôò]/o/g' \
             -e 's/[ü]/u/g' | \
         tr '[a-zæøå]' '[A-ZÆØÅ]' | \
         sort | \
@@ -179,13 +186,124 @@ $(ROOT_DICT)/no.json: /usr/share/dict/bokmaal Makefile
             { print "    \"" $$0 "\"," } \
             END { print "    \"\"\n]" }' > $@
 
-CHECKER = grep -v '^    "[A-Z]*",$$' | grep -v '\[' | grep -v '\]' | grep -v '""'
+$(ROOT_DICT)/pl.json: /usr/share/dict/polish Makefile
+       echo Building $@ from $<
+       cat $< | \
+        grep -v '^.$$' | \
+        grep -v "['-]" | \
+        sed -e 's/[ä]/a/g' \
+            -e 's/[ç]/c/g' \
+            -e 's/[é]/e/g' \
+            -e 's/[i]/i/g' \
+            -e 's/[ö]/o/g' \
+            -e 's/[šŠ]/s/g' \
+            -e 's/[û]/u/g' | \
+        tr '[a-złąęóśżćńź]' '[A-ZŁĄĘÓŚŻĆŃŹ]' | \
+        sort | \
+        uniq | \
+        awk 'BEGIN { print "[" } \
+            { print "    \"" $$0 "\"," } \
+            END { print "    \"\"\n]" }' > $@
+
+$(ROOT_DICT)/po.json: /usr/share/dict/portuguese Makefile
+       echo Building $@ from $<
+       iconv -f iso-8859-15 -t utf-8 $< | \
+        grep -v '^.$$' | \
+        grep -v "['-]" | \
+        sed -e 's/[áãâàÁÂ]/a/g' \
+            -e 's/[éêèÉ]/e/g' \
+            -e 's/[íîÍ]/i/g' \
+            -e 's/[óôõÓ]/o/g' \
+            -e 's/[úüÚ]/u/g' | \
+        tr '[a-zç]' '[A-ZÇ]' | \
+        sort | \
+        uniq | \
+        awk 'BEGIN { print "[" } \
+            { print "    \"" $$0 "\"," } \
+            END { print "    \"\"\n]" }' > $@
+
+$(ROOT_DICT)/sp.json: /usr/share/dict/spanish Makefile
+       echo Building $@ from $<
+       cat $< | \
+        grep -v '^.$$' | \
+        grep -v "['-]" | \
+        sed -e 's/[á]/a/g' \
+            -e 's/[é]/e/g' \
+                       -e 's/[í]/i/g' \
+            -e 's/[ó]/o/g' \
+            -e 's/[úü]/u/g' | \
+        tr '[a-zñ]' '[A-ZÑ]' | \
+        sort | \
+        uniq | \
+        awk 'BEGIN { print "[" } \
+            { print "    \"" $$0 "\"," } \
+            END { print "    \"\"\n]" }' > $@
+
+$(ROOT_DICT)/sv.json: /usr/share/dict/swedish Makefile
+       echo Building $@ from $<
+       iconv -f iso-8859-15 -t utf-8 $< | \
+        grep -v '^.$$' | \
+        grep -v "['-]" | \
+        sed -e 's/[æ]/ae/g' \
+            -e 's/[ç]/c/g' \
+            -e 's/[é]/e/g' \
+            -e 's/[ü]/u/g' | \
+        tr '[a-zåäö]' '[A-ZÅÄÖ]' | \
+        sort | \
+        uniq | \
+        awk 'BEGIN { print "[" } \
+            { print "    \"" $$0 "\"," } \
+            END { print "    \"\"\n]" }' > $@
+
+$(ROOT_DICT)/uk.json: /usr/share/dict/ukrainian Makefile
+       echo Building $@ from $<
+       cat $< | \
+        grep -v '^.$$' | \
+        grep -v "[-]" | \
+        tr '[абвгґдеєжзиіїйклмнопрстуфхцчшщьюя]' '[АБВГҐДЕЄЖЗИІЇЙКЛМНОПРСТУФХЦЧШЩЬЮЯ]' | \
+        sort | \
+        uniq | \
+        awk 'BEGIN { print "[" } \
+            { print "    \"" $$0 "\"," } \
+            END { print "    \"\"\n]" }' > $@
+
+CHECKER = cat $(1) | \
+    grep -v "\"[$(2)]*\"" | grep -v "\"$3\"" | \
+       grep -v '\[' | grep -v '\]' | grep -v '""' \
+       && exit 1 || echo "$< is correct"
 
 check-bg:$(ROOT_DICT)/bg.json
-        cat $< | grep -v '"[АОЕИТНПРСВМБДКЛГЪЖЗУЧЯЙХЦШЮФЩЬ]*"' | grep -v '\[' | grep -v '\]' | grep -v '""' || echo "$< is correct"
+       $(call CHECKER,$<,АОЕИТНПРСВМБДКЛГЪЖЗУЧЯЙХЦШЮФЩЬ,)
 
 check-ca:$(ROOT_DICT)/ca.json
-       cat $< | $(CHECKER) | grep -v '"[A-Z]*L·L[A-Z]*"' || echo "$< is correct"
+       $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZ,[A-Z]*L·L[A-Z]*)
+
+check-da:$(ROOT_DICT)/da.json
+       $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÆØÅ,)
+
+check-de:$(ROOT_DICT)/de.json
+       $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÄÖÜ,)
+
+check-eo:$(ROOT_DICT)/eo.json
+       $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZĈĜĤĴŜŬ,)
+
+check-no:$(ROOT_DICT)/no.json
+       $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÆØÅ,)
+
+check-pl:$(ROOT_DICT)/pl.json
+       $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZŁĄĘÓŚŻĆŃŹ,)
+
+check-po:$(ROOT_DICT)/po.json
+       $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÇ,)
+
+check-sp:$(ROOT_DICT)/sp.json
+       $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÑ,)
+
+check-sv:$(ROOT_DICT)/sv.json
+       $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÅÄÖ,)
+
+check-uk:$(ROOT_DICT)/uk.json
+       $(call CHECKER,$<,ОАИНВЕІТКРСДЛМУПЗЯЬБГЧХЄЇЙЖЦШЮҐФЩ',)
 
 check-%:$(ROOT_DICT)/%.json
-       cat $< | $(CHECKER) || echo "$< is correct"
+       $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZ,)