From: Laurent Mazet Date: Sat, 30 May 2020 22:34:00 +0000 (+0200) Subject: update dictionnary building rules X-Git-Url: https://secure.softndesign.org/git/?a=commitdiff_plain;h=064b82478d5b60ca5e388ab05757fd9167e96788;p=trivabble.git update dictionnary building rules --- diff --git a/dict-dists-extractor/Makefile b/dict-dists-extractor/Makefile index 6f2bb58..3abbaff 100644 --- a/dict-dists-extractor/Makefile +++ b/dict-dists-extractor/Makefile @@ -2,7 +2,10 @@ ROOT_DICT = ../public/dict -LANGS = bg ca da de en eo fr it nl no +LANGS = bg ca da de en eo fr it nl no po sp sv uk +# pl not working + +MAKEFLAGS = -s all: make $(addprefix check-,$(LANGS)) @@ -13,8 +16,8 @@ $(ROOT_DICT): count: wc -l $(ROOT_DICT)/* -#sed -e 's/Ѝ/И/g' $(ROOT_DICT)/bg.json: /usr/share/dict/bulgarian Makefile + echo Building $@ from $< iconv -f Windows-1251 -t utf-8 $< | \ grep -v '^.$$' | \ grep -v "['-]" | \ @@ -26,17 +29,17 @@ $(ROOT_DICT)/bg.json: /usr/share/dict/bulgarian Makefile END { print " \"\"\n]" }' > $@ $(ROOT_DICT)/ca.json: /usr/share/dict/catalan Makefile + echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ grep -v "['-]" | \ - grep -v "^[A-Z]" | \ - sed -e 's/[áàã]/a/g' \ - -e 's/[éè]/e/g' \ + sed -e 's/[áàãÁÀ]/a/g' \ + -e 's/[éèÉÈ]/e/g' \ -e 's/[ç]/c/g' \ - -e 's/[íï]/i/g' \ + -e 's/[íïÍ]/i/g' \ -e 's/[ñ]/n/g' \ - -e 's/[óöò]/o/g' \ - -e 's/[úüù]/u/g' | \ + -e 's/[óöòÓÒ]/o/g' \ + -e 's/[úüùÚ]/u/g' | \ tr '[a-z]' '[A-Z]' | \ sort | \ uniq | \ @@ -45,10 +48,10 @@ $(ROOT_DICT)/ca.json: /usr/share/dict/catalan Makefile END { print " \"\"\n]" }' > $@ $(ROOT_DICT)/da.json: /usr/share/dict/danish Makefile + echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ grep -v "['-]" | \ - grep -v "^[A-Z]" | \ sed -e 's/[áä]/a/g' \ -e 's/[ð]/d/g' \ -e 's/[éëè]/e/g' \ @@ -63,6 +66,7 @@ $(ROOT_DICT)/da.json: /usr/share/dict/danish Makefile END { print " \"\"\n]" }' > $@ $(ROOT_DICT)/de.json: /usr/share/dict/ngerman Makefile + echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ grep -v "[.'-]" | \ @@ -79,17 +83,17 @@ $(ROOT_DICT)/de.json: /usr/share/dict/ngerman Makefile END { print " \"\"\n]" }' > $@ $(ROOT_DICT)/en.json: /usr/share/dict/british-english-huge Makefile + echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ grep -v "[.'-]" | \ - grep -v "^[A-Z]" | \ - sed -e 's/[áâàäå]/a/g' \ + sed -e 's/[áâàäåÅ]/a/g' \ -e 's/[ç]/c/g' \ -e 's/[éêèë]/e/g' \ -e 's/[íîï]/i/g' \ -e 's/[ñ]/n/g' \ - -e 's/[óôöø]/o/g' \ - -e 's/[úûüù]/u/g' | \ + -e 's/[óôöøiÖ]/o/g' \ + -e 's/[úûüùiÜ]/u/g' | \ tr '[a-z]' '[A-Z]' | \ sort | \ uniq | \ @@ -98,6 +102,7 @@ $(ROOT_DICT)/en.json: /usr/share/dict/british-english-huge Makefile END { print " \"\"\n]" }' > $@ $(ROOT_DICT)/eo.json: /usr/share/dict/esperanto Makefile + echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ grep -v "[.'-]" | \ @@ -109,10 +114,10 @@ $(ROOT_DICT)/eo.json: /usr/share/dict/esperanto Makefile END { print " \"\"\n]" }' > $@ $(ROOT_DICT)/fr.json: /usr/share/dict/french Makefile + echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ grep -v "[.'ú-]" | \ - grep -v "^[A-Z]" | \ sed -e 's/[àâ]/a/g' \ -e 's/[ç]/c/g' \ -e 's/[éèêë]/e/g' \ @@ -127,10 +132,10 @@ $(ROOT_DICT)/fr.json: /usr/share/dict/french Makefile END { print " \"\"\n]" }' > $@ $(ROOT_DICT)/it.json: /usr/share/dict/italian Makefile + echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ grep -v "[.'-]" | \ - grep -v "^[A-Z]" | \ sed -e 's/[à]/a/g' \ -e 's/[éè]/e/g' \ -e 's/[ì]/i/g' \ @@ -144,10 +149,11 @@ $(ROOT_DICT)/it.json: /usr/share/dict/italian Makefile END { print " \"\"\n]" }' > $@ $(ROOT_DICT)/nl.json: /usr/share/dict/dutch Makefile + echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ grep -v "[1-9+/' .-]" | \ - sed -e 's/[áäâà]/a/g' \ + sed -e 's/[áäâàÅ]/a/g' \ -e 's/[ç]/c/g' \ -e 's/[éëèê]/e/g' \ -e 's/[íïî]/i/g' \ @@ -162,15 +168,16 @@ $(ROOT_DICT)/nl.json: /usr/share/dict/dutch Makefile END { print " \"\"\n]" }' > $@ $(ROOT_DICT)/no.json: /usr/share/dict/bokmaal Makefile - cat $< | \ + echo Building $@ from $< + iconv -f iso-8859-1 -t utf-8 $< | \ grep -v '^.$$' | \ grep -v "['-]" | \ - grep -v "^[A-Z]" | \ - sed -e 's/[áä]/a/g' \ + sed -e 's/[áäÄ]/a/g' \ + -e 's/[ç]/c/g' \ -e 's/[ð]/d/g' \ - -e 's/[éëè]/e/g' \ + -e 's/[éëèê]/e/g' \ -e 's/[íì]/i/g' \ - -e 's/[óöô]/o/g' \ + -e 's/[óöôò]/o/g' \ -e 's/[ü]/u/g' | \ tr '[a-zæøå]' '[A-ZÆØÅ]' | \ sort | \ @@ -179,13 +186,124 @@ $(ROOT_DICT)/no.json: /usr/share/dict/bokmaal Makefile { print " \"" $$0 "\"," } \ END { print " \"\"\n]" }' > $@ -CHECKER = grep -v '^ "[A-Z]*",$$' | grep -v '\[' | grep -v '\]' | grep -v '""' +$(ROOT_DICT)/pl.json: /usr/share/dict/polish Makefile + echo Building $@ from $< + cat $< | \ + grep -v '^.$$' | \ + grep -v "['-]" | \ + sed -e 's/[ä]/a/g' \ + -e 's/[ç]/c/g' \ + -e 's/[é]/e/g' \ + -e 's/[i]/i/g' \ + -e 's/[ö]/o/g' \ + -e 's/[šŠ]/s/g' \ + -e 's/[û]/u/g' | \ + tr '[a-złąęóśżćńź]' '[A-ZŁĄĘÓŚŻĆŃŹ]' | \ + sort | \ + uniq | \ + awk 'BEGIN { print "[" } \ + { print " \"" $$0 "\"," } \ + END { print " \"\"\n]" }' > $@ + +$(ROOT_DICT)/po.json: /usr/share/dict/portuguese Makefile + echo Building $@ from $< + iconv -f iso-8859-15 -t utf-8 $< | \ + grep -v '^.$$' | \ + grep -v "['-]" | \ + sed -e 's/[áãâàÁÂ]/a/g' \ + -e 's/[éêèÉ]/e/g' \ + -e 's/[íîÍ]/i/g' \ + -e 's/[óôõÓ]/o/g' \ + -e 's/[úüÚ]/u/g' | \ + tr '[a-zç]' '[A-ZÇ]' | \ + sort | \ + uniq | \ + awk 'BEGIN { print "[" } \ + { print " \"" $$0 "\"," } \ + END { print " \"\"\n]" }' > $@ + +$(ROOT_DICT)/sp.json: /usr/share/dict/spanish Makefile + echo Building $@ from $< + cat $< | \ + grep -v '^.$$' | \ + grep -v "['-]" | \ + sed -e 's/[á]/a/g' \ + -e 's/[é]/e/g' \ + -e 's/[í]/i/g' \ + -e 's/[ó]/o/g' \ + -e 's/[úü]/u/g' | \ + tr '[a-zñ]' '[A-ZÑ]' | \ + sort | \ + uniq | \ + awk 'BEGIN { print "[" } \ + { print " \"" $$0 "\"," } \ + END { print " \"\"\n]" }' > $@ + +$(ROOT_DICT)/sv.json: /usr/share/dict/swedish Makefile + echo Building $@ from $< + iconv -f iso-8859-15 -t utf-8 $< | \ + grep -v '^.$$' | \ + grep -v "['-]" | \ + sed -e 's/[æ]/ae/g' \ + -e 's/[ç]/c/g' \ + -e 's/[é]/e/g' \ + -e 's/[ü]/u/g' | \ + tr '[a-zåäö]' '[A-ZÅÄÖ]' | \ + sort | \ + uniq | \ + awk 'BEGIN { print "[" } \ + { print " \"" $$0 "\"," } \ + END { print " \"\"\n]" }' > $@ + +$(ROOT_DICT)/uk.json: /usr/share/dict/ukrainian Makefile + echo Building $@ from $< + cat $< | \ + grep -v '^.$$' | \ + grep -v "[-]" | \ + tr '[абвгґдеєжзиіїйклмнопрстуфхцчшщьюя]' '[АБВГҐДЕЄЖЗИІЇЙКЛМНОПРСТУФХЦЧШЩЬЮЯ]' | \ + sort | \ + uniq | \ + awk 'BEGIN { print "[" } \ + { print " \"" $$0 "\"," } \ + END { print " \"\"\n]" }' > $@ + +CHECKER = cat $(1) | \ + grep -v "\"[$(2)]*\"" | grep -v "\"$3\"" | \ + grep -v '\[' | grep -v '\]' | grep -v '""' \ + && exit 1 || echo "$< is correct" check-bg:$(ROOT_DICT)/bg.json - cat $< | grep -v '"[АОЕИТНПРСВМБДКЛГЪЖЗУЧЯЙХЦШЮФЩЬ]*"' | grep -v '\[' | grep -v '\]' | grep -v '""' || echo "$< is correct" + $(call CHECKER,$<,АОЕИТНПРСВМБДКЛГЪЖЗУЧЯЙХЦШЮФЩЬ,) check-ca:$(ROOT_DICT)/ca.json - cat $< | $(CHECKER) | grep -v '"[A-Z]*L·L[A-Z]*"' || echo "$< is correct" + $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZ,[A-Z]*L·L[A-Z]*) + +check-da:$(ROOT_DICT)/da.json + $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÆØÅ,) + +check-de:$(ROOT_DICT)/de.json + $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÄÖÜ,) + +check-eo:$(ROOT_DICT)/eo.json + $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZĈĜĤĴŜŬ,) + +check-no:$(ROOT_DICT)/no.json + $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÆØÅ,) + +check-pl:$(ROOT_DICT)/pl.json + $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZŁĄĘÓŚŻĆŃŹ,) + +check-po:$(ROOT_DICT)/po.json + $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÇ,) + +check-sp:$(ROOT_DICT)/sp.json + $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÑ,) + +check-sv:$(ROOT_DICT)/sv.json + $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÅÄÖ,) + +check-uk:$(ROOT_DICT)/uk.json + $(call CHECKER,$<,ОАИНВЕІТКРСДЛМУПЗЯЬБГЧХЄЇЙЖЦШЮҐФЩ',) check-%:$(ROOT_DICT)/%.json - cat $< | $(CHECKER) || echo "$< is correct" + $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZ,)