From: Laurent Mazet Date: Sun, 7 Jun 2020 12:16:38 +0000 (+0200) Subject: change dictionary format X-Git-Url: https://secure.softndesign.org/git/?a=commitdiff_plain;h=bafd01546df8d93aa4cf8e35cbfe2dc0b56e2dfd;p=trivabble.git change dictionary format build dictionary from aspell debian package --- diff --git a/dict-dists-extractor/Makefile b/dict-dists-extractor/Makefile index 67f7385..f13a7ab 100644 --- a/dict-dists-extractor/Makefile +++ b/dict-dists-extractor/Makefile @@ -2,8 +2,11 @@ ROOT_DICT = ../public/dict -LANGS = bg ca da de en eo es fr it nl no po sv uk -# pl not working +LANGS = bg ca da de en eo es fr it nl no pl pt sv uk +# to add: ar br cs cy el et eu fi? ga he hr hu hy is lv ro ru sk sl +#LANGS = af ar bg br ca cs cy da de el en eo es et eu fi fr ga he hr hu hy is it la lv ms nl no pl pt ro ru sk sl sv tr uk xk + +LOWS = $(addsuffix .low,$(LANGS)) DICTS = $(addsuffix .dict,$(LANGS)) @@ -18,53 +21,64 @@ $(ROOT_DICT): count: wc -l $(ROOT_DICT)/* -dicts: $(DICTS) +low: $(LOWS) + +INSTALL_ASPELL_DICT = apt search aspell-$(1) && \ + ( dpkg -l aspell-$(1) | grep -q ^ii || sudo apt install aspell-$(1) ) + +ASPELL_DUMP_MASTER = aspell -d $(1) dump master $(1) | \ + aspell -l $(1) expand | \ + tr -s '[:space:]' '\n' | \ + LC_ALL=C sort -S28G -u > $(1).low + +%.low: + echo Building list of word $@ + $(call INSTALL_ASPELL_DICT,$(@:.low=)) + $(call ASPELL_DUMP_MASTER,$(@:.low=)) + +no.low: + echo Building list of word $@ + $(call INSTALL_ASPELL_DICT,no) + $(call ASPELL_DUMP_MASTER,nb) + mv nb.low $@ -$(DICTS):%: - lang=$(@:.dict=); \ - apt search aspell-$$lang && \ - ( dpkg -l aspell-$$lang | grep ^ii || \ - sudo apt install aspell-$$lang ) && \ - aspell -d $$lang dump master | aspell -l $$lang expand | \ - tr -s '[:space:]' '\n' | \ - LC_ALL=C sort -S28G -u > $$lang.dict +pt.low: + echo Building list of word $@ + $(call INSTALL_ASPELL_DICT,pt-pt) + $(call ASPELL_DUMP_MASTER,pt_PT) + mv pt_PT.low $@ -$(ROOT_DICT)/bg.json: /usr/share/dict/bulgarian Makefile +$(ROOT_DICT)/bg.dict: bg.low Makefile echo Building $@ from $< - iconv -f Windows-1251 -t utf-8 $< | \ + cat $< | \ grep -v '^.$$' | \ grep -v "['-]" | \ + grep -v "^[a-zA-Z]" | \ tr '[абвгдежзийклмнопрстуфхцчшщъьюя]' '[АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЬЮЯ]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ -$(ROOT_DICT)/ca.json: /usr/share/dict/catalan Makefile +$(ROOT_DICT)/ca.dict: ca.low Makefile echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ - grep -v "['-]" | \ + grep -v "[.'-]" | \ sed -e 's/[áàãÁÀ]/a/g' \ -e 's/[éèÉÈ]/e/g' \ - -e 's/[ç]/c/g' \ + -e 's/[çÇ]/c/g' \ -e 's/[íïÍ]/i/g' \ -e 's/[ñ]/n/g' \ -e 's/[óöòÓÒ]/o/g' \ -e 's/[úüùÚ]/u/g' | \ tr '[a-z]' '[A-Z]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ -$(ROOT_DICT)/da.json: /usr/share/dict/danish Makefile +$(ROOT_DICT)/da.dict: da.low Makefile echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ - grep -v "['-]" | \ + grep -v "[.'-]" | \ sed -e 's/[áä]/a/g' \ -e 's/[ð]/d/g' \ -e 's/[éëè]/e/g' \ @@ -73,12 +87,9 @@ $(ROOT_DICT)/da.json: /usr/share/dict/danish Makefile -e 's/[ü]/u/g' | \ tr '[a-zæøå]' '[A-ZÆØÅ]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ -$(ROOT_DICT)/de.json: /usr/share/dict/ngerman Makefile +$(ROOT_DICT)/de.dict: de.low Makefile echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ @@ -90,12 +101,9 @@ $(ROOT_DICT)/de.json: /usr/share/dict/ngerman Makefile -e 's/[ß]/ss/g' | \ tr '[a-zäöü]' '[A-ZÄÖÜ]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ -$(ROOT_DICT)/en.json: /usr/share/dict/british-english-huge Makefile +$(ROOT_DICT)/en.dict: en.low Makefile echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ @@ -109,76 +117,62 @@ $(ROOT_DICT)/en.json: /usr/share/dict/british-english-huge Makefile -e 's/[úûüùiÜ]/u/g' | \ tr '[a-z]' '[A-Z]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ -$(ROOT_DICT)/eo.json: /usr/share/dict/esperanto Makefile +$(ROOT_DICT)/eo.dict: eo.low Makefile echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ grep -v "[.'-]" | \ tr '[a-zĉĝĥĵŝŭ]' '[A-ZĈĜĤĴŜŬ]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ -$(ROOT_DICT)/es.json: /usr/share/dict/spanish Makefile +$(ROOT_DICT)/es.dict: es.low Makefile echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ - grep -v "['-]" | \ - sed -e 's/[á]/a/g' \ + grep -v "[.'-]" | \ + sed -e 's/[áÁ]/a/g' \ -e 's/[é]/e/g' \ -e 's/[í]/i/g' \ -e 's/[ó]/o/g' \ -e 's/[úü]/u/g' | \ tr '[a-zñ]' '[A-ZÑ]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ -$(ROOT_DICT)/fr.json: /usr/share/dict/french Makefile +$(ROOT_DICT)/fr.dict: fr.low Makefile echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ - grep -v "[.'ú-]" | \ - sed -e 's/[àâ]/a/g' \ + grep -v "[.'-]" | \ + sed -e 's/[àâä]/a/g' \ -e 's/[ç]/c/g' \ - -e 's/[éèêë]/e/g' \ + -e 's/[Ééèêë]/e/g' \ -e 's/[ïî]/i/g' \ -e 's/[ôö]/o/g' \ - -e 's/[ùûü]/u/g' | \ + -e 's/[úùûü]/u/g' | \ tr '[a-z]' '[A-Z]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ -$(ROOT_DICT)/it.json: /usr/share/dict/italian Makefile +$(ROOT_DICT)/it.dict: it.low Makefile echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ grep -v "[.'-]" | \ - sed -e 's/[à]/a/g' \ + sed -e 's/[àâ]/a/g' \ + -e 's/[ç]/c/g' \ -e 's/[éè]/e/g' \ -e 's/[ì]/i/g' \ - -e 's/[ò]/o/g' \ + -e 's/[òiô]/o/g' \ -e 's/[ù]/u/g' | \ tr '[a-z]' '[A-Z]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ -$(ROOT_DICT)/nl.json: /usr/share/dict/dutch Makefile +$(ROOT_DICT)/nl.dict: nl.low Makefile echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ @@ -192,14 +186,11 @@ $(ROOT_DICT)/nl.json: /usr/share/dict/dutch Makefile -e 's/[úüûù]/u/g' | \ tr '[a-z]' '[A-Z]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ -$(ROOT_DICT)/no.json: /usr/share/dict/bokmaal Makefile +$(ROOT_DICT)/no.dict: no.low Makefile echo Building $@ from $< - iconv -f iso-8859-1 -t utf-8 $< | \ + cat $< | \ grep -v '^.$$' | \ grep -v "['-]" | \ sed -e 's/[áäÄ]/a/g' \ @@ -211,12 +202,9 @@ $(ROOT_DICT)/no.json: /usr/share/dict/bokmaal Makefile -e 's/[ü]/u/g' | \ tr '[a-zæøå]' '[A-ZÆØÅ]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ -$(ROOT_DICT)/pl.json: /usr/share/dict/polish Makefile +$(ROOT_DICT)/pl.dict: pl.low Makefile echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ @@ -230,14 +218,11 @@ $(ROOT_DICT)/pl.json: /usr/share/dict/polish Makefile -e 's/[û]/u/g' | \ tr '[a-złąęóśżćńź]' '[A-ZŁĄĘÓŚŻĆŃŹ]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ -$(ROOT_DICT)/po.json: /usr/share/dict/portuguese Makefile +$(ROOT_DICT)/pt.dict: pt.low Makefile echo Building $@ from $< - iconv -f iso-8859-15 -t utf-8 $< | \ + cat $< | \ grep -v '^.$$' | \ grep -v "['-]" | \ sed -e 's/[áãâàÁÂ]/a/g' \ @@ -247,14 +232,11 @@ $(ROOT_DICT)/po.json: /usr/share/dict/portuguese Makefile -e 's/[úüÚ]/u/g' | \ tr '[a-zç]' '[A-ZÇ]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ -$(ROOT_DICT)/sv.json: /usr/share/dict/swedish Makefile +$(ROOT_DICT)/sv.dict: sv.low Makefile echo Building $@ from $< - iconv -f iso-8859-15 -t utf-8 $< | \ + cat $< | \ grep -v '^.$$' | \ grep -v "['-]" | \ sed -e 's/[æ]/ae/g' \ @@ -263,60 +245,52 @@ $(ROOT_DICT)/sv.json: /usr/share/dict/swedish Makefile -e 's/[ü]/u/g' | \ tr '[a-zåäö]' '[A-ZÅÄÖ]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ -$(ROOT_DICT)/uk.json: /usr/share/dict/ukrainian Makefile +$(ROOT_DICT)/uk.dict: uk.low Makefile echo Building $@ from $< cat $< | \ grep -v '^.$$' | \ grep -v "[-]" | \ tr '[абвгґдеєжзиіїйклмнопрстуфхцчшщьюя]' '[АБВГҐДЕЄЖЗИІЇЙКЛМНОПРСТУФХЦЧШЩЬЮЯ]' | \ sort | \ - uniq | \ - awk 'BEGIN { print "[" } \ - { print " \"" $$0 "\"," } \ - END { print " \"\"\n]" }' > $@ + uniq > $@ CHECKER = cat $(1) | \ - grep -v "\"[$(2)]*\"" | grep -v "\"$3\"" | \ - grep -v '\[' | grep -v '\]' | grep -v '""' \ - && exit 1 || echo "$< is correct" + grep -v "^[$(2)]*$$" | egrep -v "$(3)" && exit 1 || echo "$< is correct" -check-bg:$(ROOT_DICT)/bg.json +check-bg:$(ROOT_DICT)/bg.dict $(call CHECKER,$<,АОЕИТНПРСВМБДКЛГЪЖЗУЧЯЙХЦШЮФЩЬ,) -check-ca:$(ROOT_DICT)/ca.json - $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZ,[A-Z]*L·L[A-Z]*) +check-ca:$(ROOT_DICT)/ca.dict + $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZ,L·L[A-Z]*(L·L)*) -check-da:$(ROOT_DICT)/da.json +check-da:$(ROOT_DICT)/da.dict $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÆØÅ,) -check-de:$(ROOT_DICT)/de.json +check-de:$(ROOT_DICT)/de.dict $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÄÖÜ,) -check-eo:$(ROOT_DICT)/eo.json +check-eo:$(ROOT_DICT)/eo.dict $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZĈĜĤĴŜŬ,) -check-es:$(ROOT_DICT)/es.json +check-es:$(ROOT_DICT)/es.dict $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÑ,) -check-no:$(ROOT_DICT)/no.json +check-no:$(ROOT_DICT)/no.dict $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÆØÅ,) -check-pl:$(ROOT_DICT)/pl.json +check-pl:$(ROOT_DICT)/pl.dict $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZŁĄĘÓŚŻĆŃŹ,) -check-po:$(ROOT_DICT)/po.json +check-po:$(ROOT_DICT)/po.dict $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÇ,) -check-sv:$(ROOT_DICT)/sv.json +check-sv:$(ROOT_DICT)/sv.dict $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZÅÄÖ,) -check-uk:$(ROOT_DICT)/uk.json +check-uk:$(ROOT_DICT)/uk.dict $(call CHECKER,$<,ОАИНВЕІТКРСДЛМУПЗЯЬБГЧХЄЇЙЖЦШЮҐФЩ',) -check-%:$(ROOT_DICT)/%.json +check-%:$(ROOT_DICT)/%.dict $(call CHECKER,$<,ABCDEFGHIJKLMNOPQRSTUVWXYZ,) diff --git a/public/trivabble.js b/public/trivabble.js index 43d5710..fdd2d19 100644 --- a/public/trivabble.js +++ b/public/trivabble.js @@ -159,12 +159,12 @@ downloadDictionary.code = code; const file = new XMLHttpRequest(); - file.overrideMimeType("application/json"); - file.open("GET", "dict/" + code + ".json", true); + file.overrideMimeType("application/dictionary"); + file.open("GET", "dict/" + code + ".dict", true); file.onreadystatechange = function () { if ((file.readyState === 4) && (file.status == "200")) { try { - dictionary = JSON.parse(file.responseText); + dictionary = file.responseText.split("\n"); document.getElementById("check-spelling").disabled = false; } catch (e) { console.error(e);