andrzej
/
micro365
1
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

definitions now concatenated, ordered correctly,

This commit is contained in:
Andrzej Stepien 2023-07-19 15:10:39 +02:00
parent d16ed6cb9a
commit 7281e50aad
1 changed files with 10 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

21
src/data/scripts/pipeline.sh
View File

@ -1,20 +1,19 @@
#!/bin/bash #!/bin/bash
cd ../processing cd ../processing
jq -r '. | select((.pos=="noun") or (.pos=="verb") or (.pos=="adj") or (.pos=="adv")) | select((.word | test("[^a-z]"))|not) | {word: .word, type: .pos, pronunciation: .sounds[0].ipa, definitions: .senses|map(.glosses)}' \ #jq -r '. | select((.pos=="noun") or (.pos=="verb") or (.pos=="adj") or (.pos=="adv")) | select((.word | test("[^a-z]"))|not) | {word: .word, type: .pos, pronunciation: .sounds[0].ipa, definitions: (try .senses|map(.glosses|join(" ")))}' \
wiktionary.json > wiktionary-processed.json #wiktionary.json > wiktionary-p1.json
jq --slurp '.' wiktionary-processed.json > wiktionary-processed-array.json #jq --slurp '. | group_by(.word)[] | {word:.[0].word, pronunciation:.[0].pronunciation, meanings:[.[]|{type:.type, definitions:[try .definitions[]]| select(.!=[]) |map({(.):1})|add|keys_unsorted}]}' \
#wiktionary-p1.json > wiktionary-p2.json
jq '. | group_by(.word)[] | {word:.[0].word, pronunciation:.[0].pronunciation, meanings:[.[]|{type:.type, definitions:[try .definitions[][]]| select(.!=[]) |map({(.):1})|add|keys_unsorted}]}' \ #jq --slurp '.' wiktionary-p2.json > wiktionary-p3.json
wiktionary-processed-array.json > wiktionary-grouped-objects.json
jq --slurp '.' wiktionary-grouped-objects.json > wiktionary-grouped-objects-array.json
#extract samples #extract samples
timestamp=$(date +%s) timestamp=$(date +%s)
jq '. | select(.word=="chocolate")' wiktionary-grouped-objects.json > ../samples/chocolate-$timestamp.json jsonl=wiktionary-p2.json
jq '. | select(.word=="write")' wiktionary-grouped-objects.json > ../samples/write-$timestamp.json jq '. | select(.word=="chocolate")' $jsonl > ../samples/chocolate-$timestamp.json
jq '. | select(.word=="terrible")' wiktionary-grouped-objects.json > ../samples/terrible-$timestamp.json jq '. | select(.word=="write")' $jsonl > ../samples/write-$timestamp.json
jq '. | select(.word=="look")' wiktionary-grouped-objects.json > ../samples/look-$timestamp.json jq '. | select(.word=="terrible")' $jsonl > ../samples/terrible-$timestamp.json
jq '. | select(.word=="look")' $jsonl > ../samples/look-$timestamp.json