Update dictionary format.

Julien LepillerSun Jul 18 18:49:34+0200 2021

cd080be

Update dictionary format.

app/src/main/java/eu/lepiller/nani/DictionaryDownloadActivity.java

6464 
        download_bar = findViewById(R.id.download_progress);
6565 
        download_button = findViewById(R.id.download_button);
6666 
67+
        Log.d("DOWNLOAD_ACTIVITY", "dictionary " + name + " is " + (d.isDownloaded()? "downloaded": "not downloaded"));
68+
6769 
        setResult(DictionaryActivity.DICO_REQUEST);
6870 
6971 
        updateLayout(false);
app/src/main/java/eu/lepiller/nani/dictionary/DictionaryFactory.java
9393 
                                    chooseLanguage(synopsis),
9494 
                                    chooseLanguage(description),
9595 
                                    cacheDir, url, size, entries, sha256);
96+
                            Log.d("FACTORY", "radk: " + (d.isDownloaded()? "downloaded": "not downloaded"));
9697 
                        } else if (type.compareTo("jmdict") == 0) {
9798 
                            d = new JMDict(name,
9899 
                                    chooseLanguage(synopsis),
app/src/main/java/eu/lepiller/nani/dictionary/FileDictionary.java
4747 
4848 
    @Override
4949 
    public boolean isDownloaded() {
50+
        Log.d("FILEDICT", getFile().getAbsolutePath());
5051 
        return getFile().exists();
5152 
    }
5253 
…
205206 
        while(c == null || !c.isEmpty()) {
206207 
            if(h instanceof ResultDictionary.HuffmanValue) {
207208 
                c = ((ResultDictionary.HuffmanValue) h).character;
208-
                Log.v(TAG, "Huffman read: " + c);
209+
                //Log.v(TAG, "Huffman read: " + c);
209210 
                b.append(c);
210211 
                h = huffman;
211212 
            } else if(h instanceof ResultDictionary.HuffmanTree) {
212213 
                if(bits.isEmpty()) {
213214 
                    byte by = file.readByte();
214-
                    Log.v(TAG, "Read byte for huffman: " + by);
215+
                    //Log.v(TAG, "Read byte for huffman: " + by);
215216 
                    for(int i = 7; i>-1; i--) {
216217 
                        bits.add((by&(1<<i))!=0);
217218 
                    }
218-
                    Log.v(TAG, "Read byte for huffman: " + bits);
219+
                    //Log.v(TAG, "Read byte for huffman: " + bits);
219220 
                }
220221 
221222 
                Boolean bo = bits.get(0);
app/src/main/java/eu/lepiller/nani/dictionary/Jibiki.java
44 
55 
import eu.lepiller.nani.R;
66 
7-
public class    Jibiki extends ResultDictionary {
7+
public class Jibiki extends ResultDictionary {
88 
    Jibiki(String name, String description, String fullDescription, File cacheDir, String url, int fileSize, int entries, String hash) {
99 
        super(name, description, fullDescription, cacheDir, url, fileSize, entries, hash);
1010 
    }
app/src/main/java/eu/lepiller/nani/dictionary/ResultDictionary.java
5353 
        for(int i=0; i<meaning_number; i++) {
5454 
            ArrayList<String> sense_references = getStringList(file);
5555 
            ArrayList<String> sense_limits = getStringList(file);
56-
            ArrayList<String> sense_infos = getStringList(file);
5756 
            ArrayList<Result.Source> sense_sources = new ArrayList<>();
5857 
            int source_number = file.readShort();
5958 
            for(int j=0; j<source_number; j++) {
6059 
                ArrayList<String> source_content = getStringList(file);
6160 
                boolean source_wasei = file.read() != 0;
62-
                String source_type = getString(file);
6361 
                String source_language = getString(file);
64-
                sense_sources.add(new Result.Source(source_content, source_wasei, source_type, source_language));
62+
                sense_sources.add(new Result.Source(source_content, source_wasei, source_language));
6563 
            }
66-
            ArrayList<Integer> sense_tags = getIntList(file);
64+
            ArrayList<String> sense_infos = getStringList(file);
6765 
            ArrayList<String> sense_glosses = getHuffmanStringList(file, meaningHuffman);
6866 
            String sense_language = getString(file);
6967 
            senses.add(new Result.Sense(sense_references, sense_limits, sense_infos, sense_sources,
70-
                    sense_tags, sense_glosses, sense_language));
68+
                    sense_glosses, sense_language));
7169 
        }
72-
        return new Result(kanjis, readings, senses);
70+
71+
        int score = file.readChar();
72+
        return new Result(kanjis, readings, senses, score);
7373 
    }
7474 
7575 
    private ArrayList<Integer> getValues(RandomAccessFile file, long triePos) throws IOException {
…
126126 
                    return null;
127127 
128128 
                // Check file format version
129-
                if(!Arrays.equals(header, "NANI_JMDICT002".getBytes())) {
130-
                    Log.d(TAG, "search: incompatible header: " + header);
129+
                if(!Arrays.equals(header, "NANI_JMDICT003".getBytes())) {
130+
                    StringBuilder error = new StringBuilder("search: incompatible header: [");
131+
                    boolean first = true;
132+
                    for(byte b: header) {
133+
                        if(first)
134+
                            first = false;
135+
                        else
136+
                            error.append(", ");
137+
                        error.append(b);
138+
                    }
139+
                    error.append("].");
140+
                    Log.d(TAG, error.toString());
131141 
                    throw new IncompatibleFormatException(getName());
132142 
                }
133143 
134144 
                byte[] search = text.toLowerCase().getBytes();
135145 
136-
                long tagsListPos = file.readInt();
146+
                file.skipBytes(4); // unused
137147 
                long kanjiTriePos = file.readInt();
138148 
                long readingTriePos = file.readInt();
139149 
                long meaningTriePos = file.readInt();
…
166176 
                if(results == null || results.isEmpty())
167177 
                    return null;
168178 
179+
                Log.d(TAG, results.size() + " result(s)");
180+
169181 
                ArrayList<Result> r = new ArrayList<>();
170182 
                ArrayList<Integer> uniqResults = new ArrayList<>();
171183 
                for(Integer i: results) {
app/src/main/java/eu/lepiller/nani/dictionary/WadokuPitchDictionary.java
2727 
        return R.drawable.wadoku;
2828 
    }
2929 
30-
    private String findPitch(String kanji, RandomAccessFile file) throws IOException {
31-
        return searchTrie(file, triePos, kanji.getBytes(), new TrieValsDecoder<String>() {
30+
    private String findPitch(String kanji, String reading, RandomAccessFile file) throws IOException {
31+
        String concat = kanji + reading;
32+
        return searchTrie(file, triePos, concat.getBytes(), new TrieValsDecoder<String>() {
3233 
            @Override
3334 
            public String decodeVals(RandomAccessFile file, long pos) throws IOException {
3435 
                file.seek(pos);
…
6869 
            }
6970 
7071 
            List<String> kanjis = r.getAlternatives();
72+
            List<Result.Reading> readings = r.getReadings();
7173 
7274 
            for(String k: kanjis) {
7375 
                Log.d(TAG, "Searching pitch for " + Arrays.toString(k.getBytes()));
74-
                String pitch = findPitch(k, file);
75-
                if(pitch != null) {
76-
                    Log.d(TAG, "Found " + pitch);
77-
                    r.addPitch(k, pitch);
76+
                for(Result.Reading read: readings) {
77+
                    List<String> readingKanjis = read.getKanjis();
78+
                    if(readingKanjis != null && !readingKanjis.contains(k)) {
79+
                        // this is not a reading for the current kanji, so continue to the next
80+
                        // reading
81+
                        continue;
82+
                    }
83+
                    List<String> innerReadings = read.getReadings();
84+
                    for(String reading: innerReadings) {
85+
                        String pitch = findPitch(k, reading, file);
86+
                        if(pitch != null) {
87+
                            Log.d(TAG, "Found " + pitch);
88+
                            read.addPitch(pitch);
89+
                        }
90+
                    }
7891 
                }
7992 
            }
8093 
        } catch (FileNotFoundException e) {
app/src/main/java/eu/lepiller/nani/result/Result.java
33 
import android.util.Log;
44 
55 
import java.util.ArrayList;
6+
import java.util.List;
67 
import java.util.regex.Matcher;
78 
import java.util.regex.Pattern;
89 
…
1213 
    public static class Source {
1314 
        private ArrayList<String> content;
1415 
        private boolean wasei;
15-
        private String type, language;
16+
        private String language;
1617 
17-
        public Source(ArrayList<String> content, boolean wasei, String type, String language) {
18+
        public Source(ArrayList<String> content, boolean wasei, String language) {
1819 
            this.content = content;
1920 
            this.wasei = wasei;
20-
            this.type = type;
2121 
            this.language = language;
2222 
        }
2323 
    }
2424 
2525 
    public static class Sense {
2626 
        private ArrayList<String> references, limits, infos, glosses;
27-
        private ArrayList<Integer> tags;
2827 
        private String language;
2928 
        private ArrayList<Source> sources;
3029 
3130 
        public Sense(ArrayList<String> references, ArrayList<String> limits, ArrayList<String> infos,
32-
                     ArrayList<Source> sources, ArrayList<Integer> tags, ArrayList<String> glosses,
31+
                     ArrayList<Source> sources, ArrayList<String> glosses,
3332 
                     String language) {
3433 
            this.references = references;
3534 
            this.limits = limits;
3635 
            this.infos = infos;
3736 
            this.sources = sources;
38-
            this.tags = tags;
3937 
            this.glosses = glosses;
4038 
            this.language = language;
4139 
        }
…
5553 
            this.pitches = new ArrayList<>();
5654 
        }
5755 
58-
        void addPitch(String pitch) {
56+
        public List<String> getKanjis() {
57+
            return kanjis;
58+
        }
59+
60+
        public List<String> getReadings() {
61+
            return readings;
62+
        }
63+
64+
        public void addPitch(String pitch) {
5965 
            pitches.add(pitch);
6066 
        }
6167 
    }
…
6369 
    private ArrayList<String> kanjis;
6470 
    private ArrayList<Reading> readings;
6571 
    private ArrayList<Sense> senses;
72+
    private int score;
6673 
67-
    public Result(ArrayList<String> kanjis, ArrayList<Reading> readings, ArrayList<Sense> senses) {
74+
    public Result(ArrayList<String> kanjis, ArrayList<Reading> readings, ArrayList<Sense> senses, int score) {
6875 
        this.kanjis = kanjis;
6976 
        this.readings = readings;
7077 
        this.senses = senses;
78+
        this.score = score;
7179 
    }
7280 
7381 
    public String getKanji() {