Add tatoeba support

Julien LepillerSun Jul 10 11:21:02+0200 2022

b2e44e9

Add tatoeba support

.gitignore

11
*.iml
22
.gradle
33
/local.properties
4-
/.idea/misc.xml
4+
.idea/misc.xml
55
/.idea/caches
66
/.idea/libraries
77
/.idea/modules.xml

.idea/misc.xml

1212
        <entry key="app/src/main/res/layout/content_main.xml" value="0.2078125" />
1313
        <entry key="app/src/main/res/layout/content_radicals.xml" value="0.1" />
1414
        <entry key="app/src/main/res/layout/fragment_results.xml" value="0.2078125" />
15+
        <entry key="app/src/main/res/layout/layout_example.xml" value="0.20677083333333332" />
1516
        <entry key="app/src/main/res/layout/layout_result.xml" value="0.2078125" />
17+
        <entry key="app/src/main/res/layout/layout_sense.xml" value="0.2078125" />
1618
        <entry key="app/src/main/res/xml/preferences.xml" value="0.2078125" />
1719
      </map>
1820
    </option>

README.md

3434
| [KanjiVG](https://kanjivg.tagaini.net/) | KanjiVG | CC-BY-SA 3.0 | Provides kanji stroke order and elements information |
3535
| [Wadoku](https://wadoku.de) | Wadoku | [Non-commercial license](https://www.wadoku.de/wiki/display/WAD/Wadoku.de-Daten+Lizenz) | Provides the main search function |
3636
| [Jibiki](https://jibiki.fr) | Jibiki | CC-0 | Provides the main search function |
37+
| [Tatoeba](https://tatoeba.org) | Tatoeba | CC-BY 2.0 FR | Provides sentence examples |
3738
3839
3940
Contributing

app/src/main/java/eu/lepiller/nani/MainActivity.java

3535
import eu.lepiller.nani.dictionary.IncompatibleFormatException;
3636
import eu.lepiller.nani.dictionary.NoDictionaryException;
3737
import eu.lepiller.nani.dictionary.NoResultDictionaryException;
38+
import eu.lepiller.nani.result.ExampleResult;
3839
import eu.lepiller.nani.result.KanjiResult;
3940
import eu.lepiller.nani.result.Result;
4041

9091
9192
        tabLayout.addTab(tabLayout.newTab().setText(R.string.tab_results));
9293
        tabLayout.addTab(tabLayout.newTab().setText(R.string.tab_kanji));
94+
        tabLayout.addTab(tabLayout.newTab().setText(R.string.tab_example));
9395
9496
        new TabLayoutMediator(tabLayout, viewPager2,
95-
                (tab, position) -> tab.setText(position == 0? R.string.tab_results: R.string.tab_kanji)
97+
                (tab, position) -> {
98+
                    switch(position) {
99+
                        case 0:
100+
                            tab.setText(R.string.tab_results);
101+
                            break;
102+
                        case 1:
103+
                            tab.setText(R.string.tab_kanji);
104+
                            break;
105+
                        case 2:
106+
                            tab.setText(R.string.tab_example);
107+
                            break;
108+
                    }
109+
                }
96110
        ).attach();
97111
98112
        try {

244258
                try {
245259
                    r = DictionaryFactory.searchKanji(c);
246260
                } catch(DictionaryException e) {
247-
                    return new SearchResult(e);
261+
                    break;
248262
                }
249263
                Log.d(TAG, "kanji " + c + ", result: " + r);
250264
                if(r != null)
251265
                    kanjiResults.add(r);
252266
            }
253267
254-
            return new SearchResult(searchResult, kanjiResults, text, converted);
268+
            List<ExampleResult> exampleResults = new ArrayList<>();
269+
            for(Result r: searchResult) {
270+
                String word = r.getKanji();
271+
272+
                List<ExampleResult> res;
273+
                try {
274+
                    res = DictionaryFactory.searchExamples(word);
275+
                } catch(DictionaryException e) {
276+
                    break;
277+
                }
278+
                exampleResults.addAll(res);
279+
            }
280+
281+
            return new SearchResult(searchResult, kanjiResults, exampleResults, text, converted);
255282
        }
256283
    }
257284

294321
295322
        List<Result> searchResult = r.getResults();
296323
        List<KanjiResult> kanjiResults = r.getKanjiResults();
297-
        Log.d(TAG, "results. Kanjis: " + r.getKanjiResults().size());
324+
        List<ExampleResult> exampleResults = r.getExampleResults();
298325
299326
        MojiDetector detector = new MojiDetector();
300327
        if(searchResult != null && searchResult.size()>0 && !r.isConverted() && detector.hasRomaji(r.getText())) {

316343
317344
        pagerAdapter.setKanjiResults(kanjiResults);
318345
        pagerAdapter.setResults(searchResult);
346+
        pagerAdapter.setExampleResults(exampleResults);
319347
        pagerAdapter.notifyItemChanged(0);
320348
        pagerAdapter.notifyItemChanged(1);
349+
        pagerAdapter.notifyItemChanged(2);
321350
    }
322351
323352
    @Override

app/src/main/java/eu/lepiller/nani/ResultPagerAdapter.java

1919
import java.util.List;
2020
import java.util.Map;
2121
22+
import eu.lepiller.nani.result.ExampleResult;
2223
import eu.lepiller.nani.views.KanjiStrokeView;
2324
import eu.lepiller.nani.views.PitchContourView;
2425
import eu.lepiller.nani.views.PitchDiagramView;

2930
public class ResultPagerAdapter extends RecyclerView.Adapter<ResultPagerAdapter.ViewHolder> {
3031
    static List<Result> results = new ArrayList<>();
3132
    static List<KanjiResult> kanjiResults = new ArrayList<>();
33+
    static List<ExampleResult> exampleResults = new ArrayList<>();
3234
    private static String readingStyle = "furigana";
3335
    private static String pitchStyle = "box";
3436
    private final Context context;

6062
            return sb.toString();
6163
        }
6264
65+
        void showExampleResults() {
66+
            result_view.removeAllViews();
67+
            if(exampleResults == null)
68+
                return;
69+
70+
            Log.d(TAG, "(show) examples: " + exampleResults.size());
71+
72+
            for(ExampleResult result: exampleResults) {
73+
                View child_result = LayoutInflater.from(context).inflate(R.layout.layout_example, result_view, false);
74+
75+
                TextView japaneseView = child_result.findViewById(R.id.japanese);
76+
                TextView translationView = child_result.findViewById(R.id.translation);
77+
                TextView langView = child_result.findViewById(R.id.lang_view);
78+
79+
                japaneseView.setText(result.getJapanese());
80+
                translationView.setText(result.getOtherLang());
81+
                langView.setText(result.getLang());
82+
83+
                result_view.addView(child_result);
84+
            }
85+
        }
86+
6387
        void showKanjiResults() {
6488
            result_view.removeAllViews();
6589
            if(kanjiResults == null)

275299
276300
    @Override
277301
    public void onBindViewHolder(@NonNull ViewHolder holder, int position) {
278-
        if(position == 0)
279-
            holder.showResults();
280-
        else
281-
            holder.showKanjiResults();
302+
        switch(position) {
303+
            case 0:
304+
                holder.showResults();
305+
                break;
306+
            case 1:
307+
                holder.showKanjiResults();
308+
                break;
309+
            case 2:
310+
                holder.showExampleResults();
311+
                break;
312+
        }
282313
    }
283314
284315
    @Override
285316
    public int getItemCount() {
286-
        return 2;
317+
        return 3;
287318
    }
288319
289320
    // Changing the style requires redrawing everything, so suppress warning

302333
    }
303334
304335
    @SuppressLint("NotifyDataSetChanged")
336+
    public void setExampleResults(List<ExampleResult> exampleResults) {
337+
        if(exampleResults == null)
338+
            return;
339+
340+
        ResultPagerAdapter.exampleResults.clear();
341+
        ResultPagerAdapter.exampleResults.addAll(exampleResults);
342+
        Log.d(TAG, "examples: " + ResultPagerAdapter.exampleResults.size());
343+
        notifyDataSetChanged();
344+
    }
345+
346+
    @SuppressLint("NotifyDataSetChanged")
305347
    public void setKanjiResults(List<KanjiResult> kanjiResults) {
306348
        if(kanjiResults == null)
307349
            return;

app/src/main/java/eu/lepiller/nani/SearchResult.java

33
import java.util.List;
44
55
import eu.lepiller.nani.dictionary.DictionaryException;
6+
import eu.lepiller.nani.result.ExampleResult;
67
import eu.lepiller.nani.result.KanjiResult;
78
import eu.lepiller.nani.result.Result;
89
910
class SearchResult {
1011
    private List<Result> results;
1112
    private List<KanjiResult> kanjiResults;
13+
    private List<ExampleResult> exampleResults;
1214
    private DictionaryException exception;
1315
    private boolean converted;
1416
    private String text;
1517
16-
    SearchResult(List<Result> results, List<KanjiResult> kanjiResults, String text, boolean converted) {
18+
    SearchResult(List<Result> results, List<KanjiResult> kanjiResults, List<ExampleResult> exampleResults, String text, boolean converted) {
1719
        this.results = results;
1820
        this.kanjiResults = kanjiResults;
21+
        this.exampleResults = exampleResults;
1922
        this.converted = converted;
2023
        this.text = text;
2124
    }

4043
        return kanjiResults;
4144
    }
4245
46+
    List<ExampleResult> getExampleResults() {
47+
        return exampleResults;
48+
    }
49+
4350
    boolean isConverted() {
4451
        return converted;
4552
    }

app/src/main/java/eu/lepiller/nani/dictionary/DictionaryFactory.java

2323
import java.util.Map;
2424
import java.util.Stack;
2525
26+
import eu.lepiller.nani.result.ExampleResult;
2627
import eu.lepiller.nani.result.KanjiResult;
2728
import eu.lepiller.nani.result.Result;
2829

149150
                                    chooseLanguage(synopsis),
150151
                                    chooseLanguage(description),
151152
                                    cacheDir, url, size, entries, sha256, lang);
153+
                        } else if (type.compareTo("tatoeba") == 0) {
154+
                            d = new TatoebaDictionary(name,
155+
                                    chooseLanguage(synopsis),
156+
                                    chooseLanguage(description),
157+
                                    cacheDir, url, size, entries, sha256, lang);
152158
                        }
153159
154160
                        if(d != null) {

312318
        return res;
313319
    }
314320
321+
    public static List<ExampleResult> searchExamples(String word) throws DictionaryException {
322+
        if(!initialized)
323+
            throw new NoDictionaryException();
324+
325+
        List<ExampleResult> results = new ArrayList<>();
326+
        for(Dictionary d: dictionaries) {
327+
            if(d instanceof ExampleDictionary && d.isDownloaded()) {
328+
                List<ExampleResult> r = ((ExampleDictionary) d).search(word);
329+
                if(r != null)
330+
                    results.addAll(r);
331+
            }
332+
        }
333+
334+
        Log.d(TAG, "search examples, " + results.size() + " results");
335+
336+
        return results;
337+
    }
338+
315339
    private static void augment(Result r) throws IncompatibleFormatException {
316340
        for(Dictionary d: dictionaries) {
317341
            if(d instanceof ResultAugmenterDictionary && d.isDownloaded()) {

app/src/main/java/eu/lepiller/nani/dictionary/ExampleDictionary.java unknown status 1

1+
package eu.lepiller.nani.dictionary;
2+
3+
import java.io.File;
4+
import java.util.List;
5+
6+
import eu.lepiller.nani.result.ExampleResult;
7+
import eu.lepiller.nani.result.KanjiResult;
8+
9+
public abstract class ExampleDictionary extends FileDictionary {
10+
    ExampleDictionary(String name, String description, String fullDescription, File cacheDir, String url, int fileSize, int entries, String hash, String lang) {
11+
        super(name, description, fullDescription, cacheDir, url, fileSize, entries, hash, lang);
12+
    }
13+
14+
    abstract List<ExampleResult> search(final String word) throws IncompatibleFormatException;
15+
}

app/src/main/java/eu/lepiller/nani/dictionary/TatoebaDictionary.java unknown status 1

1+
package eu.lepiller.nani.dictionary;
2+
3+
import android.util.Log;
4+
5+
import java.io.File;
6+
import java.io.FileNotFoundException;
7+
import java.io.IOException;
8+
import java.io.RandomAccessFile;
9+
import java.util.ArrayList;
10+
import java.util.Arrays;
11+
import java.util.Collections;
12+
import java.util.List;
13+
14+
import eu.lepiller.nani.R;
15+
import eu.lepiller.nani.result.ExampleResult;
16+
17+
public class TatoebaDictionary extends ExampleDictionary {
18+
    final private static String TAG = "TATOEBA";
19+
    private Huffman japaneseHuffman, translationHuffman;
20+
21+
    TatoebaDictionary(String name, String description, String fullDescription, File cacheDir, String url, int fileSize, int entries, String hash, String lang) {
22+
        super(name, description, fullDescription, cacheDir, url, fileSize, entries, hash, lang);
23+
    }
24+
25+
    @Override
26+
    int getDrawableId() {
27+
        return R.drawable.ic_tatoeba;
28+
    }
29+
30+
    private static class AudioParser extends Parser<byte[]> {
31+
        @Override
32+
        byte[] parse(RandomAccessFile file) throws IOException {
33+
            int size = file.readShort();
34+
            byte[] result = new byte[size];
35+
            for(int i=0; i<size; i++)
36+
                result[i] = file.readByte();
37+
            return result;
38+
        }
39+
    }
40+
41+
    private class SentenceParser extends Parser<ExampleResult> {
42+
        @Override
43+
        ExampleResult parse(RandomAccessFile file) throws IOException {
44+
            String japanese = new HuffmanStringParser(japaneseHuffman).parse(file);
45+
            String translation = new HuffmanStringParser(translationHuffman).parse(file);
46+
            List<String> tags = new ListParser<>(new HuffmanStringParser(translationHuffman)).parse(file);
47+
            byte[] audio = new AudioParser().parse(file);
48+
49+
            return new ExampleResult(japanese, translation, TatoebaDictionary.this.getLang(), tags, audio);
50+
        }
51+
    }
52+
53+
    private static class ValuesParser extends Parser<List<Integer>> {
54+
        @Override
55+
        List<Integer> parse(RandomAccessFile file) throws IOException {
56+
            ArrayList<Integer> results = new ArrayList<>();
57+
58+
            Log.v(TAG, "Getting values");
59+
            List<Integer> exactResults = new ListParser<>(new Parser<Integer>() {
60+
                @Override
61+
                Integer parse(RandomAccessFile file) throws IOException {
62+
                    return file.readInt();
63+
                }
64+
            }).parse(file);
65+
66+
            List<Integer> others = new ListParser<>(1, new Parser<Integer>() {
67+
                @Override
68+
                Integer parse(RandomAccessFile file) throws IOException {
69+
                    file.skipBytes(1);
70+
                    return file.readInt();
71+
                }
72+
            }).parse(file);
73+
74+
            for(Integer pos: others) {
75+
                file.seek(pos);
76+
                results.addAll(new ResultDictionary.ValuesParser().parse(file));
77+
            }
78+
79+
            Collections.sort(results);
80+
            Collections.sort(exactResults);
81+
82+
            Log.v(TAG, "exact result size: " + exactResults.size() + ", result size: " + results.size());
83+
            Log.v(TAG, "exact: " + Arrays.toString(exactResults.toArray()) + ", others: " + Arrays.toString(results.toArray()));
84+
            exactResults.addAll(results);
85+
            return exactResults;
86+
        }
87+
    }
88+
89+
    private List<Integer> searchTrie(RandomAccessFile file, long triePos, byte[] txt) throws IOException {
90+
        return searchTrie(file, triePos, txt, 50, new TrieParser<Integer>(new ValuesParser()) {
91+
            @Override
92+
            public void skipVals(RandomAccessFile file, long pos) throws IOException {
93+
                file.seek(pos);
94+
                int valuesLength = file.readShort();
95+
                Log.v(TAG, "number of values: " + valuesLength);
96+
                file.skipBytes(valuesLength * 4);
97+
            }
98+
        });
99+
    }
100+
101+
    @Override
102+
    List<ExampleResult> search(String word) throws IncompatibleFormatException {
103+
        if (isDownloaded()) {
104+
            try {
105+
                RandomAccessFile file = new RandomAccessFile(getFile(), "r");
106+
                byte[] header = new byte[16];
107+
                int l = file.read(header);
108+
                if (l != header.length)
109+
                    return null;
110+
111+
                // Check file format version
112+
                if (!Arrays.equals(header, "NANI_SENTENCE001".getBytes())) {
113+
                    StringBuilder error = new StringBuilder("search: incompatible header: [");
114+
                    boolean first = true;
115+
                    for (byte b : header) {
116+
                        if (first)
117+
                            first = false;
118+
                        else
119+
                            error.append(", ");
120+
                        error.append(b);
121+
                    }
122+
                    error.append("].");
123+
                    Log.d(TAG, error.toString());
124+
                    throw new IncompatibleFormatException(getName());
125+
                }
126+
127+
                byte[] search = word.toLowerCase().getBytes();
128+
129+
                long triePos = file.readInt();
130+
131+
                Log.d(TAG, "Search in: " + getFile());
132+
                Log.v(TAG, "trie: " + triePos);
133+
134+
                japaneseHuffman = new HuffmanParser().parse(file);
135+
                translationHuffman = new HuffmanParser().parse(file);
136+
137+
                // Search in Japanese
138+
                List<Integer> results = searchTrie(file, triePos, search);
139+
                Log.d(TAG, results.size() + " result(s)");
140+
141+
                List<ExampleResult> r = new ArrayList<>();
142+
                List<Integer> uniqResults = new ArrayList<>();
143+
                for(Integer i: results) {
144+
                    if(!uniqResults.contains(i))
145+
                        uniqResults.add(i);
146+
                }
147+
148+
                int num = 0;
149+
                for(int pos: uniqResults) {
150+
                    if(num > 10)
151+
                        break;
152+
                    num++;
153+
                    file.seek(pos);
154+
                    r.add(new SentenceParser().parse(file));
155+
                }
156+
                return r;
157+
            } catch (FileNotFoundException e) {
158+
                e.printStackTrace();
159+
            } catch (IOException e) {
160+
                e.printStackTrace();
161+
            }
162+
        }
163+
        return null;
164+
    }
165+
}

app/src/main/java/eu/lepiller/nani/result/ExampleResult.java unknown status 1

1+
package eu.lepiller.nani.result;
2+
3+
import java.util.List;
4+
5+
public class ExampleResult {
6+
    private final String japanese, otherLang, lang;
7+
    private final List<String> tags;
8+
    private final byte[] audio;
9+
10+
    public ExampleResult(String japanese, String otherLang, String lang, List<String> tags, byte[] audio) {
11+
        this.japanese = japanese;
12+
        this.otherLang = otherLang;
13+
        this.lang = lang;
14+
        this.tags = tags;
15+
        this.audio = audio;
16+
    }
17+
18+
    public String getLang() {
19+
        return lang;
20+
    }
21+
22+
    public String getOtherLang() {
23+
        return otherLang;
24+
    }
25+
26+
    public String getJapanese() {
27+
        return japanese;
28+
    }
29+
30+
    public List<String> getTags() {
31+
        return tags;
32+
    }
33+
34+
    public byte[] getAudio() {
35+
        return audio;
36+
    }
37+
}

app/src/main/res/layout/activity_about.xml

255255
            app:lineHeight="22dp"
256256
            android:text="@string/jibiki_license" />
257257
258+
        <TextView
259+
            android:layout_width="match_parent"
260+
            android:layout_height="wrap_content"
261+
            android:layout_marginTop="16dp"
262+
            android:text="@string/tatoeba_title"
263+
            android:textColor="@color/colorSubtitle"
264+
            android:textSize="@dimen/subtitle_size" />
265+
266+
        <TextView
267+
            android:layout_width="match_parent"
268+
            android:layout_height="wrap_content"
269+
            app:lineHeight="22dp"
270+
            android:text="@string/tatoeba_descr" />
271+
272+
        <TextView
273+
            android:layout_width="match_parent"
274+
            android:layout_height="wrap_content"
275+
            app:lineHeight="22dp"
276+
            android:text="@string/tatoeba_license" />
277+
258278
    </LinearLayout>
259279
260280
</ScrollView>
260280=
261281=
\ No newline at end of file

app/src/main/res/layout/layout_example.xml unknown status 1

1+
<?xml version="1.0" encoding="utf-8"?>
2+
<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
3+
    android:layout_width="match_parent"
4+
    android:layout_height="wrap_content"
5+
    android:orientation="vertical"
6+
    android:layout_marginBottom="16dp">
7+
8+
    <TextView
9+
        android:layout_width="match_parent"
10+
        android:layout_height="wrap_content"
11+
        android:layout_marginLeft="8dp"
12+
        android:layout_marginStart="8dp"
13+
        android:id="@+id/japanese" />
14+
    <LinearLayout
15+
        android:orientation="horizontal"
16+
        android:layout_width="match_parent"
17+
        android:layout_height="wrap_content" >
18+
        <TextView
19+
            android:id="@+id/lang_view"
20+
            android:layout_width="wrap_content"
21+
            android:layout_height="wrap_content"
22+
            android:textColor="@color/colorLang"
23+
            android:padding="4dp"
24+
            android:background="@drawable/ic_pitch_border"
25+
            android:layout_marginLeft="8dp"
26+
            android:layout_marginStart="8dp"
27+
            android:text="lang" />
28+
29+
        <TextView
30+
            android:id="@+id/translation"
31+
            android:layout_width="match_parent"
32+
            android:layout_height="wrap_content"
33+
            android:layout_margin="8dp"
34+
            android:text="definition" />
35+
    </LinearLayout>
36+
</LinearLayout>
36<
037<
\ No newline at end of file

app/src/main/res/values/strings.xml

9696
    <string name="sense_alternatives">Alternative forms: %s</string>
9797
    <string name="tab_kanji">Kanji</string>
9898
    <string name="tab_results">Word</string>
99+
    <string name="tab_example">Examples</string>
99100
    <plurals name="kanji_stroke">
100101
        <item quantity="one">%d stroke</item>
101102
        <item quantity="other">%d strokes</item>

189190
    dictionary, the French???Japanese Raguet-Martin dictionary, the Japanese???English JMdict
190191
    dictionary and Wikipedia links.</string>
191192
    <string name="jibiki_license">This source is licensed under Creative Commons 0 (public domain).</string>
193+
    <string name="tatoeba_title">Tatoeba</string>
194+
    <string name="tatoeba_descr">Tatoeba is a collection of sentences and translations. It is a
195+
    community project where anyone can contribute new sentences and new translations.</string>
196+
    <string name="tatoeba_license">This source is licensed under Creative Commons Attribution 2.0 FR.
197+
        Some sentences are under Creative Commons 0 (public domain).</string>
192198
193199
    <!-- Help Activities -->
194200
    <string name="help_intro">Welcome to Nani\'s Help Center!</string>