nani/website/tools/jmdict.scm

jmdict.scm

1
;;; Nani Project website
2
;;; Copyright © 2019 Julien Lepiller <julien@lepiller.eu>
3
;;;
4
;;; This file is part of the Nani Project website.
5
;;;
6
;;; The Nani Project website is free software; you can redistribute it and/or modify it
7
;;; under the terms of the GNU Affero General Public License as published by
8
;;; the Free Software Foundation; either version 3 of the License, or (at
9
;;; your option) any later version.
10
;;;
11
;;; The Nani Project website is distributed in the hope that it will be useful, but
12
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
13
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14
;;; GNU Affero General Public License for more details.
15
;;;
16
;;; You should have received a copy of the GNU Affero General Public License
17
;;; along with the Nani Project website.  If not, see <http://www.gnu.org/licenses/>.
18
19
(use-modules (nani result frequency))
20
(use-modules (nani result jmdict))
21
(use-modules (nani result result))
22
(use-modules (ice-9 match))
23
(use-modules (ice-9 binary-ports))
24
25
;; Break these steps to try and let the GC reclaim these big objects
26
(define (get-results1 input frq)
27
  (call-with-input-file input
28
    (lambda (port)
29
      (xml->results port frq))))
30
31
(define (get-results input meaning-filter frq)
32
  (let* ((results (get-results1 input frq))
33
         (results (map (lambda (result)
34
                         (update-result
35
                           result
36
                           #:meanings (filter meaning-filter
37
                                            (result-meanings result))))
38
                       results))
39
         (results (filter (lambda (result) (not (null? (result-meanings result))))
40
                          results)))
41
    results))
42
43
(define (compile input meaning-filter output)
44
  (let* ((results (get-results input meaning-filter
45
                               (load-frequency "dictionaries/frequency.tsv"))))
46
    (format #t "Number of entries in ~a: ~a~%" output (length results))
47
    (call-with-output-file output
48
      (lambda (port)
49
        (put-bytevector port
50
          (serialize-dictionary results))))))
51
52
(define (print word dict)
53
  #t)
54
55
(match (command-line)
56
  ((_ cmd input lang output)
57
   (cond
58
    ((equal? cmd "build")
59
     (if (equal? lang "e")
60
       (compile input (const #t) output)
61
       (compile input (lambda (meaning) (equal? (meaning-language meaning) lang)) output)))
62
    (else (format #t "Unknown cmd ~a.~%" cmd))))
63
  ((_ "print" word input)
64
   (print word input)))
65