nani/website

Julien LepillerSat Apr 20 13:34:51+0200 2019

a9fec24

sort jmdict by relevance

Makefile

 	wget --no-check-certificate \
         https://namakajiri.net/data/wikipedia-20150422-lemmas.tsv -O $@.tmp
 	sed -i 's| ||g' $@.tmp
 	mv $@.tmp $@
 	head -n20000 $@.tmp > $@
 	rm $@.tmp
 dictionaries/%.sxml: dictionaries/%.xml tools/jmdict.scm
 	guile -L modules tools/jmdict.scm convert $< nolang $@

modules/nani/frequency.scm

 (define (load-frequency file)
   (call-with-input-file file
     (lambda (port)
       (let loop ((frq '()))
       (let loop ((frq '()) (i 1))
         (let* ((line (%read-line port))
                (line (car line)))
           (if (eof-object? line)
             frq
             (let ((content (string-split line #\tab)))
               (loop (cons (cons (cadr (cdr content))
                                 (string->number (car content)))
                           frq)))))))))
               (loop (cons (cons (cadr (cdr content)) i) frq) (+ i 1)))))))))
 (define (frequency-entity frq word)
   (let ((freq (assoc-ref frq word)))

modules/nani/huffman.scm

   (define (serialize huffman)
     (match huffman
       ((((h1 h2) . weight))
        (append '(1) (serialize h1) (serialize h2) '(2)))
        (append '(1) (serialize h1) (serialize h2)))
       (((h1 h2) . weight)
        (append '(1) (serialize h1) (serialize h2) '(2)))
        (append '(1) (serialize h1) (serialize h2)))
       ((((? char? char) . weight))
        (append (bytevector->u8-list (string->utf8 (list->string (list char)))) '(0)))
       (((? char? char) . weight)

modules/nani/jmdict/serialize.scm

   (define (serialize-huffman-string huffman-code)
     (lambda (str pos bv)
       (let ((sbv (huffman-encode huffman-code str)))
         (bytevector-copy! sbv 0 bv (+ pos 4) (bytevector-length sbv))
         (bytevector-copy! sbv 0 bv pos (bytevector-length sbv))
         (+ pos (bytevector-length sbv)))))
   (define (huffman-string-size huffman-code)
     (lambda (str)

modules/nani/jmdict/xml.scm

          (('*PI* _ ...) #f)))
       (lambda (a) a)
       sxml)
     (lambda (a b) (< (result-points a) (result-points b)))))
     ;; reverse order: bigger score first
     (lambda (a b) (> (result-points a) (result-points b)))))

tools/jmdict.scm

       (lambda (port)
         (write sxml port)))))
 (define (compile input sense-filter output)
   (let* ((frq (load-frequency "dictionaries/frequency.tsv"))
          (sxml (if (equal? (substring input (- (string-length input) 4)) ".xml")
 ;; Break these steps to try and let the GC reclaim these big objects
 (define (get-results1 input frq)
   (let ((sxml (if (equal? (substring input (- (string-length input) 4)) ".xml")
                  (load-dic input)
                  (call-with-input-file input read)))
          (results (sxml->results sxml frq))
                  (call-with-input-file input read))))
     (format #t "Read xml~%")
     (sxml->results sxml frq)))
 (define (get-results input sense-filter frq)
   (let* ((results (get-results1 input frq))
          (results (map (lambda (result)
                          (update-result
                            result
                                             (result-senses result))))
                        results))
          (results (filter (lambda (result) (not (null? (result-senses result))))
                           results))
                           results)))
     results))
 (define (compile input sense-filter output)
   (let* ((results (get-results input sense-filter
                                (load-frequency "dictionaries/frequency.tsv")))
          (kanji-trie (compress-trie (make-kanji-trie results)))
          (reading-trie (compress-trie (make-reading-trie results)))
          (meaning-trie (compress-trie (make-meaning-trie results))))