;;; Nani Project website
;;; Copyright © 2022 Julien Lepiller <julien@lepiller.eu>
;;;
;;; This file is part of the Nani Project website.
;;;
;;; The Nani Project website is free software; you can redistribute it and/or modify it
;;; under the terms of the GNU Affero General Public License as published by
;;; the Free Software Foundation; either version 3 of the License, or (at
;;; your option) any later version.
;;;
;;; The Nani Project website is distributed in the hope that it will be useful, but
;;; WITHOUT ANY WARRANTY; without even the implied warranty of
;;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
;;; GNU Affero General Public License for more details.
;;;
;;; You should have received a copy of the GNU Affero General Public License
;;; along with the Nani Project website.  If not, see <http://www.gnu.org/licenses/>.

(use-modules (srfi srfi-9))
(use-modules (srfi srfi-11))
(use-modules (ice-9 match))
(use-modules (ice-9 binary-ports))
(use-modules (ice-9 textual-ports))
(use-modules (nani sentence sentence))

(define (tatoeba-file file)
  (string-append (dirname (current-filename)) "/../dictionaries/tatoeba_" file))

(define (get-sentences lang native-users)
  (call-with-input-file (tatoeba-file "sentences_detailed.csv")
    (lambda (port)
      (let loop ((jpn '()) (trans '()) (line (get-line port)))
        (if (eof-object? line)
          (values (reverse jpn) (reverse trans))
          (match (string-split line #\tab)
            ((id slang text user _ _)
             (cond
               ((equal? lang slang)
                (loop jpn (cons (cons (string->number id) text) trans)
                      (get-line port)))
               ((and (equal? slang "jpn") (member user native-users))
                (loop (cons (cons (string->number id) text) jpn) trans (get-line port)))
               (else
                 (loop jpn trans (get-line port)))))))))))

(define (get-native-jpn-users)
  (call-with-input-file (tatoeba-file "user_languages.csv")
    (lambda (port)
      (let loop ((jpn '()) (line (get-line port)))
        (if (eof-object? line)
          jpn
          (match (string-split line #\tab)
            ((ulang level user _)
             ;; consume the comment if it's on multiple lines
             (let loop2 ((line line))
               (when (string-suffix? "\\" line)
                 (loop2 (get-line port))))
             ;; Add native japanese user
             (if (and (equal? ulang "jpn") (equal? level "5"))
               (loop (cons user jpn) (get-line port))
               (loop jpn (get-line port))))))))))

(define (get-translations jpn trans)
  (define jpn-len (vector-length jpn))
  (define trans-len (vector-length trans))
  (define (member-vect elem vect len)
    (let loop ((min 0) (max (- len 1)))
      (if (> min max)
        #f
        (let* ((mid-pos (floor (+ min (/ (- max min) 2))))
               (mid-elem (vector-ref vect mid-pos)))
          (cond
            ((equal? mid-elem elem) #t)
            ((> mid-elem elem) (loop min (- mid-pos 1)))
            ((< mid-elem elem) (loop (+ mid-pos 1) max)))))))

  (call-with-input-file (tatoeba-file "sentences_base.csv")
    (lambda (port)
      (let loop ((translations '()) (line (get-line port)))
        (if (or (eof-object? line) (null? trans))
          translations
          (match (string-split line #\tab)
            ((id translation)
             (cond
               ((and (string->number id)
                     (member-vect (string->number id) trans trans-len)
                     (string->number translation)
                     (member-vect (string->number translation) jpn jpn-len))
                (loop (cons (cons (string->number translation) (string->number id)) translations)
                      (get-line port)))
               ((and (string->number id)
                     (member-vect (string->number id) jpn jpn-len)
                     (string->number translation)
                     (member-vect (string->number translation) trans trans-len))
                (loop (cons (cons (string->number id) (string->number translation)) translations)
                      (get-line port)))
               (else
                 (loop translations (get-line port)))))))))))

(define (add-tags translations trans jpn)
  (define tags
    (call-with-input-file (tatoeba-file "tags.csv")
      (lambda (port)
        (let loop ((tags '()) (line (get-line port)))
          (if (eof-object? line)
            tags
            (match (string-split line #\tab)
              ((id tag)
               (assoc-set! tags (string->number id) (cons tag (or (assoc-ref tags (string->number id)) '()))))))))))
  (map
    (match-lambda
      ((jpn-id . trans-id)
       (let ((tags (or (assoc-ref tags jpn-id) '()))
             (trans (assoc-ref trans trans-id))
             (jpn (assoc-ref jpn jpn-id)))
         (make-sentence jpn trans tags #f))))
    translations))

(define (get-tatoeba-sentences lang)
  (define native-users (get-native-jpn-users))
  (let-values (((jpn trans) (get-sentences lang native-users)))
    (format #t "jpn: ~a sentences~%" (length jpn))
    (format #t "~a: ~a sentences~%" lang (length trans))
    (let ((translations
            (get-translations
              (list->vector (sort (map car jpn) <))
              (list->vector (sort (map car trans) <)))))
      (format #t "~a pairs~%" (length translations))
      (add-tags translations trans jpn))))

(match (command-line)
  ((_ lang output)
   (let ((sentences (get-tatoeba-sentences lang)))
     (format #t "Number of entries in ~a: ~a~%" output (length sentences))
     (call-with-output-file output
       (lambda (port)
         (put-bytevector port
           (serialize-sentence-dictionary sentences)))))))