Improve format support

Julien LepillerWed Oct 10 19:57:16+0200 2018

aff36f3

Improve format support

CONTRIBUTING.md

9191 
```
9292 
9393 
Saves the list of Entry.
94+
95+
```python
96+
merge(oldformat, callback)
97+
```
98+
99+
Merge the file in the old file. The callback function is the one passed to
100+
`System.update`.
offlate/formats/gettext.py
66 
from .entry import POEntry
77 
88 
class GettextFormat:
9-
    def __init__(self, files, conf):
10-
        self.po = polib.pofile(files["file"])
9+
    def __init__(self, conf):
10+
        self.po = polib.pofile(conf["file"])
11+
        self.pot = polib.pofile(conf["pot"])
1112 
        self.conf = conf
1213 
1314 
    def content(self):
…
2021 
        self.po.metadata['Language'] = self.conf['lang']
2122 
        self.po.metadata['X-Generator'] = 'Offlate ' + self.conf['version']
2223 
        self.po.save()
24+
25+
    def merge(self, older, callback):
26+
        older.po.merge(self.pot)
27+
        self.po.save()
28+
        for oentry in older.po:
29+
            for nentry in self.po:
30+
                if oentry.msgid == nentry.msgid:
31+
                    if oentry.msgstr == nentry.msgstr:
32+
                        break
33+
                    if oentry.msgstr == "":
34+
                        break
35+
                    if nentry.msgstr == "":
36+
                        nentry.msgstr = oentry.msgstr
37+
                        break
38+
                    # otherwise, nentry and oentry have a different msgstr
39+
                    nentry.msgstr = callback(nentry.msgid, oentry.msgstr, nentry.msgstr)
40+
                    break
41+
        self.po.save()
42+
offlate/systems/tp.py
7777 
        self.updateVersion()
7878 
        self.updateFileName()
7979 
        self.updateGettextNames()
80+
        newname = self.popath
81+
        self.popath = self.popath + '.new.po'
8082 
        self.getpot()
81-
        po = polib.pofile(oldpath)
82-
        po.merge(polib.pofile(self.potpath))
83-
        po.save()
84-
        if oldname == self.filename:
85-
            self.popath = self.popath + '.new.po'
86-
            self.getpo()
87-
            self.merge(self.popath, oldpath, callback)
88-
            os.remove(oldpath)
89-
            os.rename(self.popath, oldpath)
90-
            self.popath = oldpath
91-
        else:
92-
            self.getpo()
93-
            self.merge(self.popath, oldpath, callback)
94-
            os.remove(oldpath)
95-
96-
    def merge(self, tofile, fromfile, callback):
97-
        newpo = polib.pofile(tofile)
98-
        oldpo = polib.pofile(fromfile)
99-
        # If msgid is not found in to file, it's an old one, so ignore.
100-
        # Otherwise, attempt to merge.
101-
        for oentry in oldpo:
102-
            for nentry in newpo:
103-
                if oentry.msgid == nentry.msgid:
104-
                    if oentry.msgstr == nentry.msgstr:
105-
                        break
106-
                    if oentry.msgstr == "":
107-
                        break
108-
                    if nentry.msgstr == "":
109-
                        nentry.msgstr = oentry.msgstr
110-
                        break
111-
                    # else, nentry and oentry have a different msgstr
112-
                    nentry.msgstr = callback(nentry.msgid, oentry.msgstr, nentry.msgstr)
113-
                    break
114-
        newpo.save()
83+
        self.getpo()
84+
        newcontent = GettextFormat(
85+
                {'file': self.popath,
86+
                 'pot': self.potpath,
87+
                 'version': '0.1',
88+
                 'fullname': self.conf['fullname'],
89+
                 'lang': self.lang})
90+
        content = GettextFormat(
91+
                {'file': oldpath,
92+
                 'pot': self.potpath,
93+
                 'version': '0.1',
94+
                 'fullname': self.conf['fullname'],
95+
                 'lang': self.lang})
96+
        newcontent.merge(content, callback)
97+
        self.po = newcontent
98+
        os.remove(oldpath)
99+
        os.rename(self.popath, newname)
100+
        self.popath = newname
115101 
116102 
    def send(self, interface):
117103 
        self.save()
…
131117 
        self.po.save()
132118 
133119 
    def content(self):
134-
        self.po = GettextFormat({'file': self.popath},
135-
                {'version': '0.1',
120+
        self.po = GettextFormat(
121+
                {'file': self.popath,
122+
                 'pot': self.potpath,
123+
                 'version': '0.1',
136124 
                 'fullname': self.conf['fullname'],
137125 
                 'lang': self.lang})
138126 
        return {'default': self.po.content()}
offlate/systems/transifex.py
1-
import requests
1+
""" The transifex system connector. """
2+
23 
import json
3-
from ruamel import yaml
44 
import os
5-
5+
import requests
66 
from requests.auth import HTTPBasicAuth
77 
8-
from pathlib import Path
9-
10-
from ..formats.entry import JSONEntry
11-
12-
def yaml_rec_load(path, source, dest):
13-
    ans = []
14-
    for s in source:
15-
        path2 = list(path)
16-
        path2.append(s)
17-
        if isinstance(source[s], str):
18-
            ans.append({'path': path, 'id': s, 'source_string': source[s], 'translation': dest[s]})
19-
        else:
20-
            ans.extend(yaml_rec_load(path2, source[s], dest[s]))
21-
    return ans
22-
23-
def yaml_rec_update(callback, source, old, new):
24-
    ans = {}
25-
    for i in new:
26-
        o = ''
27-
        s = ''
28-
        n = new[i]
29-
        try:
30-
            s = source[i]
31-
        except Exception:
32-
            pass
33-
        try:
34-
            o = old[i]
35-
        except Exception:
36-
            pass
37-
        if isinstance(n, str):
38-
            if o == '':
39-
                ans[i] = n
40-
            elif n == '':
41-
                ans[i] = o
42-
            else:
43-
                ans[i] = callback(s, o, n)
44-
        else:
45-
            ans[i] = yaml_rec_update(callback, s, o, n)
46-
    return ans
47-
488 
class TransifexProject:
49-
    def __init__(self, conf, name, lang, data = {}):
50-
        self.uri = "https://www.transifex.com"
9+
    def __init__(self, conf, name, lang, data={}):
5110 
        self.conf = conf
5211 
        self.name = name
5312 
        self.lang = lang
54-
        self.data = data
5513 
        self.basedir = ''
14+
        self.data = data
5615 
        self.contents = {}
5716 
5817 
    def open(self, basedir):
…
6019 
        with open(self.basedir + '/project.info') as f:
6120 
            self.files = json.load(f)
6221 
        self.slugs = [x['slug'] for x in self.files]
63-
        self.loadContent()
6422 
6523 
    def initialize(self, basedir):
6624 
        self.basedir = basedir
…
6826 
        with open(self.basedir + '/project.info', 'w') as f:
6927 
            f.write(json.dumps(self.files))
7028 
        for slug in self.slugs:
71-
            self.getFile(slug)
72-
        self.loadContent()
73-
74-
    def update(self, callback):
75-
        self.updateFileList()
76-
        for ff in self.files:
77-
            slug = ff['slug']
78-
            fname = self.filename(slug, False)
79-
            sname = self.filename(slug, True)
80-
            os.rename(fname, fname+'.old')
81-
            self.getFile(slug)
82-
            if ff['i18n_type'] == 'YML':
83-
                with open(fname+'.old') as oldf:
84-
                    with open(fname) as newf:
85-
                        with open(sname) as sourcef:
86-
                            old = yaml.safe_load(oldf)
87-
                            new = yaml.safe_load(newf)
88-
                            source = yaml.safe_load(sourcef)
89-
                            realnew = yaml_rec_update(callback, source, old, new)
90-
                            with open(fname, 'w') as f:
91-
                                yaml.dump(realnew, f, Dumper=yaml.RoundTripDumper,
92-
                                    allow_unicode=True)
29+
            self.getFiles(slug)
9330 
9431 
    def updateFileList(self):
9532 
        self.files = []
…
10239 
            l = json.loads(ans.text)
10340 
            self.slugs = [x['slug'] for x in l]
10441 
            self.files = l
105-
    
106-
    def loadContent(self):
42+
43+
    def update(self, callback):
44+
        self.updateFileList()
10745 
        for ff in self.files:
108-
            with open(self.filename(ff['slug'], True)) as f:
109-
                with open(self.filename(ff['slug'], False)) as f2:
110-
                    if ff['i18n_type'] == 'YML':
111-
                        source = yaml.safe_load(f)
112-
                        dest = yaml.safe_load(f2)
113-
                        lang1 = list(source.keys())[0]
114-
                        lang2 = list(dest.keys())[0]
115-
                        self.contents[ff['slug']] = \
116-
                            yaml_rec_load([lang2], source[lang1], dest[lang2])
46+
            slug = ff['slug']
47+
            fname = self.filename(slug, False)
48+
            sname = self.filename(slug, True)
49+
            os.rename(fname, fname+'.old')
50+
            os.rename(sname, sname+'.old')
51+
            self.getFiles(slug)
52+
            if ff['i18n_type'] == 'YML':
53+
                oldformat = YamlFormat({'dest': fname+'.old', 'source': sname+'.old'})
54+
                currentformat = YamlFormat({'dest': fname, 'source': sname})
55+
            else:
56+
                raise Exception("Unsupported format: " + ff['i18n_type'])
57+
            currentformat.merge(oldformat)
58+
59+
    def filename(self, slug, is_source):
60+
        ext = ''
61+
        for ff in self.files:
62+
            if ff['slug'] == slug:
63+
                f = ff
64+
                break
65+
        if f['i18n_type'] == 'YML':
66+
            ext = 'yml'
67+
        else:
68+
            raise Exception("Unsupported format: " + f['i18n_type'])
69+
        return self.basedir + '/' + slug + ('.source' if is_source else '') + '.' + ext
11770 
118-
    def getFile(self, slug):
71+
    def getFiles(self, slug):
11972 
        ans = requests.get('https://www.transifex.com/api/2/project/'+
12073 
                self.name+'/resource/'+slug+'/content',
12174 
                auth=HTTPBasicAuth('api', self.conf['token']))
12275 
        if ans.status_code == 200:
123-
            with open(self.filename(slug, True), 'w') as f:
124-
                f.write(json.loads(ans.text)['content'])
76+
            with open(self.filename(slug, True), 'wb') as f:
77+
                f.write(json.loads(ans.text)['content'].encode('utf-8'))
12578 
12679 
        ans = requests.get('https://www.transifex.com/api/2/project/'+self.name+
12780 
                '/resource/'+slug+'/translation/'+self.lang+'/?mode=translator',
12881 
                auth=HTTPBasicAuth('api', self.conf['token']))
12982 
        if ans.status_code == 200:
130-
            with open(self.filename(slug, False), 'w') as f:
131-
                f.write(json.loads(ans.text)['content'])
83+
            with open(self.filename(slug, False), 'wb') as f:
84+
                f.write(json.loads(ans.text)['content'].encode('utf-8'))
13285 
        else:
13386 
            print(ans.text)
13487 
135-
    def filename(self, slug, is_source):
136-
        ext = ''
137-
        for ff in self.files:
138-
            if ff['slug'] == slug:
139-
                f = ff
140-
                break
141-
        if f['i18n_type'] == 'YML':
142-
            ext = 'yml'
143-
        return self.basedir + '/' + slug + ('.source' if is_source else '') + '.' + ext
144-
14588 
    def send(self, interface):
146-
        self.save()
147-
        for ff in self.files:
148-
            print('{} => {}'.format(ff['slug'], ff['i18n_type']))
149-
            with open(self.filename(ff['slug'], False)) as f:
150-
                content = f.read()
151-
                sendcontent = {"content": content}
152-
                ans = requests.put('https://www.transifex.com/api/2/project/'+
153-
                        self.name+'/resource/'+ff['slug']+'/translation/'+self.lang+'/',
154-
                        json=sendcontent, auth=HTTPBasicAuth('api', self.conf['token']))
155-
                print(ans)
156-
                print(ans.text)
89+
        pass
15790 
15891 
    def save(self):
159-
        for slug in self.slugs:
160-
            data = {}
161-
            for d in self.contents[slug]:
162-
                path = d['path']
163-
                curr = data
164-
                for p in path:
165-
                    if p in curr:
166-
                        curr = curr[p]
167-
                    else:
168-
                        curr[p] = {}
169-
                        curr = curr[p]
170-
                curr[d['id']] = d['translation']
171-
            with open(self.filename(slug, False), 'w') as f:
172-
                yaml.dump(data, f, Dumper=yaml.RoundTripDumper, allow_unicode=True)
92+
        for slug in self.content:
93+
            slug.save()
17394 
17495 
    def content(self):
175-
        contents = {}
176-
        for content in self.contents:
177-
            contents[content] = [JSONEntry(x) for x in self.contents[content]]
178-
        return contents
96+
        content = {}
97+
        self.content = []
98+
        for slug in self.files:
99+
            if slug['i18n_type'] == 'YML':
100+
                myslug = YamlFormat(
101+
                        {'dest':   self.filename(slug['slug'], False),
102+
                         'source': self.filename(slug['slug'], True)})
103+
            else:
104+
                raise Exception("Unsupported format: " + slug['i18n_type'])
105+
            self.content.append(myslug)
106+
            content[slug['slug']] = myslug.content()
107+
        return content