xmpp.chapril.org-conversejs/trans/models.py

535 lines
18 KiB
Python
Raw Normal View History

2012-02-27 10:05:53 +01:00
from django.db import models
2012-02-28 14:50:20 +01:00
from django.db.models import Q
2012-02-29 15:26:57 +01:00
from django.contrib.auth.models import User
2012-02-27 11:39:00 +01:00
from django.conf import settings
2012-02-27 10:46:12 +01:00
from lang.models import Language
2012-03-05 13:43:33 +01:00
from django.db.models import Sum
2012-02-28 14:50:20 +01:00
from django.utils.translation import ugettext_lazy, ugettext as _
2012-02-28 16:01:07 +01:00
from django.utils.safestring import mark_safe
from glob import glob
2012-02-27 11:39:00 +01:00
import os
import os.path
2012-02-27 18:02:34 +01:00
import logging
2012-02-27 13:34:51 +01:00
import git
import traceback
2012-02-27 17:12:08 +01:00
from translate.storage import factory
2012-02-29 13:50:40 +01:00
from translate.storage import poheader
2012-02-29 13:46:25 +01:00
from datetime import datetime
2012-02-27 10:13:42 +01:00
2012-02-29 13:38:58 +01:00
import trans
from trans.managers import TranslationManager, UnitManager
2012-02-27 17:54:14 +01:00
from util import is_plural, split_plural, join_plural
2012-02-27 11:08:16 +01:00
2012-02-27 18:02:34 +01:00
logger = logging.getLogger('weblate')
2012-02-27 10:13:42 +01:00
class Project(models.Model):
name = models.CharField(max_length = 100)
slug = models.SlugField(db_index = True)
web = models.URLField()
2012-03-01 14:34:26 +01:00
mail = models.EmailField(blank = True)
instructions = models.URLField(blank = True)
2012-02-27 10:39:01 +01:00
2012-02-28 10:11:06 +01:00
class Meta:
ordering = ['name']
2012-02-27 11:11:37 +01:00
@models.permalink
def get_absolute_url(self):
2012-02-28 10:32:07 +01:00
return ('trans.views.show_project', (), {
'project': self.slug
})
2012-02-27 11:39:00 +01:00
def get_path(self):
return os.path.join(settings.GIT_ROOT, self.slug)
2012-02-27 11:45:27 +01:00
def __unicode__(self):
return self.name
2012-02-27 11:39:00 +01:00
def save(self, *args, **kwargs):
# Create filesystem directory for storing data
p = self.get_path()
if not os.path.exists(p):
os.makedirs(p)
2012-02-27 13:34:51 +01:00
2012-02-27 11:39:00 +01:00
super(Project, self).save(*args, **kwargs)
2012-02-27 11:11:37 +01:00
2012-03-05 13:43:33 +01:00
def get_translated_percent(self):
translations = Translation.objects.filter(subproject__project = self).aggregate(Sum('translated'), Sum('total'))
return round(translations['translated__sum'] * 100.0 / translations['total__sum'], 1)
2012-02-27 10:39:01 +01:00
class SubProject(models.Model):
2012-03-02 15:19:53 +01:00
name = models.CharField(max_length = 100, help_text = _('Name to display'))
2012-02-29 10:07:29 +01:00
slug = models.SlugField(db_index = True, help_text = _('Name used in URLs'))
2012-02-27 10:39:01 +01:00
project = models.ForeignKey(Project)
2012-03-01 14:34:56 +01:00
repo = models.CharField(max_length = 200, help_text = _('URL of Git repository'))
2012-02-29 10:07:29 +01:00
repoweb = models.URLField(help_text = _('Link to repository browser, use %(file)s and %(line)s as filename and line placeholders'))
branch = models.CharField(max_length = 50, help_text = _('Git branch to translate'))
filemask = models.CharField(max_length = 200, help_text = _('Mask of files to translate, use * istead of language code'))
2012-02-27 10:46:12 +01:00
2012-02-28 10:11:06 +01:00
class Meta:
ordering = ['name']
2012-02-27 11:11:37 +01:00
@models.permalink
def get_absolute_url(self):
2012-02-28 10:32:07 +01:00
return ('trans.views.show_subproject', (), {
'project': self.project.slug,
'subproject': self.slug
})
2012-02-27 11:11:37 +01:00
2012-02-27 11:45:27 +01:00
def __unicode__(self):
2012-02-27 13:34:51 +01:00
return '%s/%s' % (self.project.__unicode__(), self.name)
def get_path(self):
return os.path.join(self.project.get_path(), self.slug)
2012-02-27 15:03:26 +01:00
def get_repo(self):
'''
Gets Git repository object.
'''
2012-02-27 13:34:51 +01:00
p = self.get_path()
try:
2012-02-27 15:03:26 +01:00
return git.Repo(p)
except:
return git.Repo.init(p)
2012-02-29 10:03:37 +01:00
def get_repoweb_link(self, filename, line):
return self.repoweb % {'file': filename, 'line': line}
2012-02-27 15:03:26 +01:00
def configure_repo(self):
'''
Ensures repository is correctly configured and points to current remote.
'''
# Create/Open repo
repo = self.get_repo()
# Get/Create origin remote
try:
origin = repo.remotes.origin
except:
repo.git.remote('add', 'origin', self.repo)
origin = repo.remotes.origin
2012-02-27 15:03:26 +01:00
# Check remote source
if origin.url != self.repo:
repo.git.remote('set-url', 'origin', self.repo)
2012-02-27 15:03:26 +01:00
# Update
2012-02-28 10:17:43 +01:00
logger.info('updating repo %s', self.__unicode__())
2012-03-01 15:47:19 +01:00
try:
repo.git.remote('update', 'origin')
except Exception, e:
logger.error('Failed to update Git repo: %s', str(e))
2012-02-27 15:03:26 +01:00
def configure_branch(self):
'''
Ensures local tracking branch exists and is checkouted.
'''
repo = self.get_repo()
try:
head = repo.heads[self.branch]
except:
repo.git.branch('--track', self.branch, 'origin/%s' % self.branch)
head = repo.heads[self.branch]
repo.git.checkout(self.branch)
def update_branch(self):
'''
Updates current branch to match remote (if possible).
'''
repo = self.get_repo()
2012-03-01 16:10:08 +01:00
logger.info('pulling from remote repo %s', self.__unicode__())
2012-02-27 15:03:26 +01:00
repo.remotes.origin.pull()
try:
repo.git.merge('origin/%s' % self.branch)
2012-03-01 16:10:08 +01:00
logger.info('merged remote into repo %s', self.__unicode__())
2012-02-27 13:34:51 +01:00
except:
2012-02-27 15:03:26 +01:00
repo.git.merge('--abort')
2012-02-28 10:17:43 +01:00
logger.warning('failed merge on repo %s', self.__unicode__())
2012-02-27 15:03:26 +01:00
def get_translation_blobs(self):
'''
Scans directory for translation blobs and returns them as list.
'''
repo = self.get_repo()
tree = repo.tree()
# Glob files
prefix = os.path.join(self.get_path(), '')
2012-03-01 15:13:08 +01:00
for f in glob(os.path.join(self.get_path(), self.filemask)):
2012-03-01 15:12:11 +01:00
filename = f.replace(prefix, '')
yield (
self.get_lang_code(filename),
filename,
tree[filename]
)
2012-02-28 15:21:52 +01:00
def create_translations(self, force = False):
'''
Loads translations from git.
'''
2012-03-01 15:12:11 +01:00
for code, path, blob in self.get_translation_blobs():
2012-02-29 11:52:32 +01:00
logger.info('checking %s', path)
2012-02-28 15:21:52 +01:00
Translation.objects.update_from_blob(self, code, path, blob, force)
2012-02-27 16:09:31 +01:00
def get_lang_code(self, path):
'''
Parses language code from path.
'''
parts = self.filemask.split('*')
return path[len(parts[0]):-len(parts[1])]
2012-02-27 15:03:26 +01:00
def save(self, *args, **kwargs):
self.configure_repo()
self.configure_branch()
self.update_branch()
2012-02-27 13:34:51 +01:00
super(SubProject, self).save(*args, **kwargs)
2012-02-27 11:45:27 +01:00
self.create_translations()
2012-03-05 13:45:16 +01:00
def get_translated_percent(self):
translations = self.translation_set.aggregate(Sum('translated'), Sum('total'))
return round(translations['translated__sum'] * 100.0 / translations['total__sum'], 1)
2012-02-27 10:46:12 +01:00
class Translation(models.Model):
subproject = models.ForeignKey(SubProject)
language = models.ForeignKey(Language)
2012-02-27 16:09:31 +01:00
revision = models.CharField(max_length = 40, default = '', blank = True)
filename = models.CharField(max_length = 200)\
translated = models.IntegerField(default = 0, db_index = True)
fuzzy = models.IntegerField(default = 0, db_index = True)
total = models.IntegerField(default = 0, db_index = True)
2012-02-27 10:58:59 +01:00
2012-02-27 16:09:31 +01:00
objects = TranslationManager()
2012-02-28 10:11:06 +01:00
class Meta:
ordering = ['language__name']
def get_fuzzy_percent(self):
return round(self.fuzzy * 100.0 / self.total, 1)
def get_translated_percent(self):
return round(self.translated * 100.0 / self.total, 1)
2012-02-27 11:11:37 +01:00
@models.permalink
def get_absolute_url(self):
2012-02-28 10:32:07 +01:00
return ('trans.views.show_translation', (), {
'project': self.subproject.project.slug,
'subproject': self.subproject.slug,
'lang': self.language.code
})
2012-02-27 11:11:37 +01:00
2012-03-01 17:11:19 +01:00
@models.permalink
def get_download_url(self):
return ('trans.views.download_translation', (), {
'project': self.subproject.project.slug,
'subproject': self.subproject.slug,
'lang': self.language.code
})
2012-02-28 14:55:34 +01:00
@models.permalink
2012-02-28 14:50:20 +01:00
def get_translate_url(self):
2012-02-28 14:55:34 +01:00
return ('trans.views.translate', (), {
'project': self.subproject.project.slug,
'subproject': self.subproject.slug,
'lang': self.language.code
})
2012-02-28 14:50:20 +01:00
2012-02-27 11:45:27 +01:00
def __unicode__(self):
2012-03-02 15:31:52 +01:00
return '%s - %s' % (self.subproject.__unicode__(), _(self.language.name))
2012-02-27 11:45:27 +01:00
2012-03-01 17:11:52 +01:00
def get_filename(self):
return os.path.join(self.subproject.get_path(), self.filename)
def get_store(self):
2012-03-01 17:11:52 +01:00
return factory.getobject(self.get_filename())
def check_sync(self):
'''
Checks whether database is in sync with git and possibly does update.
'''
blob = self.get_git_blob()
self.update_from_blob(blob)
2012-02-28 15:21:52 +01:00
def update_from_blob(self, blob, force = False):
'''
Updates translation data from blob.
'''
# Check if we're not already up to date
2012-02-28 15:21:52 +01:00
if self.revision == blob.hexsha and not force:
return
2012-02-29 11:52:32 +01:00
logger.info('processing %s, revision has changed', self.filename)
oldunits = set(self.unit_set.all().values_list('id', flat = True))
2012-02-27 17:12:08 +01:00
# Load po file
store = self.get_store()
2012-02-28 15:12:41 +01:00
for pos, unit in enumerate(store.units):
2012-02-28 15:28:56 +01:00
if not unit.istranslatable():
2012-02-28 10:27:29 +01:00
continue
2012-02-28 15:12:41 +01:00
newunit = Unit.objects.update_from_unit(self, unit, pos)
try:
oldunits.remove(newunit.id)
except:
pass
# Delete not used units
Unit.objects.filter(translation = self, id__in = oldunits).delete()
2012-02-27 17:12:08 +01:00
2012-02-28 10:17:57 +01:00
# Update revision and stats
2012-02-29 11:48:16 +01:00
self.update_stats(blob)
def get_git_blob(self):
'''
Returns current Git blob for file.
'''
2012-02-29 11:50:11 +01:00
repo = self.subproject.get_repo()
2012-02-29 11:48:16 +01:00
tree = repo.tree()
return tree[self.filename]
2012-02-29 09:56:39 +01:00
2012-02-29 11:48:16 +01:00
def update_stats(self, blob = None):
if blob is None:
blob = self.get_git_blob()
self.total = self.unit_set.count()
self.fuzzy = self.unit_set.filter(fuzzy = True).count()
self.translated = self.unit_set.filter(translated = True).count()
2012-02-29 11:48:16 +01:00
self.revision = blob.hexsha
self.save()
2012-02-27 17:12:08 +01:00
2012-03-05 16:07:56 +01:00
def get_author_name(self, request):
full_name = request.user.get_full_name()
if full_name == '':
full_name = request.user.username
return '%s <%s>' % (full_name, request.user.email)
2012-02-28 14:30:09 +01:00
def git_commit(self, author):
'''
Commits translation to git.
'''
repo = self.subproject.get_repo()
status = repo.git.status('--porcelain', '--', self.filename)
if status == '':
# No changes to commit
return False
2012-02-28 14:29:24 +01:00
logger.info('Commiting %s as %s', self.filename, author)
repo.git.commit(
2012-02-28 14:04:21 +01:00
self.filename,
2012-02-28 14:29:24 +01:00
author = author,
2012-03-01 16:19:21 +01:00
m = settings.COMMIT_MESSAGE
2012-02-28 14:04:21 +01:00
)
return True
2012-02-28 14:22:11 +01:00
def update_unit(self, unit, request):
'''
Updates backend file and unit.
'''
store = self.get_store()
src = unit.get_source_plurals()[0]
2012-02-28 14:29:31 +01:00
need_save = False
2012-03-01 15:13:36 +01:00
for pounit in store.findunits(src):
if pounit.getcontext() == unit.context:
if hasattr(pounit.target, 'strings'):
potarget = join_plural(pounit.target.strings)
else:
potarget = pounit.target
if unit.target != potarget or unit.fuzzy != pounit.isfuzzy():
2012-02-28 14:36:51 +01:00
pounit.markfuzzy(unit.fuzzy)
2012-02-28 14:22:11 +01:00
if unit.is_plural():
pounit.settarget(unit.get_target_plurals())
else:
pounit.settarget(unit.target)
need_save = True
# We should have only one match
break
if need_save:
2012-03-05 16:07:56 +01:00
author = self.get_author_name(request)
if hasattr(store, 'updateheader'):
po_revision_date = datetime.now().strftime('%Y-%m-%d %H:%M') + poheader.tzstring()
store.updateheader(
add = True,
last_translator = author,
plural_forms = self.language.get_plural_form(),
language = self.language.code,
PO_Revision_Date = po_revision_date,
x_generator = 'Weblate %s' % trans.VERSION
)
store.save()
2012-02-28 14:30:09 +01:00
self.git_commit(author)
2012-02-28 14:35:31 +01:00
return need_save, pounit
2012-02-28 14:50:20 +01:00
def get_checks(self):
'''
Returns list of failing checks on current translation.
'''
2012-03-02 17:06:20 +01:00
result = [('all', _('All strings'))]
2012-02-28 15:00:23 +01:00
nottranslated = self.unit_set.filter_type('untranslated').count()
fuzzy = self.unit_set.filter_type('fuzzy').count()
2012-03-01 14:03:23 +01:00
suggestions = self.unit_set.filter_type('suggestions').count()
2012-02-28 14:50:20 +01:00
if nottranslated > 0:
result.append(('untranslated', _('Not translated strings (%d)') % nottranslated))
if fuzzy > 0:
result.append(('fuzzy', _('Fuzzy strings (%d)') % fuzzy))
2012-03-01 14:03:23 +01:00
if suggestions > 0:
result.append(('suggestions', _('Strings with suggestions (%d)') % suggestions))
2012-02-28 14:50:20 +01:00
return result
def merge_upload(self, request, fileobj, overwrite, mergefuzzy = False):
2012-03-05 10:31:11 +01:00
# Needed to behave like something what translate toolkit expects
fileobj.mode = "r"
store2 = factory.getobject(fileobj)
store1 = self.get_store()
2012-03-05 10:31:11 +01:00
store1.require_index()
for unit2 in store2.units:
if unit2.isheader():
2012-03-05 10:31:11 +01:00
if isinstance(store1, poheader.poheader):
store1.mergeheaders(store2)
continue
unit1 = store1.findid(unit2.getid())
if unit1 is None:
unit1 = store1.findunit(unit2.source)
if unit1 is None:
logger.error("The template does not contain the following unit:\n%s", str(unit2))
else:
if len(unit2.target.strip()) == 0:
continue
if not mergefuzzy:
if unit2.isfuzzy():
continue
unit1.merge(unit2, overwrite=overwrite)
store1.save()
2012-03-05 16:07:56 +01:00
author = self.get_author_name(request)
ret = self.git_commit(author)
2012-03-04 10:21:07 +01:00
self.check_sync()
return ret
2012-02-28 14:50:20 +01:00
2012-02-27 10:58:59 +01:00
class Unit(models.Model):
translation = models.ForeignKey(Translation)
2012-02-28 11:24:53 +01:00
checksum = models.CharField(max_length = 40, default = '', blank = True, db_index = True)
location = models.TextField(default = '', blank = True)
context = models.TextField(default = '', blank = True)
2012-02-28 13:25:06 +01:00
comment = models.TextField(default = '', blank = True)
flags = models.TextField(default = '', blank = True)
source = models.TextField()
target = models.TextField(default = '', blank = True)
2012-02-28 11:24:53 +01:00
fuzzy = models.BooleanField(default = False, db_index = True)
translated = models.BooleanField(default = False, db_index = True)
2012-02-28 15:12:41 +01:00
position = models.IntegerField(db_index = True)
objects = UnitManager()
2012-02-28 15:12:41 +01:00
class Meta:
ordering = ['position']
def update_from_unit(self, unit, pos, force):
location = ', '.join(unit.getlocations())
if hasattr(unit, 'typecomments'):
flags = ', '.join(unit.typecomments)
else:
flags = ''
if hasattr(unit.target, 'strings'):
target = join_plural(unit.target.strings)
else:
target = unit.target
fuzzy = unit.isfuzzy()
2012-02-28 10:13:43 +01:00
translated = unit.istranslated()
2012-02-28 13:25:06 +01:00
comment = unit.getnotes()
2012-02-28 15:12:41 +01:00
if not force and location == self.location and flags == self.flags and target == self.target and fuzzy == self.fuzzy and translated == self.translated and comment == self.comment and pos == self.position:
return
2012-02-28 15:12:41 +01:00
self.position = pos
self.location = location
self.flags = flags
self.target = target
self.fuzzy = fuzzy
2012-02-28 10:13:43 +01:00
self.translated = translated
2012-02-28 13:25:06 +01:00
self.comment = comment
2012-02-28 15:26:23 +01:00
self.save(force_insert = force, backend = True)
2012-02-27 11:08:16 +01:00
def is_plural(self):
2012-02-27 17:54:14 +01:00
return is_plural(self.source)
2012-02-27 11:08:16 +01:00
def get_source_plurals(self):
2012-02-27 17:54:14 +01:00
return split_plural(self.source)
2012-02-27 11:08:16 +01:00
def get_target_plurals(self):
if not self.is_plural():
return self.target
2012-02-27 17:54:14 +01:00
ret = split_plural(self.target)
2012-02-27 11:08:16 +01:00
plurals = self.translation.language.nplurals
if len(ret) == plurals:
return ret
while len(ret) < plurals:
ret.append('')
while len(ret) > plurals:
del(ret[-1])
return ret
def save_backend(self, request, propagate = True):
# Store to backend
2012-02-28 14:35:31 +01:00
(saved, pounit) = self.translation.update_unit(self, request)
self.translated = pounit.istranslated()
2012-03-01 16:32:11 +01:00
if hasattr(pounit, 'typecomments'):
self.flags = ', '.join(pounit.typecomments)
else:
self.flags = ''
2012-02-28 14:22:11 +01:00
self.save(backend = True)
2012-02-29 09:57:04 +01:00
self.translation.update_stats()
# Propagate to other projects
if propagate:
allunits = Unit.objects.filter(
2012-02-28 14:22:11 +01:00
checksum = self.checksum,
translation__subproject__project = self.translation.subproject.project,
2012-02-28 14:27:11 +01:00
translation__language = self.translation.language
2012-02-28 14:26:04 +01:00
).exclude(id = self.id)
for unit in allunits:
unit.target = self.target
unit.save_backend(request, False)
def save(self, *args, **kwargs):
2012-02-28 14:22:11 +01:00
if not 'backend' in kwargs:
logger.error('Unit.save called without backend sync: %s', ''.join(traceback.format_stack()))
else:
del kwargs['backend']
super(Unit, self).save(*args, **kwargs)
2012-02-28 16:01:07 +01:00
def get_location_links(self):
ret = []
2012-03-05 14:32:37 +01:00
if len(self.location) == 0:
return ''
2012-02-28 16:01:07 +01:00
for location in self.location.split(','):
location = location.strip()
filename, line = location.split(':')
2012-02-29 10:03:37 +01:00
link = self.translation.subproject.get_repoweb_link(filename, line)
2012-02-28 16:01:07 +01:00
ret.append('<a href="%s">%s</a>' % (link, location))
return mark_safe('\n'.join(ret))
2012-02-29 15:29:10 +01:00
def suggestions(self):
2012-03-01 09:59:32 +01:00
return Suggestion.objects.filter(
checksum = self.checksum,
2012-03-01 10:01:04 +01:00
project = self.translation.subproject.project,
language = self.translation.language
2012-03-01 09:59:32 +01:00
)
2012-02-29 15:29:10 +01:00
2012-02-29 15:26:57 +01:00
class Suggestion(models.Model):
checksum = models.CharField(max_length = 40, default = '', blank = True, db_index = True)
2012-02-29 15:27:23 +01:00
target = models.TextField()
2012-02-29 15:26:57 +01:00
user = models.ForeignKey(User, null = True, blank = True)
2012-03-01 09:59:32 +01:00
project = models.ForeignKey(Project)
2012-03-01 10:01:04 +01:00
language = models.ForeignKey(Language)
2012-03-01 10:03:01 +01:00
def accept(self, request):
allunits = Unit.objects.filter(
checksum = self.checksum,
translation__subproject__project = self.project,
translation__language = self.language
)
for unit in allunits:
unit.target = self.target
unit.fuzzy = False
unit.save_backend(request, False)