from django.db import models from django.db.models import Q from django.contrib.auth.models import User from django.conf import settings from lang.models import Language from django.utils.translation import ugettext_lazy, ugettext as _ from django.utils.safestring import mark_safe from glob import glob import os import os.path import logging import git import traceback from translate.storage import factory from translate.storage import poheader from datetime import datetime import trans from trans.managers import TranslationManager, UnitManager from util import is_plural, split_plural, join_plural logger = logging.getLogger('weblate') class Project(models.Model): name = models.CharField(max_length = 100) slug = models.SlugField(db_index = True) web = models.URLField() mail = models.EmailField(blank = True) instructions = models.URLField(blank = True) class Meta: ordering = ['name'] @models.permalink def get_absolute_url(self): return ('trans.views.show_project', (), { 'project': self.slug }) def get_path(self): return os.path.join(settings.GIT_ROOT, self.slug) def __unicode__(self): return self.name def save(self, *args, **kwargs): # Create filesystem directory for storing data p = self.get_path() if not os.path.exists(p): os.makedirs(p) super(Project, self).save(*args, **kwargs) class SubProject(models.Model): name = models.CharField(max_length = 100, help_text = _('Name to display')) slug = models.SlugField(db_index = True, help_text = _('Name used in URLs')) project = models.ForeignKey(Project) repo = models.CharField(max_length = 200, help_text = _('URL of Git repository')) repoweb = models.URLField(help_text = _('Link to repository browser, use %(file)s and %(line)s as filename and line placeholders')) branch = models.CharField(max_length = 50, help_text = _('Git branch to translate')) filemask = models.CharField(max_length = 200, help_text = _('Mask of files to translate, use * istead of language code')) class Meta: ordering = ['name'] @models.permalink def get_absolute_url(self): return ('trans.views.show_subproject', (), { 'project': self.project.slug, 'subproject': self.slug }) def __unicode__(self): return '%s/%s' % (self.project.__unicode__(), self.name) def get_path(self): return os.path.join(self.project.get_path(), self.slug) def get_repo(self): ''' Gets Git repository object. ''' p = self.get_path() try: return git.Repo(p) except: return git.Repo.init(p) def get_repoweb_link(self, filename, line): return self.repoweb % {'file': filename, 'line': line} def configure_repo(self): ''' Ensures repository is correctly configured and points to current remote. ''' # Create/Open repo repo = self.get_repo() # Get/Create origin remote try: origin = repo.remotes.origin except: repo.git.remote('add', 'origin', self.repo) origin = repo.remotes.origin # Check remote source if origin.url != self.repo: repo.git.remote('set-url', 'origin', self.repo) # Update logger.info('updating repo %s', self.__unicode__()) try: repo.git.remote('update', 'origin') except Exception, e: logger.error('Failed to update Git repo: %s', str(e)) def configure_branch(self): ''' Ensures local tracking branch exists and is checkouted. ''' repo = self.get_repo() try: head = repo.heads[self.branch] except: repo.git.branch('--track', self.branch, 'origin/%s' % self.branch) head = repo.heads[self.branch] repo.git.checkout(self.branch) def update_branch(self): ''' Updates current branch to match remote (if possible). ''' repo = self.get_repo() logger.info('pulling from remote repo %s', self.__unicode__()) repo.remotes.origin.pull() try: repo.git.merge('origin/%s' % self.branch) logger.info('merged remote into repo %s', self.__unicode__()) except: repo.git.merge('--abort') logger.warning('failed merge on repo %s', self.__unicode__()) def get_translation_blobs(self): ''' Scans directory for translation blobs and returns them as list. ''' repo = self.get_repo() tree = repo.tree() # Glob files prefix = os.path.join(self.get_path(), '') for f in glob(os.path.join(self.get_path(), self.filemask)): filename = f.replace(prefix, '') yield ( self.get_lang_code(filename), filename, tree[filename] ) def create_translations(self, force = False): ''' Loads translations from git. ''' for code, path, blob in self.get_translation_blobs(): logger.info('checking %s', path) Translation.objects.update_from_blob(self, code, path, blob, force) def get_lang_code(self, path): ''' Parses language code from path. ''' parts = self.filemask.split('*') return path[len(parts[0]):-len(parts[1])] def save(self, *args, **kwargs): self.configure_repo() self.configure_branch() self.update_branch() super(SubProject, self).save(*args, **kwargs) self.create_translations() class Translation(models.Model): subproject = models.ForeignKey(SubProject) language = models.ForeignKey(Language) revision = models.CharField(max_length = 40, default = '', blank = True) filename = models.CharField(max_length = 200)\ translated = models.IntegerField(default = 0, db_index = True) fuzzy = models.IntegerField(default = 0, db_index = True) total = models.IntegerField(default = 0, db_index = True) objects = TranslationManager() class Meta: ordering = ['language__name'] def get_fuzzy_percent(self): return round(self.fuzzy * 100.0 / self.total, 1) def get_translated_percent(self): return round(self.translated * 100.0 / self.total, 1) @models.permalink def get_absolute_url(self): return ('trans.views.show_translation', (), { 'project': self.subproject.project.slug, 'subproject': self.subproject.slug, 'lang': self.language.code }) @models.permalink def get_download_url(self): return ('trans.views.download_translation', (), { 'project': self.subproject.project.slug, 'subproject': self.subproject.slug, 'lang': self.language.code }) @models.permalink def get_translate_url(self): return ('trans.views.translate', (), { 'project': self.subproject.project.slug, 'subproject': self.subproject.slug, 'lang': self.language.code }) def __unicode__(self): return '%s - %s' % (self.subproject.__unicode__(), _(self.language.name)) def get_filename(self): return os.path.join(self.subproject.get_path(), self.filename) def get_store(self): return factory.getobject(self.get_filename()) def check_sync(self): ''' Checks whether database is in sync with git and possibly does update. ''' blob = self.get_git_blob() self.update_from_blob(blob) def update_from_blob(self, blob, force = False): ''' Updates translation data from blob. ''' # Check if we're not already up to date if self.revision == blob.hexsha and not force: return logger.info('processing %s, revision has changed', self.filename) oldunits = set(self.unit_set.all().values_list('id', flat = True)) # Load po file store = self.get_store() for pos, unit in enumerate(store.units): if not unit.istranslatable(): continue newunit = Unit.objects.update_from_unit(self, unit, pos) try: oldunits.remove(newunit.id) except: pass # Delete not used units Unit.objects.filter(translation = self, id__in = oldunits).delete() # Update revision and stats self.update_stats(blob) def get_git_blob(self): ''' Returns current Git blob for file. ''' repo = self.subproject.get_repo() tree = repo.tree() return tree[self.filename] def update_stats(self, blob = None): if blob is None: blob = self.get_git_blob() self.total = self.unit_set.count() self.fuzzy = self.unit_set.filter(fuzzy = True).count() self.translated = self.unit_set.filter(translated = True).count() self.revision = blob.hexsha self.save() def git_commit(self, author): ''' Commits translation to git. ''' repo = self.subproject.get_repo() logger.info('Commiting %s as %s', self.filename, author) repo.git.commit( self.filename, author = author, m = settings.COMMIT_MESSAGE ) def update_unit(self, unit, request): ''' Updates backend file and unit. ''' store = self.get_store() src = unit.get_source_plurals()[0] need_save = False for pounit in store.findunits(src): if pounit.getcontext() == unit.context: if hasattr(pounit.target, 'strings'): potarget = join_plural(pounit.target.strings) else: potarget = pounit.target if unit.target != potarget or unit.fuzzy != pounit.isfuzzy(): pounit.markfuzzy(unit.fuzzy) if unit.is_plural(): pounit.settarget(unit.get_target_plurals()) else: pounit.settarget(unit.target) need_save = True # We should have only one match break if need_save: author = '%s <%s>' % (request.user.get_full_name(), request.user.email) if hasattr(store, 'updateheader'): po_revision_date = datetime.now().strftime('%Y-%m-%d %H:%M') + poheader.tzstring() store.updateheader( add = True, last_translator = author, plural_forms = self.language.get_plural_form(), language = self.language.code, PO_Revision_Date = po_revision_date, x_generator = 'Weblate %s' % trans.VERSION ) store.save() self.git_commit(author) return need_save, pounit def get_checks(self): ''' Returns list of failing checks on current translation. ''' result = [('all', _('All strings'))] nottranslated = self.unit_set.filter_type('untranslated').count() fuzzy = self.unit_set.filter_type('fuzzy').count() suggestions = self.unit_set.filter_type('suggestions').count() if nottranslated > 0: result.append(('untranslated', _('Not translated strings (%d)') % nottranslated)) if fuzzy > 0: result.append(('fuzzy', _('Fuzzy strings (%d)') % fuzzy)) if suggestions > 0: result.append(('suggestions', _('Strings with suggestions (%d)') % suggestions)) return result def merge_upload(self, request, fileobj, overwrite, mergefuzzy = False): store2 = factory.getobjects(fileobj) store1 = self.get_store() store.require_index() for unit2 in store2.units: if unit2.isheader(): if isinstance(store1, poheader): store1.mergeheaders(store2) continue unit1 = store1.findid(unit2.getid()) if unit1 is None: unit1 = store1.findunit(unit2.source) if unit1 is None: logger.error("The template does not contain the following unit:\n%s", str(unit2)) else: if len(unit2.target.strip()) == 0: continue if not mergefuzzy: if unit2.isfuzzy(): continue unit1.merge(unit2, overwrite=overwrite) store1.save() author = '%s <%s>' % (request.user.get_full_name(), request.user.email) self.git_commit(author) self.check_sync() class Unit(models.Model): translation = models.ForeignKey(Translation) checksum = models.CharField(max_length = 40, default = '', blank = True, db_index = True) location = models.TextField(default = '', blank = True) context = models.TextField(default = '', blank = True) comment = models.TextField(default = '', blank = True) flags = models.TextField(default = '', blank = True) source = models.TextField() target = models.TextField(default = '', blank = True) fuzzy = models.BooleanField(default = False, db_index = True) translated = models.BooleanField(default = False, db_index = True) position = models.IntegerField(db_index = True) objects = UnitManager() class Meta: ordering = ['position'] def update_from_unit(self, unit, pos, force): location = ', '.join(unit.getlocations()) if hasattr(unit, 'typecomments'): flags = ', '.join(unit.typecomments) else: flags = '' if hasattr(unit.target, 'strings'): target = join_plural(unit.target.strings) else: target = unit.target fuzzy = unit.isfuzzy() translated = unit.istranslated() comment = unit.getnotes() if not force and location == self.location and flags == self.flags and target == self.target and fuzzy == self.fuzzy and translated == self.translated and comment == self.comment and pos == self.position: return self.position = pos self.location = location self.flags = flags self.target = target self.fuzzy = fuzzy self.translated = translated self.comment = comment self.save(force_insert = force, backend = True) def is_plural(self): return is_plural(self.source) def get_source_plurals(self): return split_plural(self.source) def get_target_plurals(self): if not self.is_plural(): return self.target ret = split_plural(self.target) plurals = self.translation.language.nplurals if len(ret) == plurals: return ret while len(ret) < plurals: ret.append('') while len(ret) > plurals: del(ret[-1]) return ret def save_backend(self, request, propagate = True): # Store to backend (saved, pounit) = self.translation.update_unit(self, request) self.translated = pounit.istranslated() if hasattr(pounit, 'typecomments'): self.flags = ', '.join(pounit.typecomments) else: self.flags = '' self.save(backend = True) self.translation.update_stats() # Propagate to other projects if propagate: allunits = Unit.objects.filter( checksum = self.checksum, translation__subproject__project = self.translation.subproject.project, translation__language = self.translation.language ).exclude(id = self.id) for unit in allunits: unit.target = self.target unit.save_backend(request, False) def save(self, *args, **kwargs): if not 'backend' in kwargs: logger.error('Unit.save called without backend sync: %s', ''.join(traceback.format_stack())) else: del kwargs['backend'] super(Unit, self).save(*args, **kwargs) def get_location_links(self): ret = [] for location in self.location.split(','): location = location.strip() filename, line = location.split(':') link = self.translation.subproject.get_repoweb_link(filename, line) ret.append('%s' % (link, location)) return mark_safe('\n'.join(ret)) def suggestions(self): return Suggestion.objects.filter( checksum = self.checksum, project = self.translation.subproject.project, language = self.translation.language ) class Suggestion(models.Model): checksum = models.CharField(max_length = 40, default = '', blank = True, db_index = True) target = models.TextField() user = models.ForeignKey(User, null = True, blank = True) project = models.ForeignKey(Project) language = models.ForeignKey(Language) def accept(self, request): allunits = Unit.objects.filter( checksum = self.checksum, translation__subproject__project = self.project, translation__language = self.language ) for unit in allunits: unit.target = self.target unit.fuzzy = False unit.save_backend(request, False)