-
Notifications
You must be signed in to change notification settings - Fork 0
/
models.py
3083 lines (2859 loc) · 143 KB
/
models.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
# -*- coding: utf-8 -*-"""
"""
Django models for wip application of wip project.
For more information on this file, see
https://docs.djangoproject.com/en/1.9/topics/db/models/
"""
import sys
if (sys.version_info > (3, 0)):
# Python 3 code in this block
from importlib import reload
import urllib.request as urllib2
from wip.aligner import split_alignment, split_normalize_alignment, normalized_alignment, proxy_symmetrize_alignments, proxy_eflomal_align
from io import StringIO
else:
reload(sys)
sys.setdefaultencoding('utf8')
import urllib2
import dill # required to pickle lambda functions
import pickle
# import StringIO
from StringIO import StringIO
import logging
logger = logging.getLogger('wip')
# from collections import defaultdict
import os
import json
import time
import copy
import re, regex
from collections import defaultdict
import difflib
from difflib import SequenceMatcher
import Levenshtein
# from Levenshtein.StringMatcher import StringMatcher
from lxml import html, etree
from nltk.translate import AlignedSent, Alignment, IBMModel2, IBMModel3
from django.conf import settings
from django.db import models
from django.db.models import Q
from django.db.models.expressions import RawSQL
from django.shortcuts import get_object_or_404
from django.utils import timezone
from django.utils.translation import ugettext_lazy as _
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericForeignKey
from django.utils.translation import ugettext_lazy as _
from django_extensions.db.fields import CreationDateTimeField, ModificationDateTimeField, AutoSlugField
from django_dag.models import node_factory, edge_factory
from django_diazo.models import Theme
# from django_diazo.middleware import DjangoDiazoMiddleware
from wip.wip_nltk.tokenizers import NltkTokenizer
from wip.wip_sd.sd_algorithm import SDAlgorithm
from wip.lineardoc.Doc import Doc
from wip.lineardoc.Parser import LineardocParse
from wip.lineardoc.TextBlock import TextBlock, mergeSentences
from wip.lineardoc.Utils import addCommonTag, getOpenTagHtml, getCloseTagHtml
from .vocabularies import Language, Subject, ApprovalStatus
from .aligner import tokenize, best_alignment, aer
from .settings import RESOURCES_ROOT, BLOCK_TAGS, BLOCKS_EXCLUDE_BY_XPATH, EMPTY_WORDS, BOTH_QUOTES
DEFAULT_USER = 1
from .session import get_userrole, set_userrole
from .utils import element_tostring, text_from_html, strings_from_html, elements_from_element, replace_element_content, element_signature
from .utils import compact_spaces, strip_html_comments, normalize_string, replace_segment, string_checksum, text_to_list # , non_invariant_words
from .utils import is_invariant_word as is_base_invariant_word
from .utils import get_segmenter_rules, make_segmenter
from .xliff import XLFFile, File
# import wip.srx_segmenter
import wip.srx_segmenter as srx_segmenter
UNKNOWN = NONE = 0
GOOGLE = 1
DEEPL = 2
MICROSOFT = 3
MATECAT = 4
MYMEMORY = 5
TRANSLATION_SERVICE_CHOICES = (
(NONE, ''),
(GOOGLE, _('Google')), # GoogleTranslate
(DEEPL, _('DeepL')),
(MICROSOFT, _('Microsoft')), # Microsoft Translator
(MYMEMORY, _('MyMemory')),
# (MATECAT, _('Matecat')),
)
TRANSLATION_SERVICE_DICT = dict(TRANSLATION_SERVICE_CHOICES)
TRANSLATION_SERVICE_CODE_DICT = {
None: ' ',
NONE: ' ',
GOOGLE: 'GT',
DEEPL: 'DL',
MICROSOFT: 'MT',
}
"""
The vocabulary TRANSLATION_STATE_CHOICES is shared by TranslatedVersion, Block and TranslatedBlock
but only a subset of the entries is valid for each model:
- TranslatedVersion (target language cache) currently aren't used
- Block currently are all in state 0 (NONE), but a small number in state 4 (INVARIANT);
state 5 (ALREADY) can be only a transient value computed based on target language
- TranslatedBlock ususally are created in state 2 (TRANSLATED),
but automatic pre-translation creates some of them in state 1 (PARTIALLY).
Besides that, it provides symbolic names for literal values used in function calls.
Related notes:
- Webpage - no_translate==True can be thought as state==INVARIANT
- Segment - is_invariant==True can be thought as state==INVARIANT
- Translation - info on translation state is conveyed by role and date
"""
ANY = 0
PARTIALLY = 1
TRANSLATED = 2
REVISED = 3
INVARIANT = 4
ALREADY = 5
# TO_BE_TRANSLATED = -1
TO_BE_TRANSLATED = -TRANSLATED
TO_BE_REVISED = -REVISED
TRANSLATION_STATE_CHOICES = (
(ANY, _('any'),),
(TO_BE_TRANSLATED, _('to be translated'),),
(TRANSLATED, _('translated'),),
(TO_BE_REVISED, _('to be revised'),),
(REVISED, _('revised'),),
(INVARIANT, _('invariant'),),
(ALREADY, _('in target language'),),
)
TRANSLATION_STATE_DICT = dict(TRANSLATION_STATE_CHOICES)
TRANSLATION_EXPORT_CHOICES = (
(ANY, _('all'),),
(TO_BE_TRANSLATED, _('missing'),),
(TRANSLATED, _('existing'),),
)
BLOCK_TRANSLATION_STATE_CHOICES = (
(ANY, _('any'),),
(TO_BE_TRANSLATED, _('to be translated'),),
(PARTIALLY, _('partially translated'),),
(TRANSLATED, _('translated'),),
(INVARIANT, _('invariant'),),
(ALREADY, _('in target language'),),
)
TEXT_ASC = 1
ID_ASC = 2
DATETIME_DESC = -3
DATETIME_ASC = 3
COUNT_DESC = -4
SEGMENT_SORT_CHOICES = (
(ID_ASC, _('id'),),
(TEXT_ASC, _('text'),),
(COUNT_DESC, _('count'),),
)
SEGMENT_SORT_DICT = dict(SEGMENT_SORT_CHOICES)
PARALLEL_FORMAT_NONE = 0
PARALLEL_FORMAT_XLIFF = 1
PARALLEL_FORMAT_TEXT = 2
APACHE = 1
NGINX = 2
SERVER_TYPE_CHOICES = (
(NONE, _('unknown'),),
(APACHE, _('Apache'),),
(NGINX, _('Nginx'),),
)
SERVER_TYPE_DICT = dict(SERVER_TYPE_CHOICES)
def is_linearblock_translated(block):
return block.hasCommonTag('span', attrs=['tx'])
TextBlock.isTranslated = is_linearblock_translated
def code_to_language(code):
return Language.objects.get(pk=code)
class Site(models.Model):
name = models.CharField(max_length=100)
slug = AutoSlugField(unique=True, populate_from='name', editable=True)
# language = models.ForeignKey(Language, null=True)
language = models.ForeignKey(Language, null=True, related_name='language_site')
path_prefix = models.CharField(max_length=20, default='')
url = models.CharField(max_length=100)
http_server = models.IntegerField(choices=SERVER_TYPE_CHOICES, null=True, default=NONE, verbose_name='http server type')
allowed_domains = models.TextField()
start_urls = models.TextField()
deny = models.TextField(verbose_name='Deny spyder', blank=True, null=True, help_text="Paths the spider should not follow")
extract_deny = models.TextField(verbose_name='Deny extractor', blank=True, null=True, help_text="Paths of pages the string extractor should skip" )
translate_deny = models.TextField(verbose_name='Deny translation', blank=True, null=True, help_text="Paths of pages not to be submitted to offline translation" )
checksum_deny = models.TextField(verbose_name='Exclude from checksum', blank=True, null=True, help_text="Patterns identifying lines in body not affecting page checksum" )
srx_rules = models.TextField(verbose_name='Custom SRX rules', blank=True, null=True, help_text="Custom SRX rules extending the standard set" )
srx_initials = models.TextField(verbose_name='Custom initials', blank=True, null=True, help_text="Initials to be made explicit as SRX rules" )
invariant_words = models.TextField(verbose_name='Custom invariant words', blank=True, null=True, help_text="Custom invariant words" )
themes = models.ManyToManyField(Theme, through='SiteTheme', related_name='site', blank=True, verbose_name='diazo themes')
variable_regions = models.TextField('Variable content regions', blank=True, null=True, help_text="Path-xpath combinations identifying page regions with frequently changing content" )
last_crawled = models.DateTimeField(null=True, blank=True, help_text="Last time the site was crawled")
last_fetched = models.DateTimeField(null=True, blank=True, help_text="Last time new versions of site pages were fetched")
last_block_extraction = models.DateTimeField(null=True, blank=True, help_text="Last time blocks were extracted from site pages")
last_segment_extraction = models.DateTimeField(null=True, blank=True, help_text="Last time segments were extracted from site pages")
extra_body = models.TextField(verbose_name='Extra body', blank=True, null=True, help_text="Code to be inserted before closing BODY tag")
client_key = models.CharField(max_length=20, blank=True, default='')
active = models.BooleanField(default=True)
def __init__(self, *args, **kwargs):
super(Site, self).__init__(*args, **kwargs)
self.segmenter = None
def __str__(self):
return self.name
def __unicode__(self):
return self.__str__()
def get_absolute_url(self):
return '/site/%s/' % self.slug
def get_filepath(self):
return os.path.join(settings.SITES_ROOT, self.slug)
def can_manage(self, user):
return user.is_superuser or UserRole.objects.filter(user=user, site=self, role_type__in=[OWNER, MANAGER]).count()
def can_operate(self, user):
return user.is_superuser
def can_view(self, current_role):
if not current_role:
return False
return current_role.role_type==ADMINISTRATOR or current_role.site == self or current_role.source_language == self.language
def get_allowed_domains(self):
return text_to_list(self.allowed_domains)
def get_start_urls(self):
return text_to_list(self.start_urls)
def get_deny(self):
# return ','.join(text_to_list(self.deny))
return text_to_list(self.deny)
def get_proxies(self):
return Proxy.objects.filter(site=self).order_by('language__code')
def get_proxy_languages(self):
return Language.objects.filter(language_proxy__site=self)
class Meta:
verbose_name = _('project')
verbose_name_plural = _('projects')
def make_segmenter(self, verbose=False, force=False):
if self.segmenter and not force:
if verbose:
print ('segmenter already exists')
return self.segmenter
language_code = self.language.code
current_rules = get_segmenter_rules(language_code)
if self.srx_initials:
custom_rules_list = text_to_list(self.srx_initials)
if len(custom_rules_list) == 1:
# word boundary followed by abbreviation (1)
beforebreak_text = "\\b%s" % re.escape(custom_rules_list[0])
else:
# word boundary followed by abbreviation (+)
beforebreak_text = "\\b(%s)" % '|'.join([re.escape(item) for item in custom_rules_list])
# blankspace or comma (non terminating chars)
afterbreak_text = "[\s,]"
non_breaks = current_rules['non_breaks']
non_break = (beforebreak_text, afterbreak_text)
non_breaks.append(non_break)
current_rules['non_breaks'] = non_breaks
if verbose:
for item in non_breaks:
try:
print (item)
except:
pass
if self.srx_rules:
non_breaks = current_rules['non_breaks']
custom_rules_list = text_to_list(self.srx_rules)
for item in custom_rules_list:
beforebreak_text, afterbreak_text = item.split(' ')
non_breaks.append((beforebreak_text, afterbreak_text))
if verbose:
print (beforebreak_text, afterbreak_text)
current_rules['non_breaks'] = non_breaks
# return srx_segmenter.SrxSegmenter(current_rules)
# self.segmenter = srx_segmenter.SrxSegmenter(current_rules)
self.segmenter = srx_segmenter.SrxSegmenter(rule=current_rules)
return self.segmenter
def make_tokenizer(self, return_matches=False):
""" create a tokenizer for the site language, with a list of custom regular expressions corresponding to
acronyms and abbreviations, being derived from the site configuration parameter used for segmentation """
custom_regexps = []
if self.srx_initials:
# custom_regexps = text_to_list(self.srx_initials)
for item in self.srx_initials.splitlines():
item = item.strip()
if not item:
continue
item = item.replace('.', '\\.')
custom_regexps.append(item)
if item[0].islower():
custom_regexps.append(item[0].upper() + item[1:])
# print ('custom_regexps:', custom_regexps)
return NltkTokenizer(language_code=self.language.code, custom_regexps=custom_regexps, lowercasing=False, return_matches=return_matches)
def pages_summary(self):
pages = Webpage.objects.filter(site=self).order_by('path')
proxy_codes = [proxy.language_id for proxy in Proxy.objects.filter(site=self)]
proxy_list = [[code, {'partially': 0, 'translated': 0, 'revised': 0}] for code in proxy_codes]
proxy_dict = dict(proxy_list)
total = pages.count()
invariant = 0
for page in pages:
if page.no_translate:
invariant +=1
continue
translations = TranslatedVersion.objects.filter(webpage=page)
for translation in translations:
code = translation.language_id
if translation.state == 3:
proxy_dict[code]['revised'] += 1
elif translation.state == 2:
proxy_dict[code]['translated'] += 1
else:
proxy_dict[code]['partially'] += 1
sorted_list = sorted(proxy_dict.items())
proxy_list = []
for code, t_dict in sorted_list:
language = code_to_language(code)
left = total - invariant
for value in t_dict.values():
left -= value
t_dict['left'] = left
proxy_list.append([language, t_dict])
return pages, total, invariant, proxy_list
def blocks_summary(self):
last_date = self.last_block_extraction or self.last_fetched or self.last_crawled or timezone.now()
# blocks = Block.objects.filter(Q(block_in_page__webpage__site=self) | Q(last_seen__gt=last_date)).distinct().order_by('checksum', '-time')
blocks = Block.objects.filter(site=self)
proxy_codes = [proxy.language_id for proxy in Proxy.objects.filter(site=self)]
proxy_list = [[code, {'already': 0, 'partially': 0, 'translated': 0, 'revised': 0}] for code in proxy_codes]
proxy_dict = dict(proxy_list)
total = blocks.count()
invariant = 0
for block in blocks:
if block.no_translate:
invariant +=1
continue
if block.language_id in proxy_codes:
proxy_dict[block.language_id]['already'] += 1
continue
translations = TranslatedBlock.objects.filter(block=block)
for translation in translations:
code = translation.language_id
if translation.state == 3:
proxy_dict[code]['revised'] += 1
elif translation.state == 2:
proxy_dict[code]['translated'] += 1
else:
proxy_dict[code]['partially'] += 1
sorted_list = sorted(proxy_dict.items())
proxy_list = []
for code, t_dict in sorted_list:
language = code_to_language(code)
left = total - invariant
for value in t_dict.values():
left -= value
t_dict['left'] = left
proxy_list.append([language, t_dict])
return blocks, total, invariant, proxy_list
def get_blocks_in_use(self):
# return Block.objects.filter(block_in_page__webpage__site=self).distinct()
last_date = self.last_block_extraction or self.last_fetched or self.last_crawled or timezone.now()
return Block.objects.filter(Q(block_in_page__webpage__site=self) | Q(last_seen__gt=last_date)).distinct()
def page_checksum(self, body):
deny_list = text_to_list(self.checksum_deny)
if deny_list:
l_in = body.splitlines()
l_out = []
for l in l_in:
should_skip = False
for deny_pattern in deny_list:
if l.count(deny_pattern):
should_skip = True
break
if should_skip:
continue
l_out.append(l)
body = '\n'.join(l_out)
return string_checksum(body.encode())
"""
def fetch_page(self, path, webpage=None, extract_blocks=True, extract_block=None, extract_segments=False, diff=False, dry=False, verbose=False, scan=None):
"""
def fetch_page(self, path, xpath='', extract_segments=False, dry=False, verbose=False):
site_id = self.id
page_url = self.url + path
if verbose:
print (page_url)
updated = False
request = urllib2.Request(page_url)
time_1 = time.time()
try:
response = urllib2.urlopen(request)
# except (urllib2.HTTPError, e):
except Exception as e:
if verbose:
print (page_url, ': error = ', e)
return -1
time_2 = time.time()
delay = int(round(time_2 - time_1))
if verbose:
print ('delay: ', delay)
response_code = response.getcode()
body = response.read().decode()
size = len(body)
checksum = self.page_checksum(body)
webpages = Webpage.objects.filter(site=self, path=path)
webpage = webpages and webpages[0] or None
if not dry and not webpage:
webpage = Webpage(site=self, path=path, last_checked_response_code=response_code)
webpage.save()
if not webpage:
return self.id, response_code, 0, path # ???
page_versions = PageVersion.objects.filter(webpage=webpage).order_by('-time')
page_version = page_versions and page_versions[0] or None
if page_version:
if verbose:
print ('size: ', page_version.size, '->', size)
print ('checksum: ', page_version.checksum, '->', checksum)
# if extract_block or (not dry and (not page_version or checksum != page_version.checksum)):
if xpath or (not dry and (not page_version or checksum != page_version.checksum)):
page_version = PageVersion(webpage=webpage, delay=delay, response_code=response_code, size=size, checksum=checksum, body=body, scan=None)
page_version.save()
if xpath:
webpage.extract_blocks(xpath=xpath, verbose=verbose)
n_blocks = 1
else:
extracted_blocks = webpage.extract_blocks(verbose=verbose)
webpage.create_blocks_dag()
n_blocks = len(extracted_blocks)
if verbose:
print ('blocks extracted:', n_blocks)
if verbose:
print (self.id, webpage.id, page_version and page_version.id or 0)
# return updated
return n_blocks
def purge_bips(self, verbose=False):
""" for all pages, delete all but last BlockInPage for each xpath """
webpages = Webpage.objects.filter(site=self)
n_purged = 0
for webpage in webpages:
n_purged += webpage.purge_bips(verbose=False)
if verbose:
print('purged %d old blocks' % n_purged)
def refetch_pages(self, skip_deny_path=True, extract_blocks=True, extract_segments=False, dry=False, verbose=False, user=None):
""" fetch known pages; for each, if content has changed, save the version and re-extract blocks """
MAX_UNFOUND = 3
SLEEP_SECONDS = 0.5
scan = Scan(name=self.name, site=self, scan_type=REFETCH, scan_mode=FOREGROUND, max_pages=0, extract_blocks=extract_blocks, user=user)
scan.save()
webpages = Webpage.objects.filter(site=self).order_by('path')
n_skipped = n_updated = n_unfound = 0
extract_deny_list = text_to_list(self.extract_deny)
for webpage in webpages:
# skip page if in the deny list of site configuration
path = webpage.path
if skip_deny_path:
should_skip = False
for deny_path in extract_deny_list:
if path.count(deny_path):
should_skip = True
break
if should_skip:
n_skipped += 1
continue
# skip page if last N versions are empty (fetch was unsuccessful)
page_versions = PageVersion.objects.filter(webpage=webpage).order_by('-time')
should_skip = True
for page_version in list(page_versions)[:MAX_UNFOUND]:
if page_version.size:
should_skip = False
break
if should_skip:
n_skipped += 1
continue
# updated = self.fetch_page(path, webpage=webpage, extract_blocks=extract_blocks, extract_segments=extract_segments, dry=dry, verbose=verbose, scan=scan)
updated = webpage.fetch(extract_blocks=extract_blocks, extract_segments=extract_segments, dry=dry, verbose=verbose, scan=scan)
if updated == -1:
n_unfound += 1
elif updated:
n_updated += 1
sys.stdout.write('.')
time.sleep(SLEEP_SECONDS)
scan.terminated = True
scan.save()
now = timezone.now()
self.last_fetched = now
if extract_blocks:
self.last_block_extraction = now
if extract_segments:
self.last_segment_extraction = now
self.save()
return webpages.count(), n_skipped, n_updated, n_unfound
def get_active_theme(self, request):
""" see get_active_theme(request) in module django_diazo.utils"""
if request.GET.get('theme', None):
try:
theme = request.GET.get('theme')
return Theme.objects.get(pk=theme)
except Theme.DoesNotExist:
pass
except ValueError:
pass
for theme in Theme.objects.filter(site_using_theme__site=self, enabled=True).order_by('sort'):
if theme.available(request):
return theme
return None
def add_fragment(self, text, path='', reliability=5):
added = False
segments = Segment.objects.filter(text=text, language=self.language, site=self)
if segments:
segment = segments[0]
else:
segment = Segment(text=text, is_fragment=True, site=self, language=self.language)
segment.save()
added = True
return segment, added
def get_segments(self, translation_state=ANY):
return Segment.objects.filter(site=self, language=self.language)
def get_segment_count(self):
return len(self.get_segments(translation_state=ANY))
def get_segments_in_use(self):
# return Segment.objects.filter(site=self, in_use=True)
return Segment.objects.filter(site=self, in_use__gt=0)
def refresh_segments_in_use(self):
segments_dict = defaultdict(int)
blocks_in_use = self.get_blocks_in_use()
segmenter = self.make_segmenter()
for block in blocks_in_use:
segment_texts = block.block_get_segments(segmenter)
if segment_texts:
page_count = BlockInPage.objects.filter(block=block).count()
for text in segment_texts:
for segment in Segment.objects.filter(site=self, text=text):
segments_dict[segment.id] += page_count
segments = Segment.objects.filter(site=self)
for segment in segments:
# in_use = (segments_dict[segment.id] > 0)
in_use = segments_dict[segment.id]
if in_use != segment.in_use:
segment.in_use = in_use
segment.save()
def get_token_frequency(self, lowercasing=True):
tokenizer = self.make_tokenizer()
tokens_dict = defaultdict(int)
segments = self.get_segments()
for segment in segments:
tokens = tokenizer.tokenize(segment.text)
for token in tokens:
if not is_invariant_word(token):
tokens_dict[token] += 1
return tokens_dict
def get_word_count(self, lowercasing=True):
return len(self.get_token_frequency())
def sites_user_can_view(user):
all_sites = Site.objects.all()
if user.is_superuser:
return list(all_sites)
sites = set()
user_roles = UserRole.objects.filter(user=user)
for role in user_roles:
role_type = role.role_type
if role_type in [OWNER, TRANSLATOR, CLIENT]:
sites.add(role.site)
elif role == LINGUIST:
for proxy in Proxy.objects.all():
role_language = role.source_language
if proxy.site.language==role_language or proxy.language==role_language:
sites.add(proxy.site)
return list(sites)
class SiteTheme(models.Model):
site = models.ForeignKey(Site, related_name='theme_used_for_site')
theme = models.ForeignKey(Theme, related_name='site_using_theme')
class Meta:
verbose_name = _('theme used for site')
verbose_name_plural = _('themes used for site')
def __str__(self):
return self.theme.name
class ServiceSubscription(models.Model):
site = models.ForeignKey(Site, related_name='service_subscribed_by_site')
service_type = models.IntegerField(choices=TRANSLATION_SERVICE_CHOICES, verbose_name='service type')
secret_1 = models.CharField(max_length=100, default='', blank=True)
secret_2 = models.CharField(max_length=100, default='', blank=True)
start_date = models.DateTimeField(null=True, blank=True, help_text="start date")
end_date = models.DateTimeField(null=True, blank=True, help_text="end date")
order = models.IntegerField(default=0)
class Meta:
verbose_name = _('service subscription')
verbose_name_plural = _('service subscription')
ordering = ('order', 'service_type')
def __str__(self):
return '{0} subscription for {1}'.format(TRANSLATION_SERVICE_DICT[self.service_type], self.site.name)
DISCOVER = 0
CRAWL = 1
REFETCH = 2
SCAN_TYPE_CHOICES = (
(DISCOVER, _('discover'),),
(CRAWL, _('crawl'),),
(REFETCH, _('re-fetch'),),
)
SCAN_TYPE_DICT = dict(SCAN_TYPE_CHOICES)
BACKGROUND = 0
FOREGROUND = 1
SCAN_MODE_CHOICES = (
(BACKGROUND, _('in background, asynchronously'),),
(FOREGROUND, _('in foreground, page by page'),),
)
CACHE_FOR_TRANSLATION = 0
CACHE_FOR_PUBLISHING = 1
SOURCE_CACHE_TYPE_CHOICES = (
(CACHE_FOR_TRANSLATION, _('cache for translation'),),
(CACHE_FOR_PUBLISHING, _('cache for publishing'),),
)
SOURCE_CACHE_TYPE_DICT = dict(SOURCE_CACHE_TYPE_CHOICES)
class Scan(models.Model):
name = models.CharField(max_length=20)
site = models.ForeignKey(Site, verbose_name='site/project', blank=True, null=True, help_text="leave undefined for discovery")
language = models.ForeignKey(Language, null=True)
start_urls = models.TextField()
allowed_domains = models.TextField()
allow = models.TextField()
deny = models.TextField()
max_pages = models.IntegerField()
count_words = models.BooleanField(default=False)
count_segments = models.BooleanField(default=False)
page_count = models.IntegerField(default=0)
word_count = models.IntegerField(default=0)
segment_count = models.IntegerField(default=0)
user = models.ForeignKey(User, null=True)
created = CreationDateTimeField()
modified = ModificationDateTimeField()
task_id = models.CharField(max_length=100)
terminated = models.BooleanField(default=False)
scan_type = models.IntegerField(choices=SCAN_TYPE_CHOICES, default=DISCOVER, verbose_name='scan type')
scan_mode = models.IntegerField(choices=SCAN_MODE_CHOICES, default=BACKGROUND, verbose_name='scan mode')
extract_blocks = models.BooleanField(default=False)
block_count = models.IntegerField(default=0, verbose_name='block count')
cache_type = models.IntegerField(choices=SOURCE_CACHE_TYPE_CHOICES, default=CACHE_FOR_TRANSLATION, verbose_name='cache type')
class Meta:
verbose_name = _('site scan')
verbose_name_plural = _('site scans')
def get_type(self):
return SCAN_TYPE_DICT.get(self.scan_type, DISCOVER)
def get_cache_type(self):
return SOURCE_CACHE_TYPE_DICT.get(self.cache_type, CACHE_FOR_TRANSLATION)
def get_label(self):
return '%s-%s' % (self.name, self.created.strftime("%y%m%d-%H%M"))
def get_links(self):
return Link.objects.filter(scan=self)
class Link(models.Model):
scan = models.ForeignKey(Scan)
url = models.TextField()
status = models.IntegerField()
encoding = models.TextField()
size = models.IntegerField()
title = models.TextField()
created = CreationDateTimeField()
class Meta:
verbose_name = _('followed link')
verbose_name_plural = _('followed links')
class SegmentCount(models.Model):
scan = models.ForeignKey(Scan)
segment = models.CharField(max_length=1000)
count = models.IntegerField()
class Meta:
verbose_name = _('segment count')
verbose_name_plural = _('segment counts')
class WordCount(models.Model):
scan = models.ForeignKey(Scan)
word = models.CharField(max_length=100)
count = models.IntegerField()
class Meta:
verbose_name = _('word count')
verbose_name_plural = _('word counts')
class Proxy(models.Model):
name = models.CharField(max_length=100)
slug = AutoSlugField(unique=True, populate_from='name', editable=True)
site = models.ForeignKey(Site)
language = models.ForeignKey(Language, related_name='language_proxy')
url = models.CharField(max_length=100, null=True, blank=True)
host = models.CharField(max_length=100)
base_path = models.CharField(max_length=100)
enable_live_translation = models.BooleanField(default=False)
active = models.BooleanField(default=True)
published = models.BooleanField(default=False)
translate_deny = models.TextField(verbose_name='Deny translation', blank=True, null=True, help_text="Paths of pages not to processed online by the proxy")
robots_txt = models.TextField(verbose_name='robots.txt', blank=True, null=True, help_text="The virtual content of the robots.txt page." )
custom_text = models.TextField(verbose_name='Custom text', blank=True, null=True, help_text="Custom fixed lines of text" )
head_values = models.TextField(verbose_name='Head values', blank=True, null=True, help_text="Customization of head elements" )
class Meta:
verbose_name = _('proxy site')
verbose_name_plural = _('proxy sites')
def __init__(self, *args, **kwargs):
super(Proxy, self).__init__(*args, **kwargs)
self.segmenter = None
def __str__(self):
return self.name
def get_absolute_url(self):
return '/proxy/%s/' % self.slug
def get_filepath(self):
return os.path.join(self.site.get_filepath(), self.language_id)
def get_url(self):
""" the URL at which the user should access the proxy site: to be replaced with an attribute """
site_url = self.site.url
if site_url.count(self.host):
return '%s/%s' % (site_url, self.language_id)
else:
return 'http://%s' % self.host
def can_manage(self, user):
return user.is_superuser
def can_operate(self, user):
return user.is_superuser
def can_view(self, user):
return user.is_authenticated()
def blocks_ready(self, min_state=TRANSLATED):
""" return blocks "ready for translation" """
site = self.site
language = self.language
# qs = Block.objects.filter(site=site, Block_child__isnull=True)
qs = Block.objects.filter(site=site)
qs = qs.annotate(n_pages = RawSQL("SELECT COUNT(*) FROM wip_blockinpage WHERE wip_blockinpage.block_id = wip_block.id", ()))
qs = qs.order_by('-n_pages')
blocks = []
for block in qs:
if block.no_translate:
continue
if block.compute_translation_state(language) >= min_state:
continue
children = block.get_children()
ready = True
if children:
for child in children:
if child.compute_translation_state(language) < min_state:
ready = False
break
if ready:
blocks.append(block)
return blocks
def make_segmenter(self, verbose=False):
language_code = self.language.code
current_rules = get_segmenter_rules(language_code)
self.segmenter = srx_segmenter.SrxSegmenter(rule=current_rules)
return self.segmenter
def make_tokenizer(self, return_matches=False):
""" create a tokenizer for the proxy target language ...
possibly exploit knowledge related to the target language and/or the proxy itself """
return NltkTokenizer(language_code=self.language.code, lowercasing=False, return_matches=return_matches)
def segments_summary(self):
summary = {'total': 0, 'in_use': 0, 'invariant': 0, 'already': 0, 'translated': 0, 'others': 0}
segments = Segment.objects.filter(site=self.site)
summary['total'] = segments.count()
in_use = segments.filter(in_use=True)
summary['in_use'] = in_use.count()
invariant = segments.filter(is_invariant=True)
summary['invariant'] = invariant.count()
already = segments.filter(language=self.language)
summary['already'] = already.count()
translated = segments.filter(segment_translation__language=self.language)
summary['translated'] = translated.count()
summary['others'] = summary['total'] - summary['invariant'] - summary['already'] - summary['translated']
return summary
def import_translations(self, xliff_file, request=None, user_role=None):
if not user_role:
pass # ma dovrebbe essere specificato
site = self.site
source_language = site.language
target_language = self.language
source_code = source_language.code.split('-')[0]
target_code = target_language.code.split('-')[0]
status = { 'found': 0, 'imported': 0 }
tree = etree.iterparse(xliff_file, events=("start", "end"))
opening = source = target = ''
for action, elem in tree:
tag = elem.tag
if tag.count('{') and tag.count('}'):
tag = tag.split('}')[1]
if action=='start':
if tag=='xliff' and action=='start':
xliff_version = elem.get('version')
status['version'] = xliff_version
elif tag=='file' and action=='start':
xliff_source_language = elem.get('source-language')
xliff_target_language = elem.get('target-language')
if not xliff_source_language.split('-')[0]==source_code:
status['error'] = "source language doesn't match"
break
elif not xliff_target_language.split('-')[0]==target_code:
status['error'] = "target language doesn't match"
break
elif tag=='trans-unit':
source = target = ''
elif tag in ['source', 'seg-source', 'target']:
opening = tag
elif action=='end':
text = elem.text
if text:
text = text.strip()
if text:
if opening in ['source', 'seg-source']:
source = text
elif opening == 'target':
target = text
if tag=='trans-unit':
if not source:
continue
status['found'] += 1
segments = Segment.objects.filter(text=source, language=source_language, site=site)
if segments.count():
segment = segments[0]
else:
segment = Segment(text=source, language=source_language, site=site)
segment.save()
status['imported'] += 1
sys.stdout.write('+')
if target:
translations = Translation.objects.filter(segment=segment, text=target, language=target_language)
if not translations.count():
translation = Translation(segment=segment, text=target, language=target_language, translation_type=MANUAL, timestamp=timezone.now())
translation.save()
sys.stdout.write('-')
return status
def build_xliff_export(self, translation_state=TO_BE_TRANSLATED, copy_source=False):
qs = Segment.objects.filter(site=self.site, is_invariant=False)
if translation_state == TRANSLATED:
qs = qs.filter(segment_translation__language=self.language)
elif translation_state == TO_BE_TRANSLATED:
qs = qs.exclude(segment_translation__language=self.language)
qs = qs.distinct().order_by('id')
version = '1.2'
filename = self.site.name
original = filename
attributes = {'source-language': self.site.language_id, 'target-language': self.language_id, 'datatype': 'plaintext', 'tool': 'WIP'}
file = File(original, attributes)
xlf_file = XLFFile(version=version)
xlf_file.files = { filename: file }
i = 0
for segment in qs:
i += 1
source = segment.text
target = ''
translations = Translation.objects.filter(segment=segment, language=self.language)
if translation_state == TRANSLATED:
translations = translations.exclude(text='')
elif translation_state == REVISED:
translations = translations.filter(translation_type=MANUAL)
if translations.count():
translations = translations.order_by('-translation_type', 'user_role__role_type', 'user_role__level')
target = translations[0].text
if not target and copy_source:
target = source
context = line = None
unit = xlf_file.add_unit(filename, source, target, context, line)
unit.attributes = {'id': str(i), 'datatype': 'plaintext'}
return xlf_file.to_str()
def build_parallel_text_export(self, translation_state=ANY, copy_source=False):
qs = Segment.objects.filter(site=self.site, is_invariant=False, in_use__gt=0)
if translation_state == TRANSLATED:
qs = qs.filter(segment_translation__language=self.language)
elif translation_state == TO_BE_TRANSLATED:
qs = qs.exclude(segment_translation__language=self.language)
qs = qs.order_by('text')
lines = []
for segment in qs:
source_text = segment.text
translations = Translation.objects.filter(segment=segment, language=self.language)
target_text = ''
for translation in translations:
if (translation_state==TRANSLATED and translation.text) or (translation_state==REVISED and translation.translation_type==MANUAL):
target_text = translation.text
lines.append('%s . ||| . %s' % (source_text, target_text))
if not target_text or translation_state==ANY:
if copy_source:
target_text = source_text
lines.append('%s . ||| . %s' % (source_text, target_text))
return '\n'.join(lines)
def apply_translation_memory(self):
site = self.site
site_invariants = text_to_list(site.invariant_words)
source_language = site.language
source_code = site.language_id
target_language = self.language
target_code = target_language.code
empty_words = EMPTY_WORDS[target_code]
target_codes = [target_code]
blocks_ready = self.blocks_ready()
n_ready = len(blocks_ready)
n_partially = 0
n_translated = 0
if blocks_ready:
segmenter = site.make_segmenter()
for block in blocks_ready:
translated_block = block.get_last_translation(language=target_language)
translated = True
ok_segments = 0
n_substitutions = 0
if translated_block:
body = translated_block.body
segments = translated_block.translated_block_get_segments(segmenter)
else:
body = block.body
segments = block.block_get_segments(segmenter)
body = normalize_string(body)
if not segments:
logger.info('block: %d no segments' % block.id)
continue # ???
segments.sort(key=lambda x: len(x), reverse=True)
for segment in segments:
words = segment.split()
if not non_invariant_words(words, site_invariants=site_invariants):
ok_segments +=1
logger.info('block: %d invariant segment : %s' % (block.id, segment))
continue
if Segment.objects.filter(is_invariant=True, language=source_language, site=site, text=segment):
ok_segments +=1
logger.info('block: %d invariant string, segment : %s' % (block.id, segment))
continue
translated_segment = None
# translations = Translation.objects.filter(language=target_language, segment__site=site, segment__language=source_language, segment__text=segment).distinct().order_by('-translation_type', 'user_role__role_type', 'user_role__level')
translations = Translation.objects.filter(language=target_language, segment__site=site, segment__language=source_language, segment__text=segment, translation_type=MANUAL).distinct().order_by('user_role__role_type', 'user_role__level')
if translations:
translation = translations[0]
""" add here some test on reliability """
translated_segment = translation.text
else: