translate.filters.checks

1 #!/usr/bin/env python 2 # -*- coding: utf-8 -*- 3 # 4 # Copyright 2004-2008 Zuza Software Foundation 5 # 6 # This file is part of translate. 7 # 8 # translate is free software; you can redistribute it and/or modify 9 # it under the terms of the GNU General Public License as published by 10 # the Free Software Foundation; either version 2 of the License, or 11 # (at your option) any later version. 12 # 13 # translate is distributed in the hope that it will be useful, 14 # but WITHOUT ANY WARRANTY; without even the implied warranty of 15 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 # GNU General Public License for more details. 17 # 18 # You should have received a copy of the GNU General Public License 19 # along with translate; if not, write to the Free Software 20 # Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 21 22 """This is a set of validation checks that can be performed on translation 23 units. 24 25 Derivatives of UnitChecker (like StandardUnitChecker) check translation units, 26 and derivatives of TranslationChecker (like StandardChecker) check 27 (source, target) translation pairs. 28 29 When adding a new test here, please document and explain the behaviour on the 30 U{wiki <http://translate.sourceforge.net/wiki/toolkit/pofilter_tests>}. 31 """ 32 33 from translate.filters import helpers 34 from translate.filters import decoration 35 from translate.filters import prefilters 36 from translate.filters import spelling 37 from translate.lang import factory 38 from translate.lang import data 39 # The import of xliff could fail if the user doesn't have lxml installed. For 40 # now we try to continue gracefully to help users who aren't interested in 41 # support for XLIFF or other XML formats. 42 try: 43 from translate.storage import xliff 44 except ImportError, e: 45 xliff = None 46 # The import of xliff fail silently in the absence of lxml if another module 47 # already tried to import it unsuccessfully, so let's make 100% sure: 48 if not hasattr(xliff, "xliffunit"): 49 xliff = None 50 import re 51 52 # These are some regular expressions that are compiled for use in some tests 53 54 # printf syntax based on http://en.wikipedia.org/wiki/Printf which doens't 55 # cover everything we leave \w instead of specifying the exact letters as 56 # this should capture printf types defined in other platforms. 57 # extended to support Python named format specifiers 58 printf_pat = re.compile('%((?:(?P<ord>\d+)\$|$(?P<key>\w+)$)?(?P<fullvar>[+#-]*(?:\d+)?(?:\.\d+)?(hh\|h\|l\|ll)?(?P<type>[\w%])))') 59 60 # The name of the XML tag 61 tagname_re = re.compile("<[\s]*([\w\/]*)") 62 63 # We allow escaped quotes, probably for old escaping style of OOo helpcontent 64 #TODO: remove escaped strings once usage is audited 65 property_re = re.compile(" (\w*)=((\\\\?\".*?\\\\?\")|(\\\\?'.*?\\\\?'))") 66 67 # The whole tag 68 tag_re = re.compile("<[^>]+>") 69 70 gconf_attribute_re = re.compile('"[a-z_]+?"') 71 72

73 -def tagname(string):

74 """Returns the name of the XML/HTML tag in string""" 75 return tagname_re.match(string).groups(1)[0]

76 77

78 -def intuplelist(pair, list):

79 """Tests to see if pair == (a,b,c) is in list, but handles None entries in 80 list as wildcards (only allowed in positions "a" and "c"). We take a 81 shortcut by only considering "c" if "b" has already matched.""" 82 a, b, c = pair 83 if (b, c) == (None, None): 84 #This is a tagname 85 return pair 86 for pattern in list: 87 x, y, z = pattern 88 if (x, y) in [(a, b), (None, b)]: 89 if z in [None, c]: 90 return pattern 91 return pair

92 93

94 -def tagproperties(strings, ignore):

95 """Returns all the properties in the XML/HTML tag string as 96 (tagname, propertyname, propertyvalue), but ignore those combinations 97 specified in ignore.""" 98 properties = [] 99 for string in strings: 100 tag = tagname(string) 101 properties += [(tag, None, None)] 102 #Now we isolate the attribute pairs. 103 pairs = property_re.findall(string) 104 for property, value, a, b in pairs: 105 #Strip the quotes: 106 value = value[1:-1] 107 108 canignore = False 109 if (tag, property, value) in ignore or \ 110 intuplelist((tag, property, value), ignore) != (tag, property, value): 111 canignore = True 112 break 113 if not canignore: 114 properties += [(tag, property, value)] 115 return properties

116 117

118 -class FilterFailure(Exception):

119 """This exception signals that a Filter didn't pass, and gives an 120 explanation or a comment""" 121

122 - def __init__(self, messages):

123 if not isinstance(messages, list): 124 messages = [messages] 125 assert isinstance(messages[0], unicode) # Assumption: all of same type 126 joined = u", ".join(messages) 127 Exception.__init__(self, joined) 128 # Python 2.3 doesn't have .args 129 if not hasattr(self, "args"): 130 self.args = joined

131 132

133 -class SeriousFilterFailure(FilterFailure):

134 """This exception signals that a Filter didn't pass, and the bad translation 135 might break an application (so the string will be marked fuzzy)""" 136 pass

137 138 #(tag, attribute, value) specifies a certain attribute which can be changed/ 139 #ignored if it exists inside tag. In the case where there is a third element 140 #in the tuple, it indicates a property value that can be ignored if present 141 #(like defaults, for example) 142 #If a certain item is None, it indicates that it is relevant for all values of 143 #the property/tag that is specified as None. A non-None value of "value" 144 #indicates that the value of the attribute must be taken into account. 145 common_ignoretags = [(None, "xml-lang", None)] 146 common_canchangetags = [("img", "alt", None), 147 (None, "title", None), 148 (None, "dir", None), 149 (None, "lang", None), 150 ] 151 # Actually the title tag is allowed on many tags in HTML (but probably not all) 152 153

154 -class CheckerConfig(object):

155 """object representing the configuration of a checker""" 156

157 - def __init__(self, targetlanguage=None, accelmarkers=None, varmatches=None, 158 notranslatewords=None, musttranslatewords=None, 159 validchars=None, punctuation=None, endpunctuation=None, 160 ignoretags=None, canchangetags=None, criticaltests=None, 161 credit_sources=None):

162 # Init lists 163 self.accelmarkers = self._init_list(accelmarkers) 164 self.varmatches = self._init_list(varmatches) 165 self.criticaltests = self._init_list(criticaltests) 166 self.credit_sources = self._init_list(credit_sources) 167 # Lang data 168 self.targetlanguage = targetlanguage 169 self.updatetargetlanguage(targetlanguage) 170 self.sourcelang = factory.getlanguage('en') 171 # Inits with default values 172 self.punctuation = self._init_default(data.normalized_unicode(punctuation), 173 self.lang.punctuation) 174 self.endpunctuation = self._init_default(data.normalized_unicode(endpunctuation), 175 self.lang.sentenceend) 176 self.ignoretags = self._init_default(ignoretags, common_ignoretags) 177 self.canchangetags = self._init_default(canchangetags, common_canchangetags) 178 # Other data 179 # TODO: allow user configuration of untranslatable words 180 self.notranslatewords = dict.fromkeys([data.normalized_unicode(key) for key in self._init_list(notranslatewords)]) 181 self.musttranslatewords = dict.fromkeys([data.normalized_unicode(key) for key in self._init_list(musttranslatewords)]) 182 validchars = data.normalized_unicode(validchars) 183 self.validcharsmap = {} 184 self.updatevalidchars(validchars)

185

186 - def _init_list(self, list):

187 """initialise configuration paramaters that are lists 188 189 @type list: List 190 @param list: None (we'll initialise a blank list) or a list paramater 191 @rtype: List 192 """ 193 if list is None: 194 list = [] 195 return list

196

197 - def _init_default(self, param, default):

198 """initialise parameters that can have default options 199 200 @param param: the user supplied paramater value 201 @param default: default values when param is not specified 202 @return: the paramater as specified by the user of the default settings 203 """ 204 if param is None: 205 return default 206 return param

207

208 - def update(self, otherconfig):

209 """combines the info in otherconfig into this config object""" 210 self.targetlanguage = otherconfig.targetlanguage or self.targetlanguage 211 self.updatetargetlanguage(self.targetlanguage) 212 self.accelmarkers.extend([c for c in otherconfig.accelmarkers if not c in self.accelmarkers]) 213 self.varmatches.extend(otherconfig.varmatches) 214 self.notranslatewords.update(otherconfig.notranslatewords) 215 self.musttranslatewords.update(otherconfig.musttranslatewords) 216 self.validcharsmap.update(otherconfig.validcharsmap) 217 self.punctuation += otherconfig.punctuation 218 self.endpunctuation += otherconfig.endpunctuation 219 #TODO: consider also updating in the following cases: 220 self.ignoretags = otherconfig.ignoretags 221 self.canchangetags = otherconfig.canchangetags 222 self.criticaltests.extend(otherconfig.criticaltests) 223 self.credit_sources = otherconfig.credit_sources

224

225 - def updatevalidchars(self, validchars):

226 """updates the map that eliminates valid characters""" 227 if validchars is None: 228 return True 229 validcharsmap = dict([(ord(validchar), None) for validchar in data.normalized_unicode(validchars)]) 230 self.validcharsmap.update(validcharsmap)

231

232 - def updatetargetlanguage(self, langcode):

233 """Updates the target language in the config to the given target 234 language""" 235 self.lang = factory.getlanguage(langcode)

236 237

238 -def cache_results(f):

239 240 def cached_f(self, param1): 241 key = (f.__name__, param1) 242 res_cache = self.results_cache 243 if key in res_cache: 244 return res_cache[key] 245 else: 246 value = f(self, param1) 247 res_cache[key] = value 248 return value

249 return cached_f 250 251

252 -class UnitChecker(object):

253 """Parent Checker class which does the checking based on functions available 254 in derived classes.""" 255 preconditions = {} 256

257 - def __init__(self, checkerconfig=None, excludefilters=None, 258 limitfilters=None, errorhandler=None):

259 self.errorhandler = errorhandler 260 if checkerconfig is None: 261 self.setconfig(CheckerConfig()) 262 else: 263 self.setconfig(checkerconfig) 264 # exclude functions defined in UnitChecker from being treated as tests. 265 self.helperfunctions = {} 266 for functionname in dir(UnitChecker): 267 function = getattr(self, functionname) 268 if callable(function): 269 self.helperfunctions[functionname] = function 270 self.defaultfilters = self.getfilters(excludefilters, limitfilters) 271 self.results_cache = {}

272

273 - def getfilters(self, excludefilters=None, limitfilters=None):

274 """returns dictionary of available filters, including/excluding those in 275 the given lists""" 276 filters = {} 277 if limitfilters is None: 278 # use everything available unless instructed 279 limitfilters = dir(self) 280 if excludefilters is None: 281 excludefilters = {} 282 for functionname in limitfilters: 283 if functionname in excludefilters: 284 continue 285 if functionname in self.helperfunctions: 286 continue 287 if functionname == "errorhandler": 288 continue 289 filterfunction = getattr(self, functionname, None) 290 if not callable(filterfunction): 291 continue 292 filters[functionname] = filterfunction 293 return filters

294

295 - def setconfig(self, config):

296 """sets the accelerator list""" 297 self.config = config 298 self.accfilters = [prefilters.filteraccelerators(accelmarker) for accelmarker in self.config.accelmarkers] 299 self.varfilters = [prefilters.filtervariables(startmatch, endmatch, prefilters.varname) 300 for startmatch, endmatch in self.config.varmatches] 301 self.removevarfilter = [prefilters.filtervariables(startmatch, endmatch, 302 prefilters.varnone) 303 for startmatch, endmatch in self.config.varmatches]

304

305 - def setsuggestionstore(self, store):

306 """Sets the filename that a checker should use for evaluating 307 suggestions.""" 308 self.suggestion_store = store 309 if self.suggestion_store: 310 self.suggestion_store.require_index()

311

312 - def filtervariables(self, str1):

313 """filter out variables from str1""" 314 return helpers.multifilter(str1, self.varfilters)

315 filtervariables = cache_results(filtervariables) 316

317 - def removevariables(self, str1):

318 """remove variables from str1""" 319 return helpers.multifilter(str1, self.removevarfilter)

320 removevariables = cache_results(removevariables) 321

322 - def filteraccelerators(self, str1):

323 """filter out accelerators from str1""" 324 return helpers.multifilter(str1, self.accfilters, None)

325 filteraccelerators = cache_results(filteraccelerators) 326

327 - def filteraccelerators_by_list(self, str1, acceptlist=None):

328 """filter out accelerators from str1""" 329 return helpers.multifilter(str1, self.accfilters, acceptlist)

330

331 - def filterwordswithpunctuation(self, str1):

332 """replaces words with punctuation with their unpunctuated 333 equivalents""" 334 return prefilters.filterwordswithpunctuation(str1)

335 filterwordswithpunctuation = cache_results(filterwordswithpunctuation) 336

337 - def filterxml(self, str1):

338 """filter out XML from the string so only text remains""" 339 return tag_re.sub("", str1)

340 filterxml = cache_results(filterxml) 341

342 - def run_test(self, test, unit):

343 """Runs the given test on the given unit. 344 345 Note that this can raise a FilterFailure as part of normal operation""" 346 return test(unit)

347

348 - def run_filters(self, unit):

349 """run all the tests in this suite, return failures as testname, 350 message_or_exception""" 351 self.results_cache = {} 352 failures = {} 353 ignores = self.config.lang.ignoretests[:] 354 functionnames = self.defaultfilters.keys() 355 priorityfunctionnames = self.preconditions.keys() 356 otherfunctionnames = filter(lambda functionname: functionname not in self.preconditions, functionnames) 357 for functionname in priorityfunctionnames + otherfunctionnames: 358 if functionname in ignores: 359 continue 360 filterfunction = getattr(self, functionname, None) 361 # this filterfunction may only be defined on another checker if 362 # using TeeChecker 363 if filterfunction is None: 364 continue 365 filtermessage = filterfunction.__doc__ 366 try: 367 filterresult = self.run_test(filterfunction, unit) 368 except FilterFailure, e: 369 filterresult = False 370 filtermessage = e.args[0] 371 except Exception, e: 372 if self.errorhandler is None: 373 raise ValueError("error in filter %s: %r, %r, %s" % \ 374 (functionname, unit.source, unit.target, e)) 375 else: 376 filterresult = self.errorhandler(functionname, unit.source, 377 unit.target, e) 378 if not filterresult: 379 # we test some preconditions that aren't actually a cause for 380 # failure 381 if functionname in self.defaultfilters: 382 failures[functionname] = filtermessage 383 if functionname in self.preconditions: 384 for ignoredfunctionname in self.preconditions[functionname]: 385 ignores.append(ignoredfunctionname) 386 self.results_cache = {} 387 return failures

388 389

390 -class TranslationChecker(UnitChecker):

391 """A checker that passes source and target strings to the checks, not the 392 whole unit. 393 394 This provides some speedup and simplifies testing.""" 395

396 - def __init__(self, checkerconfig=None, excludefilters=None, 397 limitfilters=None, errorhandler=None):

398 super(TranslationChecker, self).__init__(checkerconfig, excludefilters, 399 limitfilters, errorhandler)

400

401 - def run_test(self, test, unit):

402 """Runs the given test on the given unit. 403 404 Note that this can raise a FilterFailure as part of normal operation.""" 405 if self.hasplural: 406 filtermessages = [] 407 filterresult = True 408 for pluralform in unit.target.strings: 409 try: 410 if not test(self.str1, unicode(pluralform)): 411 filterresult = False 412 except FilterFailure, e: 413 filterresult = False 414 filtermessages.append(unicode(e.args)) 415 if not filterresult and filtermessages: 416 raise FilterFailure(filtermessages) 417 else: 418 return filterresult 419 else: 420 return test(self.str1, self.str2)

421

422 - def run_filters(self, unit):

423 """Do some optimisation by caching some data of the unit for the benefit 424 of run_test().""" 425 self.str1 = data.normalized_unicode(unit.source) or u"" 426 self.str2 = data.normalized_unicode(unit.target) or u"" 427 self.hasplural = unit.hasplural() 428 self.locations = unit.getlocations() 429 return super(TranslationChecker, self).run_filters(unit)

430 431

432 -class TeeChecker:

433 """A Checker that controls multiple checkers.""" 434

435 - def __init__(self, checkerconfig=None, excludefilters=None, 436 limitfilters=None, checkerclasses=None, errorhandler=None, 437 languagecode=None):

438 """construct a TeeChecker from the given checkers""" 439 self.limitfilters = limitfilters 440 if checkerclasses is None: 441 checkerclasses = [StandardChecker] 442 self.checkers = [checkerclass(checkerconfig=checkerconfig, excludefilters=excludefilters, limitfilters=limitfilters, errorhandler=errorhandler) for checkerclass in checkerclasses] 443 if languagecode: 444 for checker in self.checkers: 445 checker.config.updatetargetlanguage(languagecode) 446 # Let's hook up the language specific checker 447 lang_checker = self.checkers[0].config.lang.checker 448 if lang_checker: 449 self.checkers.append(lang_checker) 450 451 self.combinedfilters = self.getfilters(excludefilters, limitfilters) 452 self.config = checkerconfig or self.checkers[0].config

453

454 - def getfilters(self, excludefilters=None, limitfilters=None):

455 """returns dictionary of available filters, including/excluding those in 456 the given lists""" 457 if excludefilters is None: 458 excludefilters = {} 459 filterslist = [checker.getfilters(excludefilters, limitfilters) for checker in self.checkers] 460 self.combinedfilters = {} 461 for filters in filterslist: 462 self.combinedfilters.update(filters) 463 # TODO: move this somewhere more sensible (a checkfilters method?) 464 if limitfilters is not None: 465 for filtername in limitfilters: 466 if not filtername in self.combinedfilters: 467 import sys 468 print >> sys.stderr, "warning: could not find filter %s" % filtername 469 return self.combinedfilters

470

471 - def run_filters(self, unit):

472 """run all the tests in the checker's suites""" 473 failures = {} 474 for checker in self.checkers: 475 failures.update(checker.run_filters(unit)) 476 return failures

477

478 - def setsuggestionstore(self, store):

479 """Sets the filename that a checker should use for evaluating 480 suggestions.""" 481 for checker in self.checkers: 482 checker.setsuggestionstore(store)

483 484

485 -class StandardChecker(TranslationChecker):

486 """The basic test suite for source -> target translations.""" 487

488 - def untranslated(self, str1, str2):

489 """checks whether a string has been translated at all""" 490 str2 = prefilters.removekdecomments(str2) 491 return not (len(str1.strip()) > 0 and len(str2) == 0)

492

493 - def unchanged(self, str1, str2):

494 """checks whether a translation is basically identical to the original 495 string""" 496 str1 = self.filteraccelerators(self.removevariables(str1)).strip() 497 str2 = self.filteraccelerators(self.removevariables(str2)).strip() 498 if len(str1) < 2: 499 return True 500 # If the whole string is upperase, or nothing in the string can go 501 # towards uppercase, let's assume there is nothing translatable 502 # TODO: reconsider 503 if (str1.isupper() or str1.upper() == str1) and str1 == str2: 504 return True 505 if self.config.notranslatewords: 506 words1 = str1.split() 507 if len(words1) == 1 and [word for word in words1 if word in self.config.notranslatewords]: 508 #currently equivalent to: 509 # if len(words1) == 1 and words1[0] in self.config.notranslatewords: 510 #why do we only test for one notranslate word? 511 return True 512 # we could also check for things like str1.isnumeric(), but the test 513 # above (str1.upper() == str1) makes this unnecessary 514 if str1.lower() == str2.lower(): 515 raise FilterFailure(u"please translate") 516 return True

517

518 - def blank(self, str1, str2):

519 """checks whether a translation only contains spaces""" 520 len1 = len(str1.strip()) 521 len2 = len(str2.strip()) 522 return not (len1 > 0 and len(str2) != 0 and len2 == 0)

523

524 - def short(self, str1, str2):

525 """checks whether a translation is much shorter than the original 526 string""" 527 len1 = len(str1.strip()) 528 len2 = len(str2.strip()) 529 return not ((len1 > 0) and (0 < len2 < (len1 * 0.1)) or ((len1 > 1) and (len2 == 1)))

530

531 - def long(self, str1, str2):

532 """checks whether a translation is much longer than the original 533 string""" 534 len1 = len(str1.strip()) 535 len2 = len(str2.strip()) 536 return not ((len1 > 0) and (0 < len1 < (len2 * 0.1)) or ((len1 == 1) and (len2 > 1)))

537

538 - def escapes(self, str1, str2):

539 """checks whether escaping is consistent between the two strings""" 540 if not helpers.countsmatch(str1, str2, (u"\\", u"\\\\")): 541 escapes1 = u", ".join([u"'%s'" % word for word in str1.split() if u"\\" in word]) 542 escapes2 = u", ".join([u"'%s'" % word for word in str2.split() if u"\\" in word]) 543 raise SeriousFilterFailure(u"escapes in original (%s) don't match escapes in translation (%s)" % (escapes1, escapes2)) 544 else: 545 return True

546

547 - def newlines(self, str1, str2):

548 """checks whether newlines are consistent between the two strings""" 549 if not helpers.countsmatch(str1, str2, (u"\n", u"\r")): 550 raise FilterFailure(u"line endings in original don't match line endings in translation") 551 else: 552 return True

553

554 - def tabs(self, str1, str2):

555 """checks whether tabs are consistent between the two strings""" 556 if not helpers.countmatch(str1, str2, "\t"): 557 raise SeriousFilterFailure(u"tabs in original don't match tabs in translation") 558 else: 559 return True

560

561 - def singlequoting(self, str1, str2):

562 """checks whether singlequoting is consistent between the two strings""" 563 str1 = self.filterwordswithpunctuation(self.filteraccelerators(self.filtervariables(str1))) 564 str1 = self.config.lang.punctranslate(str1) 565 str2 = self.filterwordswithpunctuation(self.filteraccelerators(self.filtervariables(str2))) 566 return helpers.countsmatch(str1, str2, (u"'", u"''", u"\\'"))

567

568 - def doublequoting(self, str1, str2):

569 """checks whether doublequoting is consistent between the two strings""" 570 str1 = self.filteraccelerators(self.filtervariables(str1)) 571 str1 = self.filterxml(str1) 572 str1 = self.config.lang.punctranslate(str1) 573 str2 = self.filteraccelerators(self.filtervariables(str2)) 574 str2 = self.filterxml(str2) 575 return helpers.countsmatch(str1, str2, (u'"', u'""', u'\\"', u"«", 576 u"»", u"“", u"”"))

577

578 - def doublespacing(self, str1, str2):

579 """checks for bad double-spaces by comparing to original""" 580 str1 = self.filteraccelerators(str1) 581 str2 = self.filteraccelerators(str2) 582 return helpers.countmatch(str1, str2, u" ")

583

584 - def puncspacing(self, str1, str2):

585 """checks for bad spacing after punctuation""" 586 # Convert all nbsp to space, and just check spaces. Useful intermediate 587 # step to stricter nbsp checking? 588 str1 = self.filteraccelerators(self.filtervariables(str1)) 589 str1 = self.config.lang.punctranslate(str1) 590 str1 = str1.replace(u"\u00a0", u" ") 591 if str1.find(u" ") == -1: 592 return True 593 str2 = self.filteraccelerators(self.filtervariables(str2)) 594 str2 = str2.replace(u"\u00a0", u" ") 595 for puncchar in self.config.punctuation: 596 plaincount1 = str1.count(puncchar) 597 plaincount2 = str2.count(puncchar) 598 if not plaincount1 or plaincount1 != plaincount2: 599 continue 600 spacecount1 = str1.count(puncchar + u" ") 601 spacecount2 = str2.count(puncchar + u" ") 602 if spacecount1 != spacecount2: 603 # handle extra spaces that are because of transposed punctuation 604 if str1.endswith(puncchar) != str2.endswith(puncchar) and abs(spacecount1 - spacecount2) == 1: 605 continue 606 return False 607 return True

608

609 - def printf(self, str1, str2):

610 """checks whether printf format strings match""" 611 count1 = count2 = plural = None 612 # self.hasplural only set by run_filters, not always available 613 if 'hasplural' in self.__dict__: 614 plural = self.hasplural 615 for var_num2, match2 in enumerate(printf_pat.finditer(str2)): 616 count2 = var_num2 + 1 617 str2key = match2.group('key') 618 if match2.group('ord'): 619 for var_num1, match1 in enumerate(printf_pat.finditer(str1)): 620 count1 = var_num1 + 1 621 if int(match2.group('ord')) == var_num1 + 1: 622 if match2.group('fullvar') != match1.group('fullvar'): 623 return 0 624 elif str2key: 625 str1key = None 626 for var_num1, match1 in enumerate(printf_pat.finditer(str1)): 627 count1 = var_num1 + 1 628 if match1.group('key') and str2key == match1.group('key'): 629 str1key = match1.group('key') 630 # '%.0s' "placeholder" in plural will match anything 631 if plural and match2.group('fullvar') == '.0s': 632 continue 633 if match1.group('fullvar') != match2.group('fullvar'): 634 return 0 635 if str1key == None: 636 return 0 637 else: 638 for var_num1, match1 in enumerate(printf_pat.finditer(str1)): 639 count1 = var_num1 + 1 640 # '%.0s' "placeholder" in plural will match anything 641 if plural and match2.group('fullvar') == '.0s': 642 continue 643 if (var_num1 == var_num2) and (match1.group('fullvar') != match2.group('fullvar')): 644 return 0 645 646 if count2 is None: 647 if list(printf_pat.finditer(str1)): 648 return 0 649 650 if (count1 or count2) and (count1 != count2): 651 return 0 652 return 1

653

654 - def accelerators(self, str1, str2):

655 """checks whether accelerators are consistent between the two strings""" 656 str1 = self.filtervariables(str1) 657 str2 = self.filtervariables(str2) 658 messages = [] 659 for accelmarker in self.config.accelmarkers: 660 counter1 = decoration.countaccelerators(accelmarker, self.config.sourcelang.validaccel) 661 counter2 = decoration.countaccelerators(accelmarker, self.config.lang.validaccel) 662 count1, countbad1 = counter1(str1) 663 count2, countbad2 = counter2(str2) 664 getaccel = decoration.getaccelerators(accelmarker, self.config.lang.validaccel) 665 accel2, bad2 = getaccel(str2) 666 if count1 == count2: 667 continue 668 if count1 == 1 and count2 == 0: 669 if countbad2 == 1: 670 messages.append(u"accelerator %s appears before an invalid accelerator character '%s' (eg. space)" % (accelmarker, bad2[0])) 671 else: 672 messages.append(u"accelerator %s is missing from translation" % accelmarker) 673 elif count1 == 0: 674 messages.append(u"accelerator %s does not occur in original and should not be in translation" % accelmarker) 675 elif count1 == 1 and count2 > count1: 676 messages.append(u"accelerator %s is repeated in translation" % accelmarker) 677 else: 678 messages.append(u"accelerator %s occurs %d time(s) in original and %d time(s) in translation" % (accelmarker, count1, count2)) 679 if messages: 680 if "accelerators" in self.config.criticaltests: 681 raise SeriousFilterFailure(messages) 682 else: 683 raise FilterFailure(messages) 684 return True

685 686 # def acceleratedvariables(self, str1, str2): 687 # """checks that no variables are accelerated""" 688 # messages = [] 689 # for accelerator in self.config.accelmarkers: 690 # for variablestart, variableend in self.config.varmatches: 691 # error = accelerator + variablestart 692 # if str1.find(error) >= 0: 693 # messages.append(u"original has an accelerated variable") 694 # if str2.find(error) >= 0: 695 # messages.append(u"translation has an accelerated variable") 696 # if messages: 697 # raise FilterFailure(messages) 698 # return True 699

700 - def variables(self, str1, str2):

701 """checks whether variables of various forms are consistent between the 702 two strings""" 703 messages = [] 704 mismatch1, mismatch2 = [], [] 705 varnames1, varnames2 = [], [] 706 for startmarker, endmarker in self.config.varmatches: 707 varchecker = decoration.getvariables(startmarker, endmarker) 708 if startmarker and endmarker: 709 if isinstance(endmarker, int): 710 redecorate = lambda var: startmarker + var 711 else: 712 redecorate = lambda var: startmarker + var + endmarker 713 elif startmarker: 714 redecorate = lambda var: startmarker + var 715 else: 716 redecorate = lambda var: var 717 vars1 = varchecker(str1) 718 vars2 = varchecker(str2) 719 if vars1 != vars2: 720 # we use counts to compare so we can handle multiple variables 721 vars1, vars2 = [var for var in vars1 if vars1.count(var) > vars2.count(var)], [var for var in vars2 if vars1.count(var) < vars2.count(var)] 722 # filter variable names we've already seen, so they aren't 723 # matched by more than one filter... 724 vars1, vars2 = [var for var in vars1 if var not in varnames1], [var for var in vars2 if var not in varnames2] 725 varnames1.extend(vars1) 726 varnames2.extend(vars2) 727 vars1 = map(redecorate, vars1) 728 vars2 = map(redecorate, vars2) 729 mismatch1.extend(vars1) 730 mismatch2.extend(vars2) 731 if mismatch1: 732 messages.append(u"do not translate: %s" % u", ".join(mismatch1)) 733 elif mismatch2: 734 messages.append(u"translation contains variables not in original: %s" % u", ".join(mismatch2)) 735 if messages and mismatch1: 736 raise SeriousFilterFailure(messages) 737 elif messages: 738 raise FilterFailure(messages) 739 return True

740

741 - def functions(self, str1, str2):

742 """checks that function names are not translated""" 743 return helpers.funcmatch(str1, str2, decoration.getfunctions, self.config.punctuation)

744

745 - def emails(self, str1, str2):

746 """checks that emails are not translated""" 747 return helpers.funcmatch(str1, str2, decoration.getemails)

748

749 - def urls(self, str1, str2):

750 """checks that URLs are not translated""" 751 return helpers.funcmatch(str1, str2, decoration.geturls)

752

753 - def numbers(self, str1, str2):

754 """checks whether numbers of various forms are consistent between the 755 two strings""" 756 return helpers.countsmatch(str1, str2, decoration.getnumbers(str1))

757

758 - def startwhitespace(self, str1, str2):

759 """checks whether whitespace at the beginning of the strings matches""" 760 return helpers.funcmatch(str1, str2, decoration.spacestart)

761

762 - def endwhitespace(self, str1, str2):

763 """checks whether whitespace at the end of the strings matches""" 764 str1 = self.config.lang.punctranslate(str1) 765 return helpers.funcmatch(str1, str2, decoration.spaceend)

766

767 - def startpunc(self, str1, str2):

768 """checks whether punctuation at the beginning of the strings match""" 769 str1 = self.filterxml(self.filteraccelerators(self.filtervariables(self.filterwordswithpunctuation(str1)))) 770 str1 = self.config.lang.punctranslate(str1) 771 str2 = self.filterxml(self.filteraccelerators(self.filtervariables(self.filterwordswithpunctuation(str2)))) 772 return helpers.funcmatch(str1, str2, decoration.puncstart, self.config.punctuation)

773

774 - def endpunc(self, str1, str2):

775 """checks whether punctuation at the end of the strings match""" 776 str1 = self.filtervariables(str1) 777 str1 = self.config.lang.punctranslate(str1) 778 str2 = self.filtervariables(str2) 779 str1 = str1.rstrip() 780 str2 = str2.rstrip() 781 return helpers.funcmatch(str1, str2, decoration.puncend, self.config.endpunctuation + u":")

782

783 - def purepunc(self, str1, str2):

784 """checks that strings that are purely punctuation are not changed""" 785 # this test is a subset of startandend 786 if (decoration.ispurepunctuation(str1)): 787 return str1 == str2 788 else: 789 return not decoration.ispurepunctuation(str2)

790

791 - def brackets(self, str1, str2):

792 """checks that the number of brackets in both strings match""" 793 str1 = self.filtervariables(str1) 794 str2 = self.filtervariables(str2) 795 messages = [] 796 missing = [] 797 extra = [] 798 for bracket in (u"[", u"]", u"{", u"}", u"(", u")"): 799 count1 = str1.count(bracket) 800 count2 = str2.count(bracket) 801 if count2 < count1: 802 missing.append(u"'%s'" % bracket) 803 elif count2 > count1: 804 extra.append(u"'%s'" % bracket) 805 if missing: 806 messages.append(u"translation is missing %s" % u", ".join(missing)) 807 if extra: 808 messages.append(u"translation has extra %s" % u", ".join(extra)) 809 if messages: 810 raise FilterFailure(messages) 811 return True

812

813 - def sentencecount(self, str1, str2):

814 """checks that the number of sentences in both strings match""" 815 str1 = self.filteraccelerators(str1) 816 str2 = self.filteraccelerators(str2) 817 sentences1 = len(self.config.sourcelang.sentences(str1)) 818 sentences2 = len(self.config.lang.sentences(str2)) 819 if not sentences1 == sentences2: 820 raise FilterFailure(u"The number of sentences differ: %d versus %d" % (sentences1, sentences2)) 821 return True

822

823 - def options(self, str1, str2):

824 """checks that options are not translated""" 825 str1 = self.filtervariables(str1) 826 for word1 in str1.split(): 827 if word1 != u"--" and word1.startswith(u"--") and word1[-1].isalnum(): 828 parts = word1.split(u"=") 829 if not parts[0] in str2: 830 raise FilterFailure(u"The option %s does not occur or is translated in the translation." % parts[0]) 831 if len(parts) > 1 and parts[1] in str2: 832 raise FilterFailure(u"The parameter %(param)s in option %(option)s is not translated." % {"param": parts[1], "option": parts[0]}) 833 return True

834

835 - def startcaps(self, str1, str2):

836 """checks that the message starts with the correct capitalisation""" 837 str1 = self.filteraccelerators(str1) 838 str2 = self.filteraccelerators(str2) 839 if len(str1) > 1 and len(str2) > 1: 840 return self.config.sourcelang.capsstart(str1) == self.config.lang.capsstart(str2) 841 if len(str1) == 0 and len(str2) == 0: 842 return True 843 if len(str1) == 0 or len(str2) == 0: 844 return False 845 return True

846

847 - def simplecaps(self, str1, str2):

848 """checks the capitalisation of two strings isn't wildly different""" 849 str1 = self.removevariables(str1) 850 str2 = self.removevariables(str2) 851 # TODO: review this. The 'I' is specific to English, so it probably 852 # serves no purpose to get sourcelang.sentenceend 853 str1 = re.sub(u"[^%s]( I )" % self.config.sourcelang.sentenceend, u" i ", str1) 854 capitals1 = helpers.filtercount(str1, unicode.isupper) 855 capitals2 = helpers.filtercount(str2, unicode.isupper) 856 alpha1 = helpers.filtercount(str1, unicode.isalpha) 857 alpha2 = helpers.filtercount(str2, unicode.isalpha) 858 # Capture the all caps case 859 if capitals1 == alpha1: 860 return capitals2 == alpha2 861 # some heuristic tests to try and see that the style of capitals is 862 # vaguely the same 863 if capitals1 == 0 or capitals1 == 1: 864 return capitals2 == capitals1 865 elif capitals1 < len(str1) / 10: 866 return capitals2 <= len(str2) / 8 867 elif len(str1) < 10: 868 return abs(capitals1 - capitals2) < 3 869 elif capitals1 > len(str1) * 6 / 10: 870 return capitals2 > len(str2) * 6 / 10 871 else: 872 return abs(capitals1 - capitals2) < (len(str1) + len(str2)) / 6

873

874 - def acronyms(self, str1, str2):

875 """checks that acronyms that appear are unchanged""" 876 acronyms = [] 877 allowed = [] 878 for startmatch, endmatch in self.config.varmatches: 879 allowed += decoration.getvariables(startmatch, endmatch)(str1) 880 allowed += self.config.musttranslatewords.keys() 881 str1 = self.filteraccelerators(self.filtervariables(str1)) 882 iter = self.config.lang.word_iter(str1) 883 str2 = self.filteraccelerators(self.filtervariables(str2)) 884 #TODO: strip XML? - should provide better error messsages 885 # see mail/chrome/messanger/smime.properties.po 886 #TODO: consider limiting the word length for recognising acronyms to 887 #something like 5/6 characters 888 for word in iter: 889 if word.isupper() and len(word) > 1 and word not in allowed: 890 if str2.find(word) == -1: 891 acronyms.append(word) 892 if acronyms: 893 raise FilterFailure(u"acronyms should not be translated: " + u", ".join(acronyms)) 894 return True

895

896 - def doublewords(self, str1, str2):

897 """checks for repeated words in the translation""" 898 lastword = "" 899 without_newlines = "\n".join(str2.split("\n")) 900 words = self.filteraccelerators(self.removevariables(self.filterxml(without_newlines))).replace(u".", u"").lower().split() 901 for word in words: 902 if word == lastword and word not in self.config.lang.validdoublewords: 903 raise FilterFailure(u"The word '%s' is repeated" % word) 904 lastword = word 905 return True

906

907 - def notranslatewords(self, str1, str2):

908 """checks that words configured as untranslatable appear in the 909 translation too""" 910 if not self.config.notranslatewords: 911 return True 912 str1 = self.filtervariables(str1) 913 str2 = self.filtervariables(str2) 914 #The above is full of strange quotes and things in utf-8 encoding. 915 #single apostrophe perhaps problematic in words like "doesn't" 916 for seperator in self.config.punctuation: 917 str1 = str1.replace(seperator, u" ") 918 str2 = str2.replace(seperator, u" ") 919 words1 = self.filteraccelerators(str1).split() 920 words2 = self.filteraccelerators(str2).split() 921 stopwords = [word for word in words1 if word in self.config.notranslatewords and word not in words2] 922 if stopwords: 923 raise FilterFailure(u"do not translate: %s" % (u", ".join(stopwords))) 924 return True

925

926 - def musttranslatewords(self, str1, str2):

927 """checks that words configured as definitely translatable don't appear 928 in the translation""" 929 if not self.config.musttranslatewords: 930 return True 931 str1 = self.removevariables(str1) 932 str2 = self.removevariables(str2) 933 # The above is full of strange quotes and things in utf-8 encoding. 934 # single apostrophe perhaps problematic in words like "doesn't" 935 for seperator in self.config.punctuation: 936 str1 = str1.replace(seperator, u" ") 937 str2 = str2.replace(seperator, u" ") 938 words1 = self.filteraccelerators(str1).split() 939 words2 = self.filteraccelerators(str2).split() 940 stopwords = [word for word in words1 if word in self.config.musttranslatewords and word in words2] 941 if stopwords: 942 raise FilterFailure(u"please translate: %s" % (u", ".join(stopwords))) 943 return True

944

945 - def validchars(self, str1, str2):

946 """checks that only characters specified as valid appear in the 947 translation""" 948 if not self.config.validcharsmap: 949 return True 950 invalid1 = str1.translate(self.config.validcharsmap) 951 invalid2 = str2.translate(self.config.validcharsmap) 952 invalidchars = [u"'%s' (\\u%04x)" % (invalidchar, ord(invalidchar)) for invalidchar in invalid2 if invalidchar not in invalid1] 953 if invalidchars: 954 raise FilterFailure(u"invalid chars: %s" % (u", ".join(invalidchars))) 955 return True

956

957 - def filepaths(self, str1, str2):

958 """checks that file paths have not been translated""" 959 for word1 in self.filteraccelerators(str1).split(): 960 if word1.startswith(u"/"): 961 if not helpers.countsmatch(str1, str2, (word1,)): 962 return False 963 return True

964

965 - def xmltags(self, str1, str2):

966 """checks that XML/HTML tags have not been translated""" 967 tags1 = tag_re.findall(str1) 968 if len(tags1) > 0: 969 if (len(tags1[0]) == len(str1)) and not u"=" in tags1[0]: 970 return True 971 tags2 = tag_re.findall(str2) 972 properties1 = tagproperties(tags1, self.config.ignoretags) 973 properties2 = tagproperties(tags2, self.config.ignoretags) 974 filtered1 = [] 975 filtered2 = [] 976 for property1 in properties1: 977 filtered1 += [intuplelist(property1, self.config.canchangetags)] 978 for property2 in properties2: 979 filtered2 += [intuplelist(property2, self.config.canchangetags)] 980 981 # TODO: consider the consequences of different ordering of 982 # attributes/tags 983 if filtered1 != filtered2: 984 return False 985 else: 986 # No tags in str1, let's just check that none were added in str2. 987 # This might be useful for fuzzy strings wrongly unfuzzied. 988 tags2 = tag_re.findall(str2) 989 if len(tags2) > 0: 990 return False 991 return True

992

993 - def kdecomments(self, str1, str2):

994 """checks to ensure that no KDE style comments appear in the 995 translation""" 996 return str2.find(u"\n_:") == -1 and not str2.startswith(u"_:")

997

998 - def compendiumconflicts(self, str1, str2):

999 """checks for Gettext compendium conflicts (#-#-#-#-#)""" 1000 return str2.find(u"#-#-#-#-#") == -1

1001

1002 - def simpleplurals(self, str1, str2):

1003 """checks for English style plural(s) for you to review""" 1004 1005 def numberofpatterns(string, patterns): 1006 number = 0 1007 for pattern in patterns: 1008 number += len(re.findall(pattern, string)) 1009 return number

1010 1011 sourcepatterns = ["$s$"] 1012 targetpatterns = ["$s$"] 1013 sourcecount = numberofpatterns(str1, sourcepatterns) 1014 targetcount = numberofpatterns(str2, targetpatterns) 1015 if self.config.lang.nplurals == 1: 1016 return not targetcount 1017 return sourcecount == targetcount

1018

1019 - def spellcheck(self, str1, str2):

1020 """checks words that don't pass a spell check""" 1021 if not self.config.targetlanguage: 1022 return True 1023 if not spelling.available: 1024 return True 1025 # TODO: filterxml? 1026 str1 = self.filteraccelerators_by_list(self.filtervariables(str1), self.config.sourcelang.validaccel) 1027 str2 = self.filteraccelerators_by_list(self.filtervariables(str2), self.config.lang.validaccel) 1028 ignore1 = [] 1029 messages = [] 1030 for word, index, suggestions in spelling.check(str1, lang="en"): 1031 ignore1.append(word) 1032 for word, index, suggestions in spelling.check(str2, lang=self.config.targetlanguage): 1033 if word in self.config.notranslatewords: 1034 continue 1035 if word in ignore1: 1036 continue 1037 # hack to ignore hyphenisation rules 1038 if word in suggestions: 1039 continue 1040 messages.append(u"check spelling of %s (could be %s)" % (word, u" / ".join(suggestions[:5]))) 1041 if messages: 1042 raise FilterFailure(messages) 1043 return True

1044

1045 - def credits(self, str1, str2):

1046 """checks for messages containing translation credits instead of normal 1047 translations.""" 1048 return not str1 in self.config.credit_sources

1049 1050 # If the precondition filter is run and fails then the other tests listed are ignored 1051 preconditions = {"untranslated": ("simplecaps", "variables", "startcaps", 1052 "accelerators", "brackets", "endpunc", 1053 "acronyms", "xmltags", "startpunc", 1054 "endwhitespace", "startwhitespace", 1055 "escapes", "doublequoting", "singlequoting", 1056 "filepaths", "purepunc", "doublespacing", 1057 "sentencecount", "numbers", "isfuzzy", 1058 "isreview", "notranslatewords", "musttranslatewords", 1059 "emails", "simpleplurals", "urls", "printf", 1060 "tabs", "newlines", "functions", "options", 1061 "blank", "nplurals", "gconf"), 1062 "blank": ("simplecaps", "variables", "startcaps", 1063 "accelerators", "brackets", "endpunc", 1064 "acronyms", "xmltags", "startpunc", 1065 "endwhitespace", "startwhitespace", 1066 "escapes", "doublequoting", "singlequoting", 1067 "filepaths", "purepunc", "doublespacing", 1068 "sentencecount", "numbers", "isfuzzy", 1069 "isreview", "notranslatewords", "musttranslatewords", 1070 "emails", "simpleplurals", "urls", "printf", 1071 "tabs", "newlines", "functions", "options", 1072 "gconf"), 1073 "credits": ("simplecaps", "variables", "startcaps", 1074 "accelerators", "brackets", "endpunc", 1075 "acronyms", "xmltags", "startpunc", 1076 "escapes", "doublequoting", "singlequoting", 1077 "filepaths", "doublespacing", 1078 "sentencecount", "numbers", 1079 "emails", "simpleplurals", "urls", "printf", 1080 "tabs", "newlines", "functions", "options"), 1081 "purepunc": ("startcaps", "options"), 1082 # This is causing some problems since Python 2.6, as 1083 # startcaps is now seen as an important one to always execute 1084 # and could now be done before it is blocked by a failing 1085 # "untranslated" or "blank" test. This is probably happening 1086 # due to slightly different implementation of the internal 1087 # dict handling since Python 2.6. We should never have relied 1088 # on this ordering anyway. 1089 #"startcaps": ("simplecaps",), 1090 "endwhitespace": ("endpunc",), 1091 "startwhitespace": ("startpunc",), 1092 "unchanged": ("doublewords",), 1093 "compendiumconflicts": ("accelerators", "brackets", "escapes", 1094 "numbers", "startpunc", "long", "variables", 1095 "startcaps", "sentencecount", "simplecaps", 1096 "doublespacing", "endpunc", "xmltags", 1097 "startwhitespace", "endwhitespace", 1098 "singlequoting", "doublequoting", 1099 "filepaths", "purepunc", "doublewords", "printf")} 1100 1101 # code to actually run the tests (use unittest?) 1102 1103 openofficeconfig = CheckerConfig( 1104 accelmarkers=["~"], 1105 varmatches=[("&", ";"), ("%", "%"), ("%", None), ("%", 0), ("$(", ")"), 1106 ("$", "$"), ("${", "}"), ("#", "#"), ("#", 1), ("#", 0), 1107 ("($", ")"), ("$[", "]"), ("[", "]"), ("$", None)], 1108 ignoretags=[("alt", "xml-lang", None), ("ahelp", "visibility", "visible"), 1109 ("img", "width", None), ("img", "height", None)], 1110 canchangetags=[("link", "name", None)], 1111 ) 1112

1113 -class OpenOfficeChecker(StandardChecker):

1114

1115 - def __init__(self, **kwargs):

1116 checkerconfig = kwargs.get("checkerconfig", None) 1117 if checkerconfig is None: 1118 checkerconfig = CheckerConfig() 1119 kwargs["checkerconfig"] = checkerconfig 1120 checkerconfig.update(openofficeconfig) 1121 StandardChecker.__init__(self, **kwargs)

1122 1123 mozillaconfig = CheckerConfig( 1124 accelmarkers=["&"], 1125 varmatches=[("&", ";"), ("%", "%"), ("%", 1), ("$", "$"), ("$", None), 1126 ("#", 1), ("${", "}"), ("$(^", ")")], 1127 criticaltests=["accelerators"], 1128 ) 1129

1130 -class MozillaChecker(StandardChecker):

1131

1132 - def __init__(self, **kwargs):

1133 checkerconfig = kwargs.get("checkerconfig", None) 1134 if checkerconfig is None: 1135 checkerconfig = CheckerConfig() 1136 kwargs["checkerconfig"] = checkerconfig 1137 checkerconfig.update(mozillaconfig) 1138 StandardChecker.__init__(self, **kwargs)

1139

1140 - def credits(self, str1, str2):

1141 """checks for messages containing translation credits instead of normal 1142 translations.""" 1143 for location in self.locations: 1144 if location in ['MOZ_LANGPACK_CONTRIBUTORS', 'credit.translation']: 1145 return False 1146 return True

1147 1148 drupalconfig = CheckerConfig( 1149 varmatches=[("%", None), ("@", None), ("!", None)], 1150 ) 1151

1152 -class DrupalChecker(StandardChecker):

1153

1154 - def __init__(self, **kwargs):

1155 checkerconfig = kwargs.get("checkerconfig", None) 1156 if checkerconfig is None: 1157 checkerconfig = CheckerConfig() 1158 kwargs["checkerconfig"] = checkerconfig 1159 checkerconfig.update(drupalconfig) 1160 StandardChecker.__init__(self, **kwargs)

1161 1162 gnomeconfig = CheckerConfig( 1163 accelmarkers=["_"], 1164 varmatches=[("%", 1), ("$(", ")")], 1165 credit_sources=[u"translator-credits"], 1166 ) 1167

1168 -class GnomeChecker(StandardChecker):

1169

1170 - def __init__(self, **kwargs):

1171 checkerconfig = kwargs.get("checkerconfig", None) 1172 if checkerconfig is None: 1173 checkerconfig = CheckerConfig() 1174 kwargs["checkerconfig"] = checkerconfig 1175 checkerconfig.update(gnomeconfig) 1176 StandardChecker.__init__(self, **kwargs)

1177

1178 - def gconf(self, str1, str2):

1179 """Checks if we have any gconf config settings translated.""" 1180 for location in self.locations: 1181 if location.find('schemas.in') != -1: 1182 gconf_attributes = gconf_attribute_re.findall(str1) 1183 #stopwords = [word for word in words1 if word in self.config.notranslatewords and word not in words2] 1184 stopwords = [word for word in gconf_attributes if word[1:-1] not in str2] 1185 if stopwords: 1186 raise FilterFailure(u"do not translate gconf attribute: %s" % (u", ".join(stopwords))) 1187 return True

1188 1189 kdeconfig = CheckerConfig( 1190 accelmarkers=["&"], 1191 varmatches=[("%", 1)], 1192 credit_sources=[u"Your names", u"Your emails", u"ROLES_OF_TRANSLATORS"], 1193 ) 1194

1195 -class KdeChecker(StandardChecker):

1196

1197 - def __init__(self, **kwargs):

1198 # TODO allow setup of KDE plural and translator comments so that they do 1199 # not create false postives 1200 checkerconfig = kwargs.get("checkerconfig", None) 1201 if checkerconfig is None: 1202 checkerconfig = CheckerConfig() 1203 kwargs["checkerconfig"] = checkerconfig 1204 checkerconfig.update(kdeconfig) 1205 StandardChecker.__init__(self, **kwargs)

1206 1207 cclicenseconfig = CheckerConfig(varmatches=[("@", "@")]) 1208

1209 -class CCLicenseChecker(StandardChecker):

1210

1211 - def __init__(self, **kwargs):

1212 checkerconfig = kwargs.get("checkerconfig", None) 1213 if checkerconfig is None: 1214 checkerconfig = CheckerConfig() 1215 kwargs["checkerconfig"] = checkerconfig 1216 checkerconfig.update(cclicenseconfig) 1217 StandardChecker.__init__(self, **kwargs)

1218 1219 projectcheckers = { 1220 "openoffice": OpenOfficeChecker, 1221 "mozilla": MozillaChecker, 1222 "kde": KdeChecker, 1223 "wx": KdeChecker, 1224 "gnome": GnomeChecker, 1225 "creativecommons": CCLicenseChecker, 1226 "drupal": DrupalChecker, 1227 } 1228 1229

1230 -class StandardUnitChecker(UnitChecker):

1231 """The standard checks for common checks on translation units.""" 1232

1233 - def isfuzzy(self, unit):

1234 """Check if the unit has been marked fuzzy.""" 1235 return not unit.isfuzzy()

1236

1237 - def isreview(self, unit):

1238 """Check if the unit has been marked review.""" 1239 return not unit.isreview()

1240

1241 - def nplurals(self, unit):

1242 """Checks for the correct number of noun forms for plural 1243 translations.""" 1244 if unit.hasplural(): 1245 # if we don't have a valid nplurals value, don't run the test 1246 nplurals = self.config.lang.nplurals 1247 if nplurals > 0: 1248 return len(unit.target.strings) == nplurals 1249 return True

1250

1251 - def hassuggestion(self, unit):

1252 """Checks if there is at least one suggested translation for this 1253 unit.""" 1254 self.suggestion_store = getattr(self, 'suggestion_store', None) 1255 suggestions = [] 1256 if self.suggestion_store: 1257 suggestions = self.suggestion_store.findunits(unit.source) 1258 elif xliff and isinstance(unit, xliff.xliffunit): 1259 # TODO: we probably want to filter them somehow 1260 suggestions = unit.getalttrans() 1261 return not bool(suggestions)

1262 1263

1264 -def runtests(str1, str2, ignorelist=()):

1265 """verifies that the tests pass for a pair of strings""" 1266 from translate.storage import base 1267 str1 = data.normalized_unicode(str1) 1268 str2 = data.normalized_unicode(str2) 1269 unit = base.TranslationUnit(str1) 1270 unit.target = str2 1271 checker = StandardChecker(excludefilters=ignorelist) 1272 failures = checker.run_filters(unit) 1273 for test in failures: 1274 print "failure: %s: %s\n %r\n %r" % (test, failures[test], str1, str2) 1275 return failures

1276 1277

1278 -def batchruntests(pairs):

1279 """runs test on a batch of string pairs""" 1280 passed, numpairs = 0, len(pairs) 1281 for str1, str2 in pairs: 1282 if runtests(str1, str2): 1283 passed += 1 1284 print 1285 print "total: %d/%d pairs passed" % (passed, numpairs)

1286 1287 if __name__ == '__main__': 1288 testset = [(r"simple", r"somple"), 1289 (r"\this equals \that", r"does \this equal \that?"), 1290 (r"this \'equals\' that", r"this 'equals' that"), 1291 (r" start and end! they must match.", r"start and end! they must match."), 1292 (r"check for matching %variables marked like %this", r"%this %variable is marked"), 1293 (r"check for mismatching %variables marked like %this", r"%that %variable is marked"), 1294 (r"check for mismatching %variables% too", r"how many %variable% are marked"), 1295 (r"%% %%", r"%%"), 1296 (r"Row: %1, Column: %2", r"Mothalo: %1, Kholomo: %2"), 1297 (r"simple lowercase", r"it is all lowercase"), 1298 (r"simple lowercase", r"It Is All Lowercase"), 1299 (r"Simple First Letter Capitals", r"First Letters"), 1300 (r"SIMPLE CAPITALS", r"First Letters"), 1301 (r"SIMPLE CAPITALS", r"ALL CAPITALS"), 1302 (r"forgot to translate", r" "), 1303 ] 1304 batchruntests(testset) 1305

Source Code for Module translate.filters.checks