# -*- coding: utf-8 -*-
"""
This bot will make direct text replacements. It will retrieve information on
which pages might need changes either from an XML dump or a text file, or only
change a single page.
These command line parameters can be used to specify which pages to work on:
¶ms;
-xml Retrieve information from a local XML dump (pages-articles
or pages-meta-current, see http://download.wikimedia.org).
Argument can also be given as "-xml:filename".
-page Only edit a specific page.
Argument can also be given as "-page:pagetitle". You can
give this parameter multiple times to edit multiple pages.
Furthermore, the following command line parameters are supported:
-regex Make replacements using regular expressions. If this argument
isn't given, the bot will make simple text replacements.
-nocase Use case insensitive regular expressions.
-dotall Make the dot match any character at all, including a newline.
Without this flag, '.' will match anything except a newline.
-multiline '^' and '$' will now match begin and end of each line.
-xmlstart (Only works with -xml) Skip all articles in the XML dump
before the one specified (may also be given as
-xmlstart:Article).
-addcat:cat_name Adds "cat_name" category to every altered page.
-excepttitle:XYZ Skip pages with titles that contain XYZ. If the -regex
argument is given, XYZ will be regarded as a regular
expression.
-requiretitle:XYZ Only do pages with titles that contain XYZ. If the -regex
argument is given, XYZ will be regarded as a regular
expression.
-excepttext:XYZ Skip pages which contain the text XYZ. If the -regex
argument is given, XYZ will be regarded as a regular
expression.
-exceptinside:XYZ Skip occurences of the to-be-replaced text which lie
within XYZ. If the -regex argument is given, XYZ will be
regarded as a regular expression.
-exceptinsidetag:XYZ Skip occurences of the to-be-replaced text which lie
within an XYZ tag.
-summary:XYZ Set the summary message text for the edit to XYZ, bypassing
the predefined message texts with original and replacements
inserted.
-sleep:123 If you use -fix you can check multiple regex at the same time
in every page. This can lead to a great waste of CPU because
the bot will check every regex without waiting using all the
resources. This will slow it down between a regex and another
in order not to waste too much CPU.
-fix:XYZ Perform one of the predefined replacements tasks, which are
given in the dictionary 'fixes' defined inside the file
fixes.py.
The -regex and -nocase argument and given replacements will
be ignored if you use -fix.
Currently available predefined fixes are:
&fixes-help;
-always Don't prompt you for each replacement
-recursive Recurse replacement as long as possible. Be careful, this
might lead to an infinite loop.
-allowoverlap When occurences of the pattern overlap, replace all of them.
Be careful, this might lead to an infinite loop.
other: First argument is the old text, second argument is the new
text. If the -regex argument is given, the first argument
will be regarded as a regular expression, and the second
argument might contain expressions like \\1 or \g<name>.
Examples:
If you want to change templates from the old syntax, e.g. {{msg:Stub}}, to the
new syntax, e.g. {{Stub}}, download an XML dump file (pages-articles) from
http://download.wikimedia.org, then use this command:
python replace.py -xml -regex "{{msg:(.*?)}}" "{{\\1}}"
If you have a dump called foobar.xml and want to fix typos in articles, e.g.
Errror -> Error, use this:
python replace.py -xml:foobar.xml "Errror" "Error" -namespace:0
If you have a page called 'John Doe' and want to fix the format of ISBNs, use:
python replace.py -page:John_Doe -fix:isbn
This command will change 'referer' to 'referrer', but not in pages which
talk about HTTP, where the typo has become part of the standard:
python replace.py referer referrer -file:typos.txt -excepttext:HTTP
"""
#
# (C) Daniel Herding & the Pywikipediabot Team, 2004-2008
#
# Distributed under the terms of the MIT license.
#
from __future__ import generators
import os, codecs, re, time
from string import Template
import wikipedia, pagegenerators
import editarticle
import webbrowser
# Imports predefined replacements tasks from fixeswords.py
import fixeswords #, nesusije
# This is required for the text that is shown when you run this script
# with the parameter -help.
docuReplacements = {
'¶ms;': pagegenerators.parameterHelp,
'&fixes-help;': fixeswords.help,
}
__version__='$Id: replace.py 6412 2009-02-22 16:13:01Z nicdumz $'
# Summary messages in different languages
# NOTE: Predefined replacement tasks might use their own dictionary, see 'fixes'
# below.`v
msg = {
'ar': u'%s روبوت : استبدال تلقائي للنص',
'ca': u'Robot: Reemplaçament automàtic de text %s',
'cs': u'Robot automaticky nahradil text: %s',
'de': u'Bot: Automatisierte Textersetzung %s',
'el': u'Ρομπότ: Αυτόματη αντικατάσταση κειμένου %s',
'en': u'Robot: Automated text replacement %s',
'es': u'Robot: Reemplazo automático de texto %s',
'fa': u'ربات: تغییر خودکار متن %s',
'fr': u'Bot : Remplacement de texte automatisé %s',
'he': u'בוט: החלפת טקסט אוטומטית %s',
'hu': u'Robot: Automatikus szövegcsere %s',
'ia': u'Robot: Reimplaciamento automatic de texto %s',
'id': u'Bot: Penggantian teks otomatis %s',
'is': u'Vélmenni: breyti texta %s',
'it': u'Bot: Sostituzione automatica %s',
'ja': u'ロボットによる: 文字置き換え %s',
'ka': u'რობოტი: ტექსტის ავტომატური შეცვლა %s',
'kk': u'Бот: Мәтінді өздікті алмастырды: %s',
'ksh': u'Bot: hät outomatesch Täx jetuusch: %s',
'lt': u'robotas: Automatinis teksto keitimas %s',
'nds': u'Bot: Text automaatsch utwesselt: %s',
'nds-nl': u'Bot: autematisch tekse vervungen %s',
'nl': u'Bot: automatisch tekst vervangen %s',
'nn': u'robot: automatisk teksterstatning: %s',
'no': u'robot: automatisk teksterstatning: %s',
'pl': u'Robot automatycznie zamienia tekst %s',
'pt': u'Bot: Mudança automática %s',
'ru': u'Робот: Автоматизированная замена текста %s',
'sr': u'Бот: Аутоматска замена текста %s',
'sv': u'Bot: Automatisk textersättning: %s',
'zh': u'機器人:執行文字代換作業 %s',
}
class XmlDumpReplacePageGenerator:
"""
Iterator that will yield Pages that might contain text to replace.
These pages will be retrieved from a local XML dump file.
Arguments:
* xmlFilename - The dump's path, either absolute or relative
* xmlStart - Skip all articles in the dump before this one
* replacements - A list of 2-tuples of original text (as a
compiled regular expression) and replacement
text (as a string).
* exceptions - A dictionary which defines when to ignore an
occurence. See docu of the ReplaceRobot
constructor below.
"""
def __init__(self, xmlFilename, xmlStart, replacements, exceptions):
self.xmlFilename = xmlFilename
self.replacements = replacements
self.exceptions = exceptions
self.xmlStart = xmlStart
self.skipping = bool(xmlStart)
self.excsInside = []
if 'inside-tags' in self.exceptions:
self.excsInside += self.exceptions['inside-tags']
if 'inside' in self.exceptions:
self.excsInside += self.exceptions['inside']
import xmlreader
self.site = wikipedia.getSite()
dump = xmlreader.XmlDump(self.xmlFilename)
self.parser = dump.parse()
def __iter__(self):
try:
for entry in self.parser:
if self.skipping:
if entry.title != self.xmlStart:
continue
self.skipping = False
if not self.isTitleExcepted(entry.title) \
and not self.isTextExcepted(entry.text):
new_text = entry.text
for old, new in self.replacements:
new_text = wikipedia.replaceExcept(new_text, old, new, self.excsInside, self.site)
if new_text != entry.text:
yield wikipedia.Page(self.site, entry.title)
except KeyboardInterrupt:
try:
if not self.skipping:
wikipedia.output(
u'To resume, use "-xmlstart:%s" on the command line.'
% entry.title)
except NameError:
pass
def isTitleExcepted(self, title):
if 'title' in self.exceptions:
for exc in self.exceptions['title']:
if exc.search(title):
return True
if 'require-title' in self.exceptions:
for req in self.exceptions['require-title']:
if not req.search(title): # if not all requirements are met:
return True
return False
def isTextExcepted(self, text):
if 'text-contains' in self.exceptions:
for exc in self.exceptions['text-contains']:
if exc.search(text):
return True
if 'require-text' in self.exceptions:
ret = True
for exc in self.exceptions['require-text']:
if exc.search(text):
return False
return ret
return False
class PageCreateReader:
def __init__(self, words):
self.words = words
def run(self):
#wikipedia.output('Beginning \'%s\'...' % self.filesinfo)
for page, contents, wordsk, anton in self.words:
yield page, contents, wordsk, anton
class PageNesusijeReader:
def __init__(self, words):
self.words = words
def run(self):
#wikipedia.output('Beginning \'%s\'...' % self.filesinfo)
for page, topage in self.words:
try:
wikipedia.output('Beginning >>> \03{lightpurple}%s\03{default} <<<>>> \03{lightred}%s\03{default} <<<...' % (page, topage))
except:
wikipedia.output(u'Except on Output')
yield page, topage
class PageNesusijeRobot:
"""
Responsible for writing pages to the wiki, with the titles and contents
given by a PageFromFileReader.
"""
msg = {
'lt': u'Automated creating of articles',
}
# The following messages are added to topic when the page already exists
msg_top = {
'lt': u'append on top',
}
msg_bottom = {
'lt': u'append on bottom',
}
msg_force = {
'lt': u'existing text overwritten',
}
append = ''
def __getattr__(self, append):
return self.append
def __init__(self, reader = False, force = False, append = False, summary = 'Susiejama', minor = False,
autosummary = False, debug = False, acceptall=False, acceptallnew=False, quit=False):
self.reader = reader
self.force = force
self.append = append
self.summary = summary
self.minor = minor
self.autosummary = autosummary
self.debug = debug
self.acceptall = acceptall
self.acceptallnew = acceptallnew
self.quit = quit
def run(self):
for word, toword in self.reader.run():
self.put(word, toword)
if self.quit:
return
def put(self, word, toword):
mysite = wikipedia.getSite()
page = wikipedia.Page(mysite, word)
# Show the title of the page we're working on.
# Highlight the title in purple.
try:
wikipedia.output(u">>> \03{lightpurple}%s\03{default} <<<" % page.title())
except:
wikipedia.output(u'Except on Output')
if self.summary:
comment = self.summary
else:
comment = wikipedia.translate(mysite, self.msg)
if page.exists():
original_text = page.get()
isnew = True
new_text = original_text
needchange = True
#wikipedia.output(old)
temp = re.compile(ur'\{\{see\|(?P<parms>[^\}]*)\}\}', re.MULTILINE)
while temp.search(new_text) is not None and needchange:
for m in temp.finditer(new_text):
text = m.group()
isnew = False
parms = m.group('parms').strip()
oldsee = parms.split('|')
see = '{{see'
for wd in oldsee:
if wd.strip() == toword:
needchange = False
new_text = original_text
break
else:
see += '|' + wd.strip()
if not needchange:
break
see += '|' + toword + '}}'
new_text = new_text.replace(text, see)
break
if isnew:
new_text = '{{see|' + toword + u'}}\n' + original_text
while True:
if new_text == original_text:
try:
wikipedia.output('No changes were necessary in >>> \03{lightpurple}%s\03{default} <<< %s >>>'
% (page.aslink(), new_text[0:50]))
except:
wikipedia.output(u'Except on Output')
break
# Show the title of the page we're working on.
# Highlight the title in purple.
#wikipedia.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
# % page.title())
try:
wikipedia.showDiff(original_text, new_text)
except:
wikipedia.output(u'Except on Output')
if self.acceptall:
break
choice = wikipedia.inputChoice(
u'Do you want to accept these changes?',
['Yes', 'No', 'Edit', 'open in Browser', 'All', "Quit"],
['y', 'N', 'e', 'b', 'a', 'q'], 'N')
if choice == 'e':
editor = editarticle.TextEditor()
as_edited = editor.edit(original_text)
# if user didn't press Cancel
if as_edited and as_edited != new_text:
new_text = as_edited
continue
if choice == 'b':
webbrowser.open("http://%s%s" % (
page.site().hostname(),
page.site().nice_get_address(page.title())
))
wikipedia.input("Press Enter when finished in browser.")
original_text = page.get(get_redirect=True, force=True)
new_text = original_text
continue
if choice == 'q':
return
if choice == 'a':
self.acceptall = True
if choice == 'y':
page.put_async(new_text)
# choice must be 'N'
break
if self.acceptall and new_text != original_text:
try:
page.put(new_text, comment = comment, minorEdit = self.minor)
except wikipedia.LockedPage:
wikipedia.output(u"Page %s is locked; skipping." % title)
except wikipedia.EditConflict:
wikipedia.output(u'Skipping %s because of edit conflict' % title)
except wikipedia.SpamfilterError, error:
wikipedia.output(u'Cannot change %s because of spam blacklist entry %s' % (title, error.url))
prot = None
class PageCreateRobot:
"""
Responsible for writing pages to the wiki, with the titles and contents
given by a PageFromFileReader.
"""
msg = {
'lt': u'Automated creating of articles',
}
# The following messages are added to topic when the page already exists
msg_top = {
'lt': u'append on top',
}
msg_bottom = {
'lt': u'append on bottom',
}
msg_force = {
'lt': u'existing text overwritten',
}
append = ''
def __getattr__(self, append):
return self.append
def __init__(self, reader, force, append, summary, minor = False,
autosummary = False, debug = False, acceptall=False, acceptallnew=False, quit=False,
acceptallnewnotauto=False, prot=None, test=True, fromword = None):
self.reader = reader
self.force = force
self.append = append
self.summary = summary
self.minor = minor
self.autosummary = autosummary
self.debug = debug
self.acceptall = acceptall
self.acceptallnew = acceptallnew
self.quit = quit
self.acceptallnewnotauto=acceptallnewnotauto
self.prot = prot
self.test = test
self.fromword = fromword
self.skip = False
if self.fromword is not None:
self.skip = True
def run(self):
for word, contents, wordsk, anton in self.reader.run():
self.put(word, contents['title'], contents['contents'], contents['auto'], contents['reddir'], contents['nenaud'])
if self.quit:
return None
return self.fromword
def put(self, word, title, contents, auto, reddir, nenaud):
if self.fromword is not None:
if word == self.fromword:
self.skip = False
self.fromword = None
if self.skip:
return
wikipedia.output(u">>> \03{lightpurple}%s\03{default} <<<" % word)
#wikipedia.output(u">>> \03{lightpurple}%s\03{default} <<<\n%s" % (word,contents))
if self.test:
self.prot.write('# [[%s]]\r\n' % word)
return
mysite = wikipedia.getSite()
page = wikipedia.Page(mysite, word)
# Show the title of the page we're working on.
# Highlight the title in purple.
# wikipedia.output(u">>> \03{lightpurple}%s\03{default} <<<" % page.title())
if self.summary:
comment = self.summary
else:
comment = wikipedia.translate(mysite, self.msg)
comment_top = comment + " - " + wikipedia.translate(mysite, self.msg_top)
comment_bottom = comment + " - " + wikipedia.translate(mysite, self.msg_bottom)
comment_force = comment + " *** " + wikipedia.translate(mysite, self.msg_force) + " ***"
rezfile = u'Zodziai/'
if page.exists():
old = page.get(get_redirect=True)
try:
wikipedia.output(old)
except:
wikipedia.output(u'Except on Output')
if page.isRedirectPage():
_fnf = os.path.join('', rezfile+word+'.txt')
wprot = codecs.open(_fnf, "a+", "utf-8")
wprot.write('%s' % title + '\n' + contents)
return
## if self.test:
## self.prot.write('# [[%s]] - egzistavo, reikia patikrinti\r\n' % word)
## return
interwiki = wikipedia.interwikiFormat(wikipedia.getLanguageLinks(old))
#wikipedia.output('Interwiki: %s\n' % interwiki)
cat = wikipedia.categoryFormat(wikipedia.getCategoryLinks(old,mysite))
#wikipedia.output('Kategorijos: %s\n' % cat)
txt = wikipedia.removeLanguageLinks(old)
txt = wikipedia.removeCategoryLinks(txt,mysite)
#wikipedia.output('text: %s\n' % txt)
#wikipedia.output('text: %s\n' % (txt + '\n' + contents + '\n' + cat + '\n' + interwiki))
if self.append == None:
self.append = wikipedia.inputChoice(
u'Page <<%s>> already exists. What do?\n' % word,
['Top', 'Bottom', 'Change', 'Skip'],
['t', 'b', 'c', 's'], 's')
if self.append == "t":
wikipedia.output(u"Page %s already exists, appending on top!" % word)
contents = title + '\n' + contents + '\n' + txt + '\n'
if self.acceptallnewnotauto or self.force:
#contents += '{{patikrinti}}\n'
pass
contents += cat + '\n' + interwiki
comment = comment_top
elif self.append == "b":
wikipedia.output(u"Page %s already exists, appending on bottom!" % word)
contents = txt + '\n' + contents + '\n'
if self.acceptallnewnotauto or self.force:
#contents += '{{patikrinti}}\n'
pass
contents += cat + '\n' + interwiki
comment = comment_bottom
elif self.append == "c":
wikipedia.output(u"Page %s already exists, ***overwriting!" % word)
contents = title + '\n' + contents + '\n'
if self.acceptallnewnotauto or self.force:
contents += '{{patikrinti}}\n'
pass
contents += cat + '\n' + interwiki
comment = comment_force
else:
contents = old
if not self.acceptall or not self.force or not auto:
choice = None
if not auto:
if self.acceptallnewnotauto and not self.force:
choice = 'y'
elif self.force:
choice = 'y'
else:
choice = 'e'
else:
if self.force:
choice = 'y'
else:
choice = wikipedia.inputChoice(
u'\nDo you want to accept these changes?\n',
['Yes', 'No', 'Edit', 'Browser', 'All', 'AllOld', 'Skip', 'Change', "Quit"],
['y', 'N', 'e', 'b', 'a', 'ao', 's', 'c', 'q'], 'N')
if choice == 'e':
editor = editarticle.TextEditor()
as_edited = editor.edit(contents)
# if user didn't press Cancel
if as_edited and as_edited != contents:
contents = as_edited
elif choice == 'b':
webbrowser.open("http://%s%s" % (
page.site().hostname(),
page.site().nice_get_address(page.title())
))
wikipedia.input("Press Enter when finished in browser.")
old = page.get(get_redirect=True, force=True)
contents = old
elif choice == 'a':
self.acceptall = True
elif choice == 'ao':
self.acceptall = True
elif choice == 's':
self.append = 's'
wikipedia.output(u"Page %s already exists, not adding!" % word)
return
elif choice == 'c':
self.append = None
elif choice == 'q':
wikipedia.output(u"Page %s already exists, not adding!" % word)
self.quit = True
contents = old
return
elif choice != 'y':
wikipedia.output(u"Page %s already exists, not adding!" % word)
contents = old
return
if choice == 'c':
choice = wikipedia.inputChoice(
u'\nDo you want to accept these changes?\n',
['Yes', 'No', 'Edit', 'Browser', 'All', 'AllOld', 'Skip', "Quit"],
['y', 'N', 'e', 'b', 'a', 'ao', 's', 'q'], 'N')
if choice == 'e':
editor = editarticle.TextEditor()
as_edited = editor.edit(contents)
# if user didn't press Cancel
if as_edited and as_edited != contents:
contents = as_edited
elif choice == 'a':
self.acceptall = True
elif choice == 'ao':
self.acceptall = True
elif choice == 'b':
webbrowser.open("http://%s%s" % (
page.site().hostname(),
page.site().nice_get_address(page.title())
))
wikipedia.input("Press Enter when finished in browser.")
old = page.get(get_redirect=True, force=True)
contents = old
elif choice == 's':
wikipedia.output(u"Not adding!")
self.append = 's'
return
elif choice == 'q':
wikipedia.output(u"Not adding!")
self.quit = True
return
elif choice != 'y':
wikipedia.output(u"Not adding!")
return
if old == contents:
wikipedia.output(u"Page %s already exists, not adding!" % word)
return
else:
## if self.test: auto, reddir, nenaud
## return
if nenaud and auto:
_fnf = os.path.join('', rezfile+word+'.txt')
wprot = codecs.open(_fnf, "a+", "utf-8")
wprot.write('%s' % title + '\n' + contents)
contents = reddir
else:
contents = title + '\n' + contents
if not self.acceptallnew or not auto:
choice = None
if not auto:
if self.acceptallnewnotauto:
contents = contents #+ '{{patikrinti}}\n'
choice = 'y'
else:
choice = 'e'
else:
choice = wikipedia.inputChoice(
u'\nDo you want to accept these changes?\n',
['Yes', 'No', 'Edit', 'All', 'AllNew', 'Skip', 'Change', "Quit"],
['y', 'N', 'e', 'a', 'an', 's', 'c', 'q'], 'N')
if choice == 'e':
editor = editarticle.TextEditor()
as_edited = editor.edit(contents)
# if user didn't press Cancel
if as_edited and as_edited != contents:
contents = as_edited
elif choice == 'a':
self.acceptallnew = True
elif choice == 'an':
self.acceptallnew = True
elif choice == 's':
wikipedia.output(u"Not adding!")
self.append = 's'
return
elif choice == 'c':
self.append = None
elif choice == 'q':
wikipedia.output(u"Not adding!")
self.quit = True
return
elif choice != 'y':
wikipedia.output(u"Not adding!")
return
if choice == 'c':
choice = wikipedia.inputChoice(
u'\nDo you want to accept these changes?\n',
['Yes', 'No', 'Edit', 'All', 'AllNew', 'Skip', "Quit"],
['y', 'N', 'e', 'a', 'an', 's', 'q'], 'N')
if choice == 'e':
editor = editarticle.TextEditor()
as_edited = editor.edit(contents)
# if user didn't press Cancel
if as_edited and as_edited != contents:
contents = as_edited
elif choice == 'a':
self.acceptallnew = True
elif choice == 'an':
self.acceptallnew = True
elif choice == 's':
wikipedia.output(u"Not adding!")
self.append = 's'
return
elif choice == 'q':
wikipedia.output(u"Not adding!")
self.quit = True
return
elif choice != 'y':
wikipedia.output(u"Not adding!")
return
if self.autosummary:
comment = ''
wikipedia.setAction('')
# Remove trailing newlines (cause troubles when creating redirects)
contents = re.sub('^[\r\n]*','', contents)
if self.debug:
wikipedia.output("*** Debug mode ***\n" + \
"\03{lightpurple}word\03{default}: " + word + "\n" + \
"\03{lightpurple}contents\03{default}:\n" + contents + "\n" \
"\03{lightpurple}comment\03{default}: " + comment + "\n")
return
try:
page.put(contents, comment = comment, minorEdit = self.minor)
except wikipedia.LockedPage:
wikipedia.output(u"Page %s is locked; skipping." % title)
except wikipedia.EditConflict:
wikipedia.output(u'Skipping %s because of edit conflict' % title)
except wikipedia.SpamfilterError, error:
wikipedia.output(u'Cannot change %s because of spam blacklist entry %s' % (title, error.url))
class ReplaceRobot:
"""
A bot that can do text replacements.
"""
def __init__(self, generator, replacements, exceptions={},
acceptall=False, allowoverlap=False, recursive=False,
addedCat=None, sleep=None):
"""
Arguments:
* generator - A generator that yields Page objects.
* replacements - A list of 2-tuples of original text (as a
compiled regular expression) and replacement
text (as a string).
* exceptions - A dictionary which defines when not to change an
occurence. See below.
* acceptall - If True, the user won't be prompted before changes
are made.
* allowoverlap - If True, when matches overlap, all of them are
replaced.
* addedCat - If set to a value, add this category to every page
touched.
Structure of the exceptions dictionary:
This dictionary can have these keys:
title
A list of regular expressions. All pages with titles that
are matched by one of these regular expressions are skipped.
text-contains
A list of regular expressions. All pages with text that
contains a part which is matched by one of these regular
expressions are skipped.
inside
A list of regular expressions. All occurences are skipped which
lie within a text region which is matched by one of these
regular expressions.
inside-tags
A list of strings. These strings must be keys from the
exceptionRegexes dictionary in wikipedia.replaceExcept().
"""
self.generator = generator
self.replacements = replacements
self.exceptions = exceptions
self.acceptall = acceptall
self.allowoverlap = allowoverlap
self.recursive = recursive
if addedCat:
site = wikipedia.getSite()
cat_ns = site.category_namespaces()[0]
self.addedCat = wikipedia.Page(site,
cat_ns + ':' + addedCat)
self.sleep = sleep
#wikipedia.output(u'fix gen = %s' % self.replacements)
def isTitleExcepted(self, title):
"""
Iff one of the exceptions applies for the given title, returns True.
"""
if 'title' in self.exceptions:
for exc in self.exceptions['title']:
if exc.search(title):
return True
if 'require-title' in self.exceptions:
for req in self.exceptions['require-title']:
if not req.search(title):
return True
return False
def isTextExcepted(self, original_text):
"""
Iff one of the exceptions applies for the given page contents,
returns True.
"""
if 'text-contains' in self.exceptions:
for exc in self.exceptions['text-contains']:
if exc.search(original_text):
return True
if 'require-title' in self.exceptions:
for req in self.exceptions['require-title']:
if not req.search(title): # if not all requirements are met:
return True
if 'require-text' in self.exceptions:
ret = True
for exc in self.exceptions['require-text']:
if exc.search(original_text):
return False
return ret
return False
def doReplacements(self, original_text):
"""
Returns the text which is generated by applying all replacements to
the given text.
"""
new_text = original_text
exceptions = []
if 'inside-tags' in self.exceptions:
exceptions += self.exceptions['inside-tags']
if 'inside' in self.exceptions:
exceptions += self.exceptions['inside']
for old, new in self.replacements:
if self.sleep != None:
time.sleep(self.sleep)
new_text = wikipedia.replaceExcept(new_text, old, new, exceptions,
allowoverlap=self.allowoverlap)
return new_text
def run(self):
"""
Starts the robot.
"""
# Run the generator which will yield Pages which might need to be
# changed.
#wikipedia.output(u'fix gen = %s' % self.generator)
#return
for page in self.generator:
wikipedia.output(
u'File %s .'
% page.aslink())
if self.isTitleExcepted(page.title()):
wikipedia.output(
u'Skipping %s because the title is on the exceptions list.'
% page.aslink())
continue
try:
# Load the page's text from the wiki
original_text = page.get(get_redirect=True)
if not page.canBeEdited():
wikipedia.output(u"You can't edit page %s"
% page.aslink())
continue
except wikipedia.NoPage:
wikipedia.output(u'Page %s not found' % page.aslink())
continue
new_text = original_text
while True:
if self.isTextExcepted(new_text):
wikipedia.output(
u'Skipping %s because it contains text that is on the exceptions list.'
% page.aslink())
break
new_text = self.doReplacements(new_text)
if new_text == original_text:
wikipedia.output('No changes were necessary in %s'
% page.aslink())
break
if self.recursive:
newest_text = self.doReplacements(new_text)
while newest_text!=new_text:
new_text = newest_text
newest_text = self.doReplacements(new_text)
if hasattr(self, "addedCat"):
cats = page.categories(nofollow_redirects=True)
if self.addedCat not in cats:
cats.append(self.addedCat)
new_text = wikipedia.replaceCategoryLinks(new_text,
cats)
# Show the title of the page we're working on.
# Highlight the title in purple.
wikipedia.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<"
% page.title())
wikipedia.showDiff(original_text, new_text)
if self.acceptall:
break
choice = wikipedia.inputChoice(
u'Do you want to accept these changes?',
['Yes', 'No', 'Edit', 'open in Browser', 'All', "Quit"],
['y', 'N', 'e', 'b', 'a', 'q'], 'N')
if choice == 'e':
editor = editarticle.TextEditor()
as_edited = editor.edit(original_text)
# if user didn't press Cancel
if as_edited and as_edited != new_text:
new_text = as_edited
continue
if choice == 'b':
webbrowser.open("http://%s%s" % (
page.site().hostname(),
page.site().nice_get_address(page.title())
))
wikipedia.input("Press Enter when finished in browser.")
original_text = page.get(get_redirect=True, force=True)
new_text = original_text
continue
if choice == 'q':
return
if choice == 'a':
self.acceptall = True
if choice == 'y':
page.put_async(new_text)
# choice must be 'N'
break
if self.acceptall and new_text != original_text:
try:
page.put(new_text)
except wikipedia.EditConflict:
wikipedia.output(u'Skipping %s because of edit conflict'
% (page.title(),))
except wikipedia.SpamfilterError, e:
wikipedia.output(
u'Cannot change %s because of blacklist entry %s'
% (page.title(), e.url))
except wikipedia.PageNotSaved, error:
wikipedia.output(u'Error putting page: %s'
% (error.args,))
except wikipedia.LockedPage:
wikipedia.output(u'Skipping %s (locked page)'
% (page.title(),))
def prepareRegexForMySQL(pattern):
pattern = pattern.replace('\s', '[:space:]')
pattern = pattern.replace('\d', '[:digit:]')
pattern = pattern.replace('\w', '[:alnum:]')
pattern = pattern.replace("'", "\\" + "'")
#pattern = pattern.replace('\\', '\\\\')
#for char in ['[', ']', "'"]:
# pattern = pattern.replace(char, '\%s' % char)
return pattern
def noskiem(word):
return word.replace('~', '')
def skiem(word):
return word.replace('~~', '-').replace('~','')
def cikleWords (wordsList, force, append, acceptallnew=False, acceptallnewnotauto=False,
acceptallgroups=False, prot=None, test=True, fromword = None, zodziai={}):
acceptallgroup = acceptallgroups
for word in wordsList:
#wikipedia.output(u'word <<< %s >>>.' % word)
wordform = fixeswords.wordforms[word['dalis']][word['form']]
eti = u''
if word['fs0'] != u'':
t1 = Template(wordform[word['fs0']])
eti = t1.substitute(s1 = noskiem(word['s1']), s2 = noskiem(word['s2']),
s3 = noskiem(word['s3']), s4 = noskiem(word['s4']), s5 = noskiem(word['s5']),
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'],
fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
## if bot.append == 's':
## bot.append = None
subwordsList = []
for word2 in wordform['wordsList']:
#wikipedia.output(u'word2 <<< %s >>>.' % word2)
#wikipedia.output(u'word <<< %s >>>.' % word)
reiksmesimas = ''
reiksmesejas = ''
reiksmeseja = ''
reiksmesojas = ''
reiksmesoja = ''
if u'reiksmesimas' in word:
reiksmesimas = word[u'reiksmesimas']
if u'reiksmesejas' in word:
reiksmesejas = word[u'reiksmesejas']
if u'reiksmeseja' in word:
reiksmeseja = word[u'reiksmeseja']
if u'reiksmesojas' in word:
reiksmesojas = word[u'reiksmesojas']
if u'reiksmesoja' in word:
reiksmesoja = word[u'reiksmesoja']
lg1 = Template(word2['g1'])
lg1 = lg1.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
ls1 = Template(word2['s1'])
ls1 = ls1.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
ls2 = Template(word2['s2'])
ls2 = ls2.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
ls3 = Template(word2['s3'])
ls3 = ls3.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
ls4 = Template(word2['s4'])
ls4 = ls4.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
ls5 = Template(word2['s5'])
ls5 = ls5.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
lfs0 = Template(word2['fs0'])
lfs0 = noskiem(lfs0.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form']))
lfs1 = Template(word2['fs1'])
lfs1 = noskiem(lfs1.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form']))
lfs2 = Template(word2['fs2'])
lfs2 = noskiem(lfs2.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form']))
lfs3 = Template(word2['fs3'])
lfs3 = noskiem(lfs3.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form']))
lfs4 = Template(word2['fs4'])
lfs4 = noskiem(lfs4.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form']))
lfs5 = Template(word2['fs5'])
lfs5 = noskiem(lfs5.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form']))
ltipas = Template(word2['tipas'])
ltipas = noskiem(ltipas.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form']))
lreiksmes = Template(word2['reiksmes'])
lreiksmes = noskiem(lreiksmes.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'],
tipas = word['tipas'], form = word['form'],
reiksmesimas = reiksmesimas, reiksmesejas = reiksmesejas, reiksmeseja = reiksmeseja,
reiksmesojas = reiksmesojas, reiksmesoja = reiksmesoja))
lpozymis = Template(word2['pozymis'])
lpozymis = noskiem(lpozymis.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'],
pozymis = word['pozymis'], x = u'${x}'))
sins = word2['sinonimai']
lsinonimai = []
for sin in sins:
lsinonimai.append(sin)
isrs = word2['israiskos']
lisraiskos = []
for isr in isrs:
lisraiskos.append(isr)
newword = {'dalis': word2['dalis'],
'form': word2['form'],
'g1': lg1, 's1': ls1, 's2': ls2, 's3': ls3, 's4': ls4, 's5': ls5,
'fs0': lfs0, 'fs1': lfs1, 'fs2': lfs2,
'pozymis': lpozymis,
'israiskos': lisraiskos,
'sinonimai': lsinonimai,
'reiksmes': lreiksmes,
'fs3': lfs3, 'fs4': eti, 'fs5': lfs5, 'tipas': ltipas,
}
if u'List' in word2:
newword['List'] = word2['List']
if '[[neig.]] [[neig.]] ' not in ltipas:
subwordsList.append(newword)
#wikipedia.output(u'subwordsList <<< %s >>>.' % subwordsList)
#return
if u'List' in word:
for word2 in word['List']:
#wikipedia.output(u'word2 <<< %s >>>.' % word2)
#wikipedia.output(u'word <<< %s >>>.' % word)
reiksmesimas = ''
reiksmesejas = ''
reiksmeseja = ''
reiksmesojas = ''
reiksmesoja = ''
if u'reiksmesimas' in word2:
reiksmesimas = word2[u'reiksmesimas']
if u'reiksmesejas' in word2:
reiksmesejas = word2[u'reiksmesejas']
if u'reiksmeseja' in word2:
reiksmeseja = word2[u'reiksmeseja']
if u'reiksmesojas' in word2:
reiksmesojas = word2[u'reiksmesojas']
if u'reiksmesoja' in word2:
reiksmesoja = word2[u'reiksmesoja']
lg1 = Template(word2['g1'])
lg1 = lg1.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
ls1 = Template(word2['s1'])
ls1 = ls1.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
ls2 = Template(word2['s2'])
ls2 = ls2.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
ls3 = Template(word2['s3'])
ls3 = ls3.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
ls4 = Template(word2['s4'])
ls4 = ls4.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
ls5 = Template(word2['s5'])
ls5 = ls5.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
lfs0 = Template(word2['fs0'])
lfs0 = noskiem(lfs0.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form']))
lfs1 = Template(word2['fs1'])
lfs1 = noskiem(lfs1.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form']))
lfs2 = Template(word2['fs2'])
lfs2 = noskiem(lfs2.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form']))
lfs3 = Template(word2['fs3'])
lfs3 = noskiem(lfs3.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form']))
lfs4 = Template(word2['fs4'])
lfs4 = noskiem(lfs4.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form']))
lfs5 = Template(word2['fs5'])
lfs5 = noskiem(lfs5.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form']))
ltipas = Template(word2['tipas'])
ltipas = noskiem(ltipas.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form']))
lreiksmes = Template(word2['reiksmes'])
lreiksmes = noskiem(lreiksmes.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'],
tipas = word['tipas'], form = word['form'],
reiksmesimas = reiksmesimas, reiksmesejas = reiksmesejas, reiksmeseja = reiksmeseja,
reiksmesojas = reiksmesojas, reiksmesoja = reiksmesoja))
lpozymis = Template(word2['pozymis'])
lpozymis = noskiem(lpozymis.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'], fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'],
pozymis = word['pozymis'], x = u'${x}'))
sins = word2['sinonimai']
lsinonimai = []
for sin in sins:
lsinonimai.append(sin)
isrs = word2['israiskos']
lisraiskos = []
for isr in isrs:
lisraiskos.append(isr)
newword = {'dalis': word2['dalis'],
'form': word2['form'],
'g1': lg1, 's1': ls1, 's2': ls2, 's3': ls3, 's4': ls4, 's5': ls5,
'fs0': lfs0, 'fs1': lfs1, 'fs2': lfs2,
'pozymis': lpozymis,
'israiskos': lisraiskos,
'sinonimai': lsinonimai,
'reiksmes': lreiksmes,
'fs3': lfs3, 'fs4': eti, 'fs5': lfs5, 'tipas': ltipas,
}
if u'List' in word2:
newword['List'] = word2['List']
if '[[neig.]] [[neig.]] ' not in ltipas:
subwordsList.append(newword)
wordsll = []
for wrd in subwordsList:
wrdform = fixeswords.wordforms[wrd['dalis']][wrd['form']]
wrdref = Template(wrdform['ref'])
wrdref = wrdref.substitute(s1 = noskiem(wrd['s1']), s2 = noskiem(wrd['s2']), s3 = noskiem(wrd['s3']),
s4 = noskiem(wrd['s4']), s5 = noskiem(wrd['s5']),
fs0 = wrd['fs0'], fs1 = wrd['fs1'], fs2 = wrd['fs2'], fs3 = wrd['fs3'],
fs4 = wrd['fs4'], fs5 = wrd['fs5'], tipas = wrd['tipas'], form = wrd['form'])
if wrdref != u'':
wordsllt = u'* '+wrdref
if wrd[u'tipas'] != u'':
wordsllt += u' ('+wrd[u'tipas']+u')'
pozymis = Template(wrd['pozymis'])
pozymis = pozymis.substitute(x = u'{{x}}')
wordsllt += pozymis
wordsll.append(wordsllt)
wordsll.sort()
for wrd in wordsll:
wikipedia.output(u' <<<\03{lightpurple}%s\03{default}>>> ' % wrd)
Words = []
for wordtemp, form in wordform['forms']:
t1 = Template(wordtemp)
t1 = t1.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'],
fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
skiemt1 = skiem(t1)
noskiemt1 = noskiem(t1)
ps1 = noskiem(word['s1'])
pg1 = noskiem(word['g1'])
tf = Template(form)
tf = tf.substitute(s1 = word['s1'], s2 = word['s2'], s3 = word['s3'], s4 = word['s4'], s5 = word['s5'],
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'],
fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'])
t2 = Template(wordform[tf]['text'])
sinonimai = u''
if len(word['sinonimai']) > 0:
sinonimai = u'==== Sinonimai ====\n\n{{sin-top|kalba=lt|vardas='+noskiemt1+u'|tipas=bendrai}}\n'
for wordii,sin in enumerate(word['sinonimai']):
sinonimai += u'* {{t+|lt|'+sin+u'}}\n'
if wordii < len(word['sinonimai'])/2.0 <= wordii+1:
sinonimai += u'{{sin-mid}}\n'
sinonimai += u'{{sin-bottom}}\n\n'
antonimai = u''
antonimas = u''
if noskiemt1.find('ne') == 0 and ('[[ne-]]' in word['fs4']
or '[[ne-]]' in word['fs5']):
antonimas = noskiemt1[2:]
antonimai = u'==== Antonimai ====\n\n{{ant-top|kalba=lt|vardas='+noskiemt1+u'|tipas=bendrai}}\n'
antonimai += u'* {{t+|lt|'+noskiemt1[2:]+u'}}\n'
antonimai += u'{{ant-mid}}\n{{ant-bottom}}\n\n'
else:
if word['fs5'].find(u'{{sangrž.}}') == 0:
antonimas = 'nesi'+noskiemt1
if antonimas.endswith('si'):
antonimas = antonimas[:-2]
if antonimas.endswith('s'):
antonimas = antonimas[:-1]
antonimai = u'==== Antonimai ====\n\n{{ant-top|kalba=lt|vardas='+noskiemt1+u'|tipas=bendrai}}\n'
antonimai += u'* {{t+|lt|'+antonimas+u'}}\n'
antonimai += u'{{ant-mid}}\n{{ant-bottom}}\n\n'
else:
antonimas = 'ne'+noskiemt1
antonimai = u'==== Antonimai ====\n\n{{ant-top|kalba=lt|vardas='+noskiemt1+u'|tipas=bendrai}}\n'
antonimai += u'* {{t+|lt|ne'+noskiemt1+u'}}\n'
antonimai += u'{{ant-mid}}\n{{ant-bottom}}\n\n'
isvestiniai = u''
if len(wordsll) > 0:
isvestiniai = u'==== Išvestiniai žodžiai ====\n\n{{rel-top|kalba=lt|vardas='+noskiemt1+u'|tipas=Išvestiniai žodžiai}}\n'
for wordii,wrd in enumerate(wordsll):
isvestiniai += wrd+u'\n'
if wordii < len(wordsll)/2.0 <= wordii+1:
isvestiniai = isvestiniai + u'{{rel-mid}}\n'
isvestiniai += u'{{rel-bottom}}\n\n'
israiskos = u''
if len(word['israiskos']) > 0:
israiskos = u'==== Išraiškos arba posakiai ====\n\n{{rel-top|kalba=lt|vardas='+noskiemt1+u'|tipas=Išraiškos arba posakiai}}\n'
for wordii,isr in enumerate(word['israiskos']):
israiskos += u'* {{t+|lt|'+isr+u'}}\n'
if wordii < len(word['israiskos'])/2.0 <= wordii+1:
israiskos += u'{{rel-mid}}\n'
israiskos += u'{{rel-bottom}}\n\n'
vertimai = u''
sabl = re.compile(u'^\#+\s+(?P<tipas>.*?)(?:\:|\.)$', re.M)
inmatch = 0
match = sabl.search(word['reiksmes'], inmatch)
while match != None:
inmatch = match.end()
vtipas = match.group('tipas')
if vtipas != None:
sabl1 = re.compile(u'\[\[(?P<n1>[^\[\]\|]*?)\|(?P<n2>[^\[\]\|]*?)\]\]', re.M)
match1 = sabl1.search(vtipas, 0)
while match1 != None:
n1 = match1.group('n1')
n2 = match1.group('n2')
vtipas = vtipas[:match1.start()] + n2 + vtipas[match1.end():]
match1 = sabl1.search(vtipas, 0)
sabl1 = re.compile(u'\[\[(?P<n1>[^\[\]\|]*?)\]\]', re.M)
match1 = sabl1.search(vtipas, 0)
while match1 != None:
n1 = match1.group('n1')
vtipas = vtipas[:match1.start()] + n1 + vtipas[match1.end():]
match1 = sabl1.search(vtipas, 0)
vertimai += u'{{trans-top|kalba=lt|vardas=' + noskiemt1 + u'|tipas='+ vtipas + u'}}\n' + \
u'{{trans-mid}}\n' + \
u'{{trans-bottom}}\n'
match = sabl.search(word['reiksmes'], inmatch)
if noskiemt1 not in zodziai:
zodziai[noskiemt1] = {}
if u'pagrf' not in zodziai[noskiemt1]:
zodziai[noskiemt1][u'pagrf'] = not wordform[tf]['auto']
elif not zodziai[noskiemt1][u'pagrf']:
zodziai[noskiemt1][u'pagrf'] = not wordform[tf]['auto']
pozymis = Template(word['pozymis'])
if zodziai[noskiemt1][u'pagrf']:
pozymis = pozymis.substitute(x = u'{{x|lt|p}}')
else:
pozymis = pozymis.substitute(x = u'{{x|lt|}}')
reddir = Template(u'#REDIRECT [[${reddir}]]')
reddir = reddir.substitute(reddir = ps1+pg1)
sabl2 = re.compile(u'\{\{x\|lt\|p?\}\}', re.M)
match2 = sabl2.search(pozymis, 0)
nenaud = False
if match2 != None:
nenaud = True
t2 = t2.substitute(s1 = noskiem(word['s1']), s2 = noskiem(word['s2']),
s3 = noskiem(word['s3']), s4 = noskiem(word['s4']), s5 = noskiem(word['s5']),
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'],
fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'],
reiksmes = word['reiksmes'], pozymis = pozymis,
sinonimai = sinonimai, antonimai = antonimai, isvestiniai = isvestiniai, israiskos = israiskos,
vertimai = vertimai, word = noskiemt1, words = skiemt1)
sabl3 = re.compile(u'\[\[\]\]\, \[\[\]\]\, \[\[\]\]', re.M)
match3 = sabl3.search(t2, 0)
t3 = Template(wordform[tf]['title'])
if match3 != None:
t2 = t2[:match3.start()] + t2[match3.end():]
t3 = t3.substitute(s1 = noskiem(word['s1']), s2 = noskiem(word['s2']),
s3 = noskiem(word['s3']), s4 = noskiem(word['s4']), s5 = noskiem(word['s5']),
fs0 = word['fs0'], fs1 = word['fs1'], fs2 = word['fs2'], fs3 = word['fs3'],
fs4 = word['fs4'], fs5 = word['fs5'], tipas = word['tipas'], form = word['form'],
word = noskiemt1, words = skiemt1)
Words.append((noskiemt1, {'title':t3, 'contents':t2, 'auto':not zodziai[noskiemt1][u'pagrf'],
'reddir':reddir, 'nenaud':nenaud}, skiemt1, antonimas))
preloadingGen = PageCreateReader(Words)
bot = PageCreateRobot(preloadingGen, force, append, wordform['msg'], acceptallnew=acceptallnew,
acceptallnewnotauto=acceptallnewnotauto, prot=prot, test=test, fromword = fromword)
wikipedia.output(u'Bus formuojami: ')
for wordl, descr, wordlsk, anton in Words:
wikipedia.output(u' <<<\03{lightpurple}%s\03{default}>>> <<<\03{lightpurple}%s\03{default}>>> <<<\03{lightpurple}%s\03{default}>>> ' % (wordl, wordlsk, anton))
if acceptallgroup:
fromword = bot.run()
else:
choice = wikipedia.inputChoice(
u'Ar formuoti?',
['Create', 'CreateAll', 'No'],
['c', 'ca', 'N'], 'N')
if choice == 'c':
fromword = bot.run()
elif choice == 'ca':
acceptallgroup = True
fromword = bot.run()
(acceptallgroup, fromword, zodziai) = cikleWords (subwordsList, force, bot.append, acceptallnew=acceptallnew,
acceptallnewnotauto=acceptallnewnotauto, acceptallgroups=acceptallgroup, prot=prot,
test=test, fromword = fromword, zodziai=zodziai)
return (acceptallgroup, fromword, zodziai)
def main(*args):
add_cat = None
gen = None
# summary message
summary_commandline = None
# Array which will collect commandline parameters.
# First element is original text, second element is replacement text.
commandline_replacements = []
# A list of 2-tuples of original text and replacement text.
replacements = []
# Don't edit pages which contain certain texts.
exceptions = {
'title': [],
'text-contains': [],
'require-text': [],
'inside': [],
'inside-tags': [],
'require-title': [], # using a seperate requirements dict needs some
} # major refactoring of code.
# Should the elements of 'replacements' and 'exceptions' be interpreted
# as regular expressions?
regex = False
# Predefined fixes from dictionary 'fixes' (see above).
fix = None
force = False
append = 'b'
summary = None
minor = False
autosummary = False
debug = False
# the dump's path, either absolute or relative, which will be used
# if -xml flag is present
xmlFilename = None
useSql = False
PageTitles = []
# will become True when the user presses a ('yes to all') or uses the
# -always flag.
acceptall = False
# Will become True if the user inputs the commandline parameter -nocase
caseInsensitive = False
# Will become True if the user inputs the commandline parameter -dotall
dotall = False
# Will become True if the user inputs the commandline parameter -multiline
multiline = False
# Do all hits when they overlap
allowoverlap = False
# Do not recurse replacement
recursive = False
# This factory is responsible for processing command line arguments
# that are also used by other scripts and that determine on which pages
# to work on.
genFactory = pagegenerators.GeneratorFactory()
# Load default summary message.
# BUG WARNING: This is probably incompatible with the -lang parameter.
wikipedia.setAction(wikipedia.translate(wikipedia.getSite(), msg))
# Between a regex and another (using -fix) sleep some time (not to waste
# too much CPU
sleep = None
#Set the regular expression flags
flags = re.UNICODE
if caseInsensitive:
flags = flags | re.IGNORECASE
if dotall:
flags = flags | re.DOTALL
if multiline:
flags = flags | re.MULTILINE
# Read commandline parameters.
for arg in wikipedia.handleArgs(*args):
if arg == '-regex':
regex = True
elif arg.startswith('-xmlstart'):
if len(arg) == 9:
xmlStart = wikipedia.input(
u'Please enter the dumped article to start with:')
else:
xmlStart = arg[10:]
elif arg.startswith('-xml'):
if len(arg) == 4:
xmlFilename = wikipedia.input(
u'Please enter the XML dump\'s filename:')
else:
xmlFilename = arg[5:]
elif arg =='-sql':
useSql = True
elif arg.startswith('-page'):
if len(arg) == 5:
PageTitles.append(wikipedia.input(
u'Which page do you want to change?'))
else:
PageTitles.append(arg[6:])
elif arg.startswith('-excepttitle:'):
exceptions['title'].append(arg[13:])
elif arg.startswith('-requiretitle:'):
exceptions['require-title'].append(arg[14:])
elif arg.startswith('-excepttext:'):
exceptions['text-contains'].append(arg[12:])
elif arg.startswith('-exceptinside:'):
exceptions['inside'].append(arg[14:])
elif arg.startswith('-exceptinsidetag:'):
exceptions['inside-tags'].append(arg[17:])
elif arg == "-appendtop":
append = "t"
elif arg == "-appendbottom":
append = "b"
elif arg == "-force":
force=True
elif arg == '-minor':
minor = True
elif arg.startswith("-summary:"):
summary = arg[9:]
elif arg == '-autosummary':
autosummary = True
elif arg.startswith('-fix:'):
fix = arg[5:]
elif arg.startswith('-sleep:'):
sleep = float(arg[7:])
elif arg == '-always':
acceptall = True
elif arg == '-recursive':
recursive = True
elif arg == '-nocase':
caseInsensitive = True
elif arg == '-dotall':
dotall = True
elif arg == '-multiline':
multiline = True
elif arg.startswith('-addcat:'):
add_cat = arg[len('addcat:'):]
elif arg.startswith('-summary:'):
wikipedia.setAction(arg[len('-summary:'):])
summary_commandline = True
elif arg.startswith('-allowoverlap'):
allowoverlap = True
else:
if not genFactory.handleArg(arg):
commandline_replacements.append(arg)
if len(commandline_replacements) % 2:
raise wikipedia.Error, 'require even number of replacements.'
elif len(commandline_replacements) == 2 and fix == None:
replacements.append((commandline_replacements[0],
commandline_replacements[1]))
if summary_commandline == None:
wikipedia.setAction(wikipedia.translate(wikipedia.getSite(), msg )
% (' (-' + commandline_replacements[0] + ' +'
+ commandline_replacements[1] + ')'))
elif len(commandline_replacements) > 1:
if fix == None:
for i in xrange (0, len(commandline_replacements), 2):
replacements.append((commandline_replacements[i],
commandline_replacements[i + 1]))
if summary_commandline == None:
pairs = [( commandline_replacements[i],
commandline_replacements[i + 1] )
for i in range(0, len(commandline_replacements), 2)]
replacementsDescription = '(%s)' % ', '.join(
[('-' + pair[0] + ' +' + pair[1]) for pair in pairs])
wikipedia.setAction(
wikipedia.translate(wikipedia.getSite(), msg )
% replacementsDescription)
else:
raise wikipedia.Error(
'Specifying -fix with replacements is undefined')
elif fix == None:
old = wikipedia.input(u'Please enter the text that should be replaced:')
new = wikipedia.input(u'Please enter the new text:')
change = '(-' + old + ' +' + new
replacements.append((old, new))
while True:
old = wikipedia.input(
u'Please enter another text that should be replaced, or press Enter to start:')
if old == '':
change = change + ')'
break
new = wikipedia.input(u'Please enter the new text:')
change = change + ' & -' + old + ' +' + new
replacements.append((old, new))
if not summary_commandline == True:
default_summary_message = wikipedia.translate(wikipedia.getSite(), msg) % change
wikipedia.output(u'The summary message will default to: %s'
% default_summary_message)
summary_message = wikipedia.input(
u'Press Enter to use this default message, or enter a description of the\nchanges your bot will make:')
if summary_message == '':
summary_message = default_summary_message
wikipedia.setAction(summary_message)
elif fix not in ('word', 'nesusije') :
# Perform one of the predefined actions.
try:
fix = fixeswords.fixes[fix]
except KeyError:
wikipedia.output(u'Available predefined fixes are: %s'
% fixeswords.fixes.keys())
return
if 'regex' in fix:
regex = fix['regex']
if 'msg' in fix:
wikipedia.setAction(
wikipedia.translate(wikipedia.getSite(), fix['msg']))
if 'exceptions' in fix:
exceptions = fix['exceptions']
if 'nocase' in fix:
caseInsensitive = fix['nocase']
replacements = fix['replacements']
#wikipedia.output(u'fix get = %s' % fix)
# Pre-compile all regular expressions here to save time later
for i in range(len(replacements)):
old, new = replacements[i]
if not regex:
old = re.escape(old)
oldR = re.compile(old, flags)
replacements[i] = oldR, new
for exceptionCategory in ['title', 'require-title', 'text-contains', 'inside', 'require-text']:
if exceptionCategory in exceptions:
patterns = exceptions[exceptionCategory]
if not regex:
patterns = [re.escape(pattern) for pattern in patterns]
patterns = [re.compile(pattern, flags) for pattern in patterns]
exceptions[exceptionCategory] = patterns
#wikipedia.output(u'fix rep = %s' % replacements)
#wikipedia.output(u'fix page = %s' % PageTitles)
#return
if xmlFilename:
try:
xmlStart
except NameError:
xmlStart = None
gen = XmlDumpReplacePageGenerator(xmlFilename, xmlStart,
replacements, exceptions)
elif useSql:
whereClause = 'WHERE (%s)' % ' OR '.join(
["old_text RLIKE '%s'" % prepareRegexForMySQL(old.pattern)
for (old, new) in replacements])
if exceptions:
exceptClause = 'AND NOT (%s)' % ' OR '.join(
["old_text RLIKE '%s'" % prepareRegexForMySQL(exc.pattern)
for exc in exceptions])
else:
exceptClause = ''
query = u"""
SELECT page_namespace, page_title
FROM page
JOIN text ON (page_id = old_id)
%s
%s
LIMIT 200""" % (whereClause, exceptClause)
gen = pagegenerators.MySQLPageGenerator(query)
elif PageTitles:
pages = [wikipedia.Page(wikipedia.getSite(), PageTitle)
for PageTitle in PageTitles]
gen = iter(pages)
if fix not in ('word', 'nesusije'):
gen = genFactory.getCombinedGenerator(gen)
if fix not in ('word', 'nesusije'):
if not gen:
# syntax error, show help text from the top of this file
wikipedia.showHelp('replace')
return
if xmlFilename:
# XML parsing can be quite slow, so use smaller batches and
# longer lookahead.
preloadingGen = pagegenerators.PreloadingGenerator(gen,
pageNumber=20, lookahead=100)
elif fix not in ('word', 'nesusije'):
preloadingGen = pagegenerators.PreloadingGenerator(gen, pageNumber=60)
_fnf = os.path.join('', 'protokolas.txt')
prot = codecs.open(_fnf, "w+", "utf-8")
if fix == 'word':
fromword = None
zodziai = {}
(acceptallgroup, fromword, zodziai) = cikleWords(fixeswords.wordsList, force, append, acceptallnew=True,
acceptallnewnotauto=True, acceptallgroups=False, prot=prot,
test=True, fromword=fromword, zodziai=zodziai)
_fnf = os.path.join('', 'protokolas2.txt')
prot = codecs.open(_fnf, "w+", "utf-8")
wikipedia.output(u'<<<\03{lightpurple} Prasideda straipsnių kūrimo etapas. \03{default}>>>')
(acceptallgroup, fromword, zodziai) = cikleWords(fixeswords.wordsList, force, append, acceptallnew=True,
acceptallnewnotauto=True, acceptallgroups=False, prot=prot,
test=False, fromword=fromword, zodziai=zodziai)
elif fix == 'nesusije':
wikipedia.output(u'fix = %s' % fix)
preloadingGen = PageNesusijeReader(nesusije.wordsList)
bot = PageNesusijeRobot(preloadingGen)
bot.run()
else:
bot = ReplaceRobot(preloadingGen, replacements, exceptions, acceptall, allowoverlap, recursive, add_cat, sleep)
#wikipedia.output(u'fix gen = %s' % preloadingGen)
#return
bot.run()
if __name__ == "__main__":
try:
main()
finally:
wikipedia.stopme()
