Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
or not isinstance(v, str)]
if unicode_env:
raise TypeError(
'{}: unicode in os.environ: {!r}'.format(e, unicode_env))
child_unicode_env = [(k, v) for k, v in env.items()
if not isinstance(k, str)
or not isinstance(v, str)]
if child_unicode_env:
raise TypeError(
'{}: unicode in child env: {!r}'
.format(e, child_unicode_env))
raise
if data_in is not None:
p.stdin.write(data_in.encode(config.console_encoding))
p.stdin.flush() # _communicate() otherwise has a broken pipe
stderr_lines = b''
waited = 0
while (error or (waited < timeout)) and p.poll() is None:
# In order to kill 'shell' and others early, read only a single
# line per second, and kill the process as soon as the expected
# output has been seen.
# Additional lines will be collected later with p.communicate()
if error:
line = p.stderr.readline()
stderr_lines += line
if error in line.decode(config.console_encoding):
break
time.sleep(1)
waited += 1
"User '%s' does not have upload rights on site %s."
% (self.targetSite.user(), self.targetSite))
return
try:
if isinstance(self.url, basestring):
self._treat_counter = 1
return self.upload_file(self.url)
for file_url in self.url:
self.upload_file(file_url)
self._treat_counter += 1
except QuitKeyboardInterrupt:
pywikibot.output('\nUser quit %s bot run...' %
self.__class__.__name__)
except KeyboardInterrupt:
if config.verbose_output:
raise
else:
pywikibot.output('\nKeyboardInterrupt during %s bot run...' %
self.__class__.__name__)
finally:
self.exit()
def setDelays(self, delay=None, writedelay=None, absolute=False):
"""Set the nominal delays in seconds. Defaults to config values."""
with self.lock:
if delay is None:
delay = self.mindelay
if writedelay is None:
writedelay = config.put_throttle
if absolute:
self.maxdelay = delay
self.mindelay = delay
self.delay = delay
self.writedelay = min(max(self.mindelay, writedelay),
self.maxdelay)
# Start the delay count now, not at the next check
self.last_read = self.last_write = time.time()
def __init__(self, rebuild=False, filename='category.dump.bz2'):
"""Initializer."""
if not os.path.isabs(filename):
filename = config.datafilepath(filename)
self.filename = filename
if rebuild:
self.rebuild()
notUpdatedSites = []
# Process all languages here
self.conf.always = False
if self.conf.limittwo:
lclSite = self.originPage.site
lclSiteDone = False
frgnSiteDone = False
for siteCode in lclSite.family.languages_by_size:
site = pywikibot.Site(siteCode, lclSite.family)
if (not lclSiteDone and site == lclSite) or \
(not frgnSiteDone and site != lclSite and site in new):
if site == lclSite:
lclSiteDone = True # even if we fail the update
if (site.family.name in config.usernames
and site.code in config.usernames[
site.family.name]):
try:
if self.replaceLinks(new[site], new):
updatedSites.append(site)
if site != lclSite:
frgnSiteDone = True
except SaveError:
notUpdatedSites.append(site)
except GiveUpOnPage:
break
elif (not self.conf.strictlimittwo
and site in new and site != lclSite):
old = {}
try:
for link in new[site].iterlanglinks():
page = pywikibot.Page(link)
if hasattr(root_logger, 'captureWarnings'):
root_logger.captureWarnings(True) # introduced in Python >= 2.7
root_logger.handlers = [] # remove any old handlers
# configure handler(s) for display to user interface
ui.init_handlers(root_logger, **config.userinterface_init_kwargs)
# if user has enabled file logging, configure file handler
if moduleName in config.log or '*' in config.log:
if config.logfilename:
logfile = config.datafilepath("logs", config.logfilename)
else:
logfile = config.datafilepath("logs", "%s-bot.log" % moduleName)
file_handler = RotatingFileHandler(filename=logfile,
maxBytes=1024 * config.logfilesize,
backupCount=config.logfilecount)
file_handler.setLevel(DEBUG)
form = LoggingFormatter(
fmt="%(asctime)s %(caller_file)18s, %(caller_line)4s "
"in %(caller_name)18s: %(levelname)-8s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S"
)
file_handler.setFormatter(form)
root_logger.addHandler(file_handler)
# Turn on debugging for each component requested by user
# or for all components if nothing was specified
for component in config.debug_log:
if component:
debuglogger = logging.getLogger("pywiki." + component)
else:
debuglogger = logging.getLogger("pywiki")
uri = http_request.uri
params = http_request.params
body = http_request.body
headers = http_request.headers
if PY2 and headers:
headers = {key: str(value) for key, value in headers.items()}
auth = get_authentication(uri)
if auth is not None and len(auth) == 4:
if isinstance(requests_oauthlib, ImportError):
warn('%s' % requests_oauthlib, ImportWarning)
error('OAuth authentication not supported: %s'
% requests_oauthlib)
auth = None
else:
auth = requests_oauthlib.OAuth1(*auth)
timeout = config.socket_timeout
try:
ignore_validation = http_request.kwargs.pop(
'disable_ssl_certificate_validation', False)
# Note that the connections are pooled which mean that a future
# HTTPS request can succeed even if the certificate is invalid and
# verify=True, when a request with verify=False happened before
response = session.request(method, uri, params=params, data=body,
headers=headers, auth=auth, timeout=timeout,
verify=not ignore_validation,
**http_request.kwargs)
except Exception as e:
http_request.data = e
else:
http_request.data = response
family_sysopnames = (
config_names[self.site.family.name] or config_names['*']
)
self.username = family_sysopnames.get(self.site.code, None)
try:
self.username = self.username or family_sysopnames['*']
except KeyError:
raise NoUsername(""" \
ERROR: Sysop username for %(fam_name)s:%(wiki_code)s is undefined.
If you have a sysop account for that site, please add a line to user-config.py:
sysopnames['%(fam_name)s']['%(wiki_code)s'] = 'myUsername'"""
% {'fam_name': self.site.family.name,
'wiki_code': self.site.code})
else:
config_names = config.usernames
family_usernames = (
config_names[self.site.family.name] or config_names['*']
)
self.username = family_usernames.get(self.site.code, None)
try:
self.username = self.username or family_usernames['*']
except KeyError:
raise NoUsername(""" \
ERROR: Username for %(fam_name)s:%(wiki_code)s is undefined.
If you have an account for that site, please add a line to user-config.py:
usernames['%(fam_name)s']['%(wiki_code)s'] = 'myUsername'"""
% {'fam_name': self.site.family.name,
'wiki_code': self.site.code})
self.password = password
self.login_name = self.username
updatedSites = []
notUpdatedSites = []
# Process all languages here
self.conf.always = False
if self.conf.limittwo:
lclSite = self.originPage.site
lclSiteDone = False
frgnSiteDone = False
for siteCode in lclSite.family.languages_by_size:
site = pywikibot.Site(siteCode, lclSite.family)
if (not lclSiteDone and site == lclSite) or \
(not frgnSiteDone and site != lclSite and site in new):
if site == lclSite:
lclSiteDone = True # even if we fail the update
if (site.family.name in config.usernames and
site.code in config.usernames[site.family.name]):
try:
if self.replaceLinks(new[site], new):
updatedSites.append(site)
if site != lclSite:
frgnSiteDone = True
except SaveError:
notUpdatedSites.append(site)
except GiveUpOnPage:
break
elif (not self.conf.strictlimittwo and
site in new and
site != lclSite):
old = {}
try:
for link in new[site].iterlanglinks():
@param ignore_list: family names are mapped to dictionaries in which
language codes are mapped to lists of page titles. Each title must
be a valid regex as they are compared using L{re.search}.
@type ignore_list: dict
"""
def is_ignored(page):
if page.site.code in ignore_list.get(page.site.family.name, {}):
for ig in ignore_list[page.site.family.name][page.site.code]:
if re.search(ig, page.title()):
return True
return False
for page in generator:
if is_ignored(page):
if config.verbose_output:
pywikibot.output('Ignoring page %s' % page.title())
else:
yield page