Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def perform_query(**kwargs):
while True:
sleep(waittime)
try:
results = twitter.search(**kwargs)
except TwythonRateLimitError:
logger.warning("Rate limit reached, taking a break for a minute...\n")
sleep(60)
continue
except TwythonError as err:
logger.error("Some other error occured, taking a break for half a minute: " + str(err))
sleep(30)
continue
return results
def user(self, *args, **kwargs):
while True:
try:
super(MyStreamer, self).user(self, *args, **kwargs)
except TwythonRateLimitError as e:
log("Rate limit error, asks to retry after {0}".format(e.retry_after))
time.sleep(min(int(e.retry_after), 5))
except TwythonError as e:
log("Twython error {0}".format(e))
'headers': response.headers,
'status_code': response.status_code,
'url': response.url,
'content': response.text,
}
# greater than 304 (not modified) is an error
if response.status_code > 304:
error_message = self._get_error_message(response)
self._last_call['api_error'] = error_message
ExceptionType = TwythonError
if response.status_code == 429:
# Twitter API 1.1, always return 429 when
# rate limit is exceeded
ExceptionType = TwythonRateLimitError
elif response.status_code == 401 or 'Bad Authentication data' \
in error_message:
# Twitter API 1.1, returns a 401 Unauthorized or
# a 400 "Bad Authentication data" for invalid/expired
# app keys/user tokens
ExceptionType = TwythonAuthError
raise ExceptionType(
error_message,
error_code=response.status_code,
retry_after=response.headers.get('X-Rate-Limit-Reset'))
content = ''
try:
if response.status_code == 204:
content = response.content
else:
return
# Pagination loop: keep fetching Tweets until the desired count is
# reached while dealing with Twitter rate limits.
retries = 0
while count_from_query < limit:
try:
mcount = min(100, limit - count_from_query)
results = self.search(
q=keywords,
count=mcount,
lang=lang,
max_id=self.handler.max_id,
result_type='recent',
)
except TwythonRateLimitError as e:
print("Waiting for 15 minutes -{0}".format(e))
time.sleep(15 * 60) # wait 15 minutes
continue
except TwythonError as e:
print("Fatal error in Twython request -{0}".format(e))
if retries_after_twython_exception == retries:
raise e
retries += 1
count = len(results['statuses'])
if count == 0:
print("No more Tweets available through rest api")
return
count_from_query += count
# the max_id is also present in the Tweet metadata
# results['search_metadata']['next_results'], but as part of a
def user(self, *args, **kwargs):
while True:
try:
super(EventDispatcherSingleton, self).user(self, *args, **kwargs)
except TwythonRateLimitError as e:
log("Rate limit error, asks to retry after {0}".format(e.retry_after))
time.sleep(min(int(e.retry_after), 5))
except TwythonError as e:
log("Twython error {0}".format(e))
while cursor != 0 and retry_cnt > 1:
try:
friend_ids = self.get_friends_ids(user_id=user_id, cursor=cursor, count=200)
for handler in write_to_handlers:
handler.append(json.dumps(friend_ids), bucket=bucket, key=user_id)
for handler in cmd_handlers:
handler.append(json.dumps(friend_ids), bucket=bucket, key=user_id)
cursor = int(friend_ids['next_cursor'])
logger.debug("find #%d friend_ids... NEXT_CURSOR: %d"%(len(friend_ids["ids"]), cursor))
time.sleep(2)
except twython.exceptions.TwythonRateLimitError:
self.rate_limit_error_occured('friends', '/friends/ids')
except Exception as exc:
time.sleep(10)
logger.debug("exception: %s"%exc)
retry_cnt -= 1
if (retry_cnt == 0):
raise MaxRetryReached("max retry reached due to %s"%(exc))
logger.debug("finished find_all_friend_ids for %s..."%(user_id))
script_classes = load_scripts('scripts.standalone', INSTALLED_STANDALONE_SCRIPTS)
scripts = [script() for script in script_classes]
remaining_chunks = {}
for script in scripts:
remaining_chunks[script.__class__.__name__] = script.repeat_time
while True:
sleep(STAND_ALONE_REPEAT_TIME_CHUNKS)
try:
for script in scripts:
remaining_chunks[script.__class__.__name__] -= 1
if remaining_chunks[script.__class__.__name__] == 0:
remaining_chunks[script.__class__.__name__] = script.repeat_time
script.on_called()
except TwythonRateLimitError as e:
log("Rate limit error, asks to retry after {0}".format(e.retry_after))
time.sleep(min(int(e.retry_after), 5*60*60))
except TwythonError as e:
log("Twython error: {0}".format(e))