Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def explorer_stories_csv():
logger.info(flask_login.current_user.name)
filename = 'all-story-urls'
data = request.form
if 'searchId' in data:
solr_q, solr_fq = parse_as_sample(data['searchId'], data['uid'])
filename = filename # don't have this info + current_query['q']
# for demo users we only download 100 random stories (ie. not all matching stories)
return _stream_story_list_csv(filename, solr_q, solr_fq, 100, MediaCloud.SORT_RANDOM, 1)
else:
q = json.loads(data['q'])
filename = file_name_for_download(q['label'], filename)
# now compute total attention for all results
if (len(q['collections']) == 0) and only_queries_reddit(q['sources']):
start_date, end_date = parse_query_dates(q)
stories = pushshift.reddit_top_submissions(query=q['q'], limit=2000,
start_date=start_date, end_date=end_date,
subreddits=pushshift.NEWS_SUBREDDITS)
props = ['stories_id', 'subreddit', 'publish_date', 'score', 'last_updated', 'title', 'url', 'full_link',
'author']
return csv.stream_response(stories, props, filename)
else:
solr_q, solr_fq = parse_query_with_keywords(q)
# now page through all the stories and download them
return _stream_story_list_csv(filename, solr_q, solr_fq)
def _cached_sentence_docs(api_key, keywords, media, start, end, count=10, sort=mcapi.MediaCloud.SORT_RANDOM):
query = app.core.util.solr_query(keywords, media, start, end)
app.core.logger.debug("query: _sentence_docs %s" % query)
start_index = 0
if sort==mcapi.MediaCloud.SORT_RANDOM :
# to sort radomly, we need to offset into the results and set sort to random
# so first we need to know how many senteces there are
sentence_counts = json.loads(_sentence_numfound(api_key, keywords, media, start, end))
sentence_total = sum([day['numFound'] for day in sentence_counts])
sentence_total = min(sentence_total,5000) # don't offset too far into results otherwise query takes a LONG time to return
try:
start_index = randint(0,sentence_total-count)
except Exception as exception:
start_index = 0
res = cached_admin_sentence_list(api_key, query, '', start_index, count, sort=sort)
story_count = cached_story_count(api_key, query)
results = {
def _sentence_docs(api, keywords, media, start, end):
query = app.util.solr_query(app.util.media_to_solr(media), start, end)
res = api.sentenceList("%s AND (%s)" % (keywords, query), '', 0, 10, sort=mcapi.MediaCloud.SORT_RANDOM)
sentences = res['response']['docs']
for s in sentences:
s['totalSentences'] = res['response']['numFound'] # hack to get total sentences count to Backbone.js
return json.dumps(sentences, separators=(',',':'))
def _cached_sentence_docs(api_key, keywords, media, start, end, count=10, sort=mcapi.MediaCloud.SORT_RANDOM):
query = app.core.util.solr_query(keywords, media, start, end)
app.core.logger.debug("query: _sentence_docs %s" % query)
start_index = 0
if sort==mcapi.MediaCloud.SORT_RANDOM :
# to sort radomly, we need to offset into the results and set sort to random
# so first we need to know how many senteces there are
sentence_counts = json.loads(_sentence_numfound(api_key, keywords, media, start, end))
sentence_total = sum([day['numFound'] for day in sentence_counts])
sentence_total = min(sentence_total,5000) # don't offset too far into results otherwise query takes a LONG time to return
try:
start_index = randint(0,sentence_total-count)
except Exception as exception:
start_index = 0
res = cached_admin_sentence_list(api_key, query, '', start_index, count, sort=sort)
story_count = cached_story_count(api_key, query)
results = {
'sentences': res['response']['docs'],
'total': res['response']['numFound'],
'totalStories': story_count['count']
}
def _sentence_docs(api, keywords, media, start, end):
query = app.util.solr_query(app.util.media_to_solr(media), start, end)
res = api.sentenceList("%s AND (%s)" % (keywords, query), '', 0, 10, sort=mcapi.MediaCloud.SORT_RANDOM)
sentences = res['response']['docs']
for s in sentences:
s['totalSentences'] = res['response']['numFound'] # hack to get total sentences count to Backbone.js
return json.dumps(sentences, separators=(',',':'))