How to use the mediacloud.api.MediaCloud.SORT_RANDOM function in mediacloud

To help you get started, we’ve selected a few mediacloud examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github mitmedialab / MediaCloud-Web-Tools / server / views / explorer / story_samples.py View on Github external
def explorer_stories_csv():
    logger.info(flask_login.current_user.name)
    filename = 'all-story-urls'
    data = request.form
    if 'searchId' in data:
        solr_q, solr_fq = parse_as_sample(data['searchId'], data['uid'])
        filename = filename  # don't have this info + current_query['q']
        # for demo users we only download 100 random stories (ie. not all matching stories)
        return _stream_story_list_csv(filename, solr_q, solr_fq, 100, MediaCloud.SORT_RANDOM, 1)
    else:
        q = json.loads(data['q'])
        filename = file_name_for_download(q['label'], filename)
        # now compute total attention for all results
        if (len(q['collections']) == 0) and only_queries_reddit(q['sources']):
            start_date, end_date = parse_query_dates(q)
            stories = pushshift.reddit_top_submissions(query=q['q'], limit=2000,
                                                       start_date=start_date, end_date=end_date,
                                                       subreddits=pushshift.NEWS_SUBREDDITS)
            props = ['stories_id', 'subreddit', 'publish_date', 'score', 'last_updated', 'title', 'url', 'full_link',
                     'author']
            return csv.stream_response(stories, props, filename)
        else:
            solr_q, solr_fq = parse_query_with_keywords(q)
            # now page through all the stories and download them
            return _stream_story_list_csv(filename, solr_q, solr_fq)
github mitmedialab / MediaCloud-Dashboard / app / core / views.py View on Github external
def _cached_sentence_docs(api_key, keywords, media, start, end, count=10, sort=mcapi.MediaCloud.SORT_RANDOM):
    query = app.core.util.solr_query(keywords, media, start, end)
    app.core.logger.debug("query: _sentence_docs %s" % query)
    start_index = 0
    if sort==mcapi.MediaCloud.SORT_RANDOM :
        # to sort radomly, we need to offset into the results and set sort to random
        # so first we need to know how many senteces there are
        sentence_counts = json.loads(_sentence_numfound(api_key, keywords, media, start, end))
        sentence_total = sum([day['numFound'] for day in sentence_counts])
        sentence_total = min(sentence_total,5000)   # don't offset too far into results otherwise query takes a LONG time to return
        try:
            start_index = randint(0,sentence_total-count)
        except Exception as exception:
            start_index = 0
    res = cached_admin_sentence_list(api_key, query, '', start_index, count, sort=sort)
    story_count = cached_story_count(api_key, query)
    results = {
github mitmedialab / MediaCloud-Dashboard / app / app_views.py View on Github external
def _sentence_docs(api, keywords, media, start, end):
    query = app.util.solr_query(app.util.media_to_solr(media), start, end)
    res = api.sentenceList("%s AND (%s)" % (keywords, query), '', 0, 10, sort=mcapi.MediaCloud.SORT_RANDOM)
    sentences = res['response']['docs']
    for s in sentences:
        s['totalSentences'] = res['response']['numFound'] # hack to get total sentences count to Backbone.js
    return json.dumps(sentences, separators=(',',':'))
github mitmedialab / MediaCloud-Dashboard / app / core / views.py View on Github external
def _cached_sentence_docs(api_key, keywords, media, start, end, count=10, sort=mcapi.MediaCloud.SORT_RANDOM):
    query = app.core.util.solr_query(keywords, media, start, end)
    app.core.logger.debug("query: _sentence_docs %s" % query)
    start_index = 0
    if sort==mcapi.MediaCloud.SORT_RANDOM :
        # to sort radomly, we need to offset into the results and set sort to random
        # so first we need to know how many senteces there are
        sentence_counts = json.loads(_sentence_numfound(api_key, keywords, media, start, end))
        sentence_total = sum([day['numFound'] for day in sentence_counts])
        sentence_total = min(sentence_total,5000)   # don't offset too far into results otherwise query takes a LONG time to return
        try:
            start_index = randint(0,sentence_total-count)
        except Exception as exception:
            start_index = 0
    res = cached_admin_sentence_list(api_key, query, '', start_index, count, sort=sort)
    story_count = cached_story_count(api_key, query)
    results = {
        'sentences': res['response']['docs'],
        'total': res['response']['numFound'],
        'totalStories': story_count['count']
    }
github mitmedialab / MediaCloud-Dashboard / app / views.py View on Github external
def _sentence_docs(api, keywords, media, start, end):
    query = app.util.solr_query(app.util.media_to_solr(media), start, end)
    res = api.sentenceList("%s AND (%s)" % (keywords, query), '', 0, 10, sort=mcapi.MediaCloud.SORT_RANDOM)
    sentences = res['response']['docs']
    for s in sentences:
        s['totalSentences'] = res['response']['numFound'] # hack to get total sentences count to Backbone.js
    return json.dumps(sentences, separators=(',',':'))