Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
'''
config = ConfigParser.ConfigParser()
config.read('mc-client.config')
# set up a connection to a local DB
#db = CouchStoryDatabase('mediacloud', config.get('db','host'), config.get('db','port') )
db = MongoStoryDatabase('mediacloud')
# connect to MC and fetch some articles
mc = MediaCloud( config.get('api','user'), config.get('api','pass') )
results = mc.recentStories()
print "Fetched "+str(len(results))+" stories"
# set up my callback function that adds readability score to the story
pub.subscribe(mediacloud.examples.addFleshKincaidGradeLevelToStory, StoryDatabase.EVENT_PRE_STORY_SAVE)
# save all the stories in the db (this will fire the callback above)
saved = 0
for story in results:
worked = db.addStory(story)
if worked:
saved = saved + 1
print "Saved "+str(saved)+" stories"
max_story_id = db.getMaxStoryId()
results = mc.storiesSince( max_story_id, STORIES_TO_FETCH )
log.info("Fetched "+str(len(results))+" stories (after "+str(max_story_id)+")")
# set up my callback that adds splits up the domain nicely and adds the part of it to the story
pub.subscribe(mediacloud.examples.addDomainInfoToStory, StoryDatabase.EVENT_PRE_STORY_SAVE)
# set up my callback function that adds word count to the story
pub.subscribe(mediacloud.examples.addWordCountToStory, StoryDatabase.EVENT_PRE_STORY_SAVE)
# set up my callback function that adds the language guess to the story
pub.subscribe(mediacloud.examples.addIsEnglishToStory, StoryDatabase.EVENT_PRE_STORY_SAVE)
# set up my callback function that adds the reading grade level to the story
pub.subscribe(mediacloud.examples.addFleshKincaidGradeLevelToStory, StoryDatabase.EVENT_PRE_STORY_SAVE)
# save all the stories in the db
saved = 0
for story in results:
worked = db.addStory(story)
if worked:
saved = saved + 1
else:
log.warning(" unable to save story "+str(story['stories_id']))
max_story_id = db.getMaxStoryId()
end_time = time.time()
duration = end_time - start_time
log.info("Saved "+str(saved)+" stories - new max id "+str(max_story_id))
log.info(" took "+str(duration)+" secs ("+str(round(duration/STORIES_TO_FETCH,2))+" secs per story)")