Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
raise Exception
try:
filename_ranks = config.get(function, "output_ranks_to_filename")
max_ranks = config.get(function, "output_rank_limit")
if not max_ranks.isdigit():
max_ranks = len_
else:
max_ranks = int(max_ranks)
if max_ranks > len_:
max_ranks = len_
ranks = sort_weights(dict_of_ranks)
write_message("Ranks: %s" % str(ranks), verbose=9)
write_first_ranks_to_file(ranks, dict_of_ranks, \
max_ranks, filename_ranks)
except (ConfigParser.NoOptionError, StandardError):
write_message("If you want the ranks to be printed in a file you have \
to set output_ranks_to_filename and output_rank_limit \
parameters in the configuration file", verbose=3)
normalize_weights(dict_of_ranks)
into_db(dict_of_ranks, rank_method_code)
failed_ids = Queue.Queue()
recs_queue = Queue.Queue()
finished_queue = Queue.Queue()
suspend_resume_queue = Queue.Queue()
suspend_wait_queue = Queue.Queue()
main_syn_queue = Queue.Queue()
task_stats["semaphore"] = threading.Semaphore()
else:
insert_queue = SingleThreadQueue()
correct_queue = SingleThreadQueue()
failed_ids = SingleThreadQueue()
recs_queue = SingleThreadQueue()
task_stats["semaphore"] = None
write_message("STAGE0: Harvesting data and building the input")
# feed the input queue
total_recs = 0
for recid in hepdata_harvest_get_identifiers():
recs_queue.put_nowait(recid)
total_recs += 1
# spawn necessary number of workers (try not to spawn more than necessary)
if number_threads > 1:
for i in xrange(number_threads):
suspend_wait_queue.put(1)
ts = [RetrievalWorker(recs_queue, insert_queue, correct_queue, finished_queue, failed_ids, task_stats, suspend_wait_queue, suspend_resume_queue, main_syn_queue, number_threads, force_reupload = force_reupload) for i in xrange(number_threads)]
update_t = StatusUpdater(total_recs, finished_queue)
# start all the tasks
def bibrank_engine(run):
"""Run the indexing task.
Return 1 in case of success and 0 in case of failure.
"""
startCreate = time.time()
options["run"] = []
options["run"].append(run)
for rank_method_code in options["run"]:
task_sleep_now_if_required(can_stop_too=True)
cfg_name = getName(rank_method_code)
write_message("Running rank method: %s." % cfg_name)
config = load_config(rank_method_code)
cfg_short = rank_method_code
cfg_function = config.get("rank_method", "function") + "_exec"
cfg_repair_function = config.get("rank_method", "function") + "_repair_exec"
cfg_name = getName(cfg_short)
options["validset"] = get_valid_range(rank_method_code)
if task_get_option("collection"):
l_of_colls = string.split(task_get_option("collection"), ", ")
recIDs = perform_request_search(c=l_of_colls)
recIDs_range = []
for recID in recIDs:
recIDs_range.append([recID, recID])
options["recid_range"] = recIDs_range
elif task_get_option("id"):
options["recid_range"] = task_get_option("id")
serialized_data_dict = serialize_via_marshal(data_dict)
serialized_data_dict_ordered = serialize_via_marshal(data_dict_ordered)
serialized_data_list_sorted = serialize_via_marshal(data_list_sorted)
write_message('Serialization completed.', verbose=5)
date = strftime("%Y-%m-%d %H:%M:%S", time.localtime())
if not update_timestamp:
try:
date = run_sql('SELECT last_updated from bsrMETHODDATA WHERE id_bsrMETHOD = %s', (id_method, ))[0][0]
except IndexError:
pass # keep the generated date
write_message("Starting writing the data for method_id=%s " \
"to the database (table bsrMETHODDATA)" %id_method, verbose=5)
try:
write_message('Deleting old data..', verbose=5)
run_sql("DELETE FROM bsrMETHODDATA WHERE id_bsrMETHOD = %s", (id_method, ))
write_message('Inserting new data..', verbose=5)
run_sql("INSERT into bsrMETHODDATA \
(id_bsrMETHOD, data_dict, data_dict_ordered, data_list_sorted, last_updated) \
VALUES (%s, %s, %s, %s, %s)", \
(id_method, serialized_data_dict, serialized_data_dict_ordered, \
serialized_data_list_sorted, date, ))
except Error as err:
write_message("The error [%s] occured when inserting new bibsort data "\
"into bsrMETHODATA table" %err, sys.stderr)
return False
write_message('Writing to the bsrMETHODDATA successfully completed.', \
verbose=5)
return True
def check_method(rank_method_code):
write_message("Checking rank method...")
if len(fromDB(rank_method_code)) == 0:
write_message("Rank method not yet executed, please run it to create the necessary data.")
else:
if len(add_recIDs_by_date(rank_method_code)) > 0:
write_message("Records modified, update recommended")
else:
write_message("No records modified, update not necessary")
dump_database(dump_path, \
host=host,
port=port,
params=params, \
compress=compress, \
ignore_tables=ignore_tables)
write_message("Database dump ended")
finally:
for queue in active_queues:
enable_queue(queue)
if connection and task_get_option('dump_on_slave_helper_mode'):
write_message("Reattaching slave")
attach_slave(connection)
# prune old dump files:
task_update_progress("Pruning old dump files")
write_message("Pruning old dump files started")
_delete_old_dumps(output_dir, output_file_prefix, output_num)
write_message("Pruning old dump files ended")
# we are done:
task_update_progress("Done.")
return True
dates = get_dates_from_file(file_for_dates, dict_of_ids)
except (ConfigParser.NoOptionError, StandardError) as err:
write_message("If you want to read the dates from file set up the \
'file_for_dates' variable in the config file [%s]" %err, verbose=3)
try:
publication_year_tag = config.get(function, "publication_year_tag")
dummy = int(publication_year_tag[0:3])
except (ConfigParser.NoOptionError, StandardError):
write_message("You need to set up correctly the publication_year_tag \
in the cfg file", sys.stderr)
raise Exception
try:
creation_date_tag = config.get(function, "creation_date_tag")
dummy = int(creation_date_tag[0:3])
except (ConfigParser.NoOptionError, StandardError):
write_message("You need to set up correctly the creation_date_tag \
in the cfg file", sys.stderr)
raise Exception
dates = get_dates_from_db(dict_of_ids, publication_year_tag, \
creation_date_tag)
return dates
def clean_cache():
"""Clean the cache for expired and old files."""
write_message("""CLEANING OF OLD CACHED RSS REQUEST STARTED""")
rss_cache_dir = "%s/rss/" % CFG_CACHEDIR
try:
filenames = os.listdir(rss_cache_dir)
except OSError:
filenames = []
count = 0
for filename in filenames:
filename = os.path.join(rss_cache_dir, filename)
last_update_time = datetime.datetime.fromtimestamp(os.stat(os.path.abspath(filename)).st_mtime)
if not (datetime.datetime.now() < last_update_time + datetime.timedelta(minutes=CFG_WEBSEARCH_RSS_TTL)):
try:
os.remove(filename)
count += 1
except OSError as e:
write_message("Error: %s" % e)
write_message("""%s rss cache file pruned out of %s.""" % (count, len(filenames)))
6 send notification email for all pending linkbacks
@type mode: int
"""
mode = int(mode)
if mode == 1:
write_message("Starting to delete rejected and pending linkbacks URLs on blacklist")
delete_linkbacks_on_blacklist()
write_message("Completed to delete rejected and pending linkbacks URLs on blacklist")
elif mode == 2:
write_message("Starting to update the page titles of new linkbacks")
update_linkbacks(1)
write_message("Completed to update the page titles of new linkbacks")
elif mode == 3:
write_message("Starting to update the page titles of old linkbacks")
update_linkbacks(2)
write_message("Completed to update the page titles of old linkbacks")
elif mode == 4:
write_message("Starting to update manually set page titles")
update_linkbacks(3)
write_message("Completed to update manually set page titles")
elif mode == 5:
write_message("Starting to detect and disable broken linkbacks")
update_linkbacks(4)
write_message("Completed to detect and disable broken linkbacks")
elif mode == 6:
write_message("Starting to send notification email")
send_pending_linkbacks_notification(CFG_WEBLINKBACK_TYPE['TRACKBACK'])
write_message("Completed to send notification email")
2 update page titles of new linkbacks
3 update page titles of old linkbacks
4 update manually set page titles
5 detect and disable broken linkbacks
6 send notification email for all pending linkbacks
@type mode: int
"""
mode = int(mode)
if mode == 1:
write_message("Starting to delete rejected and pending linkbacks URLs on blacklist")
delete_linkbacks_on_blacklist()
write_message("Completed to delete rejected and pending linkbacks URLs on blacklist")
elif mode == 2:
write_message("Starting to update the page titles of new linkbacks")
update_linkbacks(1)
write_message("Completed to update the page titles of new linkbacks")
elif mode == 3:
write_message("Starting to update the page titles of old linkbacks")
update_linkbacks(2)
write_message("Completed to update the page titles of old linkbacks")
elif mode == 4:
write_message("Starting to update manually set page titles")
update_linkbacks(3)
write_message("Completed to update manually set page titles")
elif mode == 5:
write_message("Starting to detect and disable broken linkbacks")
update_linkbacks(4)
write_message("Completed to detect and disable broken linkbacks")
elif mode == 6:
write_message("Starting to send notification email")
send_pending_linkbacks_notification(CFG_WEBLINKBACK_TYPE['TRACKBACK'])
write_message("Completed to send notification email")