Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
if os.path.exists(test_features_path):
test_features = pickle.load(open(test_features_path, 'rb'))
logger.info("Loading features from: %s" % (test_features_path))
else:
test_features = convert_examples_to_features(
examples=test_examples,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,
max_query_length=args.max_query_length,
verbose_logging=args.verbose_logging,
logger=logger)
pickle.dump(test_features, open(test_features_path, 'wb'))
logger.info("Filtering features randomly")
filtered_test_features = random_filter_features(test_examples, test_features, args.n_best_size_rank,
is_training=False)
filtered_rank_logits = [0.] * len(filtered_test_features)
return build_eval_data(args, test_examples, test_features, filtered_test_features, filtered_rank_logits, logger)
if os.path.exists(eval_features_path):
eval_features = pickle.load(open(eval_features_path, 'rb'))
logger.info("Loading features from: %s" % (eval_features_path))
else:
eval_features = convert_examples_to_features(
examples=eval_examples,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,
max_query_length=args.max_query_length,
verbose_logging=args.verbose_logging,
logger=logger)
pickle.dump(eval_features, open(eval_features_path, 'wb'))
logger.info("Filtering features randomly")
filtered_eval_features = random_filter_features(eval_examples, eval_features, args.n_best_size_rank,
is_training=False)
filtered_rank_logits = [0.] * len(filtered_eval_features)
return build_eval_data(args, eval_examples, eval_features, filtered_eval_features, filtered_rank_logits, logger)
train_features = convert_examples_to_features(
examples=train_examples,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,
max_query_length=args.max_query_length,
verbose_logging=args.verbose_logging,
logger=logger)
pickle.dump(train_features, open(train_features_path, 'wb'))
if args.down_sample:
train_features = down_sample(args.sample_rate, train_features, logger)
# filter features
logger.info("Filtering features randomly")
filtered_train_features = random_filter_features(train_examples, train_features, args.n_best_size_rank,
is_training=True)
return build_train_data(args, train_examples, train_features, filtered_train_features, logger)
train_features = pickle.load(open(train_features_path, 'rb'))
logger.info("Loading features from: %s" % (train_features_path))
else:
train_features = convert_examples_to_features(
examples=train_examples,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,
max_query_length=args.max_query_length,
verbose_logging=args.verbose_logging,
logger=logger)
pickle.dump(train_features, open(train_features_path, 'wb'))
# filter features
logger.info("Filtering features randomly")
filtered_train_features = random_filter_features(train_examples, train_features, args.n_best_size_rank,
is_training=True)
return build_train_data(args, train_examples, train_features, filtered_train_features, logger)
if os.path.exists(eval_features_path):
eval_features = pickle.load(open(eval_features_path, 'rb'))
logger.info("Loading features from: %s" % (eval_features_path))
else:
eval_features = convert_examples_to_features(
examples=eval_examples,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,
max_query_length=args.max_query_length,
verbose_logging=args.verbose_logging,
logger=logger)
pickle.dump(eval_features, open(eval_features_path, 'wb'))
logger.info("Filtering features randomly")
filtered_eval_features = random_filter_features(eval_examples, eval_features, args.n_best_size_rank,
is_training=False)
filtered_rank_logits = [0.] * len(filtered_eval_features)
return build_eval_data(args, eval_examples, eval_features, filtered_eval_features, filtered_rank_logits, logger)
if os.path.exists(dev_features_path):
dev_features = pickle.load(open(dev_features_path, 'rb'))
logger.info("Loading features from: %s" % (dev_features_path))
else:
dev_features = convert_examples_to_features(
examples=dev_examples,
tokenizer=tokenizer,
max_seq_length=args.max_seq_length,
doc_stride=args.doc_stride,
max_query_length=args.max_query_length,
verbose_logging=args.verbose_logging,
logger=logger)
pickle.dump(dev_features, open(dev_features_path, 'wb'))
logger.info("Filtering features randomly")
filtered_dev_features = random_filter_features(dev_examples, dev_features, args.n_best_size_rank,
is_training=False)
filtered_rank_logits = [0.] * len(filtered_dev_features)
return build_eval_data(args, dev_examples, dev_features, filtered_dev_features, filtered_rank_logits, logger)