Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
}
}
if not dry_run:
file_id = upload_to_dataset(conn, clowder_host, clowder_user, clowder_pass, snap_dataset, img_path)
logger.debug("Created file %s [%s]" % (img_file, file_id))
file_md["file_id"] = file_id
upload_file_metadata(conn, clowder_host, clowder_key, file_id, file_md)
logger.debug("Uploaded metadata to [%s]" % file_id)
else:
logger.debug("Skipping file %s [%s]" % (img_file, "DRY RUN"))
# Submit new dataset for extraction to plantCV extractor
if not dry_run:
extractor = "terra.lemnatec.plantcv"
logger.debug("Submitting dataset [%s] to %s" % (snap_dataset, extractor))
submit_extraction(conn, clowder_host, clowder_key, snap_dataset, extractor)
logger.debug("Experiment uploading complete.")
else:
logger.debug("%s does not exist" % experiment_root)
sys.exit(1)
target_timestamps = []
query_string = targetdef["query_list"] % date
curs = psql_conn.cursor()
curs.execute(query_string)
for result in curs:
target_timestamps.append(result[0].split("/")[-2])
disp_name = Sensors("", "ua-mac").get_display_name(targetdef["parent"])
missing = list(set(parent_timestamps)-set(target_timestamps))
for ts in missing:
if ts.find("-") > -1 and ts.find("__") > -1:
dataset_name = disp_name+" - "+ts
raw_dsid = get_dsid_by_name(dataset_name)
if raw_dsid:
submit_extraction(CONN, CLOWDER_HOST, CLOWDER_KEY, raw_dsid, extractorname)
submitted.append({"name": dataset_name, "id": raw_dsid})
else:
notfound.append({"name": dataset_name})
return json.dumps({
"extractor": extractorname,
"datasets submitted": submitted,
"datasets not found": notfound
})
if not found_in_dest or self.overwrite:
fileid = upload_to_dataset(connector, host, secret_key, target_dsid, out_file)
uploaded_file_ids.append(host + ("" if host.endswith("/") else "/") + "files/" + fileid)
self.created += 1
self.bytes += os.path.getsize(out_file)
# Upload the merged result if necessary
found_in_dest = check_file_in_dataset(connector, host, secret_key, target_dsid, merged_out, remove=self.overwrite)
if not found_in_dest or self.overwrite:
fileid = upload_to_dataset(connector, host, secret_key, target_dsid, merged_out)
uploaded_file_ids.append(host + ("" if host.endswith("/") else "/") + "files/" + fileid)
self.created += 1
self.bytes += os.path.getsize(merged_out)
# Trigger las2height extractor
submit_extraction(connector, host, secret_key, target_dsid, "terra.3dscanner.las2height")
# Tell Clowder this is completed so subsequent file updates don't daisy-chain
extractor_md = build_metadata(host, self.extractor_info, resource['id'], {
"files_created": uploaded_file_ids
}, 'dataset')
self.log_info(resource, "uploading extractor metadata to Level_1 dataset")
remove_metadata(connector, host, secret_key, resource['id'], self.extractor_info['name'])
upload_metadata(connector, host, secret_key, resource['id'], extractor_md)
self.end_message(resource)
s = Sensors("", "ua-mac")
if "parent" in targetdef:
target_dir = os.path.join(sensordef[targetdef["parent"]]["path"], date)
target_timestamps = os.listdir(target_dir)
disp_name = s.get_display_name(targetdef["parent"])
for ts in target_timestamps:
if ts.find("-") > -1 and ts.find("__") > -1: # TODO: and os.listdir(os.path.join(target_dir, ts)):
# Get first populated timestamp for the date that has a Clowder ID
dataset_name = disp_name+" - "+ts
raw_dsid = get_dsid_by_name(dataset_name)
if raw_dsid:
# Submit associated Clowder ID to rulechecker
submit_extraction(CONN, CLOWDER_HOST, CLOWDER_KEY, raw_dsid, "ncsa.rulechecker.terra")
submitted.append({"name": dataset_name, "id": raw_dsid})
break
return json.dumps({
"extractor": "ncsa.rulechecker.terra",
"datasets submitted": submitted
})
parent_dir = os.path.join(parentdef["path"], date)
target_dir = os.path.join(targetdef["path"], date)
parent_timestamps = os.listdir(parent_dir)
if os.path.isdir(target_dir):
target_timestamps = os.listdir(target_dir)
else:
target_timestamps = []
disp_name = Sensors("", "ua-mac").get_display_name(targetdef["parent"])
missing = list(set(parent_timestamps)-set(target_timestamps))
for ts in missing:
if ts.find("-") > -1 and ts.find("__") > -1:
dataset_name = disp_name+" - "+ts
raw_dsid = get_dsid_by_name(dataset_name)
if raw_dsid:
submit_extraction(CONN, CLOWDER_HOST, CLOWDER_KEY, raw_dsid, extractorname)
submitted.append({"name": dataset_name, "id": raw_dsid})
else:
notfound.append({"name": dataset_name})
return json.dumps({
"extractor": extractorname,
"datasets submitted": submitted,
"datasets not found": notfound
})