How to use pyclowder - 10 common examples

}
                }
                if not dry_run:
                    file_id = upload_to_dataset(conn, clowder_host, clowder_user, clowder_pass, snap_dataset, img_path)
                    logger.debug("Created file %s [%s]" % (img_file, file_id))
                    file_md["file_id"] = file_id
                    upload_file_metadata(conn, clowder_host, clowder_key, file_id, file_md)
                    logger.debug("Uploaded metadata to [%s]" % file_id)
                else:
                    logger.debug("Skipping file %s [%s]" % (img_file, "DRY RUN"))

            # Submit new dataset for extraction to plantCV extractor
            if not dry_run:
                extractor = "terra.lemnatec.plantcv"
                logger.debug("Submitting dataset [%s] to %s" % (snap_dataset, extractor))
                submit_extraction(conn, clowder_host, clowder_key, snap_dataset, extractor)

    logger.debug("Experiment uploading complete.")

else:
    logger.debug("%s does not exist" % experiment_root)
    sys.exit(1)

target_timestamps = []
            query_string = targetdef["query_list"] % date
            curs = psql_conn.cursor()
            curs.execute(query_string)
            for result in curs:
                target_timestamps.append(result[0].split("/")[-2])

            disp_name = Sensors("", "ua-mac").get_display_name(targetdef["parent"])
            missing = list(set(parent_timestamps)-set(target_timestamps))
            for ts in missing:
                if ts.find("-") > -1 and ts.find("__") > -1:
                    dataset_name = disp_name+" - "+ts
                    raw_dsid = get_dsid_by_name(dataset_name)
                    if raw_dsid:
                        submit_extraction(CONN, CLOWDER_HOST, CLOWDER_KEY, raw_dsid, extractorname)
                        submitted.append({"name": dataset_name, "id": raw_dsid})
                    else:
                        notfound.append({"name": dataset_name})

        return json.dumps({
            "extractor": extractorname,
            "datasets submitted": submitted,
            "datasets not found": notfound
        })

'perspective' : perspective,
        'rotation_angle' : rotation_angle,
        'zoom' : zoom,
        'imager_stage_vertical_position' : stage_position,
        'camera_gain' : camera_gain,
        'camera_exposure' : camera_exposure,
        'image_id' : img_id,
        'imagedate' : snap_details['timestamp'],
        'species' : species,
        'genotype' : genotype,
        'treatment' : treatment,
        'sample_id' : barcode['unique_id']
    }


conn = Connector({}, mounted_paths={"/home/clowder/sites":"/home/clowder/sites"})
experiment_root = sys.argv[1]
experiment_name = os.path.basename(experiment_root)
if os.path.exists(experiment_root):
    logger.debug("Searching for index files in %s" % experiment_root)
    md_file  = os.path.join(experiment_root, experiment_name+"_metadata.json")
    csv_file = os.path.join(experiment_root, "SnapshotInfo.csv")

    if not os.path.isfile(md_file):
        logger.debug("%s not found" % md_file)
        sys.exit(1)
    if not os.path.isfile(csv_file):
        logger.debug("%s not found" % csv_file)
        sys.exit(1)

    logger.debug("Found index files; loading %s" % md_file)
    base_md = loadJsonFile(md_file)

logger.debug("Uploading %s" % img_file)
                img_path = os.path.join(experiment_root, snap_dir, img_file)
                img_md = formatImageMetadata(img_file, experiment_md['metadata'], snap_details)
                file_md = {
                    "@context": ["https://clowder.ncsa.illinois.edu/contexts/metadata.jsonld"],
                    "content": img_md,
                    "agent": {
                        "@type": "cat:user",
                        "user_id": "%sapi/users/%s" % (clowder_host, clowder_uid)
                    }
                }
                if not dry_run:
                    file_id = upload_to_dataset(conn, clowder_host, clowder_user, clowder_pass, snap_dataset, img_path)
                    logger.debug("Created file %s [%s]" % (img_file, file_id))
                    file_md["file_id"] = file_id
                    upload_file_metadata(conn, clowder_host, clowder_key, file_id, file_md)
                    logger.debug("Uploaded metadata to [%s]" % file_id)
                else:
                    logger.debug("Skipping file %s [%s]" % (img_file, "DRY RUN"))

            # Submit new dataset for extraction to plantCV extractor
            if not dry_run:
                extractor = "terra.lemnatec.plantcv"
                logger.debug("Submitting dataset [%s] to %s" % (snap_dataset, extractor))
                submit_extraction(conn, clowder_host, clowder_key, snap_dataset, extractor)

    logger.debug("Experiment uploading complete.")

else:
    logger.debug("%s does not exist" % experiment_root)
    sys.exit(1)

def main():
	global extractorName, messageType, rabbitmqExchange, rabbitmqURL

	# Set logging
	logging.basicConfig(format='%(levelname)-7s : %(name)s -  %(message)s', level=logging.WARN)
	logging.getLogger('pyclowder.extractors').setLevel(logging.INFO)

	# Connect to rabbitmq
	extractors.connect_message_bus(
		extractorName        = extractorName,
		messageType          = messageType,
		rabbitmqExchange     = rabbitmqExchange,
		rabbitmqURL          = rabbitmqURL,
		processFileFunction  = process_dataset,
		checkMessageFunction = check_message
	)

def main():
    global extractorName, messageType, rabbitmqExchange, rabbitmqURL, registrationEndpoints

    #set logging
    logging.basicConfig(format='%(levelname)-7s : %(name)s -  %(message)s', level=logging.WARN)
    logging.getLogger('pyclowder.extractors').setLevel(logging.INFO)

    #connect to rabbitmq
    extractors.connect_message_bus(extractorName=extractorName, messageType=messageType, processFileFunction=process_dataset,
                                   checkMessageFunction=check_message, rabbitmqExchange=rabbitmqExchange, rabbitmqURL=rabbitmqURL)

traits['treatment'], 
                    traits['imagedate'],
                    np.mean(traits['sv_area']), 
                    traits['tv_area'], 
                    np.mean(traits['hull_area']),
                    np.mean(traits['solidity']), 
                    np.mean(traits['height']),
                    np.mean(traits['perimeter'])]


    outfile = 'avg_traits.csv'
    with open(outfile, 'w') as csv:
        csv.write(','.join(map(str, fields)) + '\n')
        csv.write(','.join(map(str, trait_list)) + '\n')
        csv.flush()
        extractors.upload_file_to_dataset(outfile, parameters)
    os.remove(outfile)

    # debug
    csv_data = ','.join(map(str, fields)) + '\n' + ','.join(map(str, trait_list)) + '\n'
    print csv_data

    metadata = {
        "@context": {
            "@vocab": "https://clowder.ncsa.illinois.edu/clowder/assets/docs/api/index.html#!/files/uploadToDataset"
        },
        "dataset_id": parameters["datasetId"],
        "content": {"status": "COMPLETED", "csv": csv_data},
        "agent": {
            "@type": "cat:extractor",
            "extractor_id": parameters['host'] + "/api/extractors/" + extractorName
        }

def main():
    global extractorName, messageType, rabbitmqExchange, rabbitmqURL, registrationEndpoints

    #set logging
    logging.basicConfig(format='%(levelname)-7s : %(name)s -  %(message)s', level=logging.WARN)
    logging.getLogger('pyclowder.extractors').setLevel(logging.INFO)

    #connect to rabbitmq
    extractors.connect_message_bus(extractorName=extractorName, messageType=messageType, processFileFunction=process_dataset,
        checkMessageFunction=check_message, rabbitmqExchange=rabbitmqExchange, rabbitmqURL=rabbitmqURL)

def check_message(parameters):
    # TODO: re-enable once this is merged into Clowder: https://opensource.ncsa.illinois.edu/bitbucket/projects/CATS/repos/clowder/pull-requests/883/overview
    # fetch metadata from dataset to check if we should remove existing entry for this extractor first
    md = extractors.download_dataset_metadata_jsonld(parameters['host'], parameters['secretKey'], parameters['datasetId'], extractorName)
    for m in md:
        if 'agent' in m and 'name' in m['agent']:
            if m['agent']['name'].find(extractorName) > -1:
                print("skipping, already done")
                return False
                #extractors.remove_dataset_metadata_jsonld(parameters['host'], parameters['secretKey'], parameters['datasetId'], extractorName)

    # Check for a left and right file before beginning processing
    found_left = False
    found_right = False
    for f in parameters['filelist']:
        if 'filename' in f and f['filename'].endswith('_left.bin'):
            found_left = True
        elif 'filename' in f and f['filename'].endswith('_right.bin'):
            found_right = True

vn_traits = pcia.process_sv_images_core(vis_id, img, nir_id, nir, nir2, traits)

        print "...uploading resulting metadata"
        # upload the individual file metadata
        metadata = {
            "@context": {
                "@vocab": "https://clowder.ncsa.illinois.edu/clowder/assets/docs/api/index.html#!/files/uploadToDataset"
            },
            "content": vn_traits[0],
            "agent": {
                "@type": "cat:extractor",
                "extractor_id": parameters['host'] + "/api/extractors/" + extractorName
            }
        }
        parameters["fileid"] = vis_id
        extractors.upload_file_metadata_jsonld(mdata=metadata, parameters=parameters)
        metadata = {
            "@context": {
                "@vocab": "https://clowder.ncsa.illinois.edu/clowder/assets/docs/api/index.html#!/files/uploadToDataset"
            },
            "content": vn_traits[1],
            "agent": {
                "@type": "cat:extractor",
                "extractor_id": parameters['host'] + "/api/extractors/" + extractorName
            }
        }
        parameters["fileid"] = nir_id
        extractors.upload_file_metadata_jsonld(mdata=metadata, parameters=parameters)

    # compose the summary traits
    trait_list = pcia.generate_traits_list(traits)