How to use the dxpy.DXFile function in dxpy

To help you get started, we’ve selected a few dxpy examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github ENCODE-DCC / long-rna-seq-pipeline / dnanexus / tools / parse_property.py View on Github external
if project != None:

        try:
            if dxlink != None:
                dxfile = dxpy.get_handler(dxlink,project=project)
            else:
                dxfile = dxpy.get_handler(filePath,project=project)
        except:
            try:
                dxlink = dxpy.dxlink(filePath,project=project)
                dxfile = dxpy.get_handler(dxlink)
            except:
                try:
                    proj_id = env_get_current_project_id()
                    dxfile = dxpy.DXFile(filePath,project=proj_id)
                except:
                    sys.stderr.write('ERROR: unable to find file "' + filePath + '": \n')
                    sys.exit(0)  # Do not error on tool run in dx script

    else:

        try:
            if dxlink != None:
                dxfile = dxpy.get_handler(dxlink)
            else:
                dxfile = dxpy.get_handler(filePath)
        except:
            try:
                dxlink = dxpy.dxlink(filePath)
                dxfile = dxpy.get_handler(dxlink)
            except:
github ENCODE-DCC / chip-seq-pipeline / dnanexus / encode_idr / src / encode_idr.py View on Github external
def replicated_IDR(experiment,
                   reps_peaks, r1pr_peaks, r2pr_peaks, pooledpr_peaks,
                   rep1_ta, rep1_xcor, rep2_ta, rep2_xcor,
                   paired_end, chrom_sizes, as_file, blacklist,
                   rep1_signal, rep2_signal, pooled_signal):

    # TODO for now just taking the peak files.  This applet should actually
    # call IDR instead of putting that in the workflow populator script

    reps_peaks_file = dxpy.DXFile(reps_peaks)
    r1pr_peaks_file = dxpy.DXFile(r1pr_peaks)
    r2pr_peaks_file = dxpy.DXFile(r2pr_peaks)
    pooledpr_peaks_file = dxpy.DXFile(pooledpr_peaks)
    rep1_ta_file = dxpy.DXFile(rep1_ta)
    rep2_ta_file = dxpy.DXFile(rep2_ta)
    rep1_xcor_file = dxpy.DXFile(rep1_xcor)
    rep2_xcor_file = dxpy.DXFile(rep2_xcor)
    chrom_sizes_file = dxpy.DXFile(chrom_sizes)
    as_file_file = dxpy.DXFile(as_file)
    if blacklist is not None:
        blacklist_file = dxpy.DXFile(blacklist)
        blacklist_filename = 'blacklist_%s' % (blacklist_file.name)
        dxpy.download_dxfile(blacklist_file.get_id(), blacklist_filename)
        blacklist_filename = common.uncompress(blacklist_filename)

    # Need to prepend something to ensure the local filenames will be unique
    reps_peaks_filename = 'true_%s' % (reps_peaks_file.name)
    r1pr_peaks_filename = 'r1pr_%s' % (r1pr_peaks_file.name)
github ENCODE-DCC / chip-seq-pipeline / dnanexus / overlap_peaks / src / overlap_peaks.py View on Github external
def replicated_overlap(rep1_peaks, rep2_peaks, pooled_peaks,
                       pooledpr1_peaks, pooledpr2_peaks,
                       rep1_ta, rep1_xcor, rep2_ta, rep2_xcor,
                       paired_end, chrom_sizes, as_file, peak_type, prefix):

    rep1_peaks_file      = dxpy.DXFile(rep1_peaks)
    rep2_peaks_file      = dxpy.DXFile(rep2_peaks)
    pooled_peaks_file    = dxpy.DXFile(pooled_peaks)
    pooledpr1_peaks_file = dxpy.DXFile(pooledpr1_peaks)
    pooledpr2_peaks_file = dxpy.DXFile(pooledpr2_peaks)
    rep1_ta_file         = dxpy.DXFile(rep1_ta)
    rep2_ta_file         = dxpy.DXFile(rep2_ta)
    rep1_xcor_file       = dxpy.DXFile(rep1_xcor)
    rep2_xcor_file       = dxpy.DXFile(rep2_xcor)
    chrom_sizes_file     = dxpy.DXFile(chrom_sizes)
    as_file_file         = dxpy.DXFile(as_file)

    # Input filenames - necessary to define each explicitly because input files
    # could have the same name, in which case subsequent
    # file would overwrite previous file
    rep1_peaks_fn      = 'rep1-%s' % (rep1_peaks_file.name)
    rep2_peaks_fn      = 'rep2-%s' % (rep2_peaks_file.name)
    pooled_peaks_fn    = 'pooled-%s' % (pooled_peaks_file.name)
    pooledpr1_peaks_fn = 'pooledpr1-%s' % (pooledpr1_peaks_file.name)
    pooledpr2_peaks_fn = 'pooledpr2-%s' % (pooledpr2_peaks_file.name)
    rep1_ta_fn         = 'r1ta_%s' % (rep1_ta_file.name)
    rep2_ta_fn         = 'r2ta_%s' % (rep2_ta_file.name)
    rep1_xcor_fn       = 'r1cc_%s' % (rep1_xcor_file.name)
    rep2_xcor_fn       = 'r2cc_%s' % (rep2_xcor_file.name)
    chrom_sizes_fn     = 'chrom.sizes'
github ENCODE-DCC / chip-seq-pipeline / dnanexus / accession_analysis / src / accession_analysis.py View on Github external
def set_property(dx_fh, prop):
    try:
        dx_fh.set_properties(prop)
    except dxpy.exceptions.ResourceNotFound as e:
        logger.warning(
            '%s adding property %s to %s.  Will try in current project.'
            % (e, prop, dx_fh.name))
        try:
            current_project_fh = dxpy.DXFile(
                dx_fh.get_id(), project=dxpy.PROJECT_CONTEXT_ID)
            current_project_fh.set_properties(prop)
        except dxpy.exceptions.ResourceNotFound as e2:
            # give up
            logger.warning('%s.  Skipping saving property.' % (e2))
            pass
        except:
            raise
    except dxpy.exceptions.PermissionDenied as e:
        logger.warning(
            '%s adding property %s to %s.'
            % (e, prop, dx_fh.name))
        raise
    except:
        raise
github ENCODE-DCC / chip-seq-pipeline / dnanexus / accession_analysis / src / accession_analysis.py View on Github external
def accession_outputs(stages, keypair, server,
                      dryrun, force_patch, force_upload):
    files = []
    for (stage_name, outputs) in stages.iteritems():
        stage_metadata = outputs['stage_metadata']
        for i, file_metadata in enumerate(outputs['output_files']):
            project = stage_metadata['project']
            analysis = stage_metadata['parentAnalysis']
            dataset_accession = get_experiment_accession(analysis)
            file_id = stage_metadata['output'][file_metadata['name']]
            logger.debug(
                'in accession_outputs getting handler for file %s in %s'
                % (file_id, project))
            dx = dxpy.DXFile(file_id, project=project)
            dx_desc = dx.describe()
            surfaced_outputs = \
                [o for o in outputs['qc'] if isinstance(o, str)]  # this will be a list of strings
            calculated_outputs = \
                [o for o in outputs['qc'] if not isinstance(o, str)]  # this will be a list of functions/methods
            logger.debug(
                'in accession_outputs with stage metadata\n%s'
                % (stage_metadata.get('name')))
            logger.debug(
                'in accession_ouputs with surfaced_ouputs %s and calculated_outputs %s from project %s'
                % (surfaced_outputs, calculated_outputs, project))
            notes_qc = dict(zip(
                surfaced_outputs,
                [stage_metadata['output'][metric]
                 for metric in surfaced_outputs]))
            notes_qc.update(dict(zip(
github counsyl / stor / stor / dx.py View on Github external
def _clone(self, dest):
        """Clones the data object into the destination path.
        The original file is retained.

        Args:
            dest (Path): The destination file/folder path in a different project

        Raises:
            ValueError: If attempting to clone a project
            DNAnexusError: If cloning within same project
        """
        if not self.resource:
            raise ValueError('Cannot clone project ({})'.format(self))
        if dest.canonical_project == self.canonical_project:
            raise DNAnexusError('Cannot clone within same project')
        file_handler = dxpy.DXFile(dxid=self.canonical_resource,
                                   project=self.canonical_project)
        target_dest, should_rename = self._prep_for_copy(dest)

        with _wrap_dx_calls():
            new_file_h = file_handler.clone(project=dest.canonical_project,
                                            folder='/' + (target_dest.parent.resource or ''))
            # no need to rename if we changed destination to include original name
            if should_rename:
                new_file_h.rename(dest.name)
github counsyl / stor / stor / dx.py View on Github external
'filename MUST match object name when file_proxy_url is set'
                )
            # Append trailing slash, so that full proxy URL is included in the generated URL.
            if not file_proxy_url.endswith("/"):
                file_proxy_url += "/"
            return urllib.parse.urljoin(
                file_proxy_url,
                f'{self.virtual_project}/{self.virtual_resource}'
            )
        with _wrap_dx_calls():
            if filename is None:
                filename = self.virtual_path.name
            elif not filename:
                # e.g., set to empty string
                filename = None
            file_handler = dxpy.DXFile(dxid=self.canonical_resource,
                                       project=self.canonical_project)
            return file_handler.get_download_url(
                duration=lifetime,
                preauthenticated=True,
                filename=filename,
                project=self.canonical_project
            )[0]
github ENCODE-DCC / chip-seq-pipeline / dnanexus / chip_workflow.py View on Github external
project=project.get_id(),
            recurse=recurse,
            more_ok=False,
            zero_ok=False,
            return_handler=True)
    except dxpy.DXSearchError:
        logging.debug(
            '%s not found in project %s folder %s.  Trying as file ID'
            % (file_name, project.get_id(), folder_name))
        file_handler = None
    except:
        raise

    if not file_handler:
        try:
            file_handler = dxpy.DXFile(dxid=identifier, mode='r')
        except dxpy.DXError:
            logging.debug('%s not found as a dxid' % (identifier))
            logging.warning('Could not find file %s.' % (identifier))
            file_handler = None
        except:
            raise

    if file_handler:
        logging.info(
            "Resolved file identifier %s to %s"
            % (identifier, file_handler.get_id()))
        return file_handler
    else:
        logging.warning("Failed to resolve file identifier %s" % (identifier))
        return None
github ENCODE-DCC / chip-seq-pipeline / dnanexus / accession_histone_analysis.py View on Github external
# "Overlap gappedpeaks": {
		# 	'files': [
		# 		common.merge_dicts({'name': 'overlapping_peaks',		'derived_from': [rep1_bam, rep2_bam]},	replicated_gappedpeak_metadata),
		# 		common.merge_dicts({'name': 'overlapping_peaks_bb',		'derived_from': [rep1_bam, rep2_bam]},	replicated_gappedpeak_bb_metadata)],
		# 	'qc': ['npeaks_in', 'npeaks_out', 'npeaks_rejected']}
		}

	experiment = common.encoded_get(urlparse.urljoin(server,'/experiments/%s' %(experiment_accession)), keypair)
	rep1_bam, rep2_bam = get_rep_bams(experiment, keypair, server)

	files = []
	for (stage_name, outputs) in stage_outputs.iteritems():
		stage_metadata = next(s['execution'] for s in analysis.get('stages') if s['execution']['name'] == stage_name)
		for static_metadata in outputs['files']:
			output_name = static_metadata['name']
			dx = dxpy.DXFile(stage_metadata['output'][output_name], project=project)
			file_metadata = {
				'dx': dx,
				'notes': {
					'dx-id': dx.get_id(),
					'dx-createdBy': {
						'job': stage_metadata['id'],
						'executable': stage_metadata['executable'], #todo get applet ID
						'user': stage_metadata['launchedBy']},
					'qc': dict(zip(outputs['qc'],[stage_metadata['output'][metric] for metric in outputs['qc']]))}, #'aliases': ['ENCODE:%s-%s' %(experiment.get('accession'), static_metadata.pop('name'))],
				'dataset': experiment.get('accession'),
				'file_size': dx.describe().get('size'),
				'submitted_file_name': dx.get_proj_id() + ':' + '/'.join([dx.folder,dx.name])}
			static_metadata.pop('name')
			file_metadata.update(static_metadata)
			files.append(file_metadata)