How to use the disdat.logger.error function in disdat

To help you get started, we’ve selected a few disdat examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github kyocum / disdat / disdat / data_context.py View on Github external
elif o.scheme == 'file':
                        if dst_scheme == 's3':
                            # local to s3
                            aws_s3.put_s3_file(o.path, os.path.dirname(dst_file))
                        elif dst_scheme != 'db':  # assume 'file'
                            # local to local
                            shutil.copy(o.path, os.path.dirname(dst_file))
                        else:
                            raise Exception("copy_in_files: copy local file to unsupported scheme {}".format(dst_scheme))

                    else:
                        raise Exception("DataContext copy-in-file found bad scheme: {} from {}".format(o.scheme, o))
                else:
                    _logger.info("DataContext copy-in-file: Not adding files in directory {}".format(src_path))
            except (IOError, os.error) as why:
                _logger.error("Disdat add error: {} {} {}".format(src_path, dst_dir, str(why)))

        if return_one_file:
            return file_set[0]
        else:
            return file_set
github kyocum / disdat / disdat / run.py View on Github external
def assert_or_log(cli, msg):
        if cli:
            _logger.error(msg)
        else:
            assert False, msg
github kyocum / disdat / disdat / data_context.py View on Github external
# First, test whether any db links are used to back the current view
            no_force_required = self.rm_db_links(hfr[0])

            if no_force_required or force:
                hyperframe.update_hfr_db(self.local_engine, hyperframe.RecordState.deleted, uuid=hfr_uuid)
                self.rm_db_links(hfr[0], dry_run=False)
                shutil.rmtree(self.implicit_hframe_path(hfr_uuid))
                hyperframe.delete_hfr_db(self.local_engine, uuid=hfr_uuid)
                hyperframe.delete_fr_db(self.local_engine, hfr_uuid)
            else:
                print ("Disdat: Looks like you're trying to remove a committed bundle with a db link backing a view.")
                print ("Disdat: Removal of this bundle with db links that back a view requires '--force'")
                return False
            return True
        except (IOError, os.error) as why:
            _logger.error("Removal of hyperframe directory {} failed with error {}.".format(self.implicit_hframe_path(hfr_uuid), why))

            # Must clean up db if directory removal failed, only delete same record if state marked for removal
            hyperframe.delete_hfr_db(self.local_engine, uuid=hfr_uuid, state=hyperframe.RecordState.deleted)

            return False
github kyocum / disdat / disdat / api.py View on Github external
# if the user erroneously passes in the directory of the bundle, return same
        if dir_name == self._local_dir:
            return self._local_dir

        fqp = os.path.join(self._local_dir, dir_name.lstrip('/'))

        try:
            os.makedirs(fqp)
        except OSError as why:
            if not why.errno == errno.EEXIST:
                _logger.error("Creating directory in bundle directory failed errno {}".format(why.strerror))
                raise
            # else directory exists is OK and fall through
        except IOError as why:
            _logger.error("Creating directory in bundle directory failed {}".format(why))
            raise

        return fqp
github kyocum / disdat / disdat / pipe.py View on Github external
""" NOTE: If a user changes a task param in run(), and that param parameterizes a dependency in requires(), 
        then running requires() post run() will give different tasks.  To be safe we record the inputs before run() 
        """
        cached_bundle_inputs = self.bundle_inputs()

        try:
            start = time.time()  # P3 datetime.now().timestamp()
            user_rtn_val = self.pipe_run(**kwargs)
            stop = time.time()  # P3 datetime.now().timestamp()
        except Exception as error:
            """ If user's pipe fails for any reason, remove bundle dir and raise """
            try:
                _logger.error("User pipe_run encountered exception: {}".format(error))
                pce.bundle.abandon()
            except OSError as ose:
                _logger.error("User pipe_run encountered error, and error on remove bundle: {}".format(ose))
            raise

        try:
            # Add any output tags to the user tag dict
            if self.output_tags:
                self.user_tags.update(self.output_tags)

            # If this is the root_task, identify it as so in the tag dict
            if isinstance(self.calling_task, DriverTask):
                self.user_tags.update({'root_task': 'True'})

            """ if we have a pce, we have a new bundle that we need to add info to and close """
            pce.bundle.add_data(user_rtn_val)

            pce.bundle.add_timing(start, stop)