How to use the trains.storage.StorageHelper.get function in trains

To help you get started, we’ve selected a few trains examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github allegroai / trains / trains / model.py View on Github external
Files are uploaded separately to the destination storage (e.g. s3,gc,file) and then
        a link to the uploaded model is stored in the model object
        Notice: credentials for the upload destination will be pooled from the
        global configuration file (i.e. ~/trains.conf)

        :param uri: upload destination (string). example: 's3://bucket/directory/' or 'file:///tmp/debug/'
        :return: True if destination scheme is supported (i.e. s3:// file:// gc:// etc...)
        """
        if not uri:
            return

        # Test if we can update the model.
        self._validate_update()

        # Create the storage helper
        storage = StorageHelper.get(uri)

        # Verify that we can upload to this destination
        try:
            uri = storage.verify_upload(folder_uri=uri)
        except Exception:
            raise ValueError("Could not set destination uri to: %s [Check write permissions]" % uri)

        # store default uri
        self._get_base_model().upload_storage_uri = uri
github allegroai / trains / trains / backend_interface / model.py View on Github external
def download_model_weights(self):
        """ Download the model weights into a local file in our cache """
        uri = self.data.uri
        if not uri or not uri.strip():
            return None

        helper = StorageHelper.get(uri)
        filename = uri.split('/')[-1]
        ext = '.'.join(filename.split('.')[1:])
        fd, local_filename = mkstemp(suffix='.'+ext)
        os.close(fd)
        local_download = helper.download_to_file(uri, local_path=local_filename, overwrite_existing=True, verbose=True)
        # if we ended up without any local copy, delete the temp file
        if local_download != local_filename:
            try:
                Path(local_filename).unlink()
            except Exception:
                pass
        # save local model, so we can later query what was the original one
        Model._local_model_to_id_uri[str(local_download)] = (self.model_id, uri)
        return local_download
github allegroai / trains / trains / backend_interface / metrics / interface.py View on Github external
def _get_storage(self, storage_uri=None):
        """ Storage helper used to upload files """
        try:
            # use a lock since this storage object will be requested by thread pool threads, so we need to make sure
            # any singleton initialization will occur only once
            self._storage_lock.acquire()
            storage_uri = storage_uri or self._storage_uri
            return StorageHelper.get(storage_uri)
        except Exception as e:
            log.error('Failed getting storage helper for %s: %s' % (storage_uri, str(e)))
        finally:
            self._storage_lock.release()
github allegroai / trains / trains / logger.py View on Github external
if len([x for x in (matrix, image, local_path) if x is not None]) != 1:
            raise ValueError('Expected only one of [image, matrix, local_path]')
        if image is None:
            image = matrix
        if image is not None and not isinstance(image, (np.ndarray, Image.Image)):
            raise ValueError("Supported 'image' types are: numpy.ndarray or PIL.Image")

        # if task was not started, we have to start it
        self._start_task_if_needed()
        upload_uri = self.get_default_upload_destination()
        if not upload_uri:
            upload_uri = Path(get_cache_dir()) / 'debug_images'
            upload_uri.mkdir(parents=True, exist_ok=True)
            # Verify that we can upload to this destination
            upload_uri = str(upload_uri)
            storage = StorageHelper.get(upload_uri)
            upload_uri = storage.verify_upload(folder_uri=upload_uri)
        self._touch_title_series(title, series)

        if isinstance(image, Image.Image):
            image = np.array(image)

        self._task.reporter.report_image_and_upload(
            title=title,
            series=series,
            path=local_path,
            image=image,
            iter=iteration,
            upload_uri=upload_uri,
            max_image_history=max_image_history,
            delete_after_upload=delete_after_upload,
        )