How to use daiquiri - 10 common examples

import logging
import boto3
import daiquiri

daiquiri.setup(level=logging.INFO)
logger = daiquiri.getLogger(__name__)


def _parse_s3_file(original_file):
    """
    Convert `s3://bucketname/path/to/file.txt` to ('bucketname', 'path/to/file.txt')
    """
    bits = original_file.replace('s3://', '').split("/")
    bucket = bits[0]
    object_key = "/".join(bits[1:])
    return bucket, object_key


def file_exists(original_file):
    """
    Validate the original file is in the S3 bucket

"""

import tensorflow as tf
import tensorflow.contrib.slim as slim
from config import cfg, get_coord_add, get_dataset_size_train, get_num_classes, get_create_inputs
import time
import numpy as np
import sys
import os
import capsnet_em as net

import logging
import daiquiri

daiquiri.setup(level=logging.DEBUG)
logger = daiquiri.getLogger(__name__)


def main(args):
    """Get dataset hyperparameters."""
    assert len(args) == 2 and isinstance(args[1], str)
    dataset_name = args[1]
    logger.info('Using dataset: {}'.format(dataset_name))
    coord_add = get_coord_add(dataset_name)
    dataset_size = get_dataset_size_train(dataset_name)
    num_classes = get_num_classes(dataset_name)
    create_inputs = get_create_inputs(dataset_name, is_train=True, epochs=cfg.epoch)

    """Set reproduciable random seed"""
    tf.set_random_seed(1234)

    with tf.Graph().as_default(), tf.device('/cpu:0'):

from gnocchi import utils


OPTS = [
    cfg.IntOpt('aggregation_workers_number',
               default=1, min=1,
               help='Number of threads to process and store aggregates. '
                    'Set value roughly equal to number of aggregates to be '
                    'computed per metric'),
    cfg.StrOpt('coordination_url',
               secret=True,
               help='Coordination driver URL'),

]

LOG = daiquiri.getLogger(__name__)


class CorruptionError(ValueError):
    """Data corrupted, damn it."""

    def __init__(self, message):
        super(CorruptionError, self).__init__(message)


class CarbonaraBasedStorage(storage.StorageDriver):

    def __init__(self, conf, coord=None):
        super(CarbonaraBasedStorage, self).__init__(conf)
        self.aggregation_workers_number = conf.aggregation_workers_number
        if self.aggregation_workers_number == 1:
            # NOTE(jd) Avoid using futures at all if we don't want any threads.

import abc
import json
from multiprocessing import Pool
import os

import cv2
import daiquiri
import geopandas as gpd
import numpy as np
from osgeo import gdal
from PIL import Image

from deeposlandia import geometries

logger = daiquiri.getLogger(__name__)


AVAILABLE_DATASETS = ("shapes", "mapillary", "aerial", "tanzania")
GEOGRAPHIC_DATASETS = ("aerial", "tanzania")


class Dataset(metaclass=abc.ABCMeta):
    """Generic class that describes the behavior of a Dataset object: it is
    initialized at least with an image size, its label are added always through
    the same manner, it can be serialized (save) and deserialized (load)
    from/to a `.json` file

    Attributes
    ----------
    image_size : int
        Size of considered images (height=width), raw images will be resized

#!/usr/bin/env python3
"""Tokenizer for fabric8-analytics tagger."""

import io
import os
import re

import nltk

import daiquiri
import f8a_tagger.defaults as defaults
from f8a_tagger.errors import InstallPrepareError
from f8a_tagger.errors import InvalidInputError

_logger = daiquiri.getLogger(__name__)


class Tokenizer(object):
    """Tokenizer for fabric8-analytics."""

    _STOPWORDS_TXT = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'data',
                                  'stopwords.txt')

    def __init__(self, stopwords_file=None, ngram_size=1, lemmatizer=None, stemmer=None):
        """Construct.

        :param stopwords_file: path to stopwords file or file
        :type stopwords_file: str
        :param ngram_size: size of ngrams that should be constructed from tokens
        :type ngram_size: int
        :param lemmatizer: lemmatizer instance to be used

import logging
import daiquiri
import requests

daiquiri.setup(level=logging.INFO)
logger = daiquiri.getLogger(__name__)


class SierraAPI(object):
    def __init__(self, api_url, oauth_key, oauth_secret, sess=None):
        self.api_url = api_url
        self.oauth_key = oauth_key
        self.oauth_secret = oauth_secret
        self.sess = sess or requests.Session()

        self._refresh_auth_token()

        # This causes the Session to call ``raise_for_status()`` as soon
        # as a response is received, so any failing request will throw an
        # exception immediately.
        def raise_error(resp, *args, **kwargs):
            resp.raise_for_status()

"""Script to start offline evaluation."""

# NOTE: Currently works only with S3DataStore
from uuid import uuid1
import sys
import time

from evaluation_platform.uranus.src.evaluate_data import generate_evaluate_test_s3
import daiquiri
import logging

daiquiri.setup(level=logging.INFO)
_logger = daiquiri.getLogger(__name__)

if __name__ == "__main__":
    if len(sys.argv) < 2:
        training_data_url = "s3://dev-stack-analysis-clean-data/maven/github/"
        result_id = str(uuid1())
        _logger.info("No env provided, using default")
        _logger.info("Evalutaion result id = {}".format(result_id))
    else:
        training_data_url = sys.argv[1]
        result_id = sys.argv[2]
        _logger.info("Env Provided")

    _logger.info("S3 URL : {}".format(training_data_url))
    t0 = time.time()
    _logger.info("Kronos Evaluation started")
    generate_evaluate_test_s3(training_url=training_data_url,

""" Utilitary function for Mapillary dataset analysis
"""

import json
import os

import daiquiri
import numpy as np
import pandas as pd
from PIL import Image

from deeposlandia.datasets import GEOGRAPHIC_DATASETS


logger = daiquiri.getLogger(__name__)


def read_config(filename):
    """Read the JSON configuration file.

    Parameters
    ----------
    filename : str
        Path of the configuration file

    Returns
    -------
    dict
        Dataset glossary
    """
    with open(filename) as fobj:

import operator
import os
import subprocess
import sys
import tempfile
from urllib import parse

import daiquiri

from git_pull_request import pagure
from git_pull_request import textparse

import github


LOG = daiquiri.getLogger("git-pull-request")


def _run_shell_command(cmd, output=None, raise_on_error=True):
    if output is True:
        output = subprocess.PIPE

    LOG.debug("running %s", cmd)
    sub = subprocess.Popen(cmd, stdout=output, stderr=output)
    out = sub.communicate()
    if raise_on_error and sub.returncode:
        raise RuntimeError("%s returned %d" % (cmd, sub.returncode))

    if out[0] is not None:
        return out[0].strip().decode()

# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import daiquiri
from six.moves.urllib.parse import quote

try:
    from swiftclient import client as swclient
    from swiftclient import utils as swift_utils
except ImportError:
    swclient = None
    swift_utils = None

from gnocchi import storage

LOG = daiquiri.getLogger(__name__)


def get_connection(conf):
    if swclient is None:
        raise RuntimeError("python-swiftclient unavailable")

    os_options = {
        'endpoint_type': conf.swift_endpoint_type,
        'service_type':  conf.swift_service_type,
        'user_domain_name': conf.swift_user_domain_name,
        'project_domain_name': conf.swift_project_domain_name,
    }
    if conf.swift_region:
        os_options['region_name'] = conf.swift_region

    return swclient.Connection(