How to use the treq.collect function in treq

To help you get started, we’ve selected a few treq examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github CPChain / pdash / cpchain / proxy / client.py View on Github external
def download_proxy_file(url, file_path):

    f = open(file_path, 'wb')

    d = treq.get(url, agent=no_verify_agent())
    d.addCallback(treq.collect, f.write)
    d.addCallback(lambda _: f.close())

    return d
github CPChain / pdash / cpchain / storage-plugin / proxy.py View on Github external
def download_data(self, src, dst):

        f = open(dst, 'wb')
        resp = yield treq.get(src, agent=no_verify_agent())
        yield treq.collect(resp, f.write)
        f.close()
github CPChain / pdash / cpchain / storage_plugin / proxy.py View on Github external
def download_data(self, src, dst):

        f = open(dst, 'wb')
        resp = yield treq.get(src, agent=no_verify_agent())
        yield treq.collect(resp, f.write)
        f.close()
github gridsync / gridsync / gridsync / tahoe.py View on Github external
def download(self, cap, local_path):
        log.debug("Downloading %s...", local_path)
        yield self.await_ready()
        resp = yield treq.get("{}uri/{}".format(self.nodeurl, cap))
        if resp.code == 200:
            with atomic_write(local_path, mode="wb", overwrite=True) as f:
                yield treq.collect(resp, f.write)
            log.debug("Successfully downloaded %s", local_path)
        else:
            content = yield treq.content(resp)
            raise TahoeWebError(content.decode("utf-8"))
github leapcode / soledad / src / leap / soledad / client / _db / blobs.py View on Github external
uri = urljoin(self.remote, self.user + '/' + blob_id)
        params = {'namespace': namespace} if namespace else None
        data = yield self._client.get(uri, params=params, namespace=namespace)

        if data.code == 404:
            logger.warn("Blob not found in server: %s" % blob_id)
            defer.returnValue(None)
        elif not data.headers.hasHeader('Tag'):
            logger.error("Server didn't send a tag header for: %s" % blob_id)
            defer.returnValue(None)
        tag = data.headers.getRawHeaders('Tag')[0]
        tag = base64.urlsafe_b64decode(tag)
        buf = DecrypterBuffer(blob_id, self.secret, tag)

        # incrementally collect the body of the response
        yield treq.collect(data, buf.write)
        fd, size = buf.close()
        logger.info("Finished download: (%s, %d)" % (blob_id, size))
        defer.returnValue((fd, size))
github crossbario / crossbar / crossbar / webservice / archive.py View on Github external
def _download(reactor, url, destination_filename):
    destination = open(destination_filename, 'wb')
    d = treq.get(url)
    d.addCallback(treq.collect, destination.write)
    d.addBoth(lambda _: destination.close())
    return d
github twisted / treq / docs / examples / download_file.py View on Github external
def download_file(reactor, url, destination_filename):
    destination = open(destination_filename, 'wb')
    d = treq.get(url, unbuffered=True)
    d.addCallback(treq.collect, destination.write)
    d.addBoth(lambda _: destination.close())
    return d
github lbryio / lbry-sdk / lbrynet / core / Wallet.py View on Github external
got_406 = response.code == 406  # our file is bigger
        final_size_after_download = response.length + local_header_size
        if got_406:
            log.warning("s3 is more out of date than we are")
        # should have something to download and a final length divisible by the header size
        elif final_size_after_download and not final_size_after_download % HEADER_SIZE:
            s3_height = (final_size_after_download / HEADER_SIZE) - 1
            local_height = self.local_header_file_height()
            if s3_height > local_height:
                if local_header_size:
                    log.info("Resuming download of %i bytes from s3", response.length)
                    with open(os.path.join(self.config.path, "blockchain_headers"), "a+b") as headers_file:
                        yield treq.collect(response, headers_file.write)
                else:
                    with open(os.path.join(self.config.path, "blockchain_headers"), "wb") as headers_file:
                        yield treq.collect(response, headers_file.write)
                log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.", s3_height)
                self._check_header_file_integrity()
            else:
                log.warning("s3 is more out of date than we are")
        else:
            log.error("invalid size for headers from s3")
github leapcode / soledad / src / leap / soledad / client / _db / blobs / __init__.py View on Github external
# TODO this needs to be connected in a tube
        uri = urljoin(self.remote, self.user + '/' + blob_id)
        params = {'namespace': namespace} if namespace else None
        response = yield self._client.get(uri, params=params)
        check_http_status(response.code, blob_id=blob_id)

        if not response.headers.hasHeader('Tag'):
            msg = "Server didn't send a tag header for: %s" % blob_id
            logger.error(msg)
            raise SoledadError(msg)
        tag = response.headers.getRawHeaders('Tag')[0]
        tag = base64.urlsafe_b64decode(tag)
        buf = DecrypterBuffer(blob_id, self.secret, tag)

        # incrementally collect the body of the response
        yield treq.collect(response, buf.write)
        fd, size = buf.close()
        logger.info("Finished download: (%s, %d)" % (blob_id, size))
        defer.returnValue((fd, size))
github twisted / treq / docs / _examples / download_file.py View on Github external
def download_file(url, destination):
    d = treq.get(url)
    d.addCallback(treq.collect, destination.write)
    d.addBoth(destination.close)
    return d