Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def download_proxy_file(url, file_path):
f = open(file_path, 'wb')
d = treq.get(url, agent=no_verify_agent())
d.addCallback(treq.collect, f.write)
d.addCallback(lambda _: f.close())
return d
def download_data(self, src, dst):
f = open(dst, 'wb')
resp = yield treq.get(src, agent=no_verify_agent())
yield treq.collect(resp, f.write)
f.close()
def download_data(self, src, dst):
f = open(dst, 'wb')
resp = yield treq.get(src, agent=no_verify_agent())
yield treq.collect(resp, f.write)
f.close()
def download(self, cap, local_path):
log.debug("Downloading %s...", local_path)
yield self.await_ready()
resp = yield treq.get("{}uri/{}".format(self.nodeurl, cap))
if resp.code == 200:
with atomic_write(local_path, mode="wb", overwrite=True) as f:
yield treq.collect(resp, f.write)
log.debug("Successfully downloaded %s", local_path)
else:
content = yield treq.content(resp)
raise TahoeWebError(content.decode("utf-8"))
uri = urljoin(self.remote, self.user + '/' + blob_id)
params = {'namespace': namespace} if namespace else None
data = yield self._client.get(uri, params=params, namespace=namespace)
if data.code == 404:
logger.warn("Blob not found in server: %s" % blob_id)
defer.returnValue(None)
elif not data.headers.hasHeader('Tag'):
logger.error("Server didn't send a tag header for: %s" % blob_id)
defer.returnValue(None)
tag = data.headers.getRawHeaders('Tag')[0]
tag = base64.urlsafe_b64decode(tag)
buf = DecrypterBuffer(blob_id, self.secret, tag)
# incrementally collect the body of the response
yield treq.collect(data, buf.write)
fd, size = buf.close()
logger.info("Finished download: (%s, %d)" % (blob_id, size))
defer.returnValue((fd, size))
def _download(reactor, url, destination_filename):
destination = open(destination_filename, 'wb')
d = treq.get(url)
d.addCallback(treq.collect, destination.write)
d.addBoth(lambda _: destination.close())
return d
def download_file(reactor, url, destination_filename):
destination = open(destination_filename, 'wb')
d = treq.get(url, unbuffered=True)
d.addCallback(treq.collect, destination.write)
d.addBoth(lambda _: destination.close())
return d
got_406 = response.code == 406 # our file is bigger
final_size_after_download = response.length + local_header_size
if got_406:
log.warning("s3 is more out of date than we are")
# should have something to download and a final length divisible by the header size
elif final_size_after_download and not final_size_after_download % HEADER_SIZE:
s3_height = (final_size_after_download / HEADER_SIZE) - 1
local_height = self.local_header_file_height()
if s3_height > local_height:
if local_header_size:
log.info("Resuming download of %i bytes from s3", response.length)
with open(os.path.join(self.config.path, "blockchain_headers"), "a+b") as headers_file:
yield treq.collect(response, headers_file.write)
else:
with open(os.path.join(self.config.path, "blockchain_headers"), "wb") as headers_file:
yield treq.collect(response, headers_file.write)
log.info("fetched headers from s3 (s3 height: %i), now verifying integrity after download.", s3_height)
self._check_header_file_integrity()
else:
log.warning("s3 is more out of date than we are")
else:
log.error("invalid size for headers from s3")
# TODO this needs to be connected in a tube
uri = urljoin(self.remote, self.user + '/' + blob_id)
params = {'namespace': namespace} if namespace else None
response = yield self._client.get(uri, params=params)
check_http_status(response.code, blob_id=blob_id)
if not response.headers.hasHeader('Tag'):
msg = "Server didn't send a tag header for: %s" % blob_id
logger.error(msg)
raise SoledadError(msg)
tag = response.headers.getRawHeaders('Tag')[0]
tag = base64.urlsafe_b64decode(tag)
buf = DecrypterBuffer(blob_id, self.secret, tag)
# incrementally collect the body of the response
yield treq.collect(response, buf.write)
fd, size = buf.close()
logger.info("Finished download: (%s, %d)" % (blob_id, size))
defer.returnValue((fd, size))
def download_file(url, destination):
d = treq.get(url)
d.addCallback(treq.collect, destination.write)
d.addBoth(destination.close)
return d