Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_stream_source_read_variance(
self, original, level, streaming, source_read_size, read_sizes
):
cctx = zstd.ZstdCompressor(level=level)
if streaming:
source = io.BytesIO()
writer = cctx.stream_writer(source)
writer.write(original)
writer.flush(zstd.FLUSH_FRAME)
source.seek(0)
else:
frame = cctx.compress(original)
source = io.BytesIO(frame)
dctx = zstd.ZstdDecompressor()
chunks = []
with dctx.stream_reader(source, read_size=source_read_size) as reader:
while True:
read_size = read_sizes.draw(strategies.integers(-1, 131072))
chunk = reader.read(read_size)
if not chunk and read_size:
break
chunks.append(chunk)
self.assertEqual(b"".join(chunks), original)
def test_invalid_inputs(self):
dctx = zstd.ZstdDecompressor()
if not hasattr(dctx, "multi_decompress_to_buffer"):
self.skipTest("multi_decompress_to_buffer not available")
with self.assertRaises(TypeError):
dctx.multi_decompress_to_buffer(True)
with self.assertRaises(TypeError):
dctx.multi_decompress_to_buffer((1, 2))
with self.assertRaisesRegex(TypeError, "item 0 not a bytes like object"):
dctx.multi_decompress_to_buffer([u"foo"])
with self.assertRaisesRegex(
ValueError, "could not determine decompressed size of item 0"
):
def test_read_lines(self):
cctx = zstd.ZstdCompressor()
source = b"\n".join(("line %d" % i).encode("ascii") for i in range(1024))
frame = cctx.compress(source)
dctx = zstd.ZstdDecompressor()
reader = dctx.stream_reader(frame)
tr = io.TextIOWrapper(reader, encoding="utf-8")
lines = []
for line in tr:
lines.append(line.encode("utf-8"))
self.assertEqual(len(lines), 1024)
self.assertEqual(b"".join(lines), source)
reader = dctx.stream_reader(frame)
tr = io.TextIOWrapper(reader, encoding="utf-8")
lines = tr.readlines()
self.assertEqual(len(lines), 1024)
self.assertEqual("".join(lines).encode("utf-8"), source)
def test_read_closed(self):
dctx = zstd.ZstdDecompressor()
with dctx.stream_reader(b"foo") as reader:
reader.close()
self.assertTrue(reader.closed)
with self.assertRaisesRegex(ValueError, "stream is closed"):
reader.read(1)
def decompress_via_writer(data):
buffer = io.BytesIO()
dctx = zstd.ZstdDecompressor()
decompressor = dctx.stream_writer(buffer)
decompressor.write(data)
return buffer.getvalue()
def validate_csvs_against_metadata(metadata: MetaData, csv_dir: Path) -> None:
"""
Given a SQLAlchemy MetaData object containing models, and a folder containing
gzipped csvs that match the table names of the models, validates all rows of all
files, and outputs errors to a logfile.
"""
error_filepath: Path = csv_dir.with_name("_validation_errors").with_suffix(".csv")
make_error_file_stub(error_filepath)
for table_name, table_obj in metadata.tables.items():
print(table_name)
columns = [c.name for c in table_obj.columns]
validator = get_validator(table_obj)
data_filepath = csv_dir.joinpath(table_name).with_suffix(".csv.zstd")
with open(data_filepath, 'rb') as data_file, open(str(error_filepath), "a") as error_file:
dctx = zstd.ZstdDecompressor()
stream_reader = dctx.stream_reader(data_file)
text_stream = io.TextIOWrapper(stream_reader, encoding='utf-8')
reader = csv.DictReader(text_stream, fieldnames=columns, dialect=csv.unix_dialect)
writer = csv.writer(error_file, dialect=csv.unix_dialect)
for i, row in enumerate(reader):
results = validator.validate(row)
if results:
pk = {c.name: results.get(c.name, "MISSING PK") for c in table_obj.primary_key}
error_row = [table_name, i, pk, results]
writer.writerow(error_row)
print(row)
break
private static extern long OodleLZ_Decompress(byte[] buffer, long bufferSize, byte[] result,
long outputBufferSize, int a, int b, int c, long d, long e,
long f, long g, long h, long i, int ThreadModule);
'''
if oodle_dll:
decompressed_data = ctypes.create_string_buffer(parsed_data.header.memSz)
res = oodle_dll.OodleLZ_Decompress(data[0x20:], parsed_data.header.packedSz, decompressed_data,
parsed_data.header.memSz, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3)
if res == 0:
print("Error unpacking oodle compressed texture")
return
return dds_data.raw + decompressed_data.raw
else:
print("unsupported compression type: {}".format(dds_packed))
elif dds_packed == "zstd":
dctx = zstandard.ZstdDecompressor()
d_data = dctx.decompress(data[0x20:], max_output_size=parsed_data.header.memSz)
if d_data == 0:
print("Error unpacking zstd compressed texture")
return
return dds_data.raw + d_data
else:
print("Unknown compression type: {}".format(dds_compression_type))
return
def decompress_one_use(chunks, opts):
for chunk in chunks:
zctx = zstd.ZstdDecompressor(**opts)
zctx.decompress(chunk)
raise ValueError('Missing format version field')
if version != FORMAT_VERSION:
raise ValueError("Unsupported on disk version: " + version.decode('utf8'))
zdict = tr.get(b'zdict', None)
dbs = SimpleNamespace(main=db,
info=db_info,
groups=db.open_db(b'groups', create=False),
ds=db.open_db(b'ds', create=False),
udata=db.open_db(b'udata', create=False))
comp_params = {'dict_data': zstandard.ZstdCompressionDict(zdict)} if zdict else {}
comp = None if readonly else zstandard.ZstdCompressor(level=complevel, **comp_params)
decomp = zstandard.ZstdDecompressor(**comp_params)
state = SimpleNamespace(dbs=dbs,
comp=comp,
decomp=decomp)
return JsonBlobCache(state)
def _get_decompressor(self, dict_id=0):
thread_id = threading.get_ident()
if thread_id in self.decompressors:
return self.decompressors[thread_id]
dctx = zstandard.ZstdDecompressor()
self.decompressors[thread_id]= dctx
return dctx