Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_empty(self):
buffer = NonClosingBytesIO()
cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
with cctx.stream_writer(buffer) as compressor:
compressor.write(b"")
result = buffer.getvalue()
self.assertEqual(result, b"\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00")
params = zstd.get_frame_parameters(result)
self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
self.assertEqual(params.window_size, 524288)
self.assertEqual(params.dict_id, 0)
self.assertFalse(params.has_checksum)
# Test without context manager.
buffer = io.BytesIO()
compressor = cctx.stream_writer(buffer)
self.assertEqual(compressor.write(b""), 0)
self.assertEqual(buffer.getvalue(), b"")
self.assertEqual(compressor.flush(zstd.FLUSH_FRAME), 9)
result = buffer.getvalue()
self.assertEqual(result, b"\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00")
params = zstd.get_frame_parameters(result)
self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
def test_compress_empty(self):
cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
result = cctx.compress(b"")
self.assertEqual(result, b"\x28\xb5\x2f\xfd\x00\x48\x01\x00\x00")
params = zstd.get_frame_parameters(result)
self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
self.assertEqual(params.window_size, 524288)
self.assertEqual(params.dict_id, 0)
self.assertFalse(params.has_checksum, 0)
cctx = zstd.ZstdCompressor()
result = cctx.compress(b"")
self.assertEqual(result, b"\x28\xb5\x2f\xfd\x20\x00\x01\x00\x00")
params = zstd.get_frame_parameters(result)
self.assertEqual(params.content_size, 0)
def test_invalid_type(self):
with self.assertRaises(TypeError):
zstd.get_frame_parameters(None)
# Python 3 doesn't appear to convert unicode to Py_buffer.
if sys.version_info[0] >= 3:
with self.assertRaises(TypeError):
zstd.get_frame_parameters(u"foobarbaz")
else:
# CPython will convert unicode to Py_buffer. But CFFI won't.
if zstd.backend == "cffi":
with self.assertRaises(TypeError):
zstd.get_frame_parameters(u"foobarbaz")
else:
with self.assertRaises(zstd.ZstdError):
zstd.get_frame_parameters(u"foobarbaz")
with_size = io.BytesIO()
cctx = zstd.ZstdCompressor(level=1)
cctx.copy_stream(source, with_size)
# Source content size is unknown, so no content size written.
self.assertEqual(len(with_size.getvalue()), len(no_size.getvalue()))
source.seek(0)
with_size = io.BytesIO()
cctx.copy_stream(source, with_size, size=len(source.getvalue()))
# We specified source size, so content size header is present.
self.assertEqual(len(with_size.getvalue()), len(no_size.getvalue()) + 1)
no_params = zstd.get_frame_parameters(no_size.getvalue())
with_params = zstd.get_frame_parameters(with_size.getvalue())
self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
self.assertEqual(with_params.content_size, 1536)
self.assertEqual(no_params.dict_id, 0)
self.assertEqual(with_params.dict_id, 0)
self.assertFalse(no_params.has_checksum)
self.assertFalse(with_params.has_checksum)
d = zstd.train_dictionary(8192, samples)
h = hashlib.sha1(d.as_bytes()).hexdigest()
self.assertEqual(h, "7a2e59a876db958f74257141045af8f912e00d4e")
buffer = NonClosingBytesIO()
cctx = zstd.ZstdCompressor(level=9, dict_data=d)
with cctx.stream_writer(buffer) as compressor:
self.assertEqual(compressor.write(b"foo"), 0)
self.assertEqual(compressor.write(b"bar"), 0)
self.assertEqual(compressor.write(b"foo" * 16384), 0)
compressed = buffer.getvalue()
params = zstd.get_frame_parameters(compressed)
self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
self.assertEqual(params.window_size, 2097152)
self.assertEqual(params.dict_id, d.dict_id())
self.assertFalse(params.has_checksum)
h = hashlib.sha1(compressed).hexdigest()
self.assertEqual(h, "0a7c05635061f58039727cdbe76388c6f4cfef06")
source = b"foo" + b"bar" + (b"foo" * 16384)
dctx = zstd.ZstdDecompressor(dict_data=d)
self.assertEqual(
dctx.decompress(compressed, max_output_size=len(source)), source
)
# Upper 2 bits indicate content size.
params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b"\x40\x00\xff\x00")
self.assertEqual(params.content_size, 511)
self.assertEqual(params.window_size, 1024)
self.assertEqual(params.dict_id, 0)
self.assertFalse(params.has_checksum)
# Window descriptor is 2nd byte after frame header.
params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b"\x00\x40")
self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
self.assertEqual(params.window_size, 262144)
self.assertEqual(params.dict_id, 0)
self.assertFalse(params.has_checksum)
# Set multiple things.
params = zstd.get_frame_parameters(zstd.FRAME_HEADER + b"\x45\x40\x0f\x10\x00")
self.assertEqual(params.content_size, 272)
self.assertEqual(params.window_size, 262144)
self.assertEqual(params.dict_id, 15)
self.assertTrue(params.has_checksum)
min_match=5,
search_log=4,
target_length=10,
strategy=zstd.STRATEGY_FAST,
)
buffer = NonClosingBytesIO()
cctx = zstd.ZstdCompressor(compression_params=params)
with cctx.stream_writer(buffer) as compressor:
self.assertEqual(compressor.write(b"foo"), 0)
self.assertEqual(compressor.write(b"bar"), 0)
self.assertEqual(compressor.write(b"foobar" * 16384), 0)
compressed = buffer.getvalue()
params = zstd.get_frame_parameters(compressed)
self.assertEqual(params.content_size, zstd.CONTENTSIZE_UNKNOWN)
self.assertEqual(params.window_size, 1048576)
self.assertEqual(params.dict_id, 0)
self.assertFalse(params.has_checksum)
h = hashlib.sha1(compressed).hexdigest()
self.assertEqual(h, "dd4bb7d37c1a0235b38a2f6b462814376843ef0b")
samples.append(b"foo" * 64)
samples.append(b"bar" * 64)
samples.append(b"foobar" * 64)
d = zstd.train_dictionary(1024, samples)
cctx = zstd.ZstdCompressor(level=1, dict_data=d)
with_dict_id = cctx.compress(b"foobarfoobar")
cctx = zstd.ZstdCompressor(level=1, dict_data=d, write_dict_id=False)
no_dict_id = cctx.compress(b"foobarfoobar")
self.assertEqual(len(with_dict_id), len(no_dict_id) + 4)
no_params = zstd.get_frame_parameters(no_dict_id)
with_params = zstd.get_frame_parameters(with_dict_id)
self.assertEqual(no_params.dict_id, 0)
self.assertEqual(with_params.dict_id, 1880053135)
def test_write_content_size(self):
cctx = zstd.ZstdCompressor(level=1)
with_size = cctx.compress(b"foobar" * 256)
cctx = zstd.ZstdCompressor(level=1, write_content_size=False)
no_size = cctx.compress(b"foobar" * 256)
self.assertEqual(len(with_size), len(no_size) + 1)
no_params = zstd.get_frame_parameters(no_size)
with_params = zstd.get_frame_parameters(with_size)
self.assertEqual(no_params.content_size, zstd.CONTENTSIZE_UNKNOWN)
self.assertEqual(with_params.content_size, 1536)
def test_invalid_frame(self):
with self.assertRaisesRegex(zstd.ZstdError, "Unknown frame descriptor"):
zstd.get_frame_parameters(b"foobarbaz")