How to use the zstandard.ZstdDecompressor function in zstandard

To help you get started, we’ve selected a few zstandard examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github indygreg / python-zstandard / tests / test_decompressor_fuzzing.py View on Github external
def test_stream_source_read_variance(
        self, original, level, streaming, source_read_size, read_sizes
    ):
        cctx = zstd.ZstdCompressor(level=level)

        if streaming:
            source = io.BytesIO()
            writer = cctx.stream_writer(source)
            writer.write(original)
            writer.flush(zstd.FLUSH_FRAME)
            source.seek(0)
        else:
            frame = cctx.compress(original)
            source = io.BytesIO(frame)

        dctx = zstd.ZstdDecompressor()

        chunks = []
        with dctx.stream_reader(source, read_size=source_read_size) as reader:
            while True:
                read_size = read_sizes.draw(strategies.integers(-1, 131072))
                chunk = reader.read(read_size)
                if not chunk and read_size:
                    break

                chunks.append(chunk)

        self.assertEqual(b"".join(chunks), original)
github indygreg / python-zstandard / tests / test_decompressor.py View on Github external
def test_invalid_inputs(self):
        dctx = zstd.ZstdDecompressor()

        if not hasattr(dctx, "multi_decompress_to_buffer"):
            self.skipTest("multi_decompress_to_buffer not available")

        with self.assertRaises(TypeError):
            dctx.multi_decompress_to_buffer(True)

        with self.assertRaises(TypeError):
            dctx.multi_decompress_to_buffer((1, 2))

        with self.assertRaisesRegex(TypeError, "item 0 not a bytes like object"):
            dctx.multi_decompress_to_buffer([u"foo"])

        with self.assertRaisesRegex(
            ValueError, "could not determine decompressed size of item 0"
        ):
github indygreg / python-zstandard / tests / test_decompressor.py View on Github external
def test_read_lines(self):
        cctx = zstd.ZstdCompressor()
        source = b"\n".join(("line %d" % i).encode("ascii") for i in range(1024))

        frame = cctx.compress(source)

        dctx = zstd.ZstdDecompressor()
        reader = dctx.stream_reader(frame)
        tr = io.TextIOWrapper(reader, encoding="utf-8")

        lines = []
        for line in tr:
            lines.append(line.encode("utf-8"))

        self.assertEqual(len(lines), 1024)
        self.assertEqual(b"".join(lines), source)

        reader = dctx.stream_reader(frame)
        tr = io.TextIOWrapper(reader, encoding="utf-8")

        lines = tr.readlines()
        self.assertEqual(len(lines), 1024)
        self.assertEqual("".join(lines).encode("utf-8"), source)
github indygreg / python-zstandard / tests / test_decompressor.py View on Github external
def test_read_closed(self):
        dctx = zstd.ZstdDecompressor()

        with dctx.stream_reader(b"foo") as reader:
            reader.close()
            self.assertTrue(reader.closed)
            with self.assertRaisesRegex(ValueError, "stream is closed"):
                reader.read(1)
github indygreg / python-zstandard / tests / test_decompressor.py View on Github external
def decompress_via_writer(data):
    buffer = io.BytesIO()
    dctx = zstd.ZstdDecompressor()
    decompressor = dctx.stream_writer(buffer)
    decompressor.write(data)

    return buffer.getvalue()
github droher / boxball / transform / src / converters / validate_schemas.py View on Github external
def validate_csvs_against_metadata(metadata: MetaData, csv_dir: Path) -> None:
    """
    Given a SQLAlchemy MetaData object containing models, and a folder containing
    gzipped csvs that match the table names of the models, validates all rows of all
    files, and outputs errors to a logfile.
    """
    error_filepath: Path = csv_dir.with_name("_validation_errors").with_suffix(".csv")
    make_error_file_stub(error_filepath)

    for table_name, table_obj in metadata.tables.items():
        print(table_name)
        columns = [c.name for c in table_obj.columns]
        validator = get_validator(table_obj)
        data_filepath = csv_dir.joinpath(table_name).with_suffix(".csv.zstd")
        with open(data_filepath, 'rb') as data_file, open(str(error_filepath), "a") as error_file:
            dctx = zstd.ZstdDecompressor()
            stream_reader = dctx.stream_reader(data_file)
            text_stream = io.TextIOWrapper(stream_reader, encoding='utf-8')
            reader = csv.DictReader(text_stream, fieldnames=columns, dialect=csv.unix_dialect)
            writer = csv.writer(error_file, dialect=csv.unix_dialect)
            for i, row in enumerate(reader):
                results = validator.validate(row)
                if results:
                    pk = {c.name: results.get(c.name, "MISSING PK") for c in table_obj.primary_key}
                    error_row = [table_name, i, pk, results]
                    writer.writerow(error_row)
                    print(row)
                    break
github klensy / wt-tools / src / wt_tools / ddsx_unpack.py View on Github external
private static extern long OodleLZ_Decompress(byte[] buffer, long bufferSize, byte[] result,
            long outputBufferSize, int a, int b, int c, long d, long e,
            long f, long g, long h, long i, int ThreadModule);
        '''
        if oodle_dll:
            decompressed_data = ctypes.create_string_buffer(parsed_data.header.memSz)
            res = oodle_dll.OodleLZ_Decompress(data[0x20:], parsed_data.header.packedSz, decompressed_data,
                                               parsed_data.header.memSz, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3)
            if res == 0:
                print("Error unpacking oodle compressed texture")
                return
            return dds_data.raw + decompressed_data.raw
        else:
            print("unsupported compression type: {}".format(dds_packed))
    elif dds_packed == "zstd":
        dctx = zstandard.ZstdDecompressor()
        d_data = dctx.decompress(data[0x20:], max_output_size=parsed_data.header.memSz)
        if d_data == 0:
            print("Error unpacking zstd compressed texture")
            return
        return dds_data.raw + d_data
    else:
        print("Unknown compression type: {}".format(dds_compression_type))
        return
github indygreg / python-zstandard / bench.py View on Github external
def decompress_one_use(chunks, opts):
    for chunk in chunks:
        zctx = zstd.ZstdDecompressor(**opts)
        zctx.decompress(chunk)
github opendatacube / odc-tools / libs / dscache / odc / dscache / _jsoncache.py View on Github external
raise ValueError('Missing format version field')
        if version != FORMAT_VERSION:
            raise ValueError("Unsupported on disk version: " + version.decode('utf8'))

        zdict = tr.get(b'zdict', None)

    dbs = SimpleNamespace(main=db,
                          info=db_info,
                          groups=db.open_db(b'groups', create=False),
                          ds=db.open_db(b'ds', create=False),
                          udata=db.open_db(b'udata', create=False))

    comp_params = {'dict_data': zstandard.ZstdCompressionDict(zdict)} if zdict else {}

    comp = None if readonly else zstandard.ZstdCompressor(level=complevel, **comp_params)
    decomp = zstandard.ZstdDecompressor(**comp_params)

    state = SimpleNamespace(dbs=dbs,
                            comp=comp,
                            decomp=decomp)

    return JsonBlobCache(state)
github elemental-lf / benji / src / backy2 / data_backends / compression / zstd.py View on Github external
def _get_decompressor(self, dict_id=0):
        thread_id = threading.get_ident()

        if thread_id in self.decompressors:
            return self.decompressors[thread_id]

        dctx = zstandard.ZstdDecompressor()

        self.decompressors[thread_id]= dctx
        return dctx