How to use the xxhash.xxh64_intdigest function in xxhash

To help you get started, we’ve selected a few xxhash examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github NiceneNerd / BCML / bcml / dev.py View on Github external
print("Compiling YAML documents...")
            pool.map(_do_yml, yml_files)

        hashes = util.get_hash_table(util.get_settings("wiiu"))
        print("Packing SARCs...")
        _pack_sarcs(tmp_dir, hashes, pool)
        for folder in {d for d in tmp_dir.glob("options/*") if d.is_dir()}:
            _pack_sarcs(folder, hashes, pool)

        for option_dir in tmp_dir.glob("options/*"):
            for file in {
                f
                for f in option_dir.rglob("**/*")
                if (f.is_file() and (tmp_dir / f.relative_to(option_dir)).exists())
            }:
                xh1 = xxhash.xxh64_intdigest(
                    (tmp_dir / file.relative_to(option_dir)).read_bytes()
                )
                xh2 = xxhash.xxh64_intdigest(file.read_bytes())
                if xh1 == xh2:
                    util.vprint(
                        f"Removing {file} from option {option_dir.name}, identical to base mod"
                    )
                    file.unlink()

        if not options:
            options = {"disable": [], "options": {}}
        options["options"]["texts"] = {"user_only": False}

        try:
            _make_bnp_logs(tmp_dir, options)
            for option_dir in {d for d in tmp_dir.glob("options/*") if d.is_dir()}:
github NiceneNerd / BCML / bcml / dev.py View on Github external
try:
            old_sarc = oead.Sarc(util.unyaz_if_needed(stock_file.read_bytes()))
        except (RuntimeError, ValueError, oead.InvalidDataError):
            raise ValueError("Cannot open file from game dump")
        old_files = {f.name for f in old_sarc.get_files()}
    except (FileNotFoundError, ValueError):
        for file in {f for f in folder.rglob("**/*") if f.is_file()}:
            packed.files[file.relative_to(folder).as_posix()] = file.read_bytes()
    else:
        for file in {
            f
            for f in folder.rglob("**/*")
            if f.is_file() and not f.suffix in EXCLUDE_EXTS
        }:
            file_data = file.read_bytes()
            xhash = xxhash.xxh64_intdigest(util.unyaz_if_needed(file_data))
            file_name = file.relative_to(folder).as_posix()
            if file_name in old_files:
                old_hash = xxhash.xxh64_intdigest(
                    util.unyaz_if_needed(old_sarc.get_file(file_name).data)
                )
            if file_name not in old_files or (
                xhash != old_hash and file.suffix not in util.AAMP_EXTS
            ):
                packed.files[file_name] = file_data
    finally:
        shutil.rmtree(folder)
        if not packed.files:
            return  # pylint: disable=lost-exception
        sarc_bytes = packed.write()[1]
        folder.write_bytes(
            util.compress(sarc_bytes)
github CommunityDragon / CDTB / cdragontoolbox / rstfile.py View on Github external
def key_to_hash(key):
    if isinstance(key, str):
        return xxh64_intdigest(key.lower()) & 0xffffffffff
    else:
        return key
github iamlemec / patents / tools / simhash.py View on Github external
def murmur(x):
    return np.uint64(xxhash.xxh64_intdigest(x))
github CommunityDragon / CDTB / cdragontoolbox / hashes.py View on Github external
def check_iter(self, paths):
        """Check paths from an iterable"""

        # failsafe for common dumb error
        if isinstance(paths, str):
            raise TypeError("expected iterable of strings, got a string")
        unknown = self.unknown
        for p in paths:
            h = xxh64_intdigest(p)
            if h in unknown:
                self._add_known(h, p)
github NiceneNerd / BCML / bcml / dev.py View on Github external
hashes = util.get_hash_table(util.get_settings("wiiu"))
        print("Packing SARCs...")
        _pack_sarcs(tmp_dir, hashes, pool)
        for folder in {d for d in tmp_dir.glob("options/*") if d.is_dir()}:
            _pack_sarcs(folder, hashes, pool)

        for option_dir in tmp_dir.glob("options/*"):
            for file in {
                f
                for f in option_dir.rglob("**/*")
                if (f.is_file() and (tmp_dir / f.relative_to(option_dir)).exists())
            }:
                xh1 = xxhash.xxh64_intdigest(
                    (tmp_dir / file.relative_to(option_dir)).read_bytes()
                )
                xh2 = xxhash.xxh64_intdigest(file.read_bytes())
                if xh1 == xh2:
                    util.vprint(
                        f"Removing {file} from option {option_dir.name}, identical to base mod"
                    )
                    file.unlink()

        if not options:
            options = {"disable": [], "options": {}}
        options["options"]["texts"] = {"user_only": False}

        try:
            _make_bnp_logs(tmp_dir, options)
            for option_dir in {d for d in tmp_dir.glob("options/*") if d.is_dir()}:
                _make_bnp_logs(option_dir, options)
        except Exception as err:  # pylint: disable=broad-except
            pool.terminate()
github ooni / pipeline / af / mubench / hash64_shingles.py View on Github external
def xxhash_u64_v1(): # 100 loops, best of 100: 700 usec per loop
    for m1, m2 in prepare():
        i = xxhash.xxh64_intdigest(TEXT[m1.end():m2.start()])
github CommunityDragon / CDTB / cdragontoolbox / hashes.py View on Github external
def check(self, p):
        """Check a single hash, print and add to known on match"""

        h = xxh64_intdigest(p)
        if h in self.unknown:
            self._add_known(h, p)