How to use the tabulator.helpers.detect_encoding function in tabulator

To help you get started, we’ve selected a few tabulator examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github frictionlessdata / tabulator-py / tests / test_helpers.py View on Github external
def test_detect_encoding_utf_16_le():
    sample = u'\uFEFFthen some text'.encode('utf-16-le')
    assert helpers.detect_encoding(sample) == 'utf-16'
github frictionlessdata / tabulator-py / tests / test_helpers.py View on Github external
def test_detect_encoding_utf_16_be():
    sample = u'\uFEFFthen some text'.encode('utf-16-be')
    assert helpers.detect_encoding(sample) == 'utf-16'
github frictionlessdata / tabulator-py / tests / test_helpers.py View on Github external
def test_detect_encoding_unknown():
    sample = b'\xff\x81'
    assert helpers.detect_encoding(sample) == 'utf-8'
github frictionlessdata / tabulator-py / tabulator / loaders / stream.py View on Github external
raise exceptions.SourceError(message)

        # Prepare bytes
        bytes = source
        if self.__stats:
            bytes = helpers.BytesStatsWrapper(bytes, self.__stats)

        # Return bytes
        if mode == 'b':
            return bytes

        # Detect encoding
        if self.__bytes_sample_size:
            sample = bytes.read(self.__bytes_sample_size)
            bytes.seek(0)
            encoding = helpers.detect_encoding(sample, encoding)

        # Prepare chars
        chars = io.TextIOWrapper(bytes, encoding)

        return chars
github frictionlessdata / tabulator-py / tabulator / loaders / remote.py View on Github external
buffer.seek(0)
                bytes = buffer
            if self.__stats:
                bytes = helpers.BytesStatsWrapper(bytes, self.__stats)
        except IOError as exception:
            raise exceptions.HTTPError(str(exception))

        # Return bytes
        if mode == 'b':
            return bytes

        # Detect encoding
        if self.__bytes_sample_size:
            sample = bytes.read(self.__bytes_sample_size)
            bytes.seek(0)
            encoding = helpers.detect_encoding(sample, encoding)

        # Prepare chars
        chars = io.TextIOWrapper(bytes, encoding)

        return chars
github frictionlessdata / tabulator-py / tabulator / loaders / aws.py View on Github external
bytes.write(response['Body'].read())
            bytes.seek(0)
            if self.__stats:
                bytes = helpers.BytesStatsWrapper(bytes, self.__stats)
        except Exception as exception:
            raise exceptions.IOError(str(exception))

        # Return bytes
        if mode == 'b':
            return bytes

        # Detect encoding
        if self.__bytes_sample_size:
            sample = bytes.read(self.__bytes_sample_size)
            bytes.seek(0)
            encoding = helpers.detect_encoding(sample, encoding)

        # Prepare chars
        chars = io.TextIOWrapper(bytes, encoding)

        return chars