Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_stream_io_error():
stream = Stream('bad_path.csv')
with pytest.raises(exceptions.IOError) as excinfo:
stream.open()
assert 'bad_path.csv' in str(excinfo.value)
def test_stream_s3_non_existent_file(s3_client, bucket):
with pytest.raises(exceptions.IOError):
Stream('s3://%s/table.csv' % bucket).open()
# Prepare table
try:
stream.open()
sample = stream.sample
headers = stream.headers
if headers is None:
headers = [None] * len(sample[0]) if sample else []
if _filter_checks(checks, type='schema'):
if schema is None and self.__infer_schema:
schema = self.__infer(sample, headers)
if schema is None:
checks = _filter_checks(checks, type='schema', inverse=True)
except Exception as exception:
if _local_file_not_found(source):
message = "No such file or directory: '%s'" % source
exception = tabulator.exceptions.IOError(message)
error, fatal_error = _compose_error_from_exception(exception)
errors.append(error)
# Prepare schema
if not fatal_error:
if schema:
if schema.primary_key:
for field in schema.descriptor.get('fields', []):
if field.get('name') in schema.primary_key:
field['primaryKey'] = True
schema.commit()
for error in schema.errors:
fatal_error = True
error = _compose_error_from_schema_error(error)
errors.append(error)
# Prepare source
source = helpers.requote_uri(source)
# Prepare bytes
try:
parts = urlparse(source, allow_fragments=False)
response = self.__s3_client.get_object(Bucket=parts.netloc, Key=parts.path[1:])
# https://github.com/frictionlessdata/tabulator-py/issues/271
bytes = io.BufferedRandom(io.BytesIO())
bytes.write(response['Body'].read())
bytes.seek(0)
if self.__stats:
bytes = helpers.BytesStatsWrapper(bytes, self.__stats)
except Exception as exception:
raise exceptions.IOError(str(exception))
# Return bytes
if mode == 'b':
return bytes
# Detect encoding
if self.__bytes_sample_size:
sample = bytes.read(self.__bytes_sample_size)
bytes.seek(0)
encoding = helpers.detect_encoding(sample, encoding)
# Prepare chars
chars = io.TextIOWrapper(bytes, encoding)
return chars
def load(self, source, mode='t', encoding=None):
# Prepare source
scheme = 'file://'
if source.startswith(scheme):
source = source.replace(scheme, '', 1)
# Prepare bytes
try:
bytes = io.open(source, 'rb')
if self.__stats:
bytes = helpers.BytesStatsWrapper(bytes, self.__stats)
except IOError as exception:
raise exceptions.IOError(str(exception))
# Return bytes
if mode == 'b':
return bytes
# Detect encoding
if self.__bytes_sample_size:
sample = bytes.read(self.__bytes_sample_size)
bytes.seek(0)
encoding = helpers.detect_encoding(sample, encoding)
# Prepare chars
chars = io.TextIOWrapper(bytes, encoding)
return chars
def _compose_error_from_exception(exception):
code = 'source-error'
message = str(exception)
fatal_error = True
if isinstance(exception, tabulator.exceptions.SourceError):
code = 'source-error'
elif isinstance(exception, tabulator.exceptions.SchemeError):
code = 'scheme-error'
elif isinstance(exception, tabulator.exceptions.FormatError):
code = 'format-error'
elif isinstance(exception, tabulator.exceptions.EncodingError):
code = 'encoding-error'
elif isinstance(exception, tabulator.exceptions.IOError):
code = 'io-error'
elif isinstance(exception, tabulator.exceptions.HTTPError):
code = 'http-error'
elif isinstance(exception, tableschema.exceptions.IntegrityError):
code = 'source-error'
fatal_error = False
return (Error(code, message=message), fatal_error)
# Module API
class TabulatorException(Exception):
"""Base class for all tabulator exceptions.
"""
pass
class IOError(TabulatorException):
"""Local loading error
"""
pass
class HTTPError(IOError):
"""Remote loading error
"""
pass
class SourceError(TabulatorException):
"""The source file could not be parsed correctly.
"""
pass
class SchemeError(TabulatorException):
"""The file scheme is not supported.
"""
pass