Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
,r2c1,r2c2
sheet_name2
,col1,col2
,r1c1,r1c2
,r2c1,r2c2
Currently, it processes csv files and xls files to ensure consistent
csv delimiters, etc. for tests.
"""
if path.endswith(".csv"):
imported_sheets = csv_to_dict(path)
else:
imported_sheets = xls_to_dict(path)
foo = BytesIO()
writer = csv.writer(
foo, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL)
for sheet_name, rows in imported_sheets.items():
writer.writerow([sheet_name])
out_keys = []
out_rows = []
for row in rows:
out_row = []
for key in row.keys():
if key not in out_keys:
out_keys.append(key)
for out_key in out_keys:
out_row.append(row.get(out_key, None))
out_rows.append(out_row)
writer.writerow([None] + out_keys)
for out_row in out_rows:
writer.writerow([None] + out_row)
return foo.getvalue().decode('utf-8')
for row in self.data:
flat = self.flatten(row,
simplify=self.params.datastructure == SourceDataStructureOptions.SIMPLE)
for field in NOT_ALLOWED_FIELDS:
flat.pop(field, None)
flattened_data.append(flat)
key_set.update(flat.keys())
ordered_keys=self.params.fields or sorted(list(key_set))
ordered_keys = map(unicode,ordered_keys)
writer = csv.DictWriter(output,
ordered_keys,
restval='',
delimiter=delimiter,
quotechar='"',
quoting=csv.QUOTE_MINIMAL,
doublequote=False,
escapechar='\\',
# extrasaction='ignore',
)
writer.writeheader()
for row in flattened_data:
writer.writerow(row)
if self.data and isinstance(self.data[0], list):
writer = csv.writer(output,
delimiter=delimiter,
quotechar='"',
quoting=csv.QUOTE_MINIMAL,
doublequote=False,
escapechar='\\',
# extrasaction = 'ignore',
# Ensure destination directory exists
if not os.path.exists(constants.PATH_BUILD_ASSETS_DATA):
os.makedirs(constants.PATH_BUILD_ASSETS_DATA)
# Get keys from a complete entity
keys = None
for row in data:
if keys is None or len(keys) < len(row.keys()):
keys = row.keys()
# Write to CSV
file = open(os.path.join(constants.PATH_BUILD_ASSETS_DATA, dataset + '.csv'), 'w')
writer = csv.DictWriter(
file,
delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL, fieldnames=sorted(keys)
)
writer.writeheader()
writer.writerows(sorted(data, key=lambda k: k['$ID']))
file.close()
return super(AccountBankStatementImport, self)._parse_file(
data_file)
transactions = []
fileobj = TemporaryFile('w+')
fileobj.write(data_file)
fileobj.seek(0)
reader = unicodecsv.DictReader(
fileobj,
fieldnames=[
'company', 'division', 'empty', 'account',
'card_num', 'title', 'lastname', 'firstname',
'op_code', 'seller_type', 'seller', 'date', 'acc_date',
'hour', 'city', 'code', 'label', 'local_amount',
'local_currency', 'acc_amount', 'acc_currency'],
delimiter=';',
quoting=unicodecsv.QUOTE_MINIMAL,
encoding='latin1')
i = 0
start_balance = end_balance = 0.0
currency_code = 'EUR'
account_number = 'HSBC_CARD_EUR'
for line in reader:
i += 1
if i == 1:
continue # skip title line
_logger.debug("Line %d: %s" % (i, line))
if not line:
continue
# cleanup
for key, value in line.iteritems():
line[key] = value and value.strip() or False
if not line['date'] or not line['acc_amount']:
def fix_dialect(dialect):
if not dialect.doublequote and dialect.escapechar is None:
dialect.doublequote = True
if dialect.quoting == unicodecsv.QUOTE_MINIMAL and dialect.quotechar == "'":
# Python csv's Sniffer seems to detect a wrong quotechar when
# quoting is minimal
dialect.quotechar = '"'
def __init__(
self,
filename,
azure_data_lake_conn_id,
azure_data_lake_path,
oracle_conn_id,
sql,
sql_params=None,
delimiter=",",
encoding="utf-8",
quotechar='"',
quoting=csv.QUOTE_MINIMAL,
*args, **kwargs):
super().__init__(*args, **kwargs)
if sql_params is None:
sql_params = {}
self.filename = filename
self.oracle_conn_id = oracle_conn_id
self.sql = sql
self.sql_params = sql_params
self.azure_data_lake_conn_id = azure_data_lake_conn_id
self.azure_data_lake_path = azure_data_lake_path
self.delimiter = delimiter
self.encoding = encoding
self.quotechar = quotechar
self.quoting = quoting
,r1c1,r1c2
,r2c1,r2c2
sheet_name2
,col1,col2
,r1c1,r1c2
,r2c1,r2c2
Currently, it processes csv files and xls files to ensure consistent
csv delimiters, etc. for tests.
"""
if path.endswith(".csv"):
imported_sheets = csv_to_dict(path)
else:
imported_sheets = xls_to_dict(path)
foo = BytesIO()
writer = csv.writer(foo, delimiter=",", quotechar='"', quoting=csv.QUOTE_MINIMAL)
for sheet_name, rows in imported_sheets.items():
writer.writerow([sheet_name])
out_keys = []
out_rows = []
for row in rows:
out_row = []
for key in row.keys():
if key not in out_keys:
out_keys.append(key)
for out_key in out_keys:
out_row.append(row.get(out_key, None))
out_rows.append(out_row)
writer.writerow([None] + out_keys)
for out_row in out_rows:
writer.writerow([None] + out_row)
return foo.getvalue().decode("utf-8")
from .base import ListFormatter
from cliff import columns
import six
if sys.version_info[0] == 3:
import csv
else:
import unicodecsv as csv
class CSVLister(ListFormatter):
QUOTE_MODES = {
'all': csv.QUOTE_ALL,
'minimal': csv.QUOTE_MINIMAL,
'nonnumeric': csv.QUOTE_NONNUMERIC,
'none': csv.QUOTE_NONE,
}
def add_argument_group(self, parser):
group = parser.add_argument_group('CSV Formatter')
group.add_argument(
'--quote',
choices=sorted(self.QUOTE_MODES.keys()),
dest='quote_mode',
default='nonnumeric',
help='when to include quotes, defaults to nonnumeric',
)
def emit_list(self, column_names, data, stdout, parsed_args):
writer_kwargs = dict(
time_range_query['from']=offset
if int(size)>-1:
time_range_query['size']=size
if src_id:
time_range_query["query"]["bool"]["must"].append({"constant_score" : { "filter" : { "term" : { "origin" : src_id } } }})
app.logger.debug(time_range_query)
if self.args['format'] == "json":
app.logger.debug("json direct export")
result=storage.query(time_range_query)[0]
app.logger.debug("API: Source's News:"+json.dumps(time_range_query))
return result
elif self.args['format'] == "csv":
app.logger.debug("csv export")
start_time = time.time()
csv_output = io.BytesIO()
writer = csv.writer(csv_output, delimiter='|',quotechar='`',quoting=csv.QUOTE_MINIMAL)
writer.writerow(["collector_id","news_id","updated_date","link","title","summary","overall_score","_score"])
result_csv=storage.query(time_range_query)[0]['hits']['hits']
app.logger.debug("API: Source's News:"+json.dumps(time_range_query))
for doc in result_csv:
if isinstance(doc, list):
doc=doc[0]
app.logger.debug("fetch: "+doc['_source']['link'])
updated=doc['_source']['updated']
link=urllib2.unquote(doc['_source']['link'])
if "title" in doc['_source']:
title=doc['_source']['title']
else:
title=""
if "summary" in doc['_source']:
if ts == UNIX_TIMESTAMP_ZERO:
continue
if ts == WINDOWS_TIMESTAMP_ZERO:
continue
if ts == datetime.datetime.min:
continue
entries.append(TimelineEntry(ts, t, e))
w = unicodecsv.writer(sys.stdout, delimiter="|", quotechar="\"",
quoting=unicodecsv.QUOTE_MINIMAL, encoding="utf-8")
w.writerow(["timestamp", "timestamp_type", "path", "sha1"])
for e in sorted(entries, key=lambda e: e.timestamp):
w.writerow([e.timestamp, e.type, e.entry.path, e.entry.sha1])
else:
w = unicodecsv.writer(sys.stdout, delimiter="|", quotechar="\"",
quoting=unicodecsv.QUOTE_MINIMAL, encoding="utf-8")
w.writerow(map(lambda e: e.name, FIELDS))
for e in ee:
w.writerow(map(lambda i: getattr(e, i.name), FIELDS))