Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
resp = requests.get(the_url)
results = json_callback_param(resp.text)
kdatas = []
if results:
klines = results['data']['klines']
# TODO: ignore the last unfinished kdata now,could control it better if need
for result in klines[:-1]:
# "2000-01-28,1005.26,1012.56,1173.12,982.13,3023326,3075552000.00"
# time,open,close,high,low,volume,turnover
fields = result.split(',')
the_timestamp = to_pd_timestamp(fields[0])
the_id = generate_kdata_id(entity_id=entity.id, timestamp=the_timestamp, level=self.level)
kdatas.append(dict(id=the_id,
timestamp=the_timestamp,
entity_id=entity.id,
code=entity.code,
name=entity.name,
level=self.level.value,
open=to_float(fields[1]),
close=to_float(fields[2]),
high=to_float(fields[3]),
low=to_float(fields[4]),
volume=to_float(fields[5]),
turnover=to_float(fields[6])))
return kdatas
def generate_domain_id(self, entity, original_data):
return generate_kdata_id(entity_id=entity.id, timestamp=original_data['timestamp'], level=self.level)
def generate_domain_id(self, entity, original_data):
return generate_kdata_id(entity_id=entity.id, timestamp=original_data['timestamp'], level=self.level)
if entity.exchange == 'sh':
id_flag = "{}1".format(entity.code)
if entity.exchange == 'sz':
id_flag = "{}2".format(entity.code)
the_url = self.url.format("{}".format(id_flag), eastmoney_map_zvt_trading_level(self.level),
now_time_str(fmt=TIME_FORMAT_MINUTE), size)
resp = requests.get(the_url)
results = json_callback_param(resp.text)
kdatas = []
for result in results:
the_timestamp = to_pd_timestamp(result['time'])
the_id = generate_kdata_id(entity_id=entity.id, timestamp=the_timestamp, level=self.level)
if not data_exist(self.session, self.kdata_schema, the_id):
kdatas.append(self.kdata_schema(id=the_id,
timestamp=the_timestamp,
entity_id=entity.id,
code=entity.code,
name=entity.name,
level=self.level,
open=to_float(result['open']),
close=to_float(result['close']),
high=to_float(result['high']),
low=to_float(result['low']),
volume=to_float(result['volume']),
turnover=to_float(result['amount']),
turnover_rate=to_float(result['turnoverrate'])
))
for idx, tr in enumerate(trs):
tds = Selector(text=tr).xpath('//td//text()').extract()
tds = [x.strip() for x in tds if x.strip()]
open = tds[1]
high = tds[2]
close = tds[3]
low = tds[4]
volume = tds[5]
turnover = tds[6]
if fuquan == 'hfq':
factor = tds[7]
the_timestamp = to_pd_timestamp(tds[0])
the_id = generate_kdata_id(entity_id=entity.id, timestamp=the_timestamp, level=self.level)
if fuquan == 'hfq':
# we got bfq at first and then update hfq data
existed = [item for item in kdatas if item['id'] == the_id]
if existed:
kdata = existed[0]
else:
self.logger.error("bfq not got for:{}".format(the_id))
kdata = {
'id': the_id,
'timestamp': the_timestamp,
'name': entity.name,
'level': self.level.value,
'open': to_float(open) / to_float(factor),
'close': to_float(close) / to_float(factor),
def generate_domain_id(self, entity, original_data):
return generate_kdata_id(entity_id=entity.id, timestamp=original_data['timestamp'], level=self.level)
def generate_domain_id(self, security_item, original_data):
return generate_kdata_id(security_id=security_item.id, timestamp=original_data['timestamp'], level=self.level)
def generate_domain_id(self, entity, original_data):
return generate_kdata_id(entity_id=entity.id, timestamp=original_data['timestamp'], level=self.level)
def generate_domain_id(self, entity, original_data):
return generate_kdata_id(entity_id=entity.id, timestamp=original_data['timestamp'], level=self.level)
def generate_domain_id(self, entity, original_data):
return generate_kdata_id(entity_id=entity.id, timestamp=original_data['timestamp'], level=self.level)