Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
indices = get_entities(session=self.session, entity_type='index',
return_type='domain', filters=[
Index.category.in_(
[StockCategory.concept.value, StockCategory.industry.value])],
provider=self.provider)
for index_item in indices:
resp = requests.get(self.category_stocks_url.format(index_item.code, '1'))
try:
results = json_callback_param(resp.text)
the_list = []
for result in results:
items = result.split(',')
stock_code = items[1]
stock_id = china_stock_code_to_id(stock_code)
index_id = index_item.id
the_list.append({
'id': '{}_{}'.format(index_id, stock_id),
'index_id': index_id,
'stock_id': stock_id
})
if the_list:
df = pd.DataFrame.from_records(the_list)
df_to_db(data_schema=self.data_schema, df=df, provider=self.provider)
self.logger.info('finish recording index:{},{}'.format(index_item.category, index_item.name))
except Exception as e:
self.logger.error("error:,resp.text:", e, resp.text)
self.sleep()
response_df['stock_id'] = response_df['证券代码'].apply(lambda x: china_stock_code_to_id(str(x)))
response_df['index_id'] = index_id
response_df['stock_id'] = response_df['stock_code'].apply(lambda x: china_stock_code_to_id(str(x)))
response_df['index_id'] = index_id
lambda x: f'{index_id}_{china_stock_code_to_id(str(x))}')
response_df['entity_id'] = response_df['id']
response_df['stock_id'] = response_df['stock_code'].apply(lambda x: china_stock_code_to_id(str(x)))
response_df['index_id'] = index_id
indices = get_entities(session=self.session, entity_type='index',
return_type='domain', filters=[
Index.category.in_([StockCategory.industry.value, StockCategory.concept.value])],
provider=self.provider)
for index_item in indices:
for page in range(1, 5):
resp = requests.get(self.category_stocks_url.format(page, index_item.code))
try:
if resp.text == 'null' or resp.text is None:
break
category_jsons = demjson.decode(resp.text)
the_list = []
for category in category_jsons:
stock_code = category['code']
stock_id = china_stock_code_to_id(stock_code)
index_id = index_item.id
the_list.append({
'id': '{}_{}'.format(index_id, stock_id),
'index_id': index_id,
'stock_id': stock_id
})
if the_list:
df = pd.DataFrame.from_records(the_list)
df_to_db(data_schema=self.data_schema, df=df, provider=self.provider)
self.logger.info('finish recording index:{},{}'.format(index_item.category, index_item.name))
except Exception as e:
self.logger.error("error:,resp.text:", e, resp.text)
self.sleep()