Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def test_json_codec():
filter_str = json.dumps([Stock1dKdata.timestamp == '2017-01-01', Stock1dKdata.name == 'abcd'],
cls=CustomJsonEncoder)
assert isinstance(filter_str, str)
filter = json.loads(filter_str, cls=CustomJsonDecoder)
assert isinstance(filter[0], BinaryExpression)
def test_reader_move_on():
data_reader = DataReader(codes=['002572', '000338'], data_schema=Stock1dKdata, provider='joinquant',
start_timestamp='2019-06-13',
end_timestamp='2019-06-14', entity_provider='eastmoney')
data_reader.move_on(to_timestamp='2019-06-15')
assert ('stock_sz_002572', '2019-06-15') not in data_reader.data_df.index
assert ('stock_sz_000338', '2019-06-15') not in data_reader.data_df.index
start_time = time.time()
data_reader.move_on(to_timestamp='2019-06-20', timeout=5)
assert time.time() - start_time < 5
def test_china_stock_reader():
data_reader = DataReader(codes=['002572', '000338'], data_schema=Stock1dKdata, provider='joinquant',
start_timestamp='2019-01-01',
end_timestamp='2019-06-10', entity_provider='eastmoney')
categories = data_reader.data_df.index.levels[0].to_list()
df = data_reader.data_df
assert 'stock_sz_002572' in categories
assert 'stock_sz_000338' in categories
assert ('stock_sz_002572', '2019-01-02') in df.index
assert ('stock_sz_000338', '2019-01-02') in df.index
assert ('stock_sz_002572', '2019-06-10') in df.index
assert ('stock_sz_000338', '2019-06-10') in df.index
for timestamp in iterate_timestamps(entity_type='stock', exchange='sz',
from zvdata import IntervalLevel
from zvdata.recorder import FixedCycleDataRecorder
from zvdata.utils.time_utils import get_year_quarters, is_same_date, to_pd_timestamp
from zvdata.utils.utils import to_float
from zvt.api.common import generate_kdata_id
from zvt.api.quote import get_kdata
from zvt.domain import Stock1dKdata, Stock
# this recorder is deprecated,because sina hfq factor could not get now
class StockKdataSinaSpider(FixedCycleDataRecorder):
entity_provider = 'sina'
entity_schema = Stock
provider = 'sina'
data_schema = Stock1dKdata
def __init__(self, entity_type='stock', exchanges=['sh', 'sz'], entity_ids=None, codes=None, batch_size=10,
force_update=False, sleeping_time=10, default_size=2000, real_time=True, fix_duplicate_way='add',
start_timestamp=None, end_timestamp=None,
level=IntervalLevel.LEVEL_1DAY, kdata_use_begin_time=False, close_hour=0, close_minute=0,
one_day_trading_minutes=24 * 60) -> None:
super().__init__(entity_type, exchanges, entity_ids, codes, batch_size, force_update, sleeping_time,
default_size, real_time, fix_duplicate_way, start_timestamp, end_timestamp, close_hour,
close_minute, level, kdata_use_begin_time, one_day_trading_minutes)
self.current_factors = {}
self.latest_factors = {}
for security_item in self.entities:
kdata = get_kdata(entity_id=security_item.id, provider=self.provider,
level=self.level.value, order=Stock1dKdata.timestamp.desc(),
return_type='domain',
session=self.session)
arrowsize=1,
arrowwidth=2,
# arrowcolor='#030813',
ax=-10,
ay=-30,
bordercolor='#c7c7c7',
borderwidth=1,
bgcolor=color,
opacity=0.8
))
return annotations
if __name__ == '__main__':
df = get_data(data_schema=Stock1dKdata, provider='joinquant', entity_ids=['stock_sz_000001', 'stock_sz_000002'])
df1 = get_data(data_schema=Stock1dMaStateStats, provider='zvt', entity_ids=['stock_sz_000001', 'stock_sz_000002'],
columns=['current_count'])
drawer = Drawer(df, df1[['current_count']])
drawer.draw_kline()
def on_finish_entity(self, entity):
kdatas = get_kdata(entity_id=entity.id, level=self.level.value, order=Stock1dKdata.timestamp.asc(),
return_type='domain',
session=self.session,
filters=[Stock1dKdata.factor.is_(None),
Stock1dKdata.timestamp >= to_pd_timestamp('2005-01-01')])
if kdatas:
start = kdatas[0].timestamp
end = kdatas[-1].timestamp
# get hfq from joinquant
df = get_price(to_jq_entity_id(entity), start_date=to_time_str(start), end_date=now_time_str(),
frequency='daily',
fields=['factor', 'open', 'close', 'low', 'high'],
skip_paused=True, fq='post')
if df is not None and not df.empty:
# fill hfq data
for kdata in kdatas:
from zvdata.recorder import FixedCycleDataRecorder
from zvdata.utils import utils
from zvdata.utils.time_utils import to_time_str, TIME_FORMAT_DAY1, now_time_str, to_pd_timestamp
from zvt.api.common import generate_kdata_id
from zvt.api.quote import get_kdata
from zvt.domain import Stock1dKdata, Stock
from zvt.recorders.joinquant import to_jq_entity_id
from zvt.settings import JQ_ACCOUNT, JQ_PASSWD
class ChinaStockDayKdataRecorder(FixedCycleDataRecorder):
entity_provider = 'eastmoney'
entity_schema = Stock
provider = 'netease'
data_schema = Stock1dKdata
url = 'http://quotes.money.163.com/service/chddata.html?code={}{}&start={}&end={}&fields=TCLOSE;HIGH;LOW;TOPEN;LCLOSE;CHG;PCHG;TURNOVER;VOTURNOVER;VATURNOVER'
def __init__(self, entity_type='stock', exchanges=['sh', 'sz'], entity_ids=None, codes=None, batch_size=10,
force_update=False, sleeping_time=10, default_size=2000, real_time=True, fix_duplicate_way='add',
start_timestamp=None, end_timestamp=None, level=IntervalLevel.LEVEL_1DAY, kdata_use_begin_time=False,
close_hour=0, close_minute=0,
one_day_trading_minutes=24 * 60) -> None:
super().__init__(entity_type, exchanges, entity_ids, codes, batch_size, force_update, sleeping_time,
default_size, real_time, fix_duplicate_way, start_timestamp, end_timestamp, close_hour,
close_minute, level, kdata_use_begin_time, one_day_trading_minutes)
self.current_factors = {}
for security_item in self.entities:
kdata = get_kdata(entity_id=security_item.id, provider=self.provider,
level=self.level.value, order=Stock1dKdata.timestamp.desc(),
def __init__(self, entity_type='stock', exchanges=['sh', 'sz'], entity_ids=None, codes=None, batch_size=10,
force_update=False, sleeping_time=10, default_size=2000, real_time=True, fix_duplicate_way='add',
start_timestamp=None, end_timestamp=None,
level=IntervalLevel.LEVEL_1DAY, kdata_use_begin_time=False, close_hour=0, close_minute=0,
one_day_trading_minutes=24 * 60) -> None:
super().__init__(entity_type, exchanges, entity_ids, codes, batch_size, force_update, sleeping_time,
default_size, real_time, fix_duplicate_way, start_timestamp, end_timestamp, close_hour,
close_minute, level, kdata_use_begin_time, one_day_trading_minutes)
self.current_factors = {}
self.latest_factors = {}
for security_item in self.entities:
kdata = get_kdata(entity_id=security_item.id, provider=self.provider,
level=self.level.value, order=Stock1dKdata.timestamp.desc(),
return_type='domain',
session=self.session)
if kdata:
self.current_factors[security_item.id] = kdata[0].factor
# notify it once after registered
if pd_is_not_null(self.data_df):
listener.on_data_loaded(self.data_df)
def deregister_data_listener(self, listener):
if listener in self.data_listeners:
self.data_listeners.remove(listener)
def empty(self):
return not pd_is_not_null(self.data_df)
if __name__ == '__main__':
from zvt.domain import Stock1dKdata
data_reader = DataReader(codes=['002572', '000338'], data_schema=Stock1dKdata, provider='joinquant',
start_timestamp='2017-01-01',
end_timestamp='2019-06-10')
for timestamp in pd.date_range(start='2019-06-10', end='2019-10-10'):
data_reader.move_on(timestamp)
print(data_reader.data_df.iloc[0])
def on_finish_entity(self, entity):
kdatas = get_kdata(entity_id=entity.id, level=self.level.value, order=Stock1dKdata.timestamp.asc(),
return_type='domain',
session=self.session,
filters=[Stock1dKdata.factor.is_(None),
Stock1dKdata.timestamp >= to_pd_timestamp('2005-01-01')])
if kdatas:
start = kdatas[0].timestamp
end = kdatas[-1].timestamp
# get hfq from joinquant
df = get_price(to_jq_entity_id(entity), start_date=to_time_str(start), end_date=now_time_str(),
frequency='daily',
fields=['factor', 'open', 'close', 'low', 'high'],
skip_paused=True, fq='post')
if df is not None and not df.empty:
# fill hfq data
for kdata in kdatas:
if kdata.timestamp in df.index:
kdata.hfq_open = df.loc[kdata.timestamp, 'open']
kdata.hfq_close = df.loc[kdata.timestamp, 'close']