Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
from streamz import Stream
import asyncio
from tornado.platform.asyncio import AsyncIOMainLoop
AsyncIOMainLoop().install()
source = Stream()
s = source.sliding_window(2).map(sum)
L = s.sink_to_list() # store result in a list
s.rate_limit(0.5).sink(source.emit) # pipe output back to input
s.rate_limit(1.0).sink(lambda x: print(L)) # print state of L every second
source.emit(0) # seed with initial values
source.emit(1)
def run_asyncio_loop():
loop = asyncio.get_event_loop()
try:
loop.run_forever()
except KeyboardInterrupt:
pass
async def model_flap(nursery_url, conn, partition_size=100):
with Sub0(dial=nursery_url) as sub:
sub.subscribe(b"")
stream = Stream(asynchronous=False)
(
stream.map(ujson.loads)
.flatten()
.map(flatten_record)
.partition(partition_size)
.map(pd.DataFrame)
.map(build_game_table)
.map(partial(model_train, None, conn))
.map(partial(model_test, None, conn))
.sink(print)
)
while True:
stream.emit(await sub.arecv())
if __name__=='__main__':
# Simple example of how these should be used
# Test with: python -m numismatic.exchanges.luno
from configparser import ConfigParser
from pathlib import Path
from streamz import Stream
logging.basicConfig(level=logging.INFO)
config = ConfigParser()
config.read(Path.home() / '.coinrc')
print(list(config.keys()))
api_key_id = config['Luno'].get('api_key_id', '')
api_key_secret = config['Luno'].get('api_key_secret', '')
output_stream = Stream()
printer = output_stream.map(print)
luno = LunoExchange(output_stream=output_stream, api_key_id=api_key_id,
api_key_secret=api_key_secret)
luno_btc = luno.listen('XBTZAR')
loop = asyncio.get_event_loop()
future = asyncio.wait([luno_btc], timeout=15)
completed, pending = loop.run_until_complete(future)
for task in pending:
task.cancel()
raise NotImplemented()
def get_prices(self, assets, currencies, **kwargs):
raise NotImplemented()
def get_tickers(self, assets, currencies, **kwargs):
raise NotImplemented()
if __name__=='__main__':
# Simple example of how these should be used
# Test with: python -m numismatic.feeds.bitfinex
logging.basicConfig(level=logging.INFO)
import asyncio
from streamz import Stream
output_stream = Stream()
printer = output_stream.map(print)
bfx = BitfinexWebsocketClient(output_stream=output_stream)
bfx_btc = bfx.subscribe('BTCUSD', 'trades')
loop = asyncio.get_event_loop()
future = asyncio.wait([bfx_btc], timeout=15)
completed, pending = loop.run_until_complete(future)
for task in pending:
task.cancel()
import pandas as pd
import random
from streamz import Stream
from streamz.dataframe import DataFrame
stream = Stream()
while True:
example = pd.DataFrame([{'name': random.uniform(1,10), 'amount': random.uniform(10,20)}])
sdf = DataFrame(stream, example=example)
print(sdf)
#sdf[sdf.name == 'Alice'].amount.sum()
def __init__(self, stream=None, example=None, stream_type=None):
assert example is not None
self.example = example
if not isinstance(self.example, self._subtype):
msg = ("For streaming type %s we expect an example of type %s. "
"Got %s") % (type(self).__name__, self._subtype.__name__,
str(self.example))
raise TypeError(msg)
assert isinstance(self.example, self._subtype)
self.stream = stream or Stream()
if stream_type:
if stream_type not in ['streaming', 'updating']:
raise Exception()
self._stream_type = stream_type
#! /usr/env python
""" a recreation of spark-streaming's network_wordcount
https://spark.apache.org/docs/2.2.0/streaming-programming-guide.html#a-quick-example
"""
import time
from streamz import Stream
# absolute port on localhost for now
s = Stream.from_tcp(9999)
s.map(bytes.split).flatten().frequencies().sink(print)
print(
"""In another terminal execute
> nc 127.0.0.1 9999
and then start typing content
"""
)
s.start()
time.sleep(600)
import libkloudtrader.analysis as analysis
import libkloudtrader.stocks as stocks
import libkloudtrader.processing as processing
import libkloudtrader.algorithm as algorithm
import numpy as np
import pandas as pd
import libkloudtrader.crypto as crypto
from streamz import Stream
source = Stream()
def return_orderbook(symbol):
#data=processing.add_data_to_batch(batch_size=50,data='stocks_bid')
#a=crypto.order_book('BTC/USD',number_of_data_points=1)['bids']
#for i in a:
a=processing.add_data_to_batch(batch_size=50,data='crypto_bid')
#return 'BidPrice: {}, BidSize: {}'.format(i[0],i[1])
df=pd.DataFrame()
df['SMA_5']=analysis.ma(a,5)
df['SMA_25']=analysis.ma(a,25)
previous_5 = df['SMA_5'].shift(1)
previous_25 = df['SMA_25'].shift(1)
crossing = (((df['SMA_5'] <= df['SMA_25']) & (previous_5 >= previous_25))
| ((df['SMA_5'] >= df['SMA_25']) & (previous_5 <= previous_25)))
return crossing
def __init__(self, stream=None, example=None, columns=None):
if columns is not None and example is None:
example = pd.DataFrame({c: [] for c in columns})
assert example is not None
self.example = example
assert isinstance(self.example, self._subtype)
self.stream = stream or Stream()
if self.raw_stream is not None:
if self.raw_stream=='':
from appdirs import user_cache_dir
self.raw_stream = user_cache_dir(LIBRARY_NAME)
date = time.strftime('%Y%m%dT%H%M%S')
filename = f'{self.exchange}_{symbol}_{date}.json.gz'
raw_stream_path = str(Path(self.raw_stream) / filename)
logger.info(f'Writing raw stream to {raw_stream_path} ...')
def write_to_file(batch):
logger.debug(f'Writing batch of {len(batch)} for {symbol} ...')
with gzip.open(raw_stream_path, 'at') as f:
for packet in batch:
f.write(packet+'\n')
self.raw_stream = Stream()
(self.raw_stream
.timed_window(self.raw_interval)
.filter(len)
.sink(write_to_file)
)