How to use the streamz.Stream function in streamz

To help you get started, we’ve selected a few streamz examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github python-streamz / streamz / examples / fib_asyncio.py View on Github external
from streamz import Stream
import asyncio
from tornado.platform.asyncio import AsyncIOMainLoop
AsyncIOMainLoop().install()


source = Stream()
s = source.sliding_window(2).map(sum)
L = s.sink_to_list()                    # store result in a list

s.rate_limit(0.5).sink(source.emit)         # pipe output back to input
s.rate_limit(1.0).sink(lambda x: print(L))  # print state of L every second

source.emit(0)                          # seed with initial values
source.emit(1)


def run_asyncio_loop():
    loop = asyncio.get_event_loop()
    try:
        loop.run_forever()
    except KeyboardInterrupt:
        pass
github guillemborrell / pycones19_tut / flappystream-worker / flappystream / worker / main.py View on Github external
async def model_flap(nursery_url, conn, partition_size=100):
    with Sub0(dial=nursery_url) as sub:
        sub.subscribe(b"")

        stream = Stream(asynchronous=False)

        (
            stream.map(ujson.loads)
            .flatten()
            .map(flatten_record)
            .partition(partition_size)
            .map(pd.DataFrame)
            .map(build_game_table)
            .map(partial(model_train, None, conn))
            .map(partial(model_test, None, conn))
            .sink(print)
        )

        while True:
            stream.emit(await sub.arecv())
github snth / numismatic / numismatic / exchanges / luno.py View on Github external
if __name__=='__main__':
    # Simple example of how these should be used
    # Test with: python -m numismatic.exchanges.luno
    from configparser import ConfigParser
    from pathlib import Path
    from streamz import Stream
    logging.basicConfig(level=logging.INFO)
    config = ConfigParser()
    config.read(Path.home() / '.coinrc')
    print(list(config.keys()))
    api_key_id = config['Luno'].get('api_key_id', '')
    api_key_secret = config['Luno'].get('api_key_secret', '') 

    output_stream = Stream()
    printer = output_stream.map(print)

    luno = LunoExchange(output_stream=output_stream, api_key_id=api_key_id,
                        api_key_secret=api_key_secret)
    luno_btc = luno.listen('XBTZAR')

    loop = asyncio.get_event_loop()
    future = asyncio.wait([luno_btc], timeout=15)
    completed, pending = loop.run_until_complete(future)
    for task in pending:
        task.cancel()
github snth / numismatic / numismatic / feeds / bitfinex.py View on Github external
raise NotImplemented()
 
    def get_prices(self, assets, currencies, **kwargs):
        raise NotImplemented()       
 
    def get_tickers(self, assets, currencies, **kwargs):
        raise NotImplemented()       


if __name__=='__main__':
    # Simple example of how these should be used
    # Test with: python -m numismatic.feeds.bitfinex
    logging.basicConfig(level=logging.INFO)
    import asyncio
    from streamz import Stream
    output_stream = Stream()
    printer = output_stream.map(print)

    bfx = BitfinexWebsocketClient(output_stream=output_stream)
    bfx_btc = bfx.subscribe('BTCUSD', 'trades')

    loop = asyncio.get_event_loop()
    future = asyncio.wait([bfx_btc], timeout=15)
    completed, pending = loop.run_until_complete(future)
    for task in pending:
        task.cancel()
github KloudTrader / libkloudtrader / abc.py View on Github external
import pandas as pd
import random
from streamz import Stream
from streamz.dataframe import DataFrame
stream = Stream()
while True:
    example = pd.DataFrame([{'name': random.uniform(1,10), 'amount': random.uniform(10,20)}])
    sdf = DataFrame(stream, example=example)
    print(sdf)
#sdf[sdf.name == 'Alice'].amount.sum()
github python-streamz / streamz / streamz / collection.py View on Github external
def __init__(self, stream=None, example=None, stream_type=None):
        assert example is not None
        self.example = example
        if not isinstance(self.example, self._subtype):
            msg = ("For streaming type %s we expect an example of type %s. "
                   "Got %s") % (type(self).__name__, self._subtype.__name__,
                                str(self.example))
            raise TypeError(msg)
        assert isinstance(self.example, self._subtype)
        self.stream = stream or Stream()
        if stream_type:
            if stream_type not in ['streaming', 'updating']:
                raise Exception()
            self._stream_type = stream_type
github python-streamz / streamz / examples / network_wordcount.py View on Github external
#! /usr/env python
""" a recreation of spark-streaming's network_wordcount

https://spark.apache.org/docs/2.2.0/streaming-programming-guide.html#a-quick-example
"""
import time
from streamz import Stream

# absolute port on localhost for now
s = Stream.from_tcp(9999)
s.map(bytes.split).flatten().frequencies().sink(print)

print(
    """In another terminal execute
> nc 127.0.0.1 9999
and then start typing content
"""
)

s.start()
time.sleep(600)
github KloudTrader / libkloudtrader / algos / 1.py View on Github external
import libkloudtrader.analysis as analysis
import libkloudtrader.stocks as stocks
import libkloudtrader.processing as processing
import libkloudtrader.algorithm as algorithm
import numpy as np
import pandas as pd
import libkloudtrader.crypto as crypto



from streamz import Stream
source = Stream()
def return_orderbook(symbol):
    #data=processing.add_data_to_batch(batch_size=50,data='stocks_bid')
    #a=crypto.order_book('BTC/USD',number_of_data_points=1)['bids']
    #for i in a:
    a=processing.add_data_to_batch(batch_size=50,data='crypto_bid')
        #return 'BidPrice: {}, BidSize: {}'.format(i[0],i[1])
    df=pd.DataFrame()
    df['SMA_5']=analysis.ma(a,5)
    df['SMA_25']=analysis.ma(a,25)
    previous_5 = df['SMA_5'].shift(1)
    previous_25 = df['SMA_25'].shift(1)
    crossing = (((df['SMA_5'] <= df['SMA_25']) & (previous_5 >= previous_25))
            | ((df['SMA_5'] >= df['SMA_25']) & (previous_5 <= previous_25)))

    return crossing
github dask / pandas-streaming / pandas_streaming / core.py View on Github external
def __init__(self, stream=None, example=None, columns=None):
        if columns is not None and example is None:
            example = pd.DataFrame({c: [] for c in columns})
        assert example is not None
        self.example = example
        assert isinstance(self.example, self._subtype)
        self.stream = stream or Stream()
github snth / numismatic / numismatic / exchanges / base.py View on Github external
if self.raw_stream is not None:
            if self.raw_stream=='':
                from appdirs import user_cache_dir
                self.raw_stream = user_cache_dir(LIBRARY_NAME)
            date = time.strftime('%Y%m%dT%H%M%S')
            filename = f'{self.exchange}_{symbol}_{date}.json.gz'
            raw_stream_path = str(Path(self.raw_stream) / filename)
            logger.info(f'Writing raw stream to {raw_stream_path} ...')

            def write_to_file(batch):
                logger.debug(f'Writing batch of {len(batch)} for {symbol} ...')
                with gzip.open(raw_stream_path, 'at') as f:
                    for packet in batch:
                        f.write(packet+'\n')

            self.raw_stream = Stream()
            (self.raw_stream
             .timed_window(self.raw_interval)
             .filter(len)
             .sink(write_to_file)
             )