How to use the heronpy.api.stream.Grouping.fields function in heronpy

To help you get started, we’ve selected a few heronpy examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github apache / incubator-heron / integration_test / src / python / integration_test / topology / fields_grouping / fields_grouping.py View on Github external
def fields_grouping_builder(topology_name, http_server_url):
  builder = TestTopologyBuilder(topology_name, http_server_url)
  ab_spout = builder.add_spout("ab-spout", ABSpout, 1, max_executions=400)

  count_bolt = builder.add_bolt("count-bolt", WordCountBolt,
                                inputs={ab_spout: Grouping.fields('word')}, par=2)

  builder.add_bolt("sum-bolt", CountAggregatorBolt,
                   inputs={count_bolt: Grouping.NONE}, par=1)

  return builder.create_topology()
github apache / incubator-heron / examples / src / python / word_count_topology.py View on Github external
from examples.src.python.spout import WordSpout
from examples.src.python.bolt import CountBolt

# Topology is defined using a topology builder
# Refer to multi_stream_topology for defining a topology by subclassing Topology
# pylint: disable=superfluous-parens
if __name__ == '__main__':
  if len(sys.argv) != 2:
    print("Topology's name is not specified")
    sys.exit(1)

  builder = TopologyBuilder(name=sys.argv[1])

  word_spout = builder.add_spout("word_spout", WordSpout, par=2)
  count_bolt = builder.add_bolt("count_bolt", CountBolt, par=2,
                                inputs={word_spout: Grouping.fields('word')},
                                config={constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS: 10})

  topology_config = {constants.TOPOLOGY_RELIABILITY_MODE:
                         constants.TopologyReliabilityMode.ATLEAST_ONCE}
  builder.set_config(topology_config)

  builder.build_and_submit()
github apache / incubator-heron / examples / src / python / stateful_word_count_topology.py View on Github external
from examples.src.python.spout import StatefulWordSpout
from examples.src.python.bolt import StatefulCountBolt

# Topology is defined using a topology builder
# Refer to multi_stream_topology for defining a topology by subclassing Topology
# pylint: disable=superfluous-parens
if __name__ == '__main__':
  if len(sys.argv) != 2:
    print("Topology's name is not specified")
    sys.exit(1)

  builder = TopologyBuilder(name=sys.argv[1])

  word_spout = builder.add_spout("word_spout", StatefulWordSpout, par=2)
  count_bolt = builder.add_bolt("count_bolt", StatefulCountBolt, par=2,
                                inputs={word_spout: Grouping.fields('word')},
                                config={constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS: 10})

  topology_config = {constants.TOPOLOGY_RELIABILITY_MODE:
                         constants.TopologyReliabilityMode.EFFECTIVELY_ONCE,
                     constants.TOPOLOGY_STATEFUL_CHECKPOINT_INTERVAL_SECONDS: 30}
  builder.set_config(topology_config)

  builder.build_and_submit()
github apache / incubator-heron / examples / src / python / half_acking_topology.py View on Github external
from examples.src.python.spout import WordSpout
from examples.src.python.bolt import HalfAckBolt

# Topology is defined using a topology builder
# Refer to multi_stream_topology for defining a topology by subclassing Topology
# pylint: disable=superfluous-parens
if __name__ == '__main__':
  if len(sys.argv) != 2:
    print("Topology's name is not specified")
    sys.exit(1)

  builder = TopologyBuilder(name=sys.argv[1])

  word_spout = builder.add_spout("word_spout", WordSpout, par=2)
  half_ack_bolt = builder.add_bolt("half_ack_bolt", HalfAckBolt, par=2,
                                   inputs={word_spout: Grouping.fields('word')},
                                   config={constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS: 10})

  topology_config = {constants.TOPOLOGY_RELIABILITY_MODE:
                     constants.TopologyReliabilityMode.ATLEAST_ONCE,
                     constants.TOPOLOGY_MAX_SPOUT_PENDING: 100000000,
                     constants.TOPOLOGY_MESSAGE_TIMEOUT_SECS: 300}

  builder.set_config(topology_config)

  builder.build_and_submit()
github apache / incubator-heron / examples / src / python / window_size_topology.py View on Github external
from heron.examples.src.python.spout import WordSpout
from examples.src.python.bolt import WindowSizeBolt

# Topology is defined using a topology builder
# Refer to multi_stream_topology for defining a topology by subclassing Topology
# pylint: disable=superfluous-parens
if __name__ == '__main__':
  if len(sys.argv) != 2:
    print("Topology's name is not specified")
    sys.exit(1)

  builder = TopologyBuilder(name=sys.argv[1])

  word_spout = builder.add_spout("word_spout", WordSpout, par=2)
  count_bolt = builder.add_bolt("count_bolt", WindowSizeBolt, par=2,
                                inputs={word_spout: Grouping.fields('word')},
                                config={SlidingWindowBolt.WINDOW_DURATION_SECS: 10,
                                        SlidingWindowBolt.WINDOW_SLIDEINTERVAL_SECS: 2})

  topology_config = {constants.TOPOLOGY_RELIABILITY_MODE:
                         constants.TopologyReliabilityMode.ATLEAST_ONCE}
  builder.set_config(topology_config)

  builder.build_and_submit()
github apache / incubator-heron / examples / src / python / multi_stream_topology.py View on Github external
#  specific language governing permissions and limitations
#  under the License.

'''module for example topology: CustomGroupingTopology'''

import heronpy.api.api_constants as constants
from heronpy.api.topology import Topology
from heronpy.api.stream import Grouping

from examples.src.python.spout import MultiStreamSpout
from examples.src.python.bolt import CountBolt, StreamAggregateBolt

class MultiStream(Topology):
  spout = MultiStreamSpout.spec(par=2)
  count_bolt = CountBolt.spec(par=2,
                              inputs={spout: Grouping.fields('word')},
                              config={constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS: 10})
  stream_aggregator = StreamAggregateBolt.spec(par=1,
                                               inputs={spout: Grouping.ALL,
                                                       spout['error']: Grouping.ALL},
                                               config={constants.TOPOLOGY_TICK_TUPLE_FREQ_SECS: 15})