How to use the heronpy.api.state.stateful_component.StatefulComponent function in heronpy

To help you get started, we’ve selected a few heronpy examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github apache / incubator-heron / heronpy / streamlet / impl / unionbolt.py View on Github external
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

"""module for map bolt: UnionBolt"""
from heronpy.api.bolt.bolt import Bolt
from heronpy.api.state.stateful_component import StatefulComponent
from heronpy.api.component.component_spec import GlobalStreamId
from heronpy.api.stream import Grouping

from heronpy.streamlet.streamlet import Streamlet
from heronpy.streamlet.impl.streamletboltbase import StreamletBoltBase

# pylint: disable=unused-argument
class UnionBolt(Bolt, StatefulComponent, StreamletBoltBase):
  """UnionBolt"""
  def init_state(self, stateful_state):
    # unionBolt does not have any state
    pass

  def pre_save(self, checkpoint_id):
    # unionBolt does not have any state
    pass

  def initialize(self, config, context):
    self.logger.debug("UnionBolt's Component-specific config: \n%s" % str(config))
    self.processed = 0
    self.emitted = 0

  def process(self, tup):
    self.emit([tup.values[0]], stream='output')
github apache / incubator-heron / examples / src / python / bolt / stateful_count_bolt.py View on Github external
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

"""module for example bolt: CountBolt"""
from collections import Counter
import heronpy.api.global_metrics as global_metrics
from heronpy.api.bolt.bolt import Bolt
from heronpy.api.state.stateful_component import StatefulComponent

# pylint: disable=unused-argument
class StatefulCountBolt(Bolt, StatefulComponent):
  """CountBolt"""
  # output field declarer
  #outputs = ['word', 'count']

  # pylint: disable=attribute-defined-outside-init
  def init_state(self, stateful_state):
    self.recovered_state = stateful_state
    self.logger.info("Checkpoint Snapshot recovered : %s" % str(self.recovered_state))

  def pre_save(self, checkpoint_id):
    for (k, v) in self.counter.items():
      self.recovered_state.put(k, v)
    self.logger.info("Checkpoint Snapshot %s : %s" % (checkpoint_id, str(self.recovered_state)))

  def initialize(self, config, context):
    self.logger.info("In prepare() of CountBolt")
github apache / incubator-heron / heronpy / streamlet / impl / generatorspout.py View on Github external
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

"""module for generator spout: GeneratorSpout"""
from heronpy.api.state.stateful_component import StatefulComponent
from heronpy.api.spout.spout import Spout

from heronpy.streamlet.impl.contextimpl import ContextImpl
from heronpy.streamlet.streamlet import Streamlet
from heronpy.streamlet.impl.streamletspoutbase import StreamletSpoutBase
from heronpy.streamlet.generator import Generator

# pylint: disable=unused-argument
class GeneratorSpout(Spout, StatefulComponent, StreamletSpoutBase):
  """GeneratorSpout"""
  GENERATOR = 'generator'

  #pylint: disable=attribute-defined-outside-init
  def init_state(self, stateful_state):
    self._state = stateful_state

  def pre_save(self, checkpoint_id):
    pass

  def initialize(self, config, context):
    self.logger.debug("GeneratorSpout's Component-specific config: \n%s" % str(config))
    self.emitted = 0
    if GeneratorSpout.GENERATOR in config:
      self._generator = config[GeneratorSpout.GENERATOR]
    else:
github apache / incubator-heron / examples / src / python / spout / stateful_word_spout.py View on Github external
#
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

"""module for example spout: WordSpout"""

from itertools import cycle
from collections import Counter
from heronpy.api.spout.spout import Spout
from heronpy.api.state.stateful_component import StatefulComponent

class StatefulWordSpout(Spout, StatefulComponent):
  """StatefulWordSpout: emits a set of words repeatedly"""
  # output field declarer
  outputs = ['word']

  # pylint: disable=attribute-defined-outside-init
  def init_state(self, stateful_state):
    self.recovered_state = stateful_state
    self.logger.info("Checkpoint Snapshot recovered : %s" % str(self.recovered_state))

  def pre_save(self, checkpoint_id):
    # Purely for debugging purposes
    for (k, v) in self.counter.items():
      self.recovered_state.put(k, v)
    self.logger.info("Checkpoint Snapshot %s : %s" % (checkpoint_id, str(self.recovered_state)))

  # pylint: disable=unused-argument
github apache / incubator-heron / heronpy / streamlet / impl / supplierspout.py View on Github external
#  Unless required by applicable law or agreed to in writing,
#  software distributed under the License is distributed on an
#  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
#  KIND, either express or implied.  See the License for the
#  specific language governing permissions and limitations
#  under the License.

"""module for supplier spout: SupplierSpout"""
from heronpy.api.state.stateful_component import StatefulComponent
from heronpy.api.spout.spout import Spout

from heronpy.streamlet.streamlet import Streamlet
from heronpy.streamlet.impl.streamletspoutbase import StreamletSpoutBase

# pylint: disable=unused-argument
class SupplierSpout(Spout, StatefulComponent, StreamletSpoutBase):
  """SupplierSpout"""
  FUNCTION = 'function'

  def init_state(self, stateful_state):
    # Supplier does not have any state
    pass

  def pre_save(self, checkpoint_id):
    # Supplier does not have any state
    pass

  def initialize(self, config, context):
    self.logger.debug("SupplierSpout's Component-specific config: \n%s" % str(config))
    self.emitted = 0
    if SupplierSpout.FUNCTION in config:
      self._supplier_function = config[SupplierSpout.FUNCTION]
github apache / incubator-heron / heron / instance / src / python / basics / bolt_instance.py View on Github external
def start_component(self, stateful_state):
    context = self.pplan_helper.context
    if not self._initialized_metrics_and_tasks:
      self.bolt_metrics.register_metrics(context)
    if self.is_stateful and isinstance(self.bolt_impl, StatefulComponent):
      self.bolt_impl.init_state(stateful_state)
    self.bolt_impl.initialize(config=context.get_cluster_config(), context=context)
    # prepare tick tuple
    if not self._initialized_metrics_and_tasks:
      self._prepare_tick_tup_timer()
    self._initialized_metrics_and_tasks = True
    self.topology_state = topology_pb2.TopologyState.Value("RUNNING")