How to use the kazoo.retry.KazooRetry function in kazoo

To help you get started, we’ve selected a few kazoo examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github apache / pulsar / docker / pulsar / scripts / watch-znode.py View on Github external
sys.stderr.write("Zookeeper must be specified\n")
    usage()
    sys.exit(3)

if not znode:
    sys.stderr.write("Znode must be specified\n")
    usage()
    sys.exit(4)

if (not watch and not create and not exists):
    sys.stderr.write("Exactly one of watch (-w), create (-c) or exists (-e) must be specified\n")
    usage()
    sys.exit(5)

while True:
    zk = KazooClient(hosts=zookeeper, timeout=1, connection_retry=KazooRetry(max_tries=-1))
    try:
        zk.start()
        if create:
            try:
                zk.create(znode)
            except NodeExistsError:
                pass
            sys.exit(0)
        elif watch:
            while not zk.exists(znode):
                print("Waiting for %s" % znode)
                time.sleep(1)
            sys.exit(0)
        elif exists:
            if zk.exists(znode):
                sys.exit(0)
github python-zk / kazoo / kazoo / client.py View on Github external
old_retry_keys[key] = kwargs.pop(key)
                    warnings.warn(
                        'Passing retry configuration param %s to the '
                        'client directly is deprecated, please pass a '
                        'configured retry object (using param %s)' % (
                            key, _RETRY_COMPAT_MAPPING[key]),
                        DeprecationWarning, stacklevel=2)
                except KeyError:
                    pass

            retry_keys = {}
            for oldname, value in old_retry_keys.items():
                retry_keys[_RETRY_COMPAT_MAPPING[oldname]] = value

            if self._conn_retry is None:
                self._conn_retry = KazooRetry(
                    sleep_func=self.handler.sleep_func,
                    **retry_keys)
            if self.retry is None:
                self.retry = KazooRetry(
                    sleep_func=self.handler.sleep_func,
                    **retry_keys)

        # Managing legacy SASL options
        for scheme, auth in self.auth_data:
            if scheme != 'sasl':
                continue
            if sasl_options:
                raise ConfigurationError(
                    'Multiple SASL configurations provided'
                )
            warnings.warn(
github dcos / dcos / packages / bouncer / extra / iam-migrate-users-from-zk.py View on Github external
def create_zk_client(zk_hosts: str) -> KazooClient:
    conn_retry_policy = KazooRetry(max_tries=-1, delay=0.1, max_delay=0.1)
    cmd_retry_policy = KazooRetry(
        max_tries=3, delay=0.3, backoff=1, max_delay=1, ignore_expire=False)
    return KazooClient(
        hosts=zk_hosts,
        connection_retry=conn_retry_policy,
        command_retry=cmd_retry_policy,
    )
github Morgan-Stanley / treadmill / lib / python / treadmill / zkwatchers.py View on Github external
def __init__(self, client, path, func=None):
        """Create a data watcher for an existing path"""
        self._client = client
        self._path = path
        self._func = func
        self._stopped = False
        self._run_lock = client.handler.lock_object()
        self._version = None
        self._retry = kazoo.retry.KazooRetry(
            max_tries=None, sleep_func=client.handler.sleep_func
        )
        self._used = False

        # Register our session listener if we're going to resume
        # across session losses
        if func is not None:
            self._used = True
            self._client.add_listener(self._session_watcher)
            self._get_data()
github AppScale / appscale / AppDB / appscale / datastore / scripts / transaction_groomer.py View on Github external
def main():
  """ Starts the groomer. """
  logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)

  parser = argparse.ArgumentParser()
  parser.add_argument('-v', '--verbose', action='store_true',
                      help='Output debug-level logging')
  args = parser.parse_args()

  if args.verbose:
    logger.setLevel(logging.DEBUG)

  zk_hosts = appscale_info.get_zk_node_ips()
  zk_client = KazooClient(hosts=','.join(zk_hosts),
                          connection_retry=ZK_PERSISTENT_RECONNECTS,
                          command_retry=KazooRetry(max_tries=-1))
  zk_client.start()

  db_access = DatastoreProxy()

  thread_pool = ThreadPoolExecutor(4)

  TransactionGroomer(zk_client, db_access, thread_pool)
  logger.info('Starting transaction groomer')

  IOLoop.current().start()
github dcos / dcos / packages / cockroach / extra / register.py View on Github external
up to 3 seconds. If a command fails, that command is retried every 300ms for 3 attempts before failing.

    These values are chosen to suit a human-interactive time.

    Args:
        zk_user:
            The username to use when connecting to ZooKeeper or `None` if no authentication is necessary.
        zk_secret:
            The secret to use when connecting to ZooKeeper or `None` if no authentication is necessary.

    Returns:
        A ZooKeeper client connection in the form of a `kazoo.client.KazooClient`.
    """
    # Try to reconnect indefinitely, with time between updates going
    # exponentially to ~3s. Then every retry occurs every ~3 seconds.
    conn_retry_policy = KazooRetry(
        max_tries=-1,
        delay=0.3,
        backoff=1.3,
        max_jitter=1,
        max_delay=3,
        ignore_expire=True,
    )
    # Retry commands every 0.3 seconds, for a total of <1s (usually 0.9)
    cmd_retry_policy = KazooRetry(
        max_tries=3,
        delay=0.3,
        backoff=1,
        max_jitter=0.1,
        max_delay=1,
        ignore_expire=False,
        )
github Yelp / yelp_kafka / yelp_kafka / partitioner.py View on Github external
def start(self):
        """Create a new group and wait until the partitions have been
        acquired. This function should never be called twice.

        :raises: PartitionerError upon partitioner failures

        .. note: This is a blocking operation.
        """
        self.kazoo_retry = KazooRetry(**KAZOO_RETRY_DEFAULTS)
        self.kazoo_client = KazooClient(
            self.config.zookeeper,
            connection_retry=self.kazoo_retry,
        )
        self.kafka_client = KafkaClient(self.config.broker_list)

        self.log.debug("Starting a new group for topics %s", self.topics)
        self.released_flag = True
        self._refresh()
github AppScale / appscale / common / appscale / common / constants.py View on Github external
GO = "go"

# PHP programs.
PHP = "php"

# Location where applications are stored.
APPS_PATH = "/var/apps/"

# Locations of ZooKeeper.
ZK_LOCATIONS_FILE = "/etc/appscale/zookeeper_locations"

# Default location for connecting to ZooKeeper.
ZK_DEFAULT_CONNECTION_STR = "localhost:2181"

# A ZooKeeper reconnect policy that never stops retrying to connect.
ZK_PERSISTENT_RECONNECTS = KazooRetry(max_tries=-1, max_delay=30)

# Default location for the datastore master.
MASTERS_FILE_LOC = "/etc/appscale/masters"

# Default location for the datastore slaves.
SLAVES_FILE_LOC = "/etc/appscale/slaves"

# Application ID for AppScale Dashboard.
DASHBOARD_APP_ID = "appscaledashboard"

# Reserved application identifiers which are only internal for AppScale.
RESERVED_APP_IDS = [DASHBOARD_APP_ID]

# The seconds to wait for the schema to settle after changing it.
SCHEMA_CHANGE_TIMEOUT = 120
github yandex / mastermind / src / cocaine-app / sync / kazoo_impl / lock.py View on Github external
self.wake_event = client.handler.event_object()

        # props to Netflix Curator for this trick. It is possible for our
        # create request to succeed on the server, but for a failure to
        # prevent us from getting back the full path name. We prefix our
        # lock name with a uuid and can check for its presence on retry.
        self.prefix = uuid.uuid4().hex + self._NODE_NAME
        self.create_path = self.path + "/" + self.prefix

        self.create_tried = False
        self.is_acquired = False
        self.assured_path = False
        self.cancelled = False
        self.node = None
        self._retry = KazooRetry(max_tries=None)
github Yelp / data_pipeline / data_pipeline / zookeeper.py View on Github external
def __init__(self):
        retry_policy = KazooRetry(max_tries=self.max_tries)
        self.zk_client = self.get_kazoo_client(command_retry=retry_policy)
        self.zk_client.start()
        self.register_signal_handlers()