Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
# Consider the following scenario:
#
# * Clients 0-3 bulk index into indexA
# * Clients 4-7 bulk index into indexB
#
# Now we need to ensure that we start partitioning parameters correctly in both cases. And that means we need to start
# from (client) index 0 in both cases instead of 0 for indexA and 4 for indexB.
schedule = schedule_for(self.track, task_allocation.task, task_allocation.client_index_in_task)
executor = Executor(task, schedule, self.es, self.sampler, self.cancel, self.complete, self.abort_on_error)
final_executor = Profiler(executor, self.client_id, task.operation) if profiling_enabled else executor
self.executor_future = self.pool.submit(final_executor)
self.wakeupAfter(datetime.timedelta(seconds=self.wakeup_interval))
else:
raise exceptions.RallyAssertionError("Unknown task type [%s]" % type(task_allocation))
}
# The ES client will sometimes return string like "N/A" or "TIMEOUT" for connection errors.
if isinstance(e.status_code, int):
request_meta_data["http-status"] = e.status_code
except KeyError as e:
logger.exception("Cannot execute runner [%s]; most likely due to missing parameters." % str(runner))
msg = "Cannot execute [%s]. Provided parameters are: %s. Error: [%s]." % (str(runner), list(params.keys()), str(e))
raise exceptions.SystemSetupError(msg)
if abort_on_error and not request_meta_data["success"]:
msg = "Request returned an error:\n\n"
msg += "Error type: %s\n" % request_meta_data.get("error-type", "Unknown")
description = request_meta_data.get("error-description")
if description:
msg += "Description: %s\n" % description
raise exceptions.RallyAssertionError(msg)
return total_ops, total_ops_unit, request_meta_data
for document_set in corpus.documents:
# TODO #341: Should we allow this for snapshots too?
if document_set.is_bulk:
document_set.number_of_documents = 1000
if document_set.has_compressed_corpus():
path, ext = io.splitext(document_set.document_archive)
path_2, ext_2 = io.splitext(path)
document_set.document_archive = "%s-1k%s%s" % (path_2, ext_2, ext)
document_set.document_file = "%s-1k%s" % (path_2, ext_2)
elif document_set.has_uncompressed_corpus():
path, ext = io.splitext(document_set.document_file)
document_set.document_file = "%s-1k%s" % (path, ext)
else:
raise exceptions.RallyAssertionError("Document corpus [%s] has neither compressed nor uncompressed corpus." %
corpus.name)
# we don't want to check sizes
document_set.compressed_size_in_bytes = None
document_set.uncompressed_size_in_bytes = None
for challenge in t.challenges:
for task in challenge.schedule:
# we need iterate over leaf tasks and await iterating over possible intermediate 'parallel' elements
for leaf_task in task:
# iteration-based schedules are divided among all clients and we should provide at least one iteration for each client.
if leaf_task.warmup_iterations is not None and leaf_task.warmup_iterations > leaf_task.clients:
count = leaf_task.clients
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Resetting warmup iterations to %d for [%s]", count, str(leaf_task))
leaf_task.warmup_iterations = count
def receiveMsg_WakeupMessage(self, msg, sender):
if msg.payload == MechanicActor.WAKEUP_RESET_RELATIVE_TIME:
self.reset_relative_time()
elif msg.payload == MechanicActor.WAKEUP_FLUSH_METRICS:
self.logger.debug("Flushing cluster-wide system metrics store.")
self.metrics_store.flush(refresh=False)
self.wakeupAfter(METRIC_FLUSH_INTERVAL_SECONDS, payload=MechanicActor.WAKEUP_FLUSH_METRICS)
else:
raise exceptions.RallyAssertionError("Unknown wakeup reason [{}]".format(msg.payload))
doc_id = self.conflicting_ids[idx]
action = self.on_conflict
else:
if self.id_up_to >= len(self.conflicting_ids):
raise StopIteration()
doc_id = self.conflicting_ids[self.id_up_to]
self.id_up_to += 1
action = "index"
if action == "index":
return "index", self.meta_data_index_with_id % doc_id
elif action == "update":
return "update", self.meta_data_update_with_id % doc_id
else:
raise exceptions.RallyAssertionError("Unknown action [{}]".format(action))
else:
return "index", self.meta_data_index_no_id
def format_title(environment, track_name, flavor=None, es_license=None, suffix=None):
if flavor:
title = [environment, flavor, str(track_name)]
elif es_license:
title = [environment, es_license, str(track_name)]
elif flavor and es_license:
raise exceptions.RallyAssertionError(
"Specify either flavor [{}] or license [{}] but not both".format(flavor, es_license))
else:
title = [environment, str(track_name)]
if suffix:
title.append(suffix)
return "-".join(title)
logger.info("GOT: %s" % str(result))
logger.info("ALLOC:\n%s" % es.cat.allocation(v=True))
logger.info("RECOVERY:\n%s" % es.cat.recovery(v=True))
logger.info("SHARDS:\n%s" % es.cat.shards(v=True))
if status(reached_cluster_status) >= status(expected_cluster_status) and relocating_shards == 0:
return reached_cluster_status, relocating_shards
else:
sleep(0.5)
if status(reached_cluster_status) < status(expected_cluster_status):
msg = "Cluster did not reach status [%s]. Last reached status: [%s]" % (expected_cluster_status, reached_cluster_status)
else:
msg = "Cluster reached status [%s] which is equal or better than the expected status [%s] but there were [%d] relocating shards " \
"and we require zero relocating shards (Use the /_cat/shards API to check which shards are relocating.)" % \
(reached_cluster_status, expected_cluster_status, relocating_shards)
logger.error(msg)
raise exceptions.RallyAssertionError(msg)
self.has_expected_size(doc_path, document_set.uncompressed_size_in_bytes):
break
if document_set.has_compressed_corpus() and \
self.is_locally_available(archive_path) and \
self.has_expected_size(archive_path, document_set.compressed_size_in_bytes):
self.decompress(archive_path, doc_path, document_set.uncompressed_size_in_bytes)
else:
if document_set.has_compressed_corpus():
target_path = archive_path
expected_size = document_set.compressed_size_in_bytes
elif document_set.has_uncompressed_corpus():
target_path = doc_path
expected_size = document_set.uncompressed_size_in_bytes
else:
# this should not happen in practice as the JSON schema should take care of this
raise exceptions.RallyAssertionError("Track %s specifies documents but no corpus" % self.track_name)
# provide a specific error message in case there is no download URL
if self.is_locally_available(target_path):
# convert expected_size eagerly to a string as it might be None (but in that case we'll never see that error message)
msg = "%s is present but does not have the expected size of %s bytes" % (target_path, str(expected_size))
else:
msg = "%s is missing" % target_path
self.download(document_set.base_url, target_path, expected_size, msg)
self.create_file_offset_table(doc_path, document_set.number_of_lines)
if self.is_current_status_expected(expected_status):
self.received_responses.append(msg)
response_count = len(self.received_responses)
expected_count = len(self.children)
self.logger.debug("[%d] of [%d] child actors have responded for transition from [%s] to [%s].",
response_count, expected_count, self.status, new_status)
if response_count == expected_count:
self.logger.debug("All [%d] child actors have responded. Transitioning now from [%s] to [%s].",
expected_count, self.status, new_status)
# all nodes have responded, change status
self.status = new_status
self.received_responses = []
transition()
elif response_count > expected_count:
raise exceptions.RallyAssertionError(
"Received [%d] responses but only [%d] were expected to transition from [%s] to [%s]. The responses are: %s" %
(response_count, expected_count, self.status, new_status, self.received_responses))
else:
raise exceptions.RallyAssertionError("Received [%s] from [%s] but we are in status [%s] instead of [%s]." %
(type(msg), sender, self.status, expected_status))
def __call__(self, es, params):
source_index = mandatory(params, "source-index", self)
target_index = mandatory(params, "target-index", self)
# we need to inject additional settings so we better copy the body
target_body = deepcopy(mandatory(params, "target-body", self))
shrink_node = params.get("shrink-node")
# Choose a random data node if none is specified
if not shrink_node:
node_names = []
# choose a random data node
for node in es.nodes.info()["nodes"].values():
if "data" in node["roles"]:
node_names.append(node["name"])
if not node_names:
raise exceptions.RallyAssertionError("Could not choose a suitable shrink-node automatically. Please specify it explicitly.")
shrink_node = random.choice(node_names)
self.logger.info("Using [%s] as shrink node.", shrink_node)
self.logger.info("Preparing [%s] for shrinking.", source_index)
# prepare index for shrinking
es.indices.put_settings(index=source_index,
body={
"settings": {
"index.routing.allocation.require._name": shrink_node,
"index.blocks.write": "true"
}
},
preserve_existing=True)
self.logger.info("Waiting for relocation to finish for index [%s]...", source_index)
self._wait_for(es, source_index, "shard relocation for index [{}]".format(source_index))
self.logger.info("Shrinking [%s] to [%s].", source_index, target_index)