How to use the esrally.utils.io.normalize_path function in esrally

To help you get started, we’ve selected a few esrally examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github elastic / rally / esrally / config.py View on Github external
self.o("")
            self.o("  %s" % console.format.link(doc_link("configuration.html")))
            self.o("")
        else:
            self.o("Running simple configuration. Run the advanced configuration with:")
            self.o("")
            self.o("  %s configure --advanced-config" % PROGRAM_NAME)
            self.o("")

        if config_file.present:
            self.o("\nWARNING: Will overwrite existing config file at [%s]\n" % config_file.location)
            self.logger.debug("Detected an existing configuration file at [%s]", config_file.location)
        else:
            self.logger.debug("Did not detect a configuration file at [%s]. Running initial configuration routine.", config_file.location)

        root_dir = io.normalize_path(os.path.abspath(os.path.join(config_file.config_dir, "benchmarks")))
        if advanced_config:
            root_dir = io.normalize_path(self._ask_property("Enter the benchmark root directory", default_value=root_dir))
        else:
            self.o("* Setting up benchmark root directory in %s" % root_dir)

        # We try to autodetect an existing ES source directory
        guess = self._guess_es_src_dir()
        if guess:
            source_dir = guess
            self.logger.debug("Autodetected Elasticsearch project directory at [%s].", source_dir)
        else:
            default_src_dir = os.path.join(root_dir, "src", "elasticsearch")
            self.logger.debug("Could not autodetect Elasticsearch project directory. Providing [%s] as default.", default_src_dir)
            source_dir = default_src_dir

        if advanced_config:
github elastic / rally / esrally / reporter.py View on Github external
def write_single_report(report_file, report_format, cwd, headers, data_plain, data_rich):
    if report_format == "markdown":
        formatter = format_as_markdown
    elif report_format == "csv":
        formatter = format_as_csv
    else:
        raise exceptions.SystemSetupError("Unknown report format '%s'" % report_format)

    print_internal(formatter(headers, data_rich))
    if len(report_file) > 0:
        normalized_report_file = rio.normalize_path(report_file, cwd)
        # ensure that the parent folder already exists when we try to write the file...
        rio.ensure_dir(rio.dirname(normalized_report_file))
        with open(normalized_report_file, mode="a+", encoding="utf-8") as f:
            f.writelines(formatter(headers, data_plain))
github elastic / rally / esrally / config.py View on Github external
config["runtime"] = {}
                config["runtime"]["java9.home"] = java_9_home
            else:
                logger.debug("Could not autodetect a JDK 9 installation. Checking [java.home] already points to a JDK 9.")
                detected = False
                if "runtime" in config:
                    java_home = config["runtime"]["java.home"]
                    if jvm.major_version(java_home) == 9 and not jvm.is_early_access_release(java_home):
                        config["runtime"]["java9.home"] = java_home
                        detected = True

                if not detected:
                    logger.debug("Could not autodetect a JDK 9 installation. Asking user.")
                    raw_java_9_home = prompter.ask_property("Enter the JDK 9 root directory", check_path_exists=True, mandatory=False)
                    if raw_java_9_home and jvm.major_version(raw_java_9_home) == 9 and not jvm.is_early_access_release(raw_java_9_home):
                        java_9_home = io.normalize_path(raw_java_9_home) if raw_java_9_home else None
                        config["runtime"]["java9.home"] = java_9_home
                    else:
                        out("********************************************************************************")
                        out("You don't have a valid JDK 9 installation and cannot benchmark source builds.")
                        out("")
                        out("You can still benchmark binary distributions with e.g.:")
                        out("")
                        out("  %s --distribution-version=6.0.0" % PROGRAM_NAME)
                        out("********************************************************************************")
                        out("")

        current_version = 13
        config["meta"]["config.version"] = str(current_version)

    if current_version == 13 and target_version > current_version:
        # This version replaced java9.home with java10.home
github elastic / rally / esrally / utils / opts.py View on Github external
def to_dict(arg, default_parser=kv_to_map):
    if io.has_extension(arg, ".json"):
        with open(io.normalize_path(arg), mode="rt", encoding="utf-8") as f:
            return json.load(f)
    elif arg.startswith("{"):
        return json.loads(arg)
    else:
        return default_parser(csv_to_list(arg))
github elastic / rally / esrally / config.py View on Github external
config["runtime"] = {}
                config["runtime"]["java10.home"] = java_10_home
            else:
                logger.debug("Could not autodetect a JDK 10 installation. Checking [java.home] already points to a JDK 10.")
                detected = False
                if "runtime" in config:
                    java_home = config["runtime"]["java.home"]
                    if jvm.major_version(java_home) == 10 and not jvm.is_early_access_release(java_home):
                        config["runtime"]["java10.home"] = java_home
                        detected = True

                if not detected:
                    logger.debug("Could not autodetect a JDK 10 installation. Asking user.")
                    raw_java_10_home = prompter.ask_property("Enter the JDK 10 root directory", check_path_exists=True, mandatory=False)
                    if raw_java_10_home and jvm.major_version(raw_java_10_home) == 10 and not jvm.is_early_access_release(raw_java_10_home):
                        java_10_home = io.normalize_path(raw_java_10_home) if raw_java_10_home else None
                        config["runtime"]["java10.home"] = java_10_home
                    else:
                        out("********************************************************************************")
                        out("You don't have a valid JDK 10 installation and cannot benchmark source builds.")
                        out("")
                        out("You can still benchmark binary distributions with e.g.:")
                        out("")
                        out("  %s --distribution-version=6.0.0" % PROGRAM_NAME)
                        out("********************************************************************************")
                        out("")

        current_version = 14
        config["meta"]["config.version"] = str(current_version)

    if current_version == 14 and target_version > current_version:
        # Be agnostic about build tools. Let use specify build commands for plugins and elasticsearch
github elastic / rally / esrally / config.py View on Github external
config["runtime"] = {}
                config["runtime"]["java9.home"] = java_9_home
            else:
                logger.debug("Could not autodetect a JDK 9 installation. Checking [java.home] already points to a JDK 9.")
                detected = False
                if "runtime" in config:
                    java_home = config["runtime"]["java.home"]
                    if jvm.major_version(java_home) == 9 and not jvm.is_early_access_release(java_home):
                        config["runtime"]["java9.home"] = java_home
                        detected = True

                if not detected:
                    logger.debug("Could not autodetect a JDK 9 installation. Asking user.")
                    raw_java_9_home = prompter.ask_property("Enter the JDK 9 root directory", check_path_exists=True, mandatory=False)
                    if raw_java_9_home and jvm.major_version(raw_java_9_home) == 9 and not jvm.is_early_access_release(raw_java_9_home):
                        java_9_home = io.normalize_path(raw_java_9_home) if raw_java_9_home else None
                        config["runtime"]["java9.home"] = java_9_home
                    else:
                        out("********************************************************************************")
                        out("You don't have a valid JDK 9 installation and cannot benchmark source builds.")
                        out("")
                        out("You can still benchmark binary distributions with e.g.:")
                        out("")
                        out("  %s --distribution-version=6.0.0" % PROGRAM_NAME)
                        out("********************************************************************************")
                        out("")

        current_version = 13
        config["meta"]["config.version"] = str(current_version)

    if current_version == 13 and target_version > current_version:
        # This version replaced java9.home with java10.home
github elastic / rally / esrally / reporter.py View on Github external
def write_single_report(report_file, report_format, cwd, headers, data_plain, data_rich, write_header=True, show_also_in_console=True):
    if report_format == "markdown":
        formatter = format_as_markdown
    elif report_format == "csv":
        formatter = format_as_csv
    else:
        raise exceptions.SystemSetupError("Unknown report format '%s'" % report_format)

    if show_also_in_console:
        print_internal(formatter(headers, data_rich))
    if len(report_file) > 0:
        normalized_report_file = rio.normalize_path(report_file, cwd)
        logger.info("Writing report to [%s] (user specified: [%s]) in format [%s]" %
                    (normalized_report_file, report_file, report_format))
        # ensure that the parent folder already exists when we try to write the file...
        rio.ensure_dir(rio.dirname(normalized_report_file))
        with open(normalized_report_file, mode="a+", encoding="UTF-8") as f:
            f.writelines(formatter(headers, data_plain, write_header))
github elastic / rally / esrally / config.py View on Github external
env_name = "local"
            data_store_type = "in-memory"
            data_store_host, data_store_port, data_store_secure, data_store_user, data_store_password = "", "", "False", "", ""
            preserve_install = False

        config = configparser.ConfigParser()
        config["meta"] = {}
        config["meta"]["config.version"] = str(Config.CURRENT_CONFIG_VERSION)

        config["system"] = {}
        config["system"]["env.name"] = env_name

        config["node"] = {}
        config["node"]["root.dir"] = root_dir

        final_source_dir = io.normalize_path(os.path.abspath(os.path.join(source_dir, os.pardir)))
        config["node"]["src.root.dir"] = final_source_dir

        config["source"] = {}
        config["source"]["remote.repo.url"] = repo_url
        # the Elasticsearch directory is just the last path component (relative to the source root directory)
        config["source"]["elasticsearch.src.subdir"] = io.basename(source_dir)

        config["benchmarks"] = {}
        config["benchmarks"]["local.dataset.cache"] = os.path.join(root_dir, "data")

        config["reporting"] = {}
        config["reporting"]["datastore.type"] = data_store_type
        config["reporting"]["datastore.host"] = data_store_host
        config["reporting"]["datastore.port"] = data_store_port
        config["reporting"]["datastore.secure"] = data_store_secure
        config["reporting"]["datastore.user"] = data_store_user
github elastic / rally / esrally / log.py View on Github external
def install_default_log_config():
    """
    Ensures a log configuration file is present on this machine. The default
    log configuration is based on the template in resources/logging.json.

    It also ensures that the default log path has been created so log files
    can be successfully opened in that directory.
    """
    log_config = log_config_path()
    if not io.exists(log_config):
        io.ensure_dir(io.dirname(log_config))
        source_path = io.normalize_path(os.path.join(os.path.dirname(__file__), "resources", "logging.json"))
        with open(log_config, "w", encoding="UTF-8") as target:
            with open(source_path, "r", encoding="UTF-8") as src:
                # Ensure we have a trailing path separator as after LOG_PATH there will only be the file name
                log_path = os.path.join(paths.logs(), "")
                # the logging path might contain backslashes that we need to escape
                log_path = io.escape_path(log_path)
                contents = src.read().replace("${LOG_PATH}", log_path)
                target.write(contents)
    io.ensure_dir(paths.logs())