Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def testDefaultConfigFileParser_All(self):
p = configargparse.DefaultConfigFileParser()
# test the all syntax case
config_lines = [
"# comment1 ",
"[ some section ]",
"----",
"---------",
"_a: 3",
"; comment2 ",
"_b = c",
"_list_arg1 = [a, b, c]",
"_str_arg = true",
"_list_arg2 = [1, 2, 3]",
]
# test parse
args = parser.parse_args()
# Display all of values - and where they are coming from
print(parser.format_values())
WINDOW_SIZE = args.window_size
EMBEDDING_SIZE = args.embedding_size
DENSE_SIZE = args.dense_size
if args.create_config:
options = {}
for attr, value in args.__dict__.items():
if attr != "config" and attr != "create_config" and value is not None:
options[attr] = value
file_name = args.create_config
content = configargparse.DefaultConfigFileParser().serialize(options)
Path(file_name).write_text(content)
print("configuration saved to file: %s" % file_name)
sys.exit(0)
import tensorflow as tf
import tensorflow.keras as K
import tensorflow_datasets as tfds
# Implement simple SpaceTokenizer - built-in tokenizer in tf filter-out
# non alphanumeric tokens
# see https://www.tensorflow.org/datasets/api_docs/python/tfds/features/text/TokenTextEncoder
class SpaceTokenizer(object):
def tokenize(self, s):
toks = []
toks.extend(tf.compat.as_text(s).split(' '))
toks = [t for t in toks if t]
from collections import Counter
from pathlib import Path
import sys
import random
import math
from tqdm import tqdm
import configargparse
parser = configargparse.ArgParser(config_file_parser_class=configargparse.DefaultConfigFileParser)
parser.add('-c', '--config', is_config_file=True, help='config file path')
parser.add('-C', '--create_config', help='create config file')
parser.add("--train", help="training data", required=True)
parser.add("--test", help="testing data", required=True)
parser.add("--include_unk", "-u", action="store_true")
parser.add("--test_size", type=int, default=1000)
parser.add("--buffer_size", type=int, default=500000)
parser.add("--batch_size", type=int, default=512)
parser.add("--test_batch_size", type=int, default=32)
parser.add("--steps_per_epoch", type=int, default=4096)
parser.add("--vocab", "-v", type=int, default=20000)
parser.add("--vocab_file", help="vocabulary file", default=None)
parser.add("--verbose", type=int, default=1)
parser.add("--model", default="./lmmodel")
parser.add("--sgd_learning_rate", default=0.01, type=float)
parser.add("--sgd_decay", default=1e-6, type=float)
environment variants and defaults, and then to exit.
(eg. ["-w", "--write-out-config-file"]). Default: []
write_out_config_file_arg_help_message: The help message to use for
the args in args_for_writing_out_config_file.
"""
# This is the only way to make positional args (tested in the argparse
# main test suite) and keyword arguments work across both Python 2 and
# 3. This could be refactored to not need extra local variables.
add_config_file_help = kwargs.pop('add_config_file_help', True)
add_env_var_help = kwargs.pop('add_env_var_help', True)
auto_env_var_prefix = kwargs.pop('auto_env_var_prefix', None)
default_config_files = kwargs.pop('default_config_files', [])
ignore_unknown_config_file_keys = kwargs.pop(
'ignore_unknown_config_file_keys', False)
config_file_parser_class = kwargs.pop('config_file_parser_class',
DefaultConfigFileParser)
args_for_setting_config_path = kwargs.pop(
'args_for_setting_config_path', [])
config_arg_is_required = kwargs.pop('config_arg_is_required', False)
config_arg_help_message = kwargs.pop('config_arg_help_message',
"config file path")
args_for_writing_out_config_file = kwargs.pop(
'args_for_writing_out_config_file', [])
write_out_config_file_arg_help_message = kwargs.pop(
'write_out_config_file_arg_help_message', "takes the current "
"command line args and writes them out to a config file at the "
"given path, then exits")
self._config_file_open_func = kwargs.pop('config_file_open_func', open)
self._add_config_file_help = add_config_file_help
self._add_env_var_help = add_env_var_help
from configargparse import DefaultConfigFileParser
class CustomConfigFileParser(DefaultConfigFileParser):
def get_syntax_description(self):
"""
Overwrite string to avoid printing goo.gl URL
"""
msg = ("Config file syntax allows: key=value, flag=true, stuff=[a,b,c] "
"(for details, see syntax at https://pypi.org/project/ConfigArgParse/).")
return msg
def extract_item_from_ct_conf(
key,
user_config_dir=None,
system_config_dir=None,
exedir=None,
default=None,
verbose=0,
):
""" Extract the value for the given key from the ct.conf files.
Return the given default if no key was identified
"""
fileparser = CfgFileParser()
for cfgpath in reversed(
defaultconfigs(
user_config_dir=user_config_dir,
system_config_dir=system_config_dir,
exedir=exedir,
)
):
with open(cfgpath) as cfg:
items = fileparser.parse(cfg)
try:
value = items[key]
if verbose >= 2:
print(" ".join([cfgpath, "contains", key, "=", value]))
return value
except KeyError:
continue