Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
self.assertEqual(len(downloads), 1)
# non-existing files result in None return value
self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None)
# install broken proxy handler for opening local files
# this should make urlopen use this broken proxy for downloading from a file:// URL
proxy_handler = std_urllib.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir})
std_urllib.install_opener(std_urllib.build_opener(proxy_handler))
# downloading over a broken proxy results in None return value (failed download)
# this tests whether proxies are taken into account by download_file
self.assertEqual(ft.download_file(fn, source_url, target_location), None, "download over broken proxy fails")
# modify existing download so we can verify re-download
ft.write_file(target_location, '')
# restore a working file handler, and retest download of local file
std_urllib.install_opener(std_urllib.build_opener(std_urllib.FileHandler()))
res = ft.download_file(fn, source_url, target_location)
self.assertEqual(res, target_location, "'download' of local file works after removing broken proxy")
# existing file was re-downloaded, so a backup should have been created of the existing file
downloads = glob.glob(target_location + '*')
self.assertEqual(len(downloads), 2)
backup = [d for d in downloads if os.path.basename(d) != fn][0]
self.assertEqual(ft.read_file(backup), '')
self.assertEqual(ft.compute_checksum(target_location), ft.compute_checksum(os.path.join(toy_source_dir, fn)))
# make sure specified timeout is parsed correctly (as a float, not a string)
opts = init_config(args=['--download-timeout=5.3'])
init_config(build_options={'download_timeout': opts.download_timeout})
def run_test(custom=None, extra_params=[], fmt=None):
"""Inner function to run actual test in current setting."""
fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
os.close(fd)
avail_args = [
'-a',
'--avail-easyconfig-params',
]
for avail_arg in avail_args:
# clear log
write_file(self.logfile, '')
args = [
'--unittest-file=%s' % self.logfile,
avail_arg,
]
if fmt is not None:
args.append(fmt)
if custom is not None:
args.extend(['-e', custom])
outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True)
logtxt = read_file(self.logfile)
# check whether all parameter types are listed
par_types = [BUILD, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT,
LICENSE, MANDATORY, MODULES, OTHER, TOOLCHAIN]
elif get_module_syntax() == 'Lua':
modules_header_txt = '\n'.join([
"-- test header",
'setenv("SITE_SPECIFIC_HEADER_ENV_VAR", "foo")',
])
modules_footer_txt = '\n'.join([
"-- test footer",
'setenv("SITE_SPECIFIC_FOOTER_ENV_VAR", "bar")',
])
else:
self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())
# dump header/footer text to file
handle, modules_footer = tempfile.mkstemp(prefix='modules-footer-')
os.close(handle)
write_file(modules_footer, modules_footer_txt)
handle, modules_header = tempfile.mkstemp(prefix='modules-header-')
os.close(handle)
write_file(modules_header, modules_header_txt)
# use toy-0.0.eb easyconfig file that comes with the tests
eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb')
# check log message with --skip for existing module
args = [
eb_file,
'--sourcepath=%s' % self.test_sourcepath,
'--buildpath=%s' % self.test_buildpath,
'--installpath=%s' % self.test_installpath,
'--debug',
'--force',
'--modules-header=%s' % modules_header,
"""Test specifying external (build) dependencies."""
topdir = os.path.dirname(os.path.abspath(__file__))
ectxt = read_file(os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb'))
toy_ec = os.path.join(self.test_prefix, 'toy-0.0-external-deps.eb')
# just specify some of the test modules we ship, doesn't matter where they come from
extraectxt = "\ndependencies += [('foobar/1.2.3', EXTERNAL_MODULE)]"
extraectxt += "\nbuilddependencies = [('somebuilddep/0.1', EXTERNAL_MODULE)]"
extraectxt += "\nversionsuffix = '-external-deps'"
write_file(toy_ec, ectxt + extraectxt)
# install dummy modules
modulepath = os.path.join(self.test_prefix, 'modules')
for mod in ['intel/2018a', 'GCC/6.4.0-2.28', 'foobar/1.2.3', 'somebuilddep/0.1']:
mkdir(os.path.join(modulepath, os.path.dirname(mod)), parents=True)
write_file(os.path.join(modulepath, mod), "#%Module")
self.reset_modulepath([modulepath, os.path.join(self.test_installpath, 'modules', 'all')])
self.test_toy_build(ec_file=toy_ec, versionsuffix='-external-deps', verbose=True, raise_error=True)
self.modtool.load(['toy/0.0-external-deps'])
# note build dependency is not loaded
mods = ['intel/2018a', 'GCC/6.4.0-2.28', 'foobar/1.2.3', 'toy/0.0-external-deps']
self.assertEqual([x['mod_name'] for x in self.modtool.list()], mods)
# check behaviour when a non-existing external (build) dependency is included
err_msg = "Missing modules for dependencies marked as external modules:"
extraectxt = "\nbuilddependencies = [('nosuchbuilddep/0.0.0', EXTERNAL_MODULE)]"
extraectxt += "\nversionsuffix = '-external-deps-broken1'"
write_file(toy_ec, ectxt + extraectxt)
self.assertErrorRegex(EasyBuildError, err_msg, self.test_toy_build, ec_file=toy_ec,
def test_check_conflicts_multi_deps(self):
"""Test check_conflicts when multi_deps is used."""
test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
test_ec = os.path.join(self.test_prefix, 'test.eb')
test_ec_txt = read_file(toy_ec)
# we need to use empty dummy toolchain version to ensure dependencies are picked up...
tc_regex = re.compile(r'^toolchain = .*', re.M)
test_ec_txt = tc_regex.sub("toolchain = {'name': 'dummy', 'version': ''}", test_ec_txt)
test_ec_txt += "\nmulti_deps = {'GCC': ['4.9.2', '7.3.0-2.30']}\n"
test_ec_txt += "dependencies = [('gzip', '1.4')]\n"
write_file(test_ec, test_ec_txt)
ecs, _ = parse_easyconfigs([(test_ec, False)])
init_config(build_options={'robot_path': [test_ecs]})
# use of multi_deps should not result in false positives in check_conflicts
self.assertFalse(check_conflicts(ecs, self.modtool))
mns_regex = re.compile(r'^\s*TestIncludedMNS', re.M)
# TestIncludedMNS module naming scheme is not available by default
args = [
'--avail-module-naming-schemes',
]
logtxt, _= run_cmd("cd %s; eb %s" % (self.test_prefix, ' '.join(args)), simple=False)
self.assertFalse(mns_regex.search(logtxt), "Unexpected pattern '%s' found in: %s" % (mns_regex.pattern, logtxt))
# include extra test MNS
mns_txt = '\n'.join([
'from easybuild.tools.module_naming_scheme import ModuleNamingScheme',
'class TestIncludedMNS(ModuleNamingScheme):',
' pass',
])
write_file(os.path.join(self.test_prefix, 'test_mns.py'), mns_txt)
# clear log
write_file(self.logfile, '')
args = [
'--avail-module-naming-schemes',
'--include-module-naming-schemes=%s/*.py' % self.test_prefix,
]
logtxt, _= run_cmd("cd %s; eb %s" % (self.test_prefix, ' '.join(args)), simple=False)
self.assertTrue(mns_regex.search(logtxt), "Pattern '%s' *not* found in: %s" % (mns_regex.pattern, logtxt))
def test_toy_extension_patches(self):
"""Test install toy that includes extensions with patches."""
test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
toy_ec_txt = read_file(toy_ec)
# create file that we'll copy via 'patches'
write_file(os.path.join(self.test_prefix, 'test.txt'), 'test123')
test_ec = os.path.join(self.test_prefix, 'test.eb')
test_ec_txt = '\n'.join([
toy_ec_txt,
'exts_list = [',
' ("bar", "0.0", {',
' "buildopts": " && ls -l test.txt",',
' "patches": [',
' "bar-0.0_fix-silly-typo-in-printf-statement.patch",', # normal patch
' ("bar-0.0_fix-very-silly-typo-in-printf-statement.patch", 0),', # patch with patch level
' ("test.txt", "."),', # file to copy to build dir (not a real patch file)
' ],',
' }),',
']',
])
write_file(test_ec, test_ec_txt)
self.log.info("Running PGO steps...")
# Remove old profiles
remove_dir(self.profdir)
mkdir(self.profdir)
# Clean the old build
run_cmd('make distclean')
# Compile and run example to generate profile
print_msg("generating PGO profile...")
(out, _) = run_cmd('%s 2 hostname' % self.cfg['mpiexec_cmd'])
nodes = out.split()
if nodes[0] == nodes[1]:
raise EasyBuildError("The profile is generated with 1 node! Use 2 nodes to generate a proper profile!")
write_file('pingpong.c', PINGPONG_PGO_TEST)
run_cmd('%s/bin/mpicc pingpong.c -o pingpong' % self.installdir)
run_cmd('PSP_SHM=0 %s 2 pingpong' % self.cfg['mpiexec_cmd'])
# Check that the profiles are there
new_profs = os.listdir(self.profdir)
if not new_profs:
raise EasyBuildError("The PGO profiles where not found in the expected directory (%s)" % self.profdir)
# Change PGO related options
self.cfg['pgo'] = False
self.cfg['configopts'] = re.sub('--with-profile=gen', '--with-profile=use', self.cfg['configopts'])
# Reconfigure
print_msg("configuring with PGO...")
self.log.info("Running configure_step with PGO...")
self.configure_step()
def _check_filename(self, fn):
"""Perform sanity check on the filename, and set mechanism to set the content of the file"""
if os.path.isfile(fn):
self.get_fn = (read_file, (fn,))
self.set_fn = (write_file, (fn, self.rawcontent))
self.log.debug("Process filename %s with get function %s, set function %s" % (fn, self.get_fn, self.set_fn))
if self.get_fn is None:
raise EasyBuildError('Failed to determine get function for filename %s', fn)
if self.set_fn is None:
raise EasyBuildError('Failed to determine set function for filename %s', fn)
"""
Generate a module file.
"""
self.moduleGenerator = ModuleGenerator(self, fake)
modpath = self.moduleGenerator.create_files()
txt = ''
txt += self.make_module_description()
txt += self.make_module_dep()
txt += self.make_module_req()
txt += self.make_module_extra()
if self.cfg['exts_list']:
txt += self.make_module_extra_extensions()
txt += '\n# built with EasyBuild version %s\n' % VERBOSE_VERSION
write_file(self.moduleGenerator.filename, txt)
self.log.info("Added modulefile: %s" % (self.moduleGenerator.filename))
if not fake:
self.make_devel_module()
self.modules_tool.update()
return modpath