How to use the easybuild.tools.filetools.write_file function in easybuild

To help you get started, we’ve selected a few easybuild examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github easybuilders / easybuild-framework / test / framework / filetools.py View on Github external
self.assertEqual(len(downloads), 1)

        # non-existing files result in None return value
        self.assertEqual(ft.download_file(fn, 'file://%s/nosuchfile' % test_dir, target_location), None)

        # install broken proxy handler for opening local files
        # this should make urlopen use this broken proxy for downloading from a file:// URL
        proxy_handler = std_urllib.ProxyHandler({'file': 'file://%s/nosuchfile' % test_dir})
        std_urllib.install_opener(std_urllib.build_opener(proxy_handler))

        # downloading over a broken proxy results in None return value (failed download)
        # this tests whether proxies are taken into account by download_file
        self.assertEqual(ft.download_file(fn, source_url, target_location), None, "download over broken proxy fails")

        # modify existing download so we can verify re-download
        ft.write_file(target_location, '')

        # restore a working file handler, and retest download of local file
        std_urllib.install_opener(std_urllib.build_opener(std_urllib.FileHandler()))
        res = ft.download_file(fn, source_url, target_location)
        self.assertEqual(res, target_location, "'download' of local file works after removing broken proxy")

        # existing file was re-downloaded, so a backup should have been created of the existing file
        downloads = glob.glob(target_location + '*')
        self.assertEqual(len(downloads), 2)
        backup = [d for d in downloads if os.path.basename(d) != fn][0]
        self.assertEqual(ft.read_file(backup), '')
        self.assertEqual(ft.compute_checksum(target_location), ft.compute_checksum(os.path.join(toy_source_dir, fn)))

        # make sure specified timeout is parsed correctly (as a float, not a string)
        opts = init_config(args=['--download-timeout=5.3'])
        init_config(build_options={'download_timeout': opts.download_timeout})
github easybuilders / easybuild-framework / test / framework / options.py View on Github external
def run_test(custom=None, extra_params=[], fmt=None):
            """Inner function to run actual test in current setting."""

            fd, dummylogfn = tempfile.mkstemp(prefix='easybuild-dummy', suffix='.log')
            os.close(fd)

            avail_args = [
                '-a',
                '--avail-easyconfig-params',
            ]
            for avail_arg in avail_args:

                # clear log
                write_file(self.logfile, '')

                args = [
                    '--unittest-file=%s' % self.logfile,
                    avail_arg,
                ]
                if fmt is not None:
                    args.append(fmt)
                if custom is not None:
                    args.extend(['-e', custom])

                outtxt = self.eb_main(args, logfile=dummylogfn, verbose=True)
                logtxt = read_file(self.logfile)

                # check whether all parameter types are listed
                par_types = [BUILD, DEPENDENCIES, EXTENSIONS, FILEMANAGEMENT,
                             LICENSE, MANDATORY, MODULES, OTHER, TOOLCHAIN]
github easybuilders / easybuild-framework / test / framework / options.py View on Github external
elif get_module_syntax() == 'Lua':
            modules_header_txt = '\n'.join([
                "-- test header",
                'setenv("SITE_SPECIFIC_HEADER_ENV_VAR", "foo")',
            ])
            modules_footer_txt = '\n'.join([
                "-- test footer",
                'setenv("SITE_SPECIFIC_FOOTER_ENV_VAR", "bar")',
            ])
        else:
            self.assertTrue(False, "Unknown module syntax: %s" % get_module_syntax())

        # dump header/footer text to file
        handle, modules_footer = tempfile.mkstemp(prefix='modules-footer-')
        os.close(handle)
        write_file(modules_footer, modules_footer_txt)
        handle, modules_header = tempfile.mkstemp(prefix='modules-header-')
        os.close(handle)
        write_file(modules_header, modules_header_txt)

        # use toy-0.0.eb easyconfig file that comes with the tests
        eb_file = os.path.join(os.path.dirname(__file__), 'easyconfigs', 'toy-0.0.eb')

        # check log message with --skip for existing module
        args = [
            eb_file,
            '--sourcepath=%s' % self.test_sourcepath,
            '--buildpath=%s' % self.test_buildpath,
            '--installpath=%s' % self.test_installpath,
            '--debug',
            '--force',
            '--modules-header=%s' % modules_header,
github easybuilders / easybuild-framework / test / framework / toy_build.py View on Github external
"""Test specifying external (build) dependencies."""
        topdir = os.path.dirname(os.path.abspath(__file__))
        ectxt = read_file(os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0-deps.eb'))
        toy_ec = os.path.join(self.test_prefix, 'toy-0.0-external-deps.eb')

        # just specify some of the test modules we ship, doesn't matter where they come from
        extraectxt = "\ndependencies += [('foobar/1.2.3', EXTERNAL_MODULE)]"
        extraectxt += "\nbuilddependencies = [('somebuilddep/0.1', EXTERNAL_MODULE)]"
        extraectxt += "\nversionsuffix = '-external-deps'"
        write_file(toy_ec, ectxt + extraectxt)

        # install dummy modules
        modulepath = os.path.join(self.test_prefix, 'modules')
        for mod in ['intel/2018a', 'GCC/6.4.0-2.28', 'foobar/1.2.3', 'somebuilddep/0.1']:
            mkdir(os.path.join(modulepath, os.path.dirname(mod)), parents=True)
            write_file(os.path.join(modulepath, mod), "#%Module")

        self.reset_modulepath([modulepath, os.path.join(self.test_installpath, 'modules', 'all')])
        self.test_toy_build(ec_file=toy_ec, versionsuffix='-external-deps', verbose=True, raise_error=True)

        self.modtool.load(['toy/0.0-external-deps'])
        # note build dependency is not loaded
        mods = ['intel/2018a', 'GCC/6.4.0-2.28', 'foobar/1.2.3', 'toy/0.0-external-deps']
        self.assertEqual([x['mod_name'] for x in self.modtool.list()], mods)

        # check behaviour when a non-existing external (build) dependency is included
        err_msg = "Missing modules for dependencies marked as external modules:"

        extraectxt = "\nbuilddependencies = [('nosuchbuilddep/0.0.0', EXTERNAL_MODULE)]"
        extraectxt += "\nversionsuffix = '-external-deps-broken1'"
        write_file(toy_ec, ectxt + extraectxt)
        self.assertErrorRegex(EasyBuildError, err_msg, self.test_toy_build, ec_file=toy_ec,
github easybuilders / easybuild-framework / test / framework / robot.py View on Github external
def test_check_conflicts_multi_deps(self):
        """Test check_conflicts when multi_deps is used."""
        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')

        test_ec = os.path.join(self.test_prefix, 'test.eb')

        test_ec_txt = read_file(toy_ec)
        # we need to use empty dummy toolchain version to ensure dependencies are picked up...
        tc_regex = re.compile(r'^toolchain = .*', re.M)
        test_ec_txt = tc_regex.sub("toolchain = {'name': 'dummy', 'version': ''}", test_ec_txt)
        test_ec_txt += "\nmulti_deps = {'GCC': ['4.9.2', '7.3.0-2.30']}\n"
        test_ec_txt += "dependencies = [('gzip', '1.4')]\n"

        write_file(test_ec, test_ec_txt)
        ecs, _ = parse_easyconfigs([(test_ec, False)])

        init_config(build_options={'robot_path': [test_ecs]})

        # use of multi_deps should not result in false positives in check_conflicts
        self.assertFalse(check_conflicts(ecs, self.modtool))
github easybuilders / easybuild-framework / test / framework / options.py View on Github external
mns_regex = re.compile(r'^\s*TestIncludedMNS', re.M)

        # TestIncludedMNS module naming scheme is not available by default
        args = [
            '--avail-module-naming-schemes',
        ]
        logtxt, _= run_cmd("cd %s; eb %s" % (self.test_prefix, ' '.join(args)), simple=False)
        self.assertFalse(mns_regex.search(logtxt), "Unexpected pattern '%s' found in: %s" % (mns_regex.pattern, logtxt))

        # include extra test MNS
        mns_txt = '\n'.join([
            'from easybuild.tools.module_naming_scheme import ModuleNamingScheme',
            'class TestIncludedMNS(ModuleNamingScheme):',
            '   pass',
        ])
        write_file(os.path.join(self.test_prefix, 'test_mns.py'), mns_txt)

        # clear log
        write_file(self.logfile, '')

        args = [
            '--avail-module-naming-schemes',
            '--include-module-naming-schemes=%s/*.py' % self.test_prefix,
        ]
        logtxt, _= run_cmd("cd %s; eb %s" % (self.test_prefix, ' '.join(args)), simple=False)
        self.assertTrue(mns_regex.search(logtxt), "Pattern '%s' *not* found in: %s" % (mns_regex.pattern, logtxt))
github easybuilders / easybuild-framework / test / framework / toy_build.py View on Github external
def test_toy_extension_patches(self):
        """Test install toy that includes extensions with patches."""
        test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')
        toy_ec = os.path.join(test_ecs, 't', 'toy', 'toy-0.0.eb')
        toy_ec_txt = read_file(toy_ec)

        # create file that we'll copy via 'patches'
        write_file(os.path.join(self.test_prefix, 'test.txt'), 'test123')

        test_ec = os.path.join(self.test_prefix, 'test.eb')
        test_ec_txt = '\n'.join([
            toy_ec_txt,
            'exts_list = [',
            '   ("bar", "0.0", {',
            '       "buildopts": " && ls -l test.txt",',
            '       "patches": [',
            '           "bar-0.0_fix-silly-typo-in-printf-statement.patch",',  # normal patch
            '           ("bar-0.0_fix-very-silly-typo-in-printf-statement.patch", 0),',  # patch with patch level
            '           ("test.txt", "."),',  # file to copy to build dir (not a real patch file)
            '       ],',
            '   }),',
            ']',
        ])
        write_file(test_ec, test_ec_txt)
github easybuilders / JSC / Custom_EasyBlocks / 2018b / psmpi.py View on Github external
self.log.info("Running PGO steps...")
        # Remove old profiles
        remove_dir(self.profdir)
        mkdir(self.profdir)

        # Clean the old build
        run_cmd('make distclean')

        # Compile and run example to generate profile
        print_msg("generating PGO profile...")
        (out, _) = run_cmd('%s 2 hostname' % self.cfg['mpiexec_cmd'])
        nodes = out.split()
        if nodes[0] == nodes[1]:
            raise EasyBuildError("The profile is generated with 1 node! Use 2 nodes to generate a proper profile!")

        write_file('pingpong.c', PINGPONG_PGO_TEST)
        run_cmd('%s/bin/mpicc pingpong.c -o pingpong' % self.installdir)
        run_cmd('PSP_SHM=0 %s 2 pingpong' % self.cfg['mpiexec_cmd'])

        # Check that the profiles are there
        new_profs = os.listdir(self.profdir)
        if not new_profs:
            raise EasyBuildError("The PGO profiles where not found in the expected directory (%s)" % self.profdir)

        # Change PGO related options
        self.cfg['pgo'] = False
        self.cfg['configopts'] = re.sub('--with-profile=gen', '--with-profile=use', self.cfg['configopts'])

        # Reconfigure
        print_msg("configuring with PGO...")
        self.log.info("Running configure_step with PGO...")
        self.configure_step()
github easybuilders / easybuild-framework / easybuild / framework / easyconfig / parser.py View on Github external
def _check_filename(self, fn):
        """Perform sanity check on the filename, and set mechanism to set the content of the file"""
        if os.path.isfile(fn):
            self.get_fn = (read_file, (fn,))
            self.set_fn = (write_file, (fn, self.rawcontent))

        self.log.debug("Process filename %s with get function %s, set function %s" % (fn, self.get_fn, self.set_fn))

        if self.get_fn is None:
            raise EasyBuildError('Failed to determine get function for filename %s', fn)
        if self.set_fn is None:
            raise EasyBuildError('Failed to determine set function for filename %s', fn)
github easybuilders / easybuild-framework / easybuild / framework / easyblock.py View on Github external
"""
        Generate a module file.
        """
        self.moduleGenerator = ModuleGenerator(self, fake)
        modpath = self.moduleGenerator.create_files()

        txt = ''
        txt += self.make_module_description()
        txt += self.make_module_dep()
        txt += self.make_module_req()
        txt += self.make_module_extra()
        if self.cfg['exts_list']:
            txt += self.make_module_extra_extensions()
        txt += '\n# built with EasyBuild version %s\n' % VERBOSE_VERSION

        write_file(self.moduleGenerator.filename, txt)

        self.log.info("Added modulefile: %s" % (self.moduleGenerator.filename))

        if not fake:
            self.make_devel_module()

        self.modules_tool.update()

        return modpath