How to use the omniduct.utils.processes.run_in_subprocess function in omniduct

To help you get started, we’ve selected a few omniduct examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github airbnb / omniduct / omniduct / databases / hiveserver2.py View on Github external
def _run_in_hivecli(self, cmd):
        """Run a query using hive cli in a subprocess."""
        # Turn hive command into quotable string.
        double_escaped = re.sub('\\' * 2, '\\' * 4, cmd)
        backtick_escape = '\\\\\\`' if self.remote else '\\`'
        sys_cmd = 'hive -e "{0}"'.format(re.sub('"', '\\"', double_escaped)) \
                                 .replace('`', backtick_escape)
        # Execute command in a subprocess.
        if self.remote:
            proc = self.remote.execute(sys_cmd)
        else:
            proc = run_in_subprocess(sys_cmd, check_output=True)
        return proc
github airbnb / omniduct / omniduct / remotes / ssh.py View on Github external
are handled using the standard implementation.
        """
        from ..filesystems.local import LocalFsClient

        if fs is None or isinstance(fs, LocalFsClient):
            logger.info('Copying file to local...')
            dest = dest or posixpath.basename(source)
            cmd = (
                "scp -r -o ControlPath={socket} {login}:'{remote_file}' '{local_file}'".format(
                    socket=self._socket_path,
                    login=self._login_info,
                    remote_file=dest.replace('"', r'\"'),
                    local_file=source.replace('"', r'\"'),  # quote escaped for bash
                )
            )
            proc = run_in_subprocess(cmd, check_output=True)
            logger.info(proc.stderr or 'Success')
        else:
            return super(RemoteClient, self).download(source, dest, overwrite, fs)
github airbnb / omniduct / omniduct / remotes / ssh.py View on Github external
def _port_forward_stop(self, local_port, remote_host, remote_port, connection):
        logger.info('Cancelling port forward...')
        cmd_template = 'ssh {login} -T -O cancel -S {socket} -L localhost:{local_port}:{remote_host}:{remote_port}'
        cmd = cmd_template.format(login=self._login_info,
                                  socket=self._socket_path,
                                  local_port=local_port,
                                  remote_host=remote_host,
                                  remote_port=remote_port)
        proc = run_in_subprocess(cmd)
        logger.info('Port forward succesfully stopped.' if proc.returncode == 0 else 'Failed to stop port forwarding.')
github airbnb / omniduct / omniduct / remotes / ssh.py View on Github external
def _execute(self, cmd, skip_cwd=False, **kwargs):
        """
        Additional Args:
            skip_cwd (bool): Whether to skip changing to the current working
                directory associated with this client before executing the
                command. This is mainly useful to methods internal to this
                class.
        """
        template = 'ssh {login} -T -o ControlPath={socket} << EOF\n{cwd}{cmd}\nEOF'
        config = dict(self._subprocess_config)
        config.update(kwargs)

        cwd = 'cd "{path}"\n'.format(path=escape_path(self.path_cwd)) if not skip_cwd else ''
        return run_in_subprocess(template.format(login=self._login_info,
                                                 socket=self._socket_path,
                                                 cwd=cwd,
                                                 cmd=cmd),
                                 check_output=True,
                                 **config)
github airbnb / omniduct / omniduct / remotes / ssh.py View on Github external
are handled using the standard implementation.
        """
        from ..filesystems.local import LocalFsClient

        if fs is None or isinstance(fs, LocalFsClient):
            logger.info('Copying file from local...')
            dest = dest or posixpath.basename(source)
            cmd = (
                "scp -r -o ControlPath={socket} '{local_file}' {login}:'{remote_file}'".format(
                    socket=self._socket_path,
                    local_file=source.replace('"', r'\"'),  # quote escaped for bash
                    login=self._login_info,
                    remote_file=dest.replace('"', r'\"'),
                )
            )
            proc = run_in_subprocess(cmd, check_output=True)
            logger.info(proc.stderr or 'Success')
        else:
            return super(RemoteClient, self).upload(source, dest, overwrite, fs)
github airbnb / omniduct / omniduct / remotes / ssh.py View on Github external
def update_host_keys(self):
        """
        Update host keys associated with this remote.

        This method updates the SSH host-keys stored in `~/.ssh/known_hosts`,
        allowing one to successfully connect to hosts when servers are,
        for example, redeployed and have different host keys.
        """
        assert not self.remote, "Updating host key only works for local connections."
        cmd = "ssh-keygen -R {host} && ssh-keyscan {host} >> ~/.ssh/known_hosts".format(host=self.host)
        proc = run_in_subprocess(cmd, True)
        if proc.returncode != 0:
            raise RuntimeError(
                "Could not update host keys! Please handle this manually. The "
                "error was:\n" + '\n'.join([proc.stdout.decode('utf-8'), proc.stderr.decode('utf-8')])
            )
github airbnb / omniduct / omniduct / remotes / ssh.py View on Github external
def _port_forward_start(self, local_port, remote_host, remote_port):
        logger.info('Establishing port forward...')
        cmd_template = 'ssh {login} -T -O forward -S {socket} -L localhost:{local_port}:{remote_host}:{remote_port}'
        cmd = cmd_template.format(login=self._login_info,
                                  socket=self._socket_path,
                                  local_port=local_port,
                                  remote_host=remote_host,
                                  remote_port=remote_port)
        proc = run_in_subprocess(cmd)
        if proc.returncode != 0:
            raise Exception('Unable to port forward with command: {}'.format(cmd))
        logger.info(proc.stderr or 'Success')
        return proc
github airbnb / omniduct / omniduct / remotes / ssh.py View on Github external
def _is_connected(self):
        cmd = "ssh {login} -T -S {socket} -O check".format(login=self._login_info,
                                                           socket=self._socket_path)
        proc = run_in_subprocess(cmd)

        if proc.returncode != 0:
            if os.path.exists(self._socket_path):
                os.remove(self._socket_path)
            return False
        return True