How to use the sparkmagic.utils.configuration function in sparkmagic

To help you get started, we’ve selected a few sparkmagic examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github jupyter-incubator / sparkmagic / sparkmagic / sparkmagic / livyclientlib / livyreliablehttpclient.py View on Github external
def _get_retry_policy():
        policy = conf.retry_policy()
        
        if policy == LINEAR_RETRY:
            return LinearRetryPolicy(seconds_to_sleep=5, max_retries=5)
        elif policy == CONFIGURABLE_RETRY:
            return ConfigurableRetryPolicy(retry_seconds_to_sleep_list=conf.retry_seconds_to_sleep_list(), max_retries=conf.configurable_retry_policy_max_retries())
        else:
            raise BadUserConfigurationException(u"Retry policy '{}' not supported".format(policy))
github jupyter-incubator / sparkmagic / sparkmagic / sparkmagic / livyclientlib / sqlquery.py View on Github external
def _pyspark_command(self, sql_context_variable_name, encode_result=True):
        command = u'{}.sql(u"""{} """).toJSON()'.format(sql_context_variable_name, self.query)
        if self.samplemethod == u'sample':
            command = u'{}.sample(False, {})'.format(command, self.samplefraction)
        if self.maxrows >= 0:
            command = u'{}.take({})'.format(command, self.maxrows)
        else:
            command = u'{}.collect()'.format(command)
        # Unicode support has improved in Python 3 so we don't need to encode.
        if encode_result:
            print_command = '{}.encode("{}")'.format(constants.LONG_RANDOM_VARIABLE_NAME,
                                                     conf.pyspark_sql_encoding())
        else:
            print_command = constants.LONG_RANDOM_VARIABLE_NAME
        command = u'for {} in {}: print({})'.format(constants.LONG_RANDOM_VARIABLE_NAME,
                                                    command,
                                                    print_command)
        return Command(command)
github jupyter-incubator / sparkmagic / sparkmagic / sparkmagic / livyclientlib / reliablehttpclient.py View on Github external
def __init__(self, endpoint, headers, retry_policy):
        self._endpoint = endpoint
        self._headers = headers
        self._retry_policy = retry_policy
        if self._endpoint.auth == constants.AUTH_KERBEROS:
            self._auth = HTTPKerberosAuth(mutual_authentication=REQUIRED)
        elif self._endpoint.auth == constants.AUTH_BASIC:
            self._auth = (self._endpoint.username, self._endpoint.password)
        elif self._endpoint.auth != constants.NO_AUTH:
            raise BadUserConfigurationException(u"Unsupported auth %s" %self._endpoint.auth)

        self.logger = SparkLog(u"ReliableHttpClient")

        self.verify_ssl = not conf.ignore_ssl_errors()
        if not self.verify_ssl:
            self.logger.debug(u"ATTENTION: Will ignore SSL errors. This might render you vulnerable to attacks.")
            requests.packages.urllib3.disable_warnings()
github jupyter-incubator / sparkmagic / sparkmagic / sparkmagic / controllerwidget / magicscontrollerwidget.py View on Github external
def _get_default_endpoints():
        default_endpoints = set()

        for kernel_type in LANGS_SUPPORTED:
            endpoint_config = getattr(conf, 'kernel_%s_credentials' % kernel_type)()
            if all([p in endpoint_config for p in ["url", "password", "username"]]) and endpoint_config["url"] != "":
                user = endpoint_config["username"]
                passwd = endpoint_config["password"]
                
                authentication = endpoint_config.get("auth", None)
                if authentication is None:
                    authentication = conf.get_auth_value(user, passwd)

                default_endpoints.add(Endpoint(
                    username=user,
                    password=passwd,
                    auth=authentication,
                    url=endpoint_config["url"],
                    implicitly_added=True))

        return default_endpoints
github jupyter-incubator / sparkmagic / sparkmagic / sparkmagic / controllerwidget / createsessionwidget.py View on Github external
self.refresh_method = refresh_method

        self.endpoints_dropdown_widget = endpoints_dropdown_widget

        self.session_widget = self.ipywidget_factory.get_text(
            description='Name:',
            value='session-name'
        )
        self.lang_widget = self.ipywidget_factory.get_toggle_buttons(
            description='Language:',
            options=[LANG_SCALA, LANG_PYTHON, LANG_PYTHON3],
        )
        self.properties = self.ipywidget_factory.get_text(
            description='Properties:',
            value=json.dumps(conf.session_configs())
        )
        self.submit_widget = self.ipywidget_factory.get_submit_button(
            description='Create Session'
        )

        self.children = [self.ipywidget_factory.get_html(value="<br>", width="600px"), self.endpoints_dropdown_widget,
                         self.session_widget, self.lang_widget, self.properties,
                         self.ipywidget_factory.get_html(value="<br>", width="600px"), self.submit_widget]

        for child in self.children:
            child.parent_widget = self