How to use the urllib3.util.parse_url function in urllib3

To help you get started, we’ve selected a few urllib3 examples, based on popular ways it is used in public projects.

Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.

github zrzka / blackmamba / blackmamba / script / forum-copy-code.py View on Github external
def get_code_elements(url):
    parsed = urllib3.util.parse_url(url)

    if not parsed.host == 'forum.omz-software.com':
        raise ValueError(f'{url} does not point to the Pythonista Forum')

    path = parsed.path
    if not path.startswith('/api/'):
        path = f'/api{path}'

    url = f'{parsed.scheme}://{parsed.host}{path}'

    j = json.loads(requests.get(url).text)

    try:
        post = int(url.split('/')[-1])
        content = j['posts'][post-1]['content']
        return re.findall(CODE_PATTERN, content)
github microsoft / msticpy / msticpy / sectools / tiproviders / ti_provider_base.py View on Github external
Tuple[Optional[str], Optional[str], Optional[str]
        Tuple of URL, scheme, host

    """
    clean_url = None
    scheme = None
    host = None
    try:
        scheme, _, host, _, _, _, _ = parse_url(url)
        clean_url = url
    except LocationParseError:
        # Try to clean URL and re-check
        cleaned_url = _clean_url(url)
        if cleaned_url is not None:
            try:
                scheme, _, host, _, _, _, _ = parse_url(cleaned_url)
                clean_url = cleaned_url
            except LocationParseError:
                pass
    if require_url_encoding and clean_url:
        clean_url = quote_plus(clean_url)
    return clean_url, scheme, host
github sarugaku / requirementslib / src / requirementslib / models / url.py View on Github external
def _get_parsed_url(url):
    # type: (S) -> Url
    """
    This is a stand-in function for `urllib3.util.parse_url`

    The orignal function doesn't handle special characters very well, this simply splits
    out the authentication section, creates the parsed url, then puts the authentication
    section back in, bypassing validation.

    :return: The new, parsed URL object
    :rtype: :class:`~urllib3.util.url.Url`
    """

    try:
        parsed = urllib3_parse(url)
    except ValueError:
        scheme, _, url = url.partition("://")
        auth, _, url = url.rpartition("@")
        url = "{scheme}://{url}".format(scheme=scheme, url=url)
        parsed = urllib3_parse(url)._replace(auth=auth)
    if parsed.auth and unquote_plus(parsed.auth) != parsed.auth:
        return parsed._replace(auth=unquote_plus(parsed.auth))
    return parsed
github RedHatQE / pulp-automation / misc / fake_consumer.py View on Github external
def run(url, consumer_name, reporting, polling_frequency, key_file, pulp_url, register):
    from gevent import monkey
    monkey.patch_all(select=False, thread=False)

    if not pulp_url:
        # use http://admin:admin@/"
        amqp_url = urllib3.util.parse_url(url)
        pulp_url = 'https://admin:admin@' + amqp_url.host + '/'

    rsa = None
    pem = ""
    if key_file:
        # load rsa and set-up authentication
        rsa = RSA.load_key(key_file)
        bio_fd = BIO.MemoryBuffer()
        rsa.save_pub_key_bio(bio_fd)
        pem = bio_fd.getvalue()

    pulp = Pulp(pulp_url)
    consumer = None
    if register:
        consumer = Consumer.register(pulp, consumer_name, rsa_pub=pem)
        log.info("registered: " + str(consumer))
github joaomatosf / jexboss / jexboss.py View on Github external
def check_vul(url):
    """
    Test if a GET to a URL is successful
    :param url: The URL to test
    :return: A dict with the exploit type as the keys, and the HTTP status code as the value
    """
    url_check = parse_url(url)
    if '443' in str(url_check.port) and url_check.scheme != 'https':
        url = "https://"+str(url_check.host)+":"+str(url_check.port)+str(url_check.path)

    print_and_flush(GREEN + "\n ** Checking Host: %s **\n" % url)
    logging.info("Checking Host: %s" % url)

    headers = {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
               "Connection": "keep-alive",
               "User-Agent": get_random_user_agent()}

    paths = {"jmx-console": "/jmx-console/HtmlAdaptor?action=inspectMBean&name=jboss.system:type=ServerInfo",
             "web-console": "/web-console/Invoker",
             "JMXInvokerServlet": "/invoker/JMXInvokerServlet",
             "admin-console": "/admin-console/",
             "Application Deserialization": "",
             "Servlet Deserialization" : "",
github dcs4cop / xcube / xcube / core / dsio.py View on Github external
def split_obs_url(path: str) -> Tuple[Optional[str], str]:
    """
    If *path* is a URL, return tuple (endpoint_url, root), otherwise (None, *path*)
    """
    url = urllib3.util.parse_url(path)
    if all((url.scheme, url.host, url.path)) and url.scheme != 's3':
        if url.port is not None:
            endpoint_url = f'{url.scheme}://{url.host}:{url.port}'
        else:
            endpoint_url = f'{url.scheme}://{url.host}'
        root = url.path
        if root.startswith('/'):
            root = root[1:]
        return endpoint_url, root
    return None, path
github lukasc-ubc / SiEPIC-Tools / Python_packages_for_KLayout / python / requests / models.py View on Github external
else:
            url = unicode(url) if is_py2 else str(url)

        # Remove leading whitespaces from url
        url = url.lstrip()

        # Don't do any URL preparation for non-HTTP schemes like `mailto`,
        # `data` etc to work around exceptions from `url_parse`, which
        # handles RFC 3986 only.
        if ':' in url and not url.lower().startswith('http'):
            self.url = url
            return

        # Support for unicode domain names and paths.
        try:
            scheme, auth, host, port, path, query, fragment = parse_url(url)
        except LocationParseError as e:
            raise InvalidURL(*e.args)

        if not scheme:
            error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?")
            error = error.format(to_native_string(url, 'utf8'))

            raise MissingSchema(error)

        if not host:
            raise InvalidURL("Invalid URL %r: No host supplied" % url)

        # In general, we want to try IDNA encoding the hostname if the string contains
        # non-ASCII characters. This allows users to automatically get the correct IDNA
        # behaviour. For strings containing only ASCII characters, we need to also verify
        # it doesn't start with a wildcard (*), before allowing the unencoded hostname.
github opencord / voltha / voltha / adapters / adtran_olt / download.py View on Github external
def parse_url(self):
        from urllib3 import util, exceptions
        try:
            results = util.parse_url(self._url)

            # Server info
            self._scheme = results.scheme.lower()
            if self._scheme not in self._supported_protocols:
                self._failure_reason = ImageDownload.INVALID_URL
                self._additional_info = "Unsupported file transfer protocol: {}".format(results.scheme)
                return False

            self._host = results.host
            self._port = results.port
            self._path = results.path
            self._auth = results.auth
            return True

        except exceptions.LocationValueError as e:
            self._failure_reason = ImageDownload.INVALID_URL