Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def get_code_elements(url):
parsed = urllib3.util.parse_url(url)
if not parsed.host == 'forum.omz-software.com':
raise ValueError(f'{url} does not point to the Pythonista Forum')
path = parsed.path
if not path.startswith('/api/'):
path = f'/api{path}'
url = f'{parsed.scheme}://{parsed.host}{path}'
j = json.loads(requests.get(url).text)
try:
post = int(url.split('/')[-1])
content = j['posts'][post-1]['content']
return re.findall(CODE_PATTERN, content)
Tuple[Optional[str], Optional[str], Optional[str]
Tuple of URL, scheme, host
"""
clean_url = None
scheme = None
host = None
try:
scheme, _, host, _, _, _, _ = parse_url(url)
clean_url = url
except LocationParseError:
# Try to clean URL and re-check
cleaned_url = _clean_url(url)
if cleaned_url is not None:
try:
scheme, _, host, _, _, _, _ = parse_url(cleaned_url)
clean_url = cleaned_url
except LocationParseError:
pass
if require_url_encoding and clean_url:
clean_url = quote_plus(clean_url)
return clean_url, scheme, host
def _get_parsed_url(url):
# type: (S) -> Url
"""
This is a stand-in function for `urllib3.util.parse_url`
The orignal function doesn't handle special characters very well, this simply splits
out the authentication section, creates the parsed url, then puts the authentication
section back in, bypassing validation.
:return: The new, parsed URL object
:rtype: :class:`~urllib3.util.url.Url`
"""
try:
parsed = urllib3_parse(url)
except ValueError:
scheme, _, url = url.partition("://")
auth, _, url = url.rpartition("@")
url = "{scheme}://{url}".format(scheme=scheme, url=url)
parsed = urllib3_parse(url)._replace(auth=auth)
if parsed.auth and unquote_plus(parsed.auth) != parsed.auth:
return parsed._replace(auth=unquote_plus(parsed.auth))
return parsed
def run(url, consumer_name, reporting, polling_frequency, key_file, pulp_url, register):
from gevent import monkey
monkey.patch_all(select=False, thread=False)
if not pulp_url:
# use http://admin:admin@/"
amqp_url = urllib3.util.parse_url(url)
pulp_url = 'https://admin:admin@' + amqp_url.host + '/'
rsa = None
pem = ""
if key_file:
# load rsa and set-up authentication
rsa = RSA.load_key(key_file)
bio_fd = BIO.MemoryBuffer()
rsa.save_pub_key_bio(bio_fd)
pem = bio_fd.getvalue()
pulp = Pulp(pulp_url)
consumer = None
if register:
consumer = Consumer.register(pulp, consumer_name, rsa_pub=pem)
log.info("registered: " + str(consumer))
def check_vul(url):
"""
Test if a GET to a URL is successful
:param url: The URL to test
:return: A dict with the exploit type as the keys, and the HTTP status code as the value
"""
url_check = parse_url(url)
if '443' in str(url_check.port) and url_check.scheme != 'https':
url = "https://"+str(url_check.host)+":"+str(url_check.port)+str(url_check.path)
print_and_flush(GREEN + "\n ** Checking Host: %s **\n" % url)
logging.info("Checking Host: %s" % url)
headers = {"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Connection": "keep-alive",
"User-Agent": get_random_user_agent()}
paths = {"jmx-console": "/jmx-console/HtmlAdaptor?action=inspectMBean&name=jboss.system:type=ServerInfo",
"web-console": "/web-console/Invoker",
"JMXInvokerServlet": "/invoker/JMXInvokerServlet",
"admin-console": "/admin-console/",
"Application Deserialization": "",
"Servlet Deserialization" : "",
def split_obs_url(path: str) -> Tuple[Optional[str], str]:
"""
If *path* is a URL, return tuple (endpoint_url, root), otherwise (None, *path*)
"""
url = urllib3.util.parse_url(path)
if all((url.scheme, url.host, url.path)) and url.scheme != 's3':
if url.port is not None:
endpoint_url = f'{url.scheme}://{url.host}:{url.port}'
else:
endpoint_url = f'{url.scheme}://{url.host}'
root = url.path
if root.startswith('/'):
root = root[1:]
return endpoint_url, root
return None, path
else:
url = unicode(url) if is_py2 else str(url)
# Remove leading whitespaces from url
url = url.lstrip()
# Don't do any URL preparation for non-HTTP schemes like `mailto`,
# `data` etc to work around exceptions from `url_parse`, which
# handles RFC 3986 only.
if ':' in url and not url.lower().startswith('http'):
self.url = url
return
# Support for unicode domain names and paths.
try:
scheme, auth, host, port, path, query, fragment = parse_url(url)
except LocationParseError as e:
raise InvalidURL(*e.args)
if not scheme:
error = ("Invalid URL {0!r}: No schema supplied. Perhaps you meant http://{0}?")
error = error.format(to_native_string(url, 'utf8'))
raise MissingSchema(error)
if not host:
raise InvalidURL("Invalid URL %r: No host supplied" % url)
# In general, we want to try IDNA encoding the hostname if the string contains
# non-ASCII characters. This allows users to automatically get the correct IDNA
# behaviour. For strings containing only ASCII characters, we need to also verify
# it doesn't start with a wildcard (*), before allowing the unencoded hostname.
def parse_url(self):
from urllib3 import util, exceptions
try:
results = util.parse_url(self._url)
# Server info
self._scheme = results.scheme.lower()
if self._scheme not in self._supported_protocols:
self._failure_reason = ImageDownload.INVALID_URL
self._additional_info = "Unsupported file transfer protocol: {}".format(results.scheme)
return False
self._host = results.host
self._port = results.port
self._path = results.path
self._auth = results.auth
return True
except exceptions.LocationValueError as e:
self._failure_reason = ImageDownload.INVALID_URL