Secure your code as it's written. Use Snyk Code to scan source code in minutes - no build needed - and fix issues immediately.
def _make_req(string, item):
try:
return Requirement(string)
except Exception:
raise YAMLException("Could not parse requirement {!r} from {!r}", string, item)
# Install the primary dependencies
sp.run(
(pip_exe_pth, *PIP_INSTALL_ARGS, *map(str, reqs_dict.values())),
stdout=sp.DEVNULL,
check=True,
)
# Get pinned primary+secondary dependencies from pip freeze
proc = sp.run(
(pip_exe_pth, *PIP_FREEZE_ARGS), stdout=sp.PIPE, check=True, encoding="utf-8"
)
# Return Requirement objects
ret = []
for req_obj in map(packaging.requirements.Requirement, proc.stdout.strip().split("\n")):
dep_name = req_obj.name.lower()
# Don't include core dependencies if these are extra dependencies
if dep_name in all_core_deps:
if req_obj.specifier != all_core_deps[dep_name].specifier:
print(
f"[WARNING] {dep_name} is listed as both a core requirement and an extra "
f"requirement, and it's possible that their versions conflict!",
file=sys.stderr,
)
continue
# Preserve environment markers
if dep_name in reqs_dict:
req_obj.marker = reqs_dict[dep_name].marker
ret.append(req_obj)
def pre_download_packages(memory, specs, verbose=False, index_url=DEFAULT_INDEX_URL):
futures = {}
with concurrent.futures.ThreadPoolExecutor(max_workers=MAX_WORKERS) as executor:
for spec in specs:
package, _, _ = _explode_package_spec(spec)
req = Requirement(package)
futures[
executor.submit(get_package_data, req.name, index_url, verbose=verbose)
] = req.name
for future in concurrent.futures.as_completed(futures):
content = future.result()
memory[futures[future]] = content
def _is_same_dep(cls, dep1, dep2):
"""Definitions are the same if they refer to the same package, even if versions differ."""
dep1_name = canonicalize_name(requirements.Requirement(dep1).name)
try:
dep2_name = canonicalize_name(requirements.Requirement(dep2).name)
except requirements.InvalidRequirement:
# we couldn't parse a version, probably a URL
return False
return dep1_name == dep2_name
def parse_markers(self):
# type: () -> None
if self.markers:
marker_str = self.markers.replace('"', "'")
markers = PackagingRequirement("fakepkg; {0}".format(marker_str)).marker
self.parsed_marker = markers
[('pep8', '1.5.7'),
('flake8', '2.4.0'),
('mccabe', '0.3'),
('pyflakes', '0.8.1')]
Example output:
{'pep8': ['flake8'],
'flake8': [],
'mccabe': ['flake8'],
'pyflakes': ['flake8']}
"""
# First, collect all the dependencies into a sequence of (parent, child) tuples, like [('flake8', 'pep8'),
# ('flake8', 'mccabe'), ...]
return lookup_table((key_from_req(Requirement(dep_name)), name)
for name, version_and_extras in cache_keys
for dep_name in self.cache[name][version_and_extras])
def _requirement_name(self, full_requirement: str) -> str:
requirement = Requirement(full_requirement)
return requirement.name
def add_markers_to_dep(d, marker_str):
# type: (str, Union[str, Marker]) -> str
req = PackagingRequirement(d)
existing_marker = getattr(req, "marker", None)
if isinstance(marker_str, Marker):
marker_str = str(marker_str)
if existing_marker is not None:
marker_str = str(merge_markers(existing_marker, marker_str))
if marker_str:
marker_str = marker_str.replace("'", '"')
req.marker = Marker(marker_str)
return str(req)
def get_restricted_packages():
req_list = ["tensorflow==1.14.0"]
req_files = glob.glob("/src/**/requirements.txt", recursive=True)
for req_file in req_files:
# clean requirements file, like removing comments
with open(req_file) as f:
for req in requirements.parse(f):
specifiers = [op + version for op, version in req.specs]
req_list.append(req.name + ",".join(specifiers))
cortex_packages = {}
for req_line in req_list:
parsed_req = Requirement(req_line)
if cortex_packages.get(parsed_req.name) is None:
cortex_packages[parsed_req.name] = parsed_req.specifier
else:
cortex_packages[parsed_req.name] = (
cortex_packages[parsed_req.name] & parsed_req.specifier
)
return cortex_packages
def _is_same_dep(cls, dep1, dep2):
"""Definitions are the same if they refer to the same package, even if versions differ."""
dep1_name = canonicalize_name(requirements.Requirement(dep1).name)
try:
dep2_name = canonicalize_name(requirements.Requirement(dep2).name)
except requirements.InvalidRequirement:
# we couldn't parse a version, probably a URL
return False
return dep1_name == dep2_name