Repository URL to install this package:
|
Version:
0.1.31 ▾
|
from __future__ import annotations
import contextlib
import functools
import glob
import logging
import os
from pathlib import Path
from typing import TYPE_CHECKING
from typing import Any
import pkginfo
from poetry.core.factory import Factory
from poetry.core.packages.dependency import Dependency
from poetry.core.packages.package import Package
from poetry.core.utils.helpers import parse_requires
from poetry.core.utils.helpers import temporary_directory
from poetry.core.version.markers import InvalidMarker
from poetry.core.version.requirements import InvalidRequirement
from poetry.pyproject.toml import PyProjectTOML
from poetry.utils.env import EnvCommandError
from poetry.utils.env import ephemeral_environment
from poetry.utils.helpers import extractall
from poetry.utils.setup_reader import SetupReader
if TYPE_CHECKING:
from collections.abc import Iterator
from poetry.core.packages.project_package import ProjectPackage
logger = logging.getLogger(__name__)
PEP517_META_BUILD = """\
import build
import build.env
import pyproject_hooks
source = '{source}'
dest = '{dest}'
with build.env.DefaultIsolatedEnv() as env:
builder = build.ProjectBuilder(
source_dir=source,
python_executable=env.python_executable,
runner=pyproject_hooks.quiet_subprocess_runner,
)
env.install(builder.build_system_requires)
env.install(builder.get_requires_for_build('wheel'))
builder.metadata_path(dest)
"""
PEP517_META_BUILD_DEPS = ["build==1.0.3", "pyproject_hooks==1.0.0"]
class PackageInfoError(ValueError):
def __init__(self, path: Path, *reasons: BaseException | str) -> None:
reasons = (f"Unable to determine package info for path: {path!s}", *reasons)
super().__init__("\n\n".join(str(msg).strip() for msg in reasons if msg))
class PackageInfo:
def __init__(
self,
*,
name: str | None = None,
version: str | None = None,
summary: str | None = None,
requires_dist: list[str] | None = None,
requires_python: str | None = None,
files: list[dict[str, str]] | None = None,
yanked: str | bool = False,
cache_version: str | None = None,
) -> None:
self.name = name
self.version = version
self.summary = summary
self.requires_dist = requires_dist
self.requires_python = requires_python
self.files = files or []
self.yanked = yanked
self._cache_version = cache_version
self._source_type: str | None = None
self._source_url: str | None = None
self._source_reference: str | None = None
@property
def cache_version(self) -> str | None:
return self._cache_version
def update(self, other: PackageInfo) -> PackageInfo:
self.name = other.name or self.name
self.version = other.version or self.version
self.summary = other.summary or self.summary
self.requires_dist = other.requires_dist or self.requires_dist
self.requires_python = other.requires_python or self.requires_python
self.files = other.files or self.files
self._cache_version = other.cache_version or self._cache_version
return self
def asdict(self) -> dict[str, Any]:
"""
Helper method to convert package info into a dictionary used for caching.
"""
return {
"name": self.name,
"version": self.version,
"summary": self.summary,
"requires_dist": self.requires_dist,
"requires_python": self.requires_python,
"files": self.files,
"yanked": self.yanked,
"_cache_version": self._cache_version,
}
@classmethod
def load(cls, data: dict[str, Any]) -> PackageInfo:
"""
Helper method to load data from a dictionary produced by `PackageInfo.asdict()`.
:param data: Data to load. This is expected to be a `dict` object output by
`asdict()`.
"""
cache_version = data.pop("_cache_version", None)
return cls(cache_version=cache_version, **data)
def to_package(
self,
name: str | None = None,
extras: list[str] | None = None,
root_dir: Path | None = None,
) -> Package:
"""
Create a new `poetry.core.packages.package.Package` instance using metadata from
this instance.
:param name: Name to use for the package, if not specified name from this
instance is used.
:param extras: Extras to activate for this package.
:param root_dir: Optional root directory to use for the package. If set,
dependency strings will be parsed relative to this directory.
"""
name = name or self.name
if not name:
raise RuntimeError("Unable to create package with no name")
if not self.version:
# The version could not be determined, so we raise an error since it is
# mandatory.
raise RuntimeError(f"Unable to retrieve the package version for {name}")
package = Package(
name=name,
version=self.version,
source_type=self._source_type,
source_url=self._source_url,
source_reference=self._source_reference,
yanked=self.yanked,
)
if self.summary is not None:
package.description = self.summary
package.root_dir = root_dir
package.python_versions = self.requires_python or "*"
package.files = self.files
# If this is a local poetry project, we can extract "richer" requirement
# information, eg: development requirements etc.
if root_dir is not None:
path = root_dir
elif self._source_type == "directory" and self._source_url is not None:
path = Path(self._source_url)
else:
path = None
if path is not None:
poetry_package = self._get_poetry_package(path=path)
if poetry_package:
package.extras = poetry_package.extras
for dependency in poetry_package.requires:
package.add_dependency(dependency)
return package
seen_requirements = set()
for req in self.requires_dist or []:
try:
# Attempt to parse the PEP-508 requirement string
dependency = Dependency.create_from_pep_508(req, relative_to=root_dir)
except InvalidMarker:
# Invalid marker, We strip the markers hoping for the best
logger.warning(
"Stripping invalid marker (%s) found in %s-%s dependencies",
req,
package.name,
package.version,
)
req = req.split(";")[0]
dependency = Dependency.create_from_pep_508(req, relative_to=root_dir)
except InvalidRequirement:
# Unable to parse requirement so we skip it
logger.warning(
"Invalid requirement (%s) found in %s-%s dependencies, skipping",
req,
package.name,
package.version,
)
continue
if dependency.in_extras:
# this dependency is required by an extra package
for extra in dependency.in_extras:
if extra not in package.extras:
# this is the first time we encounter this extra for this
# package
package.extras[extra] = []
package.extras[extra].append(dependency)
req = dependency.to_pep_508(with_extras=True)
if req not in seen_requirements:
package.add_dependency(dependency)
seen_requirements.add(req)
return package
@classmethod
def _from_distribution(
cls, dist: pkginfo.BDist | pkginfo.SDist | pkginfo.Wheel
) -> PackageInfo:
"""
Helper method to parse package information from a `pkginfo.Distribution`
instance.
:param dist: The distribution instance to parse information from.
"""
requirements = None
if dist.requires_dist:
requirements = list(dist.requires_dist)
else:
requires = Path(dist.filename) / "requires.txt"
if requires.exists():
with requires.open(encoding="utf-8") as f:
requirements = parse_requires(f.read())
info = cls(
name=dist.name,
version=dist.version,
summary=dist.summary,
requires_dist=requirements,
requires_python=dist.requires_python,
)
info._source_type = "file"
info._source_url = Path(dist.filename).resolve().as_posix()
return info
@classmethod
def _from_sdist_file(cls, path: Path) -> PackageInfo:
"""
Helper method to parse package information from an sdist file. We attempt to
first inspect the file using `pkginfo.SDist`. If this does not provide us with
package requirements, we extract the source and handle it as a directory.
:param path: The sdist file to parse information from.
"""
info = None
try:
info = cls._from_distribution(pkginfo.SDist(str(path)))
except ValueError:
# Unable to determine dependencies
# We pass and go deeper
pass
else:
if info.requires_dist is not None:
# we successfully retrieved dependencies from sdist metadata
return info
# Still not dependencies found
# So, we unpack and introspect
suffix = path.suffix
zip = suffix == ".zip"
if suffix == ".bz2":
suffixes = path.suffixes
if len(suffixes) > 1 and suffixes[-2] == ".tar":
suffix = ".tar.bz2"
elif not zip:
suffix = ".tar.gz"
with temporary_directory() as tmp_str:
tmp = Path(tmp_str)
extractall(source=path, dest=tmp, zip=zip)
# a little bit of guess work to determine the directory we care about
elements = list(tmp.glob("*"))
if len(elements) == 1 and elements[0].is_dir():
sdist_dir = elements[0]
else:
sdist_dir = tmp / path.name.rstrip(suffix)
if not sdist_dir.is_dir():
sdist_dir = tmp
# now this is an unpacked directory we know how to deal with
new_info = cls.from_directory(path=sdist_dir)
if not info:
return new_info
return info.update(new_info)
@staticmethod
def has_setup_files(path: Path) -> bool:
return any((path / f).exists() for f in SetupReader.FILES)
@classmethod
def from_setup_files(cls, path: Path) -> PackageInfo:
"""
Mechanism to parse package information from a `setup.[py|cfg]` file. This uses
the implementation at `poetry.utils.setup_reader.SetupReader` in order to parse
the file. This is not reliable for complex setup files and should only attempted
as a fallback.
:param path: Path to `setup.py` file
"""
if not cls.has_setup_files(path):
raise PackageInfoError(
path, "No setup files (setup.py, setup.cfg) was found."
)
try:
result = SetupReader.read_from_directory(path)
except Exception as e:
raise PackageInfoError(path, e)
python_requires = result["python_requires"]
if python_requires is None:
python_requires = "*"
requires = "".join(dep + "\n" for dep in result["install_requires"])
if result["extras_require"]:
requires += "\n"
for extra_name, deps in result["extras_require"].items():
requires += f"[{extra_name}]\n"
for dep in deps:
requires += dep + "\n"
requires += "\n"
requirements = parse_requires(requires)
info = cls(
name=result.get("name"),
version=result.get("version"),
summary=result.get("description", ""),
requires_dist=requirements or None,
requires_python=python_requires,
)
if not (info.name and info.version) and not info.requires_dist:
# there is nothing useful here
raise PackageInfoError(
path,
"No core metadata (name, version, requires-dist) could be retrieved.",
)
return info
@staticmethod
def _find_dist_info(path: Path) -> Iterator[Path]:
"""
Discover all `*.*-info` directories in a given path.
:param path: Path to search.
"""
pattern = "**/*.*-info"
# Sometimes pathlib will fail on recursive symbolic links, so we need to work
# around it and use the glob module instead. Note that this does not happen with
# pathlib2 so it's safe to use it for Python < 3.4.
directories = glob.iglob(path.joinpath(pattern).as_posix(), recursive=True)
for d in directories:
yield Path(d)
@classmethod
def from_metadata(cls, path: Path) -> PackageInfo | None:
"""
Helper method to parse package information from an unpacked metadata directory.
:param path: The metadata directory to parse information from.
"""
if path.suffix in {".dist-info", ".egg-info"}:
directories = [path]
else:
directories = list(cls._find_dist_info(path=path))
dist: pkginfo.BDist | pkginfo.SDist | pkginfo.Wheel
for directory in directories:
try:
if directory.suffix == ".egg-info":
dist = pkginfo.UnpackedSDist(directory.as_posix())
elif directory.suffix == ".dist-info":
dist = pkginfo.Wheel(directory.as_posix())
else:
continue
break
except ValueError:
continue
else:
try:
# handle PKG-INFO in unpacked sdist root
dist = pkginfo.UnpackedSDist(path.as_posix())
except ValueError:
return None
return cls._from_distribution(dist=dist)
@classmethod
def from_package(cls, package: Package) -> PackageInfo:
"""
Helper method to inspect a `Package` object, in order to generate package info.
:param package: This must be a poetry package instance.
"""
requires = {dependency.to_pep_508() for dependency in package.requires}
for extra_requires in package.extras.values():
for dependency in extra_requires:
requires.add(dependency.to_pep_508())
return cls(
name=package.name,
version=str(package.version),
summary=package.description,
requires_dist=list(requires),
requires_python=package.python_versions,
files=package.files,
yanked=package.yanked_reason if package.yanked else False,
)
@staticmethod
def _get_poetry_package(path: Path) -> ProjectPackage | None:
# Note: we ignore any setup.py file at this step
# TODO: add support for handling non-poetry PEP-517 builds
if PyProjectTOML(path.joinpath("pyproject.toml")).is_poetry_project():
with contextlib.suppress(RuntimeError):
return Factory().create_poetry(path).package
return None
@classmethod
def from_directory(cls, path: Path, disable_build: bool = False) -> PackageInfo:
"""
Generate package information from a package source directory. If `disable_build`
is not `True` and introspection of all available metadata fails, the package is
attempted to be built in an isolated environment so as to generate required
metadata.
:param path: Path to generate package information from.
:param disable_build: If not `True` and setup reader fails, PEP 517 isolated
build is attempted in order to gather metadata.
"""
project_package = cls._get_poetry_package(path)
info: PackageInfo | None
if project_package:
info = cls.from_package(project_package)
else:
info = cls.from_metadata(path)
if not info or info.requires_dist is None:
try:
if disable_build:
info = cls.from_setup_files(path)
else:
info = get_pep517_metadata(path)
except PackageInfoError:
if not info:
raise
# we discovered PkgInfo but no requirements were listed
info._source_type = "directory"
info._source_url = path.as_posix()
return info
@classmethod
def from_sdist(cls, path: Path) -> PackageInfo:
"""
Gather package information from an sdist file, packed or unpacked.
:param path: Path to an sdist file or unpacked directory.
"""
if path.is_file():
return cls._from_sdist_file(path=path)
# if we get here then it is neither an sdist instance nor a file
# so, we assume this is an directory
return cls.from_directory(path=path)
@classmethod
def from_wheel(cls, path: Path) -> PackageInfo:
"""
Gather package information from a wheel.
:param path: Path to wheel.
"""
try:
return cls._from_distribution(pkginfo.Wheel(str(path)))
except ValueError:
return PackageInfo()
@classmethod
def from_bdist(cls, path: Path) -> PackageInfo:
"""
Gather package information from a bdist (wheel etc.).
:param path: Path to bdist.
"""
if isinstance(path, (pkginfo.BDist, pkginfo.Wheel)):
cls._from_distribution(dist=path)
if path.suffix == ".whl":
return cls.from_wheel(path=path)
try:
return cls._from_distribution(pkginfo.BDist(str(path)))
except ValueError as e:
raise PackageInfoError(path, e)
@classmethod
def from_path(cls, path: Path) -> PackageInfo:
"""
Gather package information from a given path (bdist, sdist, directory).
:param path: Path to inspect.
"""
try:
return cls.from_bdist(path=path)
except PackageInfoError:
return cls.from_sdist(path=path)
@functools.lru_cache(maxsize=None)
def get_pep517_metadata(path: Path) -> PackageInfo:
"""
Helper method to use PEP-517 library to build and read package metadata.
:param path: Path to package source to build and read metadata for.
"""
info = None
with contextlib.suppress(PackageInfoError):
info = PackageInfo.from_setup_files(path)
if all([info.version, info.name, info.requires_dist]):
return info
with ephemeral_environment(
flags={"no-pip": False, "setuptools": "bundle", "wheel": "bundle"}
) as venv:
# TODO: cache PEP 517 build environment corresponding to each project venv
dest_dir = venv.path.parent / "dist"
dest_dir.mkdir()
pep517_meta_build_script = PEP517_META_BUILD.format(
source=path.as_posix(), dest=dest_dir.as_posix()
)
try:
venv.run_pip(
"install",
"--disable-pip-version-check",
"--ignore-installed",
"--no-input",
*PEP517_META_BUILD_DEPS,
)
venv.run_python_script(pep517_meta_build_script)
info = PackageInfo.from_metadata(dest_dir)
except EnvCommandError as e:
# something went wrong while attempting pep517 metadata build
# fallback to egg_info if setup.py available
logger.debug("PEP517 build failed: %s", e)
setup_py = path / "setup.py"
if not setup_py.exists():
raise PackageInfoError(
path,
e,
"No fallback setup.py file was found to generate egg_info.",
)
cwd = Path.cwd()
os.chdir(path)
try:
venv.run("python", "setup.py", "egg_info")
info = PackageInfo.from_metadata(path)
except EnvCommandError as fbe:
raise PackageInfoError(
path, e, "Fallback egg_info generation failed.", fbe
)
finally:
os.chdir(cwd)
if info:
logger.debug("Falling back to parsed setup.py file for %s", path)
return info
# if we reach here, everything has failed and all hope is lost
raise PackageInfoError(path, "Exhausted all core metadata sources.")