diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..55dbe85 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b8394ac..5d02cb9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,53 +9,65 @@ on: - v* pull_request: +env: + UV_SYSTEM_PYTHON: 1 + jobs: - dowsing: + test: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: ["3.6", "3.7", "3.8", "3.9"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] os: [macOS-latest, ubuntu-latest, windows-latest] steps: - name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Set Up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - uses: astral-sh/setup-uv@v3 - name: Install run: | - python -m pip install --upgrade pip - make setup - pip install -U . + uv pip install -e .[test] - name: Test run: make test - name: Lint - run: make lint - if: ${{ matrix.python-version != '3.9' }} - - check-deps: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - python-version: ["3.6", "3.7", "3.8", "3.9"] - os: [ubuntu-latest] + run: | + uv pip install -e .[test,dev] + make lint + if: ${{ matrix.python-version != '3.9' && matrix.python-version != '3.8' }} + build: + needs: test + runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v1 - - name: Set Up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.12" + - uses: astral-sh/setup-uv@v3 - name: Install - run: | - python -m pip install --upgrade pip - pip install 'pessimist>=0.8.0' - echo 'importall>=0.2.1' > importall.txt - - name: Check Deps - run: | - python -m pessimist --requirements=importall.txt --fast -c 'importall --root=. --exclude=tests,_demo_pep517.py,check_source_mapping.py dowsing' . + run: uv pip install build + - name: Build + run: python -m build + - name: Upload + uses: actions/upload-artifact@v3 + with: + name: sdist + path: dist + + publish: + needs: build + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v3 + with: + name: sdist + path: dist + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b7ccc0..8ff4ed4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +## v0.9.0b3 + +* Support PEP 639 style metadata (#76) +* Support more `setup.py` assignments (#57) +* 3.12 compat (depends on setuptools) +* Fix tests to work on modern Python + +## v0.9.0b2 + +* `source_mapping` bugfixes + * `packages` being an empty string (#20) + * `py_modules` containing dots (#22) + * Flit modules instead of packages (#24) + * `setup.py` parsing addition operator (#25) + ## v0.9.0b1 * Includes package data in `source_mapping` all the time. diff --git a/Makefile b/Makefile index cfd7927..da69f1f 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PYTHON?=python +PYTHON?=python3 SOURCES=dowsing setup.py .PHONY: venv @@ -12,7 +12,7 @@ venv: .PHONY: setup setup: - python -m pip install -U -r requirements.txt -r requirements-dev.txt + python -m pip install -Ue .[dev,test] .PHONY: test test: @@ -21,13 +21,11 @@ test: .PHONY: format format: - python -m usort format $(SOURCES) - python -m black $(SOURCES) + python -m ufmt format $(SOURCES) .PHONY: lint lint: - python -m usort check $(SOURCES) - python -m black --check $(SOURCES) + python -m ufmt check $(SOURCES) python -m flake8 $(SOURCES) mypy --strict dowsing diff --git a/dowsing/_demo_pep517.py b/dowsing/_demo_pep517.py index 967d9c7..e66cb8b 100644 --- a/dowsing/_demo_pep517.py +++ b/dowsing/_demo_pep517.py @@ -1,6 +1,7 @@ """ For testing, dump the requirements that we find using the pep517 project. """ + import json import sys @@ -19,12 +20,12 @@ def main(path: str) -> None: d = {} with BuildEnvironment() as env: env.pip_install(requires) - d[ - "get_requires_for_build_sdist" - ] = requires + hooks.get_requires_for_build_sdist(None) - d[ - "get_requires_for_build_wheel" - ] = requires + hooks.get_requires_for_build_wheel(None) + d["get_requires_for_build_sdist"] = ( + requires + hooks.get_requires_for_build_sdist(None) + ) + d["get_requires_for_build_wheel"] = ( + requires + hooks.get_requires_for_build_wheel(None) + ) print(json.dumps(d)) diff --git a/dowsing/check_source_mapping.py b/dowsing/check_source_mapping.py index 9d0c292..efe38f3 100644 --- a/dowsing/check_source_mapping.py +++ b/dowsing/check_source_mapping.py @@ -6,7 +6,7 @@ from honesty.archive import extract_and_get_names from honesty.cache import Cache from honesty.cmdline import select_versions, wrap_async -from honesty.releases import FileType, async_parse_index +from honesty.releases import async_parse_index, FileType from moreorless.click import echo_color_unified_diff from dowsing.pep517 import get_metadata @@ -70,7 +70,9 @@ async def main(packages: List[str]) -> None: ) md_blob = "".join(sorted(f"{f}\n" for f in metadata.source_mapping.keys())) - if md_blob == wheel_blob: + if metadata.source_mapping == {}: + print(f"{package_name}: empty dict") + elif md_blob == wheel_blob: print(f"{package_name}: ok") elif md_blob in ("", "?.py\n"): print(f"{package_name}: COMPLETELY MISSING") diff --git a/dowsing/flit.py b/dowsing/flit.py index 45c6a93..60907d0 100644 --- a/dowsing/flit.py +++ b/dowsing/flit.py @@ -4,10 +4,11 @@ import tomlkit from setuptools import find_packages -from .types import BaseReader, Distribution +from .pep621 import Pep621Reader +from .types import Distribution -class FlitReader(BaseReader): +class FlitReader(Pep621Reader): def __init__(self, path: Path): self.path = path @@ -21,23 +22,30 @@ def get_metadata(self) -> Distribution: pyproject = self.path / "pyproject.toml" doc = tomlkit.parse(pyproject.read_text()) - d = Distribution() - d.metadata_version = "2.1" - d.project_urls = {} - d.entry_points = {} + d = self.get_pep621_metadata() + d.entry_points = dict(d.entry_points) or {} + d.project_urls = list(d.project_urls) - for k, v in doc["tool"]["flit"]["metadata"].items(): + assert isinstance(d.project_urls, list) + + flit = doc.get("tool", {}).get("flit", {}) + metadata = flit.get("metadata", {}) + for k, v in metadata.items(): # TODO description-file -> long_description # TODO home-page -> urls # TODO requires -> requires_dist # TODO tool.flit.metadata.urls if k == "home-page": - d.project_urls["Homepage"] = v + d.project_urls.append("Homepage={v}") continue elif k == "module": - k = "packages" - v = find_packages(self.path.as_posix(), include=(f"{v}.*")) - d.packages_dict = {i: i.replace(".", "/") for i in v} + if (self.path / f"{v}.py").exists(): + k = "py_modules" + v = [v] + else: + k = "packages" + v = find_packages(self.path.as_posix(), include=(f"{v}.*")) + d.packages_dict = {i: i.replace(".", "/") for i in v} elif k == "description-file": k = "description" v = f"file: {v}" @@ -48,10 +56,10 @@ def get_metadata(self) -> Distribution: if k2 in d: setattr(d, k2, v) - for k, v in doc["tool"]["flit"]["metadata"].get("urls", {}).items(): - d.project_urls[k] = v + for k, v in metadata.get("urls", {}).items(): + d.project_urls.append(f"{k}={v}") - for k, v in doc["tool"]["flit"].get("scripts", {}).items(): + for k, v in flit.get("scripts", {}).items(): d.entry_points[k] = v # TODO extras-require @@ -68,8 +76,7 @@ def _get_requires(self) -> Sequence[str]: https://github.com/takluyver/flit/issues/141 """ - pyproject = self.path / "pyproject.toml" - doc = tomlkit.parse(pyproject.read_text()) - seq = doc["tool"]["flit"]["metadata"].get("requires", ()) + dist = self.get_metadata() + seq = dist.requires_dist assert isinstance(seq, (list, tuple)) return seq diff --git a/dowsing/maturin.py b/dowsing/maturin.py index 4576b34..299ea51 100644 --- a/dowsing/maturin.py +++ b/dowsing/maturin.py @@ -25,7 +25,8 @@ def get_metadata(self) -> Distribution: cargo = self.path / "Cargo.toml" doc = tomlkit.parse(cargo.read_text()) - for k, v in doc["package"].items(): + package = doc.get("package", {}) + for k, v in package.items(): if k == "name": d.name = v elif k == "version": @@ -39,7 +40,8 @@ def get_metadata(self) -> Distribution: # homepage # readme (filename) - for k, v in doc["package"]["metadata"]["maturin"].items(): + maturin = package.get("metadata", {}).get("maturin", {}) + for k, v in maturin.items(): if k == "requires-python": d.requires_python = v elif k == "classifier": diff --git a/dowsing/pep517.py b/dowsing/pep517.py index 26a7e99..57ee59a 100644 --- a/dowsing/pep517.py +++ b/dowsing/pep517.py @@ -11,6 +11,7 @@ KNOWN_BACKENDS: Dict[str, str] = { "setuptools.build_meta:__legacy__": "dowsing.setuptools:SetuptoolsReader", "setuptools.build_meta": "dowsing.setuptools:SetuptoolsReader", + "jupyter_packaging.build_api": "dowsing.setuptools:SetuptoolsReader", "flit_core.buildapi": "dowsing.flit:FlitReader", "flit.buildapi": "dowsing.flit:FlitReader", "maturin": "dowsing.maturin:MaturinReader", @@ -26,13 +27,14 @@ def get_backend(path: Path) -> Tuple[List[str], BaseReader]: requires: List[str] = [] if pyproject.exists(): doc = tomlkit.parse(pyproject.read_text()) - if "build-system" in doc: - # 1b. include any build-system requires - if "requires" in doc["build-system"]: - requires.extend(doc["build-system"]["requires"]) - if "build-backend" in doc["build-system"]: - backend = doc["build-system"]["build-backend"] - # TODO backend-path + table = doc.get("build-system", {}) + + # 1b. include any build-system requires + if "requires" in table: + requires.extend(table["requires"]) + if "build-backend" in table: + backend = table["build-backend"] + # TODO backend-path try: backend_path = KNOWN_BACKENDS[backend] @@ -74,10 +76,12 @@ def _default(obj: Any) -> Any: def main(path: Path) -> None: + metadata = get_metadata(path) d = { "get_requires_for_build_sdist": get_requires_for_build_sdist(path), "get_requires_for_build_wheel": get_requires_for_build_wheel(path), - "get_metadata": get_metadata(path).asdict(), + "get_metadata": metadata.asdict(), + "source_mapping": metadata.source_mapping, } print(json.dumps(d, default=_default)) diff --git a/dowsing/pep621.py b/dowsing/pep621.py new file mode 100644 index 0000000..47697f8 --- /dev/null +++ b/dowsing/pep621.py @@ -0,0 +1,53 @@ +import tomlkit +from setuptools import find_packages + +from .types import BaseReader, Distribution + + +class Pep621Reader(BaseReader): + def get_pep621_metadata(self) -> Distribution: + pyproject = self.path / "pyproject.toml" + doc = tomlkit.parse(pyproject.read_text()) + + d = Distribution() + d.metadata_version = "2.1" + d.project_urls = [] + d.entry_points = {} + d.requires_dist = [] + d.packages = [] + d.packages_dict = {} + + assert isinstance(d.project_urls, list) + + table = doc.get("project", None) + if table: + for k, v in table.items(): + if k == "name": + if (self.path / f"{v}.py").exists(): + d.py_modules = [v] + else: + d.packages = find_packages( + self.path.as_posix(), include=(f"{v}.*") + ) + d.packages_dict = {i: i.replace(".", "/") for i in d.packages} + elif k == "license": + if isinstance(v, str): + pass # PEP 639 proposes `license = "MIT"` style metadata + elif "text" in v: + v = v["text"] + elif "file" in v: + v = f"file: {v['file']}" + else: + raise ValueError("no known license field values") + elif k == "dependencies": + k = "requires_dist" + elif k == "optional-dependencies": + pass + elif k == "urls": + d.project_urls.extend([f"{x}={y}" for x, y in v.items()]) + + k2 = k.replace("-", "_") + if k2 in d: + setattr(d, k2, v) + + return d diff --git a/dowsing/poetry.py b/dowsing/poetry.py index 37b6157..f6dc977 100644 --- a/dowsing/poetry.py +++ b/dowsing/poetry.py @@ -36,15 +36,18 @@ def get_metadata(self) -> Distribution: d = Distribution() d.metadata_version = "2.1" - d.project_urls = {} + d.project_urls = [] d.entry_points = {} d.requires_dist = [] d.packages = [] d.packages_dict = {} - for k, v in doc["tool"]["poetry"].items(): + assert isinstance(d.project_urls, list) + + poetry = doc.get("tool", {}).get("poetry", {}) + for k, v in poetry.items(): if k in ("homepage", "repository", "documentation"): - d.project_urls[k] = v + d.project_urls.append(f"{k}={v}") elif k == "packages": # TODO improve and add tests; this works for tf2_utils and # poetry itself but include can be a glob and there are excludes @@ -64,16 +67,16 @@ def get_metadata(self) -> Distribution: d.packages_dict[p] = p.replace(".", "/") d.packages.append(p) - for k, v in doc["tool"]["poetry"].get("dependencies", {}).items(): + for k, v in poetry.get("dependencies", {}).items(): if k == "python": pass # TODO translate to requires_python else: d.requires_dist.append(k) # TODO something with version - for k, v in doc["tool"]["poetry"].get("urls", {}).items(): - d.project_urls[k] = v + for k, v in poetry.get("urls", {}).items(): + d.project_urls.append(f"{k}={v}") - for k, v in doc["tool"]["poetry"].get("scripts", {}).items(): + for k, v in poetry.get("scripts", {}).items(): d.entry_points[k] = v d.source_mapping = d._source_mapping(self.path) diff --git a/dowsing/setuptools/__init__.py b/dowsing/setuptools/__init__.py index 57bb7cb..ce8452e 100644 --- a/dowsing/setuptools/__init__.py +++ b/dowsing/setuptools/__init__.py @@ -43,11 +43,25 @@ def get_metadata(self) -> Distribution: if getattr(d2, k): setattr(d1, k, getattr(d2, k)) + # This is the bare minimum to get pbr projects to show as having any + # sources. I don't want to use pbr.util.cfg_to_args because it appears + # to import and run arbitrary code. + if d1.pbr or (d1.pbr__files__packages and not d1.packages): + where = "." + if d1.pbr__files__packages_root: + d1.package_dir = {"": d1.pbr__files__packages_root} + where = d1.pbr__files__packages_root + + if d1.pbr__files__packages: + d1.packages = d1.pbr__files__packages + else: + d1.packages = FindPackages(where, (), ("*",)) # type: ignore + # package_dir can both add and remove components, see docs # https://docs.python.org/2/distutils/setupscript.html#listing-whole-packages package_dir: Mapping[str, str] = d1.package_dir # If there was an error, we might have written "??" - if package_dir != "??": + if package_dir != "??": # type: ignore if not package_dir: package_dir = {"": "."} @@ -82,10 +96,13 @@ def mangle(package: str) -> str: d1.find_packages_include, ): d1.packages_dict[p] = mangle(p) - elif d1.packages != "??": - assert isinstance(d1.packages, (list, tuple)) + elif d1.packages not in ("??", "????"): + assert isinstance( + d1.packages, (list, tuple) + ), f"{d1.packages!r} is not a list/tuple" for p in d1.packages: - d1.packages_dict[p] = mangle(p) + if p: + d1.packages_dict[p] = mangle(p) d1.source_mapping = d1._source_mapping(self.path) return d1 diff --git a/dowsing/setuptools/setup_and_metadata.py b/dowsing/setuptools/setup_and_metadata.py index 9c44087..ff64c4c 100644 --- a/dowsing/setuptools/setup_and_metadata.py +++ b/dowsing/setuptools/setup_and_metadata.py @@ -41,14 +41,22 @@ ), ConfigField("author", SetupCfg("metadata", "author"), Metadata("Author")), ConfigField( - "author_email", SetupCfg("metadata", "author_email"), Metadata("Author-email"), + "author_email", + SetupCfg("metadata", "author_email"), + Metadata("Author-email"), + ), + ConfigField( + "license", + SetupCfg("metadata", "license"), + Metadata("License"), ), - ConfigField("license", SetupCfg("metadata", "license"), Metadata("License"),), # TODO licence (alternate spelling) # TODO license_file, license_files (setuptools-specific) ConfigField("url", SetupCfg("metadata", "url"), Metadata("Home-page")), ConfigField( - "description", SetupCfg("metadata", "description"), Metadata("Summary"), + "description", + SetupCfg("metadata", "description"), + Metadata("Summary"), ), ConfigField( "long_description", @@ -167,7 +175,7 @@ ConfigField( "packages", SetupCfg("options", "packages", writer_cls=ListCommaWriter), - sample_value=["a", "b"], + sample_value=["a"], ), ConfigField( "package_dir", @@ -222,4 +230,19 @@ SetupCfg("options.packages.find", "include", writer_cls=ListCommaWriter), sample_value=None, ), + ConfigField( + "pbr", + SetupCfg("--unused--", "--unused--"), + sample_value=None, + ), + ConfigField( + "pbr__files__packages_root", + SetupCfg("files", "packages_root"), + sample_value=None, + ), + ConfigField( + "pbr__files__packages", + SetupCfg("files", "packages", writer_cls=ListCommaWriter), + sample_value=None, + ), ] diff --git a/dowsing/setuptools/setup_py_parsing.py b/dowsing/setuptools/setup_py_parsing.py index 8d9819b..8774e6d 100644 --- a/dowsing/setuptools/setup_py_parsing.py +++ b/dowsing/setuptools/setup_py_parsing.py @@ -8,7 +8,12 @@ from typing import Any, Dict, Optional import libcst as cst -from libcst.metadata import ParentNodeProvider, QualifiedNameProvider, ScopeProvider +from libcst.metadata import ( + ParentNodeProvider, + PositionProvider, + QualifiedNameProvider, + ScopeProvider, +) from ..types import Distribution from .setup_and_metadata import SETUP_ARGS @@ -92,7 +97,7 @@ def __init__(self, filename: str) -> None: class SetupCallTransformer(cst.CSTTransformer): - METADATA_DEPENDENCIES = (ScopeProvider, ParentNodeProvider, QualifiedNameProvider) # type: ignore + METADATA_DEPENDENCIES = (ScopeProvider, ParentNodeProvider, QualifiedNameProvider) def __init__( self, @@ -124,7 +129,12 @@ def leave_Call( class SetupCallAnalyzer(cst.CSTVisitor): - METADATA_DEPENDENCIES = (ScopeProvider, ParentNodeProvider, QualifiedNameProvider) # type: ignore + METADATA_DEPENDENCIES = ( + ScopeProvider, + ParentNodeProvider, + QualifiedNameProvider, + PositionProvider, + ) # TODO names resulting from other than 'from setuptools import setup' # TODO wrapper funcs that modify args @@ -141,7 +151,13 @@ def visit_Call(self, node: cst.Call) -> Optional[bool]: # TODO sometimes there is more than one setup call, we might # prioritize/merge... if any( - q.name in ("setuptools.setup", "distutils.core.setup", "setup3lib") + q.name + in ( + "setuptools.setup", + "distutils.core.setup", + "setup3lib", + "skbuild.setup", + ) for q in names ): self.found_setup = True @@ -172,30 +188,50 @@ def visit_Call(self, node: cst.Call) -> Optional[bool]: BOOL_NAMES = {"True": True, "False": False, "None": None} PRETEND_ARGV = ["setup.py", "bdist_wheel"] - def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: + def evaluate_in_scope( + self, item: cst.CSTNode, scope: Any, target_line: int = 0 + ) -> Any: qnames = self.get_metadata(QualifiedNameProvider, item) if isinstance(item, cst.SimpleString): return item.evaluated_value - # TODO int/float/etc + elif isinstance(item, (cst.Integer, cst.Float)): + return int(item.value) elif isinstance(item, cst.Name) and item.value in self.BOOL_NAMES: return self.BOOL_NAMES[item.value] elif isinstance(item, cst.Name): name = item.value assignments = scope[name] - for a in assignments: - # TODO: Only assignments "before" this node matter if in the - # same scope; really if we had a call graph and walked the other - # way, we could have a better idea of what has already happened. + assignment_nodes = sorted( + ( + (self.get_metadata(PositionProvider, a.node).start.line, a.node) + for a in assignments + if a.node + ), + reverse=True, + ) + # Walk assignments from bottom to top, evaluating them recursively. + for lineno, node in assignment_nodes: + + # When recursing, only look at assignments above the "target line". + if target_line and lineno >= target_line: + continue # Assign( # targets=[AssignTarget(target=Name(value="v"))], # value=SimpleString(value="'x'"), # ) + # + # AugAssign( + # target=Name(value="v"), + # operator=AddAssign(...), + # value=SimpleString(value="'x'"), + # ) + # # TODO or an import... # TODO builtins have BuiltinAssignment + try: - node = a.node if node: parent = self.get_metadata(ParentNodeProvider, node) if parent: @@ -205,25 +241,37 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: else: raise KeyError except (KeyError, AttributeError): - return "??" - - # This presumes a single assignment - if not isinstance(gp, cst.Assign) or len(gp.targets) != 1: - return "??" # TooComplicated(repr(gp)) + continue try: scope = self.get_metadata(ScopeProvider, gp) except KeyError: # module scope isn't in the dict - return "??" + continue - return self.evaluate_in_scope(gp.value, scope) + # This presumes a single assignment + if isinstance(gp, cst.Assign) and len(gp.targets) == 1: + result = self.evaluate_in_scope(gp.value, scope, lineno) + elif isinstance(parent, cst.AugAssign): + result = self.evaluate_in_scope(parent, scope, lineno) + else: + # too complicated? + continue + + # keep trying assignments until we get something other than ?? + if result != "??": + return result + + # give up + return "??" elif isinstance(item, (cst.Tuple, cst.List)): lst = [] for el in item.elements: lst.append( self.evaluate_in_scope( - el.value, self.get_metadata(ScopeProvider, el) + el.value, + self.get_metadata(ScopeProvider, el), + target_line, ) ) if isinstance(item, cst.Tuple): @@ -241,10 +289,10 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: for arg in item.args: if isinstance(arg.keyword, cst.Name): args[names.index(arg.keyword.value)] = self.evaluate_in_scope( - arg.value, scope + arg.value, scope, target_line ) else: - args[i] = self.evaluate_in_scope(arg.value, scope) + args[i] = self.evaluate_in_scope(arg.value, scope, target_line) i += 1 # TODO clear ones that are still default @@ -257,7 +305,9 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: d = {} for arg in item.args: if isinstance(arg.keyword, cst.Name): - d[arg.keyword.value] = self.evaluate_in_scope(arg.value, scope) + d[arg.keyword.value] = self.evaluate_in_scope( + arg.value, scope, target_line + ) # TODO something with **kwargs return d elif isinstance(item, cst.Dict): @@ -265,22 +315,55 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: for el2 in item.elements: if isinstance(el2, cst.DictElement): d[self.evaluate_in_scope(el2.key, scope)] = self.evaluate_in_scope( - el2.value, scope + el2.value, scope, target_line ) return d elif isinstance(item, cst.Subscript): - lhs = self.evaluate_in_scope(item.value, scope) + lhs = self.evaluate_in_scope(item.value, scope, target_line) if isinstance(lhs, str): # A "??" entry, propagate return "??" # TODO: Figure out why this is Sequence if isinstance(item.slice[0].slice, cst.Index): - rhs = self.evaluate_in_scope(item.slice[0].slice.value, scope) - return lhs.get(rhs, "??") + rhs = self.evaluate_in_scope( + item.slice[0].slice.value, scope, target_line + ) + try: + if isinstance(lhs, dict): + return lhs.get(rhs, "??") + else: + return lhs[rhs] + except Exception: + return "??" + else: # LOG.warning(f"Omit2 {type(item.slice[0].slice)!r}") return "??" + elif isinstance(item, cst.BinaryOperation): + lhs = self.evaluate_in_scope(item.left, scope, target_line) + rhs = self.evaluate_in_scope(item.right, scope, target_line) + if lhs == "??" or rhs == "??": + return "??" + if isinstance(item.operator, cst.Add): + try: + return lhs + rhs + except Exception: + return "??" + else: + return "??" + elif isinstance(item, cst.AugAssign): + lhs = self.evaluate_in_scope(item.target, scope, target_line) + rhs = self.evaluate_in_scope(item.value, scope, target_line) + if lhs == "??" or rhs == "??": + return "??" + if isinstance(item.operator, cst.AddAssign): + try: + return lhs + rhs + except Exception: + return "??" + else: + return "??" else: # LOG.warning(f"Omit1 {type(item)!r}") return "??" diff --git a/dowsing/tests/__init__.py b/dowsing/tests/__init__.py index 7eb139d..80335e6 100644 --- a/dowsing/tests/__init__.py +++ b/dowsing/tests/__init__.py @@ -2,6 +2,7 @@ from .flit import FlitReaderTest from .maturin import MaturinReaderTest from .pep517 import Pep517Test +from .pep621 import Pep621ReaderTest from .poetry import PoetryReaderTest from .setuptools import SetuptoolsReaderTest from .setuptools_metadata import SetupArgsTest @@ -12,6 +13,7 @@ "FlitReaderTest", "MaturinReaderTest", "Pep517Test", + "Pep621ReaderTest", "PoetryReaderTest", "SetuptoolsReaderTest", "WriterTest", diff --git a/dowsing/tests/flit.py b/dowsing/tests/flit.py index ae15ad6..98a15b0 100644 --- a/dowsing/tests/flit.py +++ b/dowsing/tests/flit.py @@ -23,8 +23,8 @@ def test_simplest(self) -> None: # handle missing metadata appropriately. r = FlitReader(dp) - self.assertEqual((), r.get_requires_for_build_sdist()) - self.assertEqual((), r.get_requires_for_build_wheel()) + self.assertEqual([], r.get_requires_for_build_sdist()) + self.assertEqual([], r.get_requires_for_build_wheel()) md = r.get_metadata() self.assertEqual("Name", md.name) @@ -65,7 +65,48 @@ def test_normal(self) -> None: "packages": ["foo", "foo.tests"], "packages_dict": {"foo": "foo", "foo.tests": "foo/tests"}, "requires_dist": ["abc", "def"], - "project_urls": {"Foo": "https://"}, + "project_urls": ["Foo=https://"], + }, + md.asdict(), + ) + + def test_pep621(self) -> None: + with volatile.dir() as d: + dp = Path(d) + (dp / "pyproject.toml").write_text( + """\ +[build-system] +requires = ["flit_core >=2,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "foo" +dependencies = ["abc", "def"] + +[project.urls] +Foo = "https://" +""" + ) + (dp / "foo").mkdir() + (dp / "foo" / "tests").mkdir() + (dp / "foo" / "__init__.py").write_text("") + (dp / "foo" / "tests" / "__init__.py").write_text("") + + r = FlitReader(dp) + # Notably these do not include flit itself; that's handled by + # dowsing.pep517 + self.assertEqual(["abc", "def"], r.get_requires_for_build_sdist()) + self.assertEqual(["abc", "def"], r.get_requires_for_build_wheel()) + md = r.get_metadata() + self.assertEqual("foo", md.name) + self.assertEqual( + { + "metadata_version": "2.1", + "name": "foo", + "packages": ["foo", "foo.tests"], + "packages_dict": {"foo": "foo", "foo.tests": "foo/tests"}, + "requires_dist": ["abc", "def"], + "project_urls": ["Foo=https://"], }, md.asdict(), ) diff --git a/dowsing/tests/pep621.py b/dowsing/tests/pep621.py new file mode 100644 index 0000000..408f3ae --- /dev/null +++ b/dowsing/tests/pep621.py @@ -0,0 +1,73 @@ +import unittest +from pathlib import Path + +import volatile + +from ..pep621 import Pep621Reader + + +class Pep621ReaderTest(unittest.TestCase): + def test_simplest(self) -> None: + with volatile.dir() as d: + dp = Path(d) + (dp / "pyproject.toml").write_text( + """\ +[project] +name = "Name" +""" + ) + + r = Pep621Reader(dp) + md = r.get_pep621_metadata() + self.assertEqual("Name", md.name) + + def test_normal(self) -> None: + with volatile.dir() as d: + dp = Path(d) + (dp / "pyproject.toml").write_text( + """\ +[project] +name = "foo" +dependencies = ["abc", "def"] +license = {text = "MIT"} + +[project.urls] +Foo = "https://" +""" + ) + (dp / "foo").mkdir() + (dp / "foo" / "tests").mkdir() + (dp / "foo" / "__init__.py").write_text("") + (dp / "foo" / "tests" / "__init__.py").write_text("") + + r = Pep621Reader(dp) + md = r.get_pep621_metadata() + self.assertEqual("foo", md.name) + self.assertEqual( + { + "metadata_version": "2.1", + "name": "foo", + "license": "MIT", + "packages": ["foo", "foo.tests"], + "packages_dict": {"foo": "foo", "foo.tests": "foo/tests"}, + "requires_dist": ["abc", "def"], + "project_urls": ["Foo=https://"], + }, + md.asdict(), + ) + + def test_pep639(self) -> None: + with volatile.dir() as d: + dp = Path(d) + (dp / "pyproject.toml").write_text( + """\ +[project] +name = "Name" +license = "MIT" +""" + ) + + r = Pep621Reader(dp) + md = r.get_pep621_metadata() + self.assertEqual("Name", md.name) + self.assertEqual("MIT", md.license) diff --git a/dowsing/tests/poetry.py b/dowsing/tests/poetry.py index ae8b2d4..f8a1f9e 100644 --- a/dowsing/tests/poetry.py +++ b/dowsing/tests/poetry.py @@ -41,10 +41,10 @@ def test_basic(self) -> None: self.assertEqual("1.5.2", md.version) self.assertEqual("BSD-3-Clause", md.license) self.assertEqual( - { - "homepage": "http://example.com", - "Bug Tracker": "https://github.com/python-poetry/poetry/issues", - }, + [ + "homepage=http://example.com", + "Bug Tracker=https://github.com/python-poetry/poetry/issues", + ], md.project_urls, ) self.assertEqual(["Not a real classifier"], md.classifiers) diff --git a/dowsing/tests/setuptools.py b/dowsing/tests/setuptools.py index 5295975..88840a7 100644 --- a/dowsing/tests/setuptools.py +++ b/dowsing/tests/setuptools.py @@ -1,5 +1,6 @@ import unittest from pathlib import Path +from typing import Dict, Optional import volatile @@ -63,10 +64,18 @@ def test_setup_py(self) -> None: ("setuptools", "wheel", "def"), r.get_requires_for_build_wheel() ) - def _read(self, data: str, src_dir: str = ".") -> Distribution: + def _read( + self, + data: str, + src_dir: str = ".", + extra_files: Optional[Dict[str, str]] = None, + ) -> Distribution: with volatile.dir() as d: sp = Path(d, "setup.py") sp.write_text(data) + if extra_files: + for k, v in extra_files.items(): + Path(d, k).write_text(v) Path(d, src_dir, "pkg").mkdir(parents=True) Path(d, src_dir, "pkg", "__init__.py").touch() Path(d, src_dir, "pkg", "sub").mkdir() @@ -232,3 +241,179 @@ def test_invalid_packages(self) -> None: ) # TODO wish this were None self.assertEqual(d.source_mapping, {}) + + def test_pbr_properly_enabled(self) -> None: + d = self._read( + """\ +from setuptools import setup + +setup( + setup_requires=['pbr>=1.9', 'setuptools>=17.1'], + pbr=True, +)""", + extra_files={ + "setup.cfg": """\ +[metadata] +name = pbr +author = OpenStack Foundation + +[files] +packages = + pkg +""" + }, + ) + self.assertEqual( + d.source_mapping, + { + "pkg/__init__.py": "pkg/__init__.py", + "pkg/sub/__init__.py": "pkg/sub/__init__.py", + "pkg/tests/__init__.py": "pkg/tests/__init__.py", + }, + ) + + def test_pbr_properly_enabled_src(self) -> None: + d = self._read( + """\ +from setuptools import setup + +setup( + setup_requires=['pbr>=1.9', 'setuptools>=17.1'], + pbr=True, +)""", + src_dir="src", + extra_files={ + "setup.cfg": """\ +[metadata] +name = pbr +author = OpenStack Foundation + +[files] +packages = + pkg +packages_root = src +""" + }, + ) + self.assertEqual( + d.source_mapping, + { + "pkg/__init__.py": "src/pkg/__init__.py", + "pkg/sub/__init__.py": "src/pkg/sub/__init__.py", + "pkg/tests/__init__.py": "src/pkg/tests/__init__.py", + }, + ) + + def test_pbr_improperly_enabled(self) -> None: + # pbr itself is something like this. + d = self._read( + """\ +from setuptools import setup + +setup()""", + extra_files={ + "setup.cfg": """\ +[metadata] +name = pbr +author = OpenStack Foundation + +[files] +packages = + pkg +""" + }, + ) + self.assertEqual( + d.source_mapping, + { + "pkg/__init__.py": "pkg/__init__.py", + "pkg/sub/__init__.py": "pkg/sub/__init__.py", + "pkg/tests/__init__.py": "pkg/tests/__init__.py", + }, + ) + + def test_add_items(self) -> None: + d = self._read( + """\ +from setuptools import setup +a = "aaaa" +p = ["a", "b", "c"] +setup(name = a + "1111", packages=[] + p, classifiers=a + p) + """ + ) + self.assertEqual(d.name, "aaaa1111") + self.assertEqual(d.packages, ["a", "b", "c"]) + self.assertEqual(d.classifiers, "??") + + def test_self_reference_assignments(self) -> None: + d = self._read( + """\ +from setuptools import setup + +version = "base" +name = "foo" +name += "bar" +version = version + ".suffix" + +classifiers = [ + "123", + "abc", +] + +if True: + classifiers = classifiers + ["xyz"] + +setup( + name=name, + version=version, + classifiers=classifiers, +) + """ + ) + self.assertEqual(d.name, "foobar") + self.assertEqual(d.version, "base.suffix") + self.assertSequenceEqual(d.classifiers, ["123", "abc", "xyz"]) + + def test_circular_references(self) -> None: + d = self._read( + """\ +from setuptools import setup + +name = "foo" + +foo = bar +bar = version +version = foo + +classifiers = classifiers + +setup( + name=name, + version=version, +) + """ + ) + self.assertEqual(d.name, "foo") + self.assertEqual(d.version, "??") + self.assertEqual(d.classifiers, ()) + + def test_redefines_builtin(self) -> None: + d = self._read( + """\ +import setuptools +with open("CREDITS.txt", "r", encoding="utf-8") as fp: + credits = fp.read() + +long_desc = "a" + credits + "b" +name = "foo" + +kwargs = dict( + long_description = long_desc, + name = name, +) + +setuptools.setup(**kwargs) +""" + ) + self.assertEqual(d.name, "foo") + self.assertEqual(d.description, "??") diff --git a/dowsing/tests/setuptools_metadata.py b/dowsing/tests/setuptools_metadata.py index d1ac9b5..3e3c0ac 100644 --- a/dowsing/tests/setuptools_metadata.py +++ b/dowsing/tests/setuptools_metadata.py @@ -31,20 +31,20 @@ def egg_info(files: Dict[str, str]) -> Tuple[Message, Distribution]: os.chdir(d) sys.stdout = io.StringIO() - dist = run_setup(f"setup.py", ["egg_info"]) + dist = run_setup("setup.py", ["egg_info"]) finally: os.chdir(cwd) sys.stdout = stdout sources = list(Path(d).rglob("PKG-INFO")) - assert len(sources) == 1 + assert len(sources) == 1, sources with open(sources[0]) as f: parser = email.parser.Parser() info = parser.parse(f) reader = SetuptoolsReader(Path(d)) - dist = reader.get_metadata() - return info, dist + dist = reader.get_metadata() # type: ignore[assignment] + return info, dist # type: ignore[return-value] # These tests do not increase coverage, and just verify that we have the right @@ -63,7 +63,6 @@ def test_arg_mapping(self) -> None: "setup.py": "from setuptools import setup\n" f"setup({field.keyword}={foo!r})\n", "a/__init__.py": "", - "b/__init__.py": "", } ) @@ -74,25 +73,33 @@ def test_arg_mapping(self) -> None: f"{field.cfg.key} = {cfg_format_foo}\n", "setup.py": "from setuptools import setup\n" "setup()\n", "a/__init__.py": "", - "b/__init__.py": "", } ) name = field.get_distribution_key() self.assertNotEqual( - getattr(setup_py_dist, name), None, + getattr(setup_py_dist, name), + None, ) self.assertEqual( - foo, getattr(setup_py_dist, name), + foo, + getattr(setup_py_dist, name), ) self.assertEqual( - foo, getattr(setup_cfg_dist, name), + foo, + getattr(setup_cfg_dist, name), ) if field.metadata: a = setup_py_info.get_all(field.metadata.key) b = setup_cfg_info.get_all(field.metadata.key) + # setuptools>=57 writes long_description to the body/payload + # of PKG-INFO, and skips the description field entirely. + if field.keyword == "long_description" and a is None: + a = setup_py_info.get_payload() # type: ignore[assignment] + b = setup_cfg_info.get_payload() # type: ignore[assignment] + # install_requires gets written out to *.egg-info/requires.txt # instead if field.keyword != "install_requires": diff --git a/dowsing/tests/setuptools_types.py b/dowsing/tests/setuptools_types.py index b73cf8d..dc51813 100644 --- a/dowsing/tests/setuptools_types.py +++ b/dowsing/tests/setuptools_types.py @@ -19,7 +19,10 @@ class WriterTest(unittest.TestCase): @parameterized.expand( # type: ignore - [(False,), (True,),] + [ + (False,), + (True,), + ] ) def test_bool_writer(self, arg: bool) -> None: c = ConfigFile() @@ -32,7 +35,10 @@ def test_bool_writer(self, arg: bool) -> None: self.assertEqual(str(arg).lower(), rcp["a"]["b"]) @parameterized.expand( # type: ignore - [("hello",), ("a\nb\nc",),] + [ + ("hello",), + ("a\nb\nc",), + ] ) def test_str_writer(self, arg: str) -> None: c = ConfigFile() diff --git a/dowsing/types.py b/dowsing/types.py index 349c456..7c93096 100644 --- a/dowsing/types.py +++ b/dowsing/types.py @@ -37,9 +37,8 @@ def get_metadata(self) -> "Distribution": DEFAULT_EMPTY_DICT: Mapping[str, Any] = MappingProxyType({}) -# TODO: pkginfo isn't typed, and is doing to require a yak-shave to send a PR -# since it's on launchpad. -class Distribution(pkginfo.distribution.Distribution): # type: ignore + +class Distribution(pkginfo.distribution.Distribution): # These are not actually part of the metadata, see PEP 566 setup_requires: Sequence[str] = () tests_require: Sequence[str] = () @@ -60,11 +59,16 @@ class Distribution(pkginfo.distribution.Distribution): # type: ignore find_packages_exclude: Sequence[str] = () find_packages_include: Sequence[str] = ("*",) source_mapping: Optional[Mapping[str, str]] = None + pbr: Optional[bool] = None + pbr__files__packages_root: Optional[str] = None + pbr__files__packages: Optional[str] = None + provides_extra: Optional[Sequence[str]] = () def _getHeaderAttrs(self) -> Sequence[Tuple[str, str, bool]]: # Until I invent a metadata version to include this, do so # unconditionally. - return tuple(super()._getHeaderAttrs()) + ( + # Stubs are wrong, this does too exist. + return tuple(super()._getHeaderAttrs()) + ( # type: ignore[misc] ("X-Setup-Requires", "setup_requires", True), ("X-Tests-Require", "tests_require", True), ("???", "extras_require", False), @@ -80,6 +84,9 @@ def _getHeaderAttrs(self) -> Sequence[Tuple[str, str, bool]]: ("X-Packages-Dict", "packages_dict", False), ("X-Py-Modules", "py_modules", True), ("X-Entry-Points", "entry_points", False), + ("X-Pbr", "pbr", False), + ("X-pbr__files__packages_root", "pbr__files__packages_root", False), + ("X-pbr__files__packages", "pbr__files__packages", True), ) def asdict(self) -> Dict[str, Any]: @@ -100,6 +107,7 @@ def _source_mapping(self, root: Path) -> Optional[Dict[str, str]]: for m in self.py_modules: if m == "?": return None + m = m.replace(".", "/") d[f"{m}.py"] = f"{m}.py" try: diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 0b2bf04..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,11 +0,0 @@ -black==19.10b0 -click==7.1.2 -coverage==4.5.4 -flake8==3.7.9 -mypy==0.750 -tox==3.14.1 -twine==3.1.1 -usort==0.6.2 -volatile==2.1.0 -wheel==0.33.6 -honesty==0.3.0a1 diff --git a/requirements.txt b/requirements.txt index 578b979..7afca69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ -highlighter==0.1.1 +highlighter==0.2.0 imperfect==0.3.0 -LibCST==0.3.12 -tomlkit==0.7.0 -pkginfo==1.5.0.1 +LibCST==1.5.1 +tomlkit==0.13.2 +pkginfo==1.11.2 diff --git a/setup.cfg b/setup.cfg index e6b7df7..db662a1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,13 +16,30 @@ packages = setup_requires = setuptools_scm setuptools >= 38.3.0 -python_requires = >=3.6 +python_requires = >=3.7 install_requires = highlighter>=0.1.1 imperfect>=0.1.0 LibCST>=0.3.7 tomlkit>=0.2.0 pkginfo>=1.4.2 + setuptools >= 38.3.0 + +[options.extras_require] +dev = + black==24.10.0 + click==8.1.7 + flake8==7.1.1 + mypy==1.13.0 + tox==4.23.2 + twine==5.1.1 + ufmt==2.8.0 + usort==1.0.8.post1 + wheel==0.45.1 + honesty==0.3.0b1 +test = + coverage >= 6 + volatile==2.1.0 [check] metadata = true @@ -48,17 +65,19 @@ use_parentheses = True [mypy] ignore_missing_imports = True +python_version = 3.8 +strict = True [tox:tox] -envlist = py36, py37, py38 +envlist = py{38,39,310,311,312,313}-tests [testenv] -deps = -rrequirements-dev.txt -whitelist_externals = make +deps = .[test] +allowlist_externals = make commands = make test setenv = - py{36,37,38}: COVERAGE_FILE={envdir}/.coverage + tests: COVERAGE_FILE={envdir}/.coverage [flake8] ignore = E203, E231, E266, E302, E501, W503