From 16d4eb89fd1974bdfa45c177a08f3d714e3379c9 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 1 Oct 2020 21:00:02 -0700 Subject: [PATCH 01/71] Update README --- README.md | 62 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index b510735..a9b5bda 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,34 @@ # dowsing -TODO: Reword so it flows better. +Short version: + +``` +python -m dowsing.pep517 /path/to/repo | jq . +``` + +or + +``` +from dowsing.pep517 import get_metadata +dist = get_metadata(Path("/path/to/repo")) +``` ## Basic reasoning +I don't want to execute arbitrary `setup.py` in order to find out their basic +metadata. I don't want to use the pep517 module in a sandbox, because commonly +packages forget to list their build-time dependencies. + +This project is one step better than grepping source files, but also understands +`build-system` in `pyproject.toml` (from PEP 517/518). It does pretty well run +on a sampling of pypi projects, but does fail on some notable ones (including +setuptools). + +When it fails, a key will be `"??"` and due to some quirks in list context, this +can be `["?", "?"]`. + +## A rant + The reality of python packaging, even with recent PEPs, is that most nontrivial python packages do moderately interesting stuff in their `setup.py`: @@ -13,27 +38,24 @@ python packages do moderately interesting stuff in their `setup.py`: * Making sure native libs are installed, or there's a working C compiler * Choosing deps based on platform -The disappointing part of several of these from the perspective of basically -running a distro, is that they produce messages intended for humans, rather than -actually using the mechanisms that we have in PEP 508 (environment markers) and -518 (pyproject.toml requires). +From the perspective of basically running a distro, they produce messages +intended for humans, rather than actually using the mechanisms that we have in +PEP 508 (environment markers) and 518 (pyproject.toml requires). There is also +no well-specified way to request native libs, and many projects choose to fail +to run `setup.py` when libs are missing. ## Goals This project is a bridge to find several things out, about primarily setup.py -but also understanding PEP 517/518 as a one-stop-shop, about: - -* for cases where the package's version is stored within, but has external - requirements that are not listed at build-time, currently returns an unknown - value and moves on -* potential imports, to guess at what should have been in the build-time - requirements (e.g `numpy.distutils` is pretty clear) -* doesn't actually execute, so fetches or execs can't cause it to fail -* Gives the PEP 517 APIs `get_requirements_for_sdist` and - `get_requirements_for_build_wheel`, even on a different platform through - simulated execution, with no sandboxing required. -* A lower-level api suitable for making edits to the place where the setup args - are defined. +but also understanding some popular PEP 517/518 builders as a one-stop-shop, about: + +* doesn't actually execute, so fetches or execs can't cause it to fail [done] +* cases where we could find out the version string, but it fails to import [done] +* lets you simulate the `pep517` module's output on different platforms [done] +* a lower-level api suitable for making edits to the place where the setup args + are defined [done] +* to list potential imports, and guess at missing build-time deps (something + like `numpy.distutils` is pretty clear) [todo] ## Doing this "right" @@ -46,6 +68,10 @@ If you're willing to run the code and have it take longer, take a look at the pep517 api `get_requires_for_*` or have it generate the metadata (assuming what you want is in there). An example is in `dowsing/_demo_pep517.py` +This project's `dowsing.pep517` api is designed to do something similar, but not +fail on missing build-time requirements. + + # Further Reading * PEP 241, Metadata 1.0 From e7766ec3bf815d21a6b5031172f0c8eaa8df35aa Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 1 Oct 2020 21:06:17 -0700 Subject: [PATCH 02/71] Include py.typed --- dowsing/py.typed | 0 setup.cfg | 10 +++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) create mode 100644 dowsing/py.typed diff --git a/dowsing/py.typed b/dowsing/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/setup.cfg b/setup.cfg index 18b446a..618364a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -9,7 +9,11 @@ author = Tim Hatch author_email = tim@timhatch.com [options] -packages = dowsing +packages = + dowsing + dowsing.setuptools + dowsing.tests +include_package_data = true setup_requires = setuptools_scm setuptools >= 38.3.0 @@ -59,3 +63,7 @@ setenv = [flake8] ignore = E203, E231, E266, E302, E501, W503 max-line-length = 88 + +[options.package_data] +dowsing = + py.typed From 648717d586626fa5776201635378e895945ec5ff Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 3 Oct 2020 16:17:00 -0700 Subject: [PATCH 03/71] Enable dependency validation using pessimist --- .github/workflows/build.yml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 366a8d3..9eac462 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,3 +34,26 @@ jobs: run: make test - name: Lint run: make lint + check-deps: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + python-version: [3.6, 3.7, 3.8] + os: [ubuntu-latest] + + steps: + - name: Checkout + uses: actions/checkout@v1 + - name: Set Up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + - name: Install + run: | + python -m pip install --upgrade pip + pip install 'pessimist>=0.8.0' + echo 'importall>=0.2.1' > importall.txt + - name: Check Deps + run: | + python -m pessimist --requirements=importall.txt --fast -c 'importall --root=. --exclude=tests,_demo_pep517.py dowsing' . From 25347c02043c95f3bc78076f0244378bd4f1d931 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 3 Oct 2020 16:23:33 -0700 Subject: [PATCH 04/71] Correct min deps highlighter 0.1.0 is buggy because it doesn't have a packaging dep. pkginfo was outright missing. --- setup.cfg | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 618364a..7c04954 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,10 +19,11 @@ setup_requires = setuptools >= 38.3.0 python_requires = >=3.6 install_requires = - highlighter + highlighter>=0.1.1 imperfect LibCST>=0.3.1 tomlkit>=0.2.0 + pkginfo>=0.6 [check] metadata = true From 6c1a9b0d09bfb74be491d1abdafd5a87cb3d9ebb Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sun, 4 Oct 2020 20:34:09 -0700 Subject: [PATCH 05/71] Add setuptools_metadata tests, fix bugs it found :/ --- Makefile | 2 +- dowsing/setuptools/setup_and_metadata.py | 43 ++++-- dowsing/setuptools/setup_cfg_parsing.py | 28 ++-- dowsing/setuptools/setup_py_parsing.py | 9 +- dowsing/setuptools/types.py | 29 +++- dowsing/tests/__init__.py | 4 + dowsing/tests/setuptools.py | 53 ++++--- dowsing/tests/setuptools_metadata.py | 95 +++++++++++++ dowsing/tests/setuptools_types.py | 167 +++++++++++++++++++++++ dowsing/types.py | 24 +++- requirements.txt | 3 +- setup.cfg | 1 - 12 files changed, 379 insertions(+), 79 deletions(-) create mode 100644 dowsing/tests/setuptools_metadata.py create mode 100644 dowsing/tests/setuptools_types.py diff --git a/Makefile b/Makefile index 4c166f6..dee08c4 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ venv: .PHONY: setup setup: - python -m pip install -Ur requirements-dev.txt + python -m pip install -U -r requirements.txt -r requirements-dev.txt .PHONY: test test: diff --git a/dowsing/setuptools/setup_and_metadata.py b/dowsing/setuptools/setup_and_metadata.py index 0d478d8..7c6f30d 100644 --- a/dowsing/setuptools/setup_and_metadata.py +++ b/dowsing/setuptools/setup_and_metadata.py @@ -33,7 +33,12 @@ # but doesn't really tell you what they do or what the metadata keys are or # what metadata version they correspond to. ConfigField("name", SetupCfg("metadata", "name"), Metadata("Name")), - ConfigField("version", SetupCfg("metadata", "version"), Metadata("Version")), + ConfigField( + "version", + SetupCfg("metadata", "version"), + Metadata("Version"), + sample_value="1.5.1", + ), ConfigField("author", SetupCfg("metadata", "author"), Metadata("Author")), ConfigField( "author_email", SetupCfg("metadata", "author_email"), Metadata("Author-email"), @@ -54,6 +59,7 @@ "keywords", SetupCfg("metadata", "keywords", writer_cls=ListCommaWriterCompat), Metadata("Keywords"), + sample_value=["abc", "def"], ), # but not repeated # platforms # fullname @@ -67,7 +73,11 @@ "classifiers", SetupCfg("metadata", "classifiers", writer_cls=ListSemiWriter), Metadata("Classifier", repeated=True), - sample_value=None, + sample_value=[ + "License :: OSI Approved :: MIT License", + "Intended Audience :: Developers", + ], + distribution_key="classifiers", ), # download_url # Metadata 1.1 @@ -94,9 +104,9 @@ "project_urls", SetupCfg("metadata", "project_urls", writer_cls=DictWriter), Metadata("Project-URL"), - sample_value=None, # {"Bugtracker": "http://example.com"}, + sample_value={"Bugtracker": "http://example.com"}, + distribution_key="project_urls", ), - # requires_dist # provides_dist (rarely used) # obsoletes_dist (rarely used) # Metadata 2.1 @@ -113,27 +123,29 @@ ConfigField( "zip_safe", SetupCfg("options", "zip_safe", writer_cls=BoolWriter), - sample_value=None, + sample_value=True, ), ConfigField( "setup_requires", SetupCfg("options", "setup_requires", writer_cls=ListSemiWriter), - sample_value=None, + sample_value=["setuptools"], ), ConfigField( "install_requires", - SetupCfg("options", "install_requires", writer_cls=ListSemiWriter), - sample_value=None, + SetupCfg("options", "install_requires", writer_cls=ListCommaWriter), + Metadata("Requires-Dist"), + sample_value=["a", "b ; python_version < '3'"], + distribution_key="requires_dist", ), ConfigField( "tests_require", SetupCfg("options", "tests_require", writer_cls=ListSemiWriter), - sample_value=None, + sample_value=["pytest"], ), ConfigField( "include_package_data", SetupCfg("options", "include_package_data", writer_cls=BoolWriter), - sample_value=None, # True, + sample_value=True, ), # ConfigField( @@ -155,7 +167,7 @@ ConfigField( "packages", SetupCfg("options", "packages", writer_cls=ListCommaWriter), - sample_value=None, + sample_value=["a", "b"], ), ConfigField( "package_dir", @@ -184,8 +196,13 @@ SetupCfg("options.data_files", "UNUSED", writer_cls=SectionWriter), sample_value=None, ), + ConfigField( + "entry_points", + SetupCfg("options.entry_points", "UNUSED", writer_cls=SectionWriter), + sample_value=None, + ), # # Documented, but not in the table... - ConfigField("test_suite", SetupCfg("options", "test_suite"), sample_value=None,), - ConfigField("test_loader", SetupCfg("options", "test_loader"), sample_value=None,), + ConfigField("test_suite", SetupCfg("options", "test_suite")), + ConfigField("test_loader", SetupCfg("options", "test_loader")), ] diff --git a/dowsing/setuptools/setup_cfg_parsing.py b/dowsing/setuptools/setup_cfg_parsing.py index af4d309..ab38653 100644 --- a/dowsing/setuptools/setup_cfg_parsing.py +++ b/dowsing/setuptools/setup_cfg_parsing.py @@ -5,6 +5,7 @@ from ..types import Distribution from .setup_and_metadata import SETUP_ARGS +from .types import SectionWriter def from_setup_cfg(path: Path, markers: Dict[str, Any]) -> Distribution: @@ -15,21 +16,24 @@ def from_setup_cfg(path: Path, markers: Dict[str, Any]) -> Distribution: d.metadata_version = "2.1" for field in SETUP_ARGS: - # Until there's a better representation... - if not field.metadata and field.keyword not in ("setup_requires",): + name = field.get_distribution_key() + if not hasattr(d, name): continue - try: - raw_data = cfg[field.cfg.section][field.cfg.key] - except KeyError: - continue cls = field.cfg.writer_cls - parsed = cls().from_ini(raw_data) + if cls is SectionWriter: + try: + raw_section_data = cfg[field.cfg.section] + except KeyError: + continue + # ConfigSection behaves like a Dict[str, str] so this is fine + parsed = SectionWriter().from_ini_section(raw_section_data) # type: ignore + else: + try: + raw_data = cfg[field.cfg.section][field.cfg.key] + except KeyError: + continue + parsed = cls().from_ini(raw_data) - name = ( - (field.metadata.key if field.metadata else field.keyword) - .lower() - .replace("-", "_") - ) setattr(d, name, parsed) return d diff --git a/dowsing/setuptools/setup_py_parsing.py b/dowsing/setuptools/setup_py_parsing.py index d974759..4b55583 100644 --- a/dowsing/setuptools/setup_py_parsing.py +++ b/dowsing/setuptools/setup_py_parsing.py @@ -48,15 +48,10 @@ def from_setup_py(path: Path, markers: Dict[str, Any]) -> Distribution: raise SyntaxError("No simple setup call found") for field in SETUP_ARGS: - # Until there's a better representation... - if not field.metadata and field.keyword not in ("setup_requires",): + name = field.get_distribution_key() + if not hasattr(d, name): continue - name = ( - (field.metadata.key if field.metadata else field.keyword) - .lower() - .replace("-", "_") - ) if field.keyword in analyzer.saved_args: v = analyzer.saved_args[field.keyword] if isinstance(v, Literal): diff --git a/dowsing/setuptools/types.py b/dowsing/setuptools/types.py index d53e394..4618ca0 100644 --- a/dowsing/setuptools/types.py +++ b/dowsing/setuptools/types.py @@ -24,11 +24,11 @@ class ListCommaWriter(BaseWriter): def to_ini(self, value: List[str]) -> str: if not value: return "" - return "".join(f"\n{k}" for k in value) + return "".join(f"\n {k}" for k in value) def from_ini(self, value: str) -> List[str]: # TODO, on all of these, handle other separators, \r, and stripping - return value.strip().split("\n") + return [line.strip() for line in value.strip().split("\n")] class ListCommaWriterCompat(BaseWriter): @@ -37,20 +37,20 @@ def to_ini(self, value: Union[str, List[str]]) -> str: return "" if isinstance(value, str): value = [value] - return "".join(f"\n{k}" for k in value) + return "".join(f"\n {k}" for k in value) def from_ini(self, value: str) -> List[str]: - return value.strip().split("\n") + return [line.strip() for line in value.strip().split("\n")] class ListSemiWriter(BaseWriter): def to_ini(self, value: List[str]) -> str: if not value: return "" - return "".join(f"\n{k}" for k in value) + return "".join(f"\n {k}" for k in value) def from_ini(self, value: str) -> List[str]: - return value.strip().split("\n") + return [line.strip() for line in value.strip().split("\n")] # This class is also specialcased @@ -60,6 +60,9 @@ def to_ini(self, value: List[str]) -> str: return "" return "".join(f"\n{k}" for k in value) + def from_ini_section(self, section: Dict[str, str]) -> Dict[str, List[str]]: + return {k: section[k].strip().split("\n") for k in section.keys()} + class BoolWriter(BaseWriter): def to_ini(self, value: bool) -> str: @@ -74,7 +77,7 @@ class DictWriter(BaseWriter): def to_ini(self, value: Dict[str, str]) -> str: if not value: return "" - return "".join(f"\n{k}={v}" for k, v in value.items()) + return "".join(f"\n {k}={v}" for k, v in value.items()) def from_ini(self, value: str) -> Dict[str, str]: d = {} @@ -128,3 +131,15 @@ class ConfigField: # Not all kwargs end up in metadata. We have a modified Distribution that # keeps them for now, but looking for something better (even if it's just # using ConfigField objects as events in a stream). + distribution_key: Optional[str] = None + + def get_distribution_key(self) -> str: + # Returns the member name of pkginfo.Distribution (or our subclasS) + if self.metadata is not None: + return ( + (self.distribution_key or self.metadata.key or self.keyword) + .replace("-", "_") + .lower() + ) + else: + return (self.distribution_key or self.keyword).replace("-", "_").lower() diff --git a/dowsing/tests/__init__.py b/dowsing/tests/__init__.py index d9b9540..cde6be9 100644 --- a/dowsing/tests/__init__.py +++ b/dowsing/tests/__init__.py @@ -2,10 +2,14 @@ from .flit import FlitReaderTest from .pep517 import Pep517Test from .setuptools import SetuptoolsReaderTest +from .setuptools_metadata import SetupArgsTest +from .setuptools_types import WriterTest __all__ = [ "ApiTest", "FlitReaderTest", "Pep517Test", "SetuptoolsReaderTest", + "WriterTest", + "SetupArgsTest", ] diff --git a/dowsing/tests/setuptools.py b/dowsing/tests/setuptools.py index 770a23d..0c543ff 100644 --- a/dowsing/tests/setuptools.py +++ b/dowsing/tests/setuptools.py @@ -4,14 +4,8 @@ import volatile from dowsing.setuptools import SetuptoolsReader -from dowsing.setuptools.types import ( - BoolWriter, - DictWriter, - ListCommaWriter, - ListCommaWriterCompat, - ListSemiWriter, - StrWriter, -) +from dowsing.setuptools.setup_py_parsing import from_setup_py +from dowsing.types import Distribution class SetuptoolsReaderTest(unittest.TestCase): @@ -51,26 +45,25 @@ def test_setup_py(self) -> None: ("setuptools", "wheel", "def"), r.get_requires_for_build_wheel() ) - def test_writer_classes_roundtrip_str(self) -> None: - s = "abc" - inst = StrWriter() - self.assertEqual(s, inst.from_ini(inst.to_ini(s))) - - def test_writer_classes_roundtrip_lists(self) -> None: - lst = ["a", "bc"] - inst = ListSemiWriter() - self.assertEqual(lst, inst.from_ini(inst.to_ini(lst))) - inst2 = ListCommaWriter() - self.assertEqual(lst, inst2.from_ini(inst2.to_ini(lst))) - inst3 = ListCommaWriterCompat() - self.assertEqual(lst, inst3.from_ini(inst3.to_ini(lst))) - - def test_writer_classes_roundtrip_dict(self) -> None: - d = {"a": "bc", "d": "ef"} - inst = DictWriter() - self.assertEqual(d, inst.from_ini(inst.to_ini(d))) + def _read(self, data: str) -> Distribution: + with volatile.dir() as d: + sp = Path(d, "setup.py") + sp.write_text(data) + return from_setup_py(Path(d), {}) - def test_writer_classes_roundtrip_bool(self) -> None: - for b in (True, False): - inst = BoolWriter() - self.assertEqual(b, inst.from_ini(inst.to_ini(b))) + def test_smoke(self) -> None: + d = self._read( + """\ +from setuptools import setup +setup( + name="foo", + version="0.1", + classifiers=["CLASSIFIER"], + install_requires=["abc"], +) +""" + ) + self.assertEqual("foo", d.name) + self.assertEqual("0.1", d.version) + self.assertEqual(["CLASSIFIER"], d.classifiers) + self.assertEqual(["abc"], d.requires_dist) diff --git a/dowsing/tests/setuptools_metadata.py b/dowsing/tests/setuptools_metadata.py new file mode 100644 index 0000000..cdef42b --- /dev/null +++ b/dowsing/tests/setuptools_metadata.py @@ -0,0 +1,95 @@ +import email.parser +import io +import os +import sys +import tempfile +import unittest +from distutils.core import run_setup +from email.message import Message +from pathlib import Path +from typing import Dict, Tuple + +import setuptools # noqa: F401 patchers gotta patch + +from dowsing.setuptools import SetuptoolsReader +from dowsing.setuptools.setup_and_metadata import SETUP_ARGS +from dowsing.types import Distribution + + +def egg_info(files: Dict[str, str]) -> Tuple[Message, Distribution]: + # TODO consider + # https://docs.python.org/3/distutils/apiref.html#distutils.core.run_setup + # and whether that gives a Distribution that knows setuptools-only options + with tempfile.TemporaryDirectory() as d: + for relname, contents in files.items(): + (Path(d) / relname).write_text(contents) + + try: + cwd = os.getcwd() + stdout = sys.stdout + + os.chdir(d) + sys.stdout = io.StringIO() + dist = run_setup(f"setup.py", ["egg_info"]) + finally: + os.chdir(cwd) + sys.stdout = stdout + + sources = list(Path(d).rglob("PKG-INFO")) + assert len(sources) == 1 + + with open(sources[0]) as f: + parser = email.parser.Parser() + info = parser.parse(f) + reader = SetuptoolsReader(Path(d)) + dist = reader.get_metadata() + return info, dist + + +# These tests do not increase coverage, and just verify that we have the right +# static data. +class SetupArgsTest(unittest.TestCase): + def test_arg_mapping(self) -> None: + for field in SETUP_ARGS: + if field.sample_value is None: + continue + with self.subTest(field.keyword): + # Tests that the same arg from setup.py or setup.cfg makes it into + # metadata in the same way. + foo = field.sample_value + setup_py_info, setup_py_dist = egg_info( + { + "setup.py": "from setuptools import setup\n" + f"setup({field.keyword}={foo!r})\n", + } + ) + + cfg_format_foo = field.cfg.writer_cls().to_ini(foo) + setup_cfg_info, setup_cfg_dist = egg_info( + { + "setup.cfg": f"[{field.cfg.section}]\n" + f"{field.cfg.key} = {cfg_format_foo}\n", + "setup.py": "from setuptools import setup\n" "setup()\n", + } + ) + + name = field.get_distribution_key() + self.assertNotEqual( + getattr(setup_py_dist, name), None, + ) + self.assertEqual( + foo, getattr(setup_py_dist, name), + ) + self.assertEqual( + foo, getattr(setup_cfg_dist, name), + ) + + if field.metadata: + a = setup_py_info.get_all(field.metadata.key) + b = setup_cfg_info.get_all(field.metadata.key) + + # install_requires gets written out to *.egg-info/requires.txt + # instead + if field.keyword != "install_requires": + self.assertEqual(a, b) + self.assertNotEqual(a, None) diff --git a/dowsing/tests/setuptools_types.py b/dowsing/tests/setuptools_types.py new file mode 100644 index 0000000..b73cf8d --- /dev/null +++ b/dowsing/tests/setuptools_types.py @@ -0,0 +1,167 @@ +import unittest +from configparser import RawConfigParser +from io import StringIO +from typing import Dict, List, Union + +from imperfect import ConfigFile +from parameterized import parameterized + +from dowsing.setuptools.types import ( + BoolWriter, + DictWriter, + ListCommaWriter, + ListCommaWriterCompat, + ListSemiWriter, + SectionWriter, + StrWriter, +) + + +class WriterTest(unittest.TestCase): + @parameterized.expand( # type: ignore + [(False,), (True,),] + ) + def test_bool_writer(self, arg: bool) -> None: + c = ConfigFile() + c.set_value("a", "b", BoolWriter().to_ini(arg)) + buf = StringIO() + c.build(buf) + + rcp = RawConfigParser(strict=False) + rcp.read_string(buf.getvalue()) + self.assertEqual(str(arg).lower(), rcp["a"]["b"]) + + @parameterized.expand( # type: ignore + [("hello",), ("a\nb\nc",),] + ) + def test_str_writer(self, arg: str) -> None: + c = ConfigFile() + c.set_value("a", "b", StrWriter().to_ini(arg)) + buf = StringIO() + c.build(buf) + + rcp = RawConfigParser(strict=False) + rcp.read_string(buf.getvalue()) + self.assertEqual(arg, rcp["a"]["b"]) + + @parameterized.expand( # type: ignore + [ + ([], ""), + (["a"], "\na"), + (["a", "b"], "\na\nb"), + (["a", "b", "c"], "\na\nb\nc"), + ] + ) + def test_list_comma_writer(self, arg: List[str], expected: str) -> None: + c = ConfigFile() + c.set_value("a", "b", ListCommaWriter().to_ini(arg)) + buf = StringIO() + c.build(buf) + + rcp = RawConfigParser(strict=False) + rcp.read_string(buf.getvalue()) + self.assertEqual(expected, rcp["a"]["b"]) + + @parameterized.expand( # type: ignore + [ + ([], ""), + (["a"], "\na"), + (["a", "b"], "\na\nb"), + (["a", "b", "c"], "\na\nb\nc"), + ] + ) + def test_list_semi_writer(self, arg: List[str], expected: str) -> None: + c = ConfigFile() + c.set_value("a", "b", ListSemiWriter().to_ini(arg)) + buf = StringIO() + c.build(buf) + + rcp = RawConfigParser(strict=False) + rcp.read_string(buf.getvalue()) + self.assertEqual(expected, rcp["a"]["b"]) + + @parameterized.expand( # type: ignore + # fmt: off + [ + ({}, ""), + ({"x": "y"}, "\nx=y"), + ({"x": "y", "z": "zz"}, "\nx=y\nz=zz"), + ] + # fmt: on + ) + def test_dict_writer(self, arg: Dict[str, str], expected: str) -> None: + c = ConfigFile() + c.set_value("a", "b", DictWriter().to_ini(arg)) + buf = StringIO() + c.build(buf) + + rcp = RawConfigParser(strict=False) + rcp.read_string(buf.getvalue()) + # I would prefer this be dangling lines + self.assertEqual(expected, rcp["a"]["b"]) + + @parameterized.expand( # type: ignore + # fmt: off + [ + ([], ""), + ("abc", "\nabc"), + (["a"], "\na"), + (["a", "b"], "\na\nb"), + (["a", "b", "c"], "\na\nb\nc"), + ] + # fmt: on + ) + def test_list_comma_writer_compat( + self, arg: Union[str, List[str]], expected: str + ) -> None: + c = ConfigFile() + c.set_value("a", "b", ListCommaWriterCompat().to_ini(arg)) + buf = StringIO() + c.build(buf) + + rcp = RawConfigParser(strict=False) + rcp.read_string(buf.getvalue()) + # I would prefer this be dangling lines + self.assertEqual(expected, rcp["a"]["b"]) + + @parameterized.expand( # type: ignore + [ + ([], ""), + (["a"], "\na"), + (["a", "b"], "\na\nb"), + (["a", "b", "c"], "\na\nb\nc"), + ] + ) + def test_section_writer(self, arg: List[str], expected: str) -> None: + c = ConfigFile() + c.set_value("a", "b", SectionWriter().to_ini(arg)) + buf = StringIO() + c.build(buf) + + rcp = RawConfigParser(strict=False) + rcp.read_string(buf.getvalue()) + self.assertEqual(expected, rcp["a"]["b"]) + + def test_roundtrip_str(self) -> None: + s = "abc" + inst = StrWriter() + self.assertEqual(s, inst.from_ini(inst.to_ini(s))) + + def test_roundtrip_lists(self) -> None: + lst = ["a", "bc"] + inst = ListSemiWriter() + self.assertEqual(lst, inst.from_ini(inst.to_ini(lst))) + inst2 = ListCommaWriter() + self.assertEqual(lst, inst2.from_ini(inst2.to_ini(lst))) + inst3 = ListCommaWriterCompat() + self.assertEqual(lst, inst3.from_ini(inst3.to_ini(lst))) + + def test_roundtrip_dict(self) -> None: + d = {"a": "bc", "d": "ef"} + inst = DictWriter() + self.assertEqual(d, inst.from_ini(inst.to_ini(d))) + + def test_roundtrip_bool(self) -> None: + for b in (True, False): + inst = BoolWriter() + self.assertEqual(b, inst.from_ini(inst.to_ini(b))) diff --git a/dowsing/types.py b/dowsing/types.py index bfbfae1..e64853b 100644 --- a/dowsing/types.py +++ b/dowsing/types.py @@ -45,20 +45,30 @@ class Distribution(pkginfo.distribution.Distribution): # type: ignore zip_safe: Optional[bool] = None include_package_data: Optional[bool] = None test_suite: str = "" + test_loader: str = "" namespace_packages: Sequence[str] = () + package_data: Dict[str, Sequence[str]] = {} + packages: Sequence[str] = () + package_dir: Optional[str] = None + entry_points: Dict[str, Sequence[str]] = {} def _getHeaderAttrs(self) -> Sequence[Tuple[str, str, bool]]: # Until I invent a metadata version to include this, do so # unconditionally. return tuple(super()._getHeaderAttrs()) + ( - ("Setup-Requires", "setup_requires", True), - ("Tests-Require", "tests_require", True), + ("X-Setup-Requires", "setup_requires", True), + ("X-Tests-Require", "tests_require", True), ("???", "extras_require", False), - ("Use-SCM-Version", "use_scm_version", False), - ("Zip-Safe", "zip_safe", False), - ("Test-Suite", "test_suite", False), - ("Include-Package-Data", "include_package_data", False), - ("Namespace-Package", "namespace_packages", True), + ("X-Use-SCM-Version", "use_scm_version", False), + ("x-Zip-Safe", "zip_safe", False), + ("X-Test-Suite", "test_suite", False), + ("X-Test-Loader", "test_loader", False), + ("X-Include-Package-Data", "include_package_data", False), + ("X-Namespace-Package", "namespace_packages", True), + ("X-Package-Data", "package_data", False), + ("X-Packages", "packages", True), + ("X-Package-Dir", "package_dir", False), + ("X-Entry-Points", "entry_points", False), ) def asdict(self) -> Dict[str, Any]: diff --git a/requirements.txt b/requirements.txt index 757f549..578b979 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ -highlighter==0.1.0 +highlighter==0.1.1 imperfect==0.3.0 LibCST==0.3.12 tomlkit==0.7.0 +pkginfo==1.5.0.1 diff --git a/setup.cfg b/setup.cfg index 7c04954..19d0687 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,7 +13,6 @@ packages = dowsing dowsing.setuptools dowsing.tests -include_package_data = true setup_requires = setuptools_scm setuptools >= 38.3.0 From b8796ab45b5b6bf7d45164fa1b148ce1f418938a Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Mon, 5 Oct 2020 08:33:23 -0700 Subject: [PATCH 06/71] Better support for Flit emulation --- dowsing/flit.py | 21 +++++++++++++++++++++ dowsing/tests/flit.py | 15 ++++++++++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/dowsing/flit.py b/dowsing/flit.py index c0f8231..83350f8 100644 --- a/dowsing/flit.py +++ b/dowsing/flit.py @@ -2,6 +2,7 @@ from typing import Sequence import tomlkit +from setuptools import find_packages from .types import BaseReader, Distribution @@ -22,12 +23,32 @@ def get_metadata(self) -> Distribution: d = Distribution() d.metadata_version = "2.1" + d.project_urls = {} for k, v in doc["tool"]["flit"]["metadata"].items(): + # TODO description-file -> long_description + # TODO home-page -> urls + # TODO requires -> requires_dist + # TODO tool.flit.metadata.urls + if k == "home-page": + d.project_urls["Homepage"] = v + continue + elif k == "module": + k = "packages" + v = find_packages(self.path.as_posix(), include=(f"{v}.*")) + elif k == "description-file": + k = "description" + v = f"file: {v}" + elif k == "requires": + k = "requires_dist" + k2 = k.replace("-", "_") if k2 in d: setattr(d, k2, v) + for k, v in doc["tool"]["flit"]["metadata"].get("urls", {}).items(): + d.project_urls[k] = v + # TODO extras-require return d diff --git a/dowsing/tests/flit.py b/dowsing/tests/flit.py index 1bc74b9..281bc29 100644 --- a/dowsing/tests/flit.py +++ b/dowsing/tests/flit.py @@ -41,8 +41,15 @@ def test_normal(self) -> None: name = "Name" module = "foo" requires = ["abc", "def"] + +[tool.flit.metadata.urls] +Foo = "https://" """ ) + (dp / "foo").mkdir() + (dp / "foo" / "tests").mkdir() + (dp / "foo" / "__init__.py").write_text("") + (dp / "foo" / "tests" / "__init__.py").write_text("") r = FlitReader(dp) # Notably these do not include flit itself; that's handled by @@ -52,6 +59,12 @@ def test_normal(self) -> None: md = r.get_metadata() self.assertEqual("Name", md.name) self.assertEqual( - {"metadata_version": "2.1", "name": "Name", "requires": ["abc", "def"]}, + { + "metadata_version": "2.1", + "name": "Name", + "packages": ["foo", "foo.tests"], + "requires_dist": ["abc", "def"], + "project_urls": {"Foo": "https://"}, + }, md.asdict(), ) From 1dfeabe55b62bb50fc3c31d8989ccfc7c9517f95 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Mon, 5 Oct 2020 08:40:02 -0700 Subject: [PATCH 07/71] Get ready for v0.6.0 --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..b6e2801 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,7 @@ +## v0.6.0 + +* Fix many bugs in Flit and Setuptools support, better test coverage. + +## v0.5.0 + +* Initial code extracted from Opine From c53f3e80f9f21fa27766c06ee0cbf535e06d0521 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Mon, 5 Oct 2020 10:41:01 -0700 Subject: [PATCH 08/71] Initial support for poetry --- dowsing/pep517.py | 1 + dowsing/poetry.py | 64 +++++++++++++++++++++++++++++++++++++++ dowsing/tests/__init__.py | 2 ++ dowsing/tests/poetry.py | 51 +++++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+) create mode 100644 dowsing/poetry.py create mode 100644 dowsing/tests/poetry.py diff --git a/dowsing/pep517.py b/dowsing/pep517.py index 50fbd9b..58be8af 100644 --- a/dowsing/pep517.py +++ b/dowsing/pep517.py @@ -12,6 +12,7 @@ "setuptools.build_meta:__legacy__": "dowsing.setuptools:SetuptoolsReader", "setuptools.build_meta": "dowsing.setuptools:SetuptoolsReader", "flit_core.buildapi": "dowsing.flit:FlitReader", + "poetry.core.masonry.api": "dowsing.poetry:PoetryReader", } diff --git a/dowsing/poetry.py b/dowsing/poetry.py new file mode 100644 index 0000000..234809d --- /dev/null +++ b/dowsing/poetry.py @@ -0,0 +1,64 @@ +from pathlib import Path +from typing import Sequence + +import tomlkit +from setuptools import find_packages + +from .types import BaseReader, Distribution + +METADATA_MAPPING = { + "name": "name", + "version": "version", + "description": "summary", + "license": "license", # SPDX short name + # authors + # maintainers + # readme -> long desc? w/ content type rst/md + "keywords": "keywords", + "classifiers": "classifiers", +} + + +class PoetryReader(BaseReader): + def __init__(self, path: Path): + self.path = path + + def get_requires_for_build_sdist(self) -> Sequence[str]: + return () # TODO + + def get_requires_for_build_wheel(self) -> Sequence[str]: + return () # TODO + + def get_metadata(self) -> Distribution: + pyproject = self.path / "pyproject.toml" + doc = tomlkit.parse(pyproject.read_text()) + + d = Distribution() + d.metadata_version = "2.1" + d.project_urls = {} + d.requires_dist = [] + d.packages = [] + + for k, v in doc["tool"]["poetry"].items(): + if k in ("homepage", "repository", "documentation"): + d.project_urls[k] = v + elif k == "packages": + for x in v: + d.packages.extend( + p + for p in find_packages(self.path.as_posix()) + if p == x["include"] or p.startswith(f"{x['include']}.") + ) + elif k in METADATA_MAPPING: + setattr(d, METADATA_MAPPING[k], v) + + for k, v in doc["tool"]["poetry"].get("dependencies", {}).items(): + if k == "python": + pass # TODO translate to requires_python + else: + d.requires_dist.append(k) + + for k, v in doc["tool"]["poetry"].get("urls", {}).items(): + d.project_urls[k] = v + + return d diff --git a/dowsing/tests/__init__.py b/dowsing/tests/__init__.py index cde6be9..9e24cfe 100644 --- a/dowsing/tests/__init__.py +++ b/dowsing/tests/__init__.py @@ -1,6 +1,7 @@ from .api import ApiTest from .flit import FlitReaderTest from .pep517 import Pep517Test +from .poetry import PoetryReaderTest from .setuptools import SetuptoolsReaderTest from .setuptools_metadata import SetupArgsTest from .setuptools_types import WriterTest @@ -9,6 +10,7 @@ "ApiTest", "FlitReaderTest", "Pep517Test", + "PoetryReaderTest", "SetuptoolsReaderTest", "WriterTest", "SetupArgsTest", diff --git a/dowsing/tests/poetry.py b/dowsing/tests/poetry.py new file mode 100644 index 0000000..ae8b2d4 --- /dev/null +++ b/dowsing/tests/poetry.py @@ -0,0 +1,51 @@ +import unittest +from pathlib import Path + +import volatile + +from dowsing.poetry import PoetryReader + + +class PoetryReaderTest(unittest.TestCase): + def test_basic(self) -> None: + with volatile.dir() as d: + dp = Path(d) + (dp / "pyproject.toml").write_text( + """\ +[build-system] +requires = ["poetry-core>=1.0.0a9"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry] +name = "Name" +version = "1.5.2" +description = "Short Desc" +authors = ["Author "] +license = "BSD-3-Clause" +homepage = "http://example.com" +classifiers = [ + "Not a real classifier", +] + +[tool.poetry.dependencies] +python = "~2.7 || ^3.5" +functools32 = { version = "^3.2.3", python = "~2.7" } + +[tool.poetry.urls] +"Bug Tracker" = "https://github.com/python-poetry/poetry/issues" +""" + ) + r = PoetryReader(dp) + md = r.get_metadata() + self.assertEqual("Name", md.name) + self.assertEqual("1.5.2", md.version) + self.assertEqual("BSD-3-Clause", md.license) + self.assertEqual( + { + "homepage": "http://example.com", + "Bug Tracker": "https://github.com/python-poetry/poetry/issues", + }, + md.project_urls, + ) + self.assertEqual(["Not a real classifier"], md.classifiers) + self.assertEqual(["functools32"], md.requires_dist) From 09f4199d1ec35ec298389dea3ac901417d242cc2 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Mon, 19 Oct 2020 19:49:51 -0700 Subject: [PATCH 09/71] Add support for FindPackages in setuptools --- dowsing/pep517.py | 10 ++- dowsing/setuptools/__init__.py | 51 ++++++++++- dowsing/setuptools/setup_and_metadata.py | 17 ++++ dowsing/setuptools/setup_py_parsing.py | 28 ++++++ dowsing/tests/setuptools.py | 104 ++++++++++++++++++++++- dowsing/types.py | 9 +- 6 files changed, 210 insertions(+), 9 deletions(-) diff --git a/dowsing/pep517.py b/dowsing/pep517.py index 58be8af..2b4476e 100644 --- a/dowsing/pep517.py +++ b/dowsing/pep517.py @@ -2,7 +2,7 @@ import json import sys from pathlib import Path -from typing import Dict, List, Tuple, Type +from typing import Any, Dict, List, Tuple, Type import tomlkit @@ -64,13 +64,19 @@ def get_metadata(path: Path) -> Distribution: return backend.get_metadata() +def _default(obj: Any) -> Any: + if obj.__class__.__name__ == "FindPackages": + return f"FindPackages({obj.where!r}, {obj.exclude!r}, {obj.include!r}" + raise TypeError(obj) + + def main(path: Path) -> None: d = { "get_requires_for_build_sdist": get_requires_for_build_sdist(path), "get_requires_for_build_wheel": get_requires_for_build_wheel(path), "get_metadata": get_metadata(path).asdict(), } - print(json.dumps(d)) + print(json.dumps(d, default=_default)) if __name__ == "__main__": diff --git a/dowsing/setuptools/__init__.py b/dowsing/setuptools/__init__.py index f4954e1..deef847 100644 --- a/dowsing/setuptools/__init__.py +++ b/dowsing/setuptools/__init__.py @@ -1,9 +1,18 @@ +import posixpath from pathlib import Path -from typing import Sequence, Tuple +from typing import Generator, Mapping, Sequence, Tuple + +from setuptools import find_packages from ..types import BaseReader, Distribution from .setup_cfg_parsing import from_setup_cfg -from .setup_py_parsing import from_setup_py +from .setup_py_parsing import FindPackages, from_setup_py + + +def _prefixes(dotted_name: str) -> Generator[Tuple[str, str], None, None]: + parts = dotted_name.split(".") + for i in range(len(parts), -1, -1): + yield ".".join(parts[:i]), "/".join(parts[i:]) class SetuptoolsReader(BaseReader): @@ -34,6 +43,44 @@ def get_metadata(self) -> Distribution: if getattr(d2, k): setattr(d1, k, getattr(d2, k)) + # package_dir can both add and remove components, see docs + # https://docs.python.org/2/distutils/setupscript.html#listing-whole-packages + package_dir: Mapping[str, str] = d1.package_dir + # If there was an error, we might have written "??" + if package_dir != "??": + if not package_dir: + package_dir = {"": "."} + + assert isinstance(package_dir, dict) + + def mangle(package: str) -> str: + for x, rest in _prefixes(package): + if x in package_dir: + return posixpath.normpath(posixpath.join(package_dir[x], rest)) + raise Exception("Should have stopped by now") + + d1.packages_dict = {} # Break shared class-level dict + if isinstance(d1.packages, FindPackages): + # This encodes a lot of sketchy logic, and deserves more test cases, + # plus some around py_modules + for p in find_packages( + self.path / d1.packages.where, + d1.packages.exclude, + d1.packages.include, + ): + d1.packages_dict[p] = mangle(p) + elif d1.packages == ["find:"]: + for p in find_packages( + self.path / d1.find_packages_where, + d1.find_packages_exclude, + d1.find_packages_include, + ): + d1.packages_dict[p] = mangle(p) + elif d1.packages != "??": + assert isinstance(d1.packages, (list, tuple)) + for p in d1.packages: + d1.packages_dict[p] = mangle(p) + return d1 def _get_requires(self) -> Tuple[str, ...]: diff --git a/dowsing/setuptools/setup_and_metadata.py b/dowsing/setuptools/setup_and_metadata.py index 7c6f30d..49e47d6 100644 --- a/dowsing/setuptools/setup_and_metadata.py +++ b/dowsing/setuptools/setup_and_metadata.py @@ -205,4 +205,21 @@ # Documented, but not in the table... ConfigField("test_suite", SetupCfg("options", "test_suite")), ConfigField("test_loader", SetupCfg("options", "test_loader")), + # + # FindPackages + ConfigField( + "find_packages_where", + SetupCfg("options.packages.find", "where"), + sample_value=None, + ), + ConfigField( + "find_packages_exclude", + SetupCfg("options.packages.find", "exclude", writer_cls=ListCommaWriter), + sample_value=None, + ), + ConfigField( + "find_packages_include", + SetupCfg("options.packages.find", "include", writer_cls=ListCommaWriter), + sample_value=None, + ), ] diff --git a/dowsing/setuptools/setup_py_parsing.py b/dowsing/setuptools/setup_py_parsing.py index 4b55583..8d9819b 100644 --- a/dowsing/setuptools/setup_py_parsing.py +++ b/dowsing/setuptools/setup_py_parsing.py @@ -79,6 +79,13 @@ class Literal: cst_node: Optional[cst.CSTNode] +@dataclass +class FindPackages: + where: Any = None + exclude: Any = None + include: Any = None + + class FileReference: def __init__(self, filename: str) -> None: self.filename = filename @@ -166,6 +173,8 @@ def visit_Call(self, node: cst.Call) -> Optional[bool]: PRETEND_ARGV = ["setup.py", "bdist_wheel"] def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: + qnames = self.get_metadata(QualifiedNameProvider, item) + if isinstance(item, cst.SimpleString): return item.evaluated_value # TODO int/float/etc @@ -221,6 +230,25 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: return tuple(lst) else: return lst + elif isinstance(item, cst.Call) and any( + q.name == "setuptools.find_packages" for q in qnames + ): + default_args = [".", (), ("*",)] + args = default_args.copy() + + names = ("where", "exclude", "include") + i = 0 + for arg in item.args: + if isinstance(arg.keyword, cst.Name): + args[names.index(arg.keyword.value)] = self.evaluate_in_scope( + arg.value, scope + ) + else: + args[i] = self.evaluate_in_scope(arg.value, scope) + i += 1 + + # TODO clear ones that are still default + return FindPackages(*args) elif ( isinstance(item, cst.Call) and isinstance(item.func, cst.Name) diff --git a/dowsing/tests/setuptools.py b/dowsing/tests/setuptools.py index 0c543ff..f3ac136 100644 --- a/dowsing/tests/setuptools.py +++ b/dowsing/tests/setuptools.py @@ -4,7 +4,7 @@ import volatile from dowsing.setuptools import SetuptoolsReader -from dowsing.setuptools.setup_py_parsing import from_setup_py +from dowsing.setuptools.setup_py_parsing import FindPackages from dowsing.types import Distribution @@ -45,11 +45,17 @@ def test_setup_py(self) -> None: ("setuptools", "wheel", "def"), r.get_requires_for_build_wheel() ) - def _read(self, data: str) -> Distribution: + def _read(self, data: str, src_dir: str = ".") -> Distribution: with volatile.dir() as d: sp = Path(d, "setup.py") sp.write_text(data) - return from_setup_py(Path(d), {}) + Path(d, src_dir, "pkg").mkdir(parents=True) + Path(d, src_dir, "pkg", "__init__.py").touch() + Path(d, src_dir, "pkg", "sub").mkdir() + Path(d, src_dir, "pkg", "sub", "__init__.py").touch() + Path(d, src_dir, "pkg", "tests").mkdir() + Path(d, src_dir, "pkg", "tests", "__init__.py").touch() + return SetuptoolsReader(Path(d)).get_metadata() def test_smoke(self) -> None: d = self._read( @@ -67,3 +73,95 @@ def test_smoke(self) -> None: self.assertEqual("0.1", d.version) self.assertEqual(["CLASSIFIER"], d.classifiers) self.assertEqual(["abc"], d.requires_dist) + + def test_packages_dict_literal(self) -> None: + d = self._read( + """\ +from setuptools import setup, find_packages +setup( + packages=["pkg", "pkg.tests"], +) +""" + ) + self.assertEqual(d.packages, ["pkg", "pkg.tests"]) + self.assertEqual(d.packages_dict, {"pkg": "pkg", "pkg.tests": "pkg/tests"}) + + def test_packages_find_packages_call(self) -> None: + d = self._read( + """\ +from setuptools import setup, find_packages +setup( + packages=find_packages(exclude=("pkg.sub",)), +) + """ + ) + self.assertEqual(d.packages, FindPackages(".", ("pkg.sub",), ("*",))) + self.assertEqual(d.packages_dict, {"pkg": "pkg", "pkg.tests": "pkg/tests"}) + + def test_packages_find_packages_call_package_dir(self) -> None: + d = self._read( + """\ +from setuptools import setup, find_packages +setup( + package_dir={'': '.'}, + packages=find_packages(exclude=("pkg.sub",)), +) + """ + ) + self.assertEqual(d.packages, FindPackages(".", ("pkg.sub",), ("*",))) + self.assertEqual(d.packages_dict, {"pkg": "pkg", "pkg.tests": "pkg/tests"}) + + def test_packages_find_packages_call_package_dir_src(self) -> None: + d = self._read( + """\ +from setuptools import setup, find_packages +setup( + package_dir={'': 'src'}, + packages=find_packages("src", exclude=("pkg.sub",)), +) + """, + "src", + ) + self.assertEqual(d.packages, FindPackages("src", ("pkg.sub",), ("*",))) + self.assertEqual( + d.packages_dict, {"pkg": "src/pkg", "pkg.tests": "src/pkg/tests"} + ) + + def test_packages_find_packages_call_package_dir2(self) -> None: + d = self._read( + """\ +from setuptools import setup, find_packages +setup( + package_dir={'pkg': 'pkg'}, + packages=find_packages(exclude=("pkg.sub",)), +) + """ + ) + self.assertEqual(d.packages, FindPackages(".", ("pkg.sub",), ("*",))) + self.assertEqual(d.packages_dict, {"pkg": "pkg", "pkg.tests": "pkg/tests"}) + + def test_packages_find_packages_call_package_dir3(self) -> None: + d = self._read( + """\ +from setuptools import setup, find_packages +setup( + package_dir={'': 'pkg'}, + packages=find_packages("pkg"), +) + """ + ) + self.assertEqual(d.packages, FindPackages("pkg", (), ("*",))) + self.assertEqual(d.packages_dict, {"sub": "pkg/sub", "tests": "pkg/tests"}) + + def test_packages_find_packages_include(self) -> None: + # This is weird behavior but documented. + d = self._read( + """\ +from setuptools import setup, find_packages +setup( + packages=find_packages(include=("pkg",)), +) + """ + ) + self.assertEqual(d.packages, FindPackages(".", (), ("pkg",))) + self.assertEqual(d.packages_dict, {"pkg": "pkg"}) diff --git a/dowsing/types.py b/dowsing/types.py index e64853b..b596a1c 100644 --- a/dowsing/types.py +++ b/dowsing/types.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any, Dict, Optional, Sequence, Tuple +from typing import Any, Dict, Mapping, Optional, Sequence, Tuple import pkginfo.distribution @@ -49,8 +49,12 @@ class Distribution(pkginfo.distribution.Distribution): # type: ignore namespace_packages: Sequence[str] = () package_data: Dict[str, Sequence[str]] = {} packages: Sequence[str] = () - package_dir: Optional[str] = None + package_dir: Mapping[str, str] = {} + packages_dict: Mapping[str, str] = {} entry_points: Dict[str, Sequence[str]] = {} + find_packages_where: str = "." + find_packages_exclude: Sequence[str] = () + find_packages_include: Sequence[str] = ("*",) def _getHeaderAttrs(self) -> Sequence[Tuple[str, str, bool]]: # Until I invent a metadata version to include this, do so @@ -68,6 +72,7 @@ def _getHeaderAttrs(self) -> Sequence[Tuple[str, str, bool]]: ("X-Package-Data", "package_data", False), ("X-Packages", "packages", True), ("X-Package-Dir", "package_dir", False), + ("X-Packages-Dict", "packages_dict", False), ("X-Entry-Points", "entry_points", False), ) From b0a5bdb124a59ea01e7a85c2b553469f7bbe5474 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Mon, 19 Oct 2020 20:14:18 -0700 Subject: [PATCH 10/71] Poetry packages_dict improvements --- dowsing/pep517.py | 1 + dowsing/poetry.py | 27 +++++++++++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/dowsing/pep517.py b/dowsing/pep517.py index 2b4476e..b13be01 100644 --- a/dowsing/pep517.py +++ b/dowsing/pep517.py @@ -13,6 +13,7 @@ "setuptools.build_meta": "dowsing.setuptools:SetuptoolsReader", "flit_core.buildapi": "dowsing.flit:FlitReader", "poetry.core.masonry.api": "dowsing.poetry:PoetryReader", + "poetry.masonry.api": "dowsing.poetry:PoetryReader", } diff --git a/dowsing/poetry.py b/dowsing/poetry.py index 234809d..7f164cc 100644 --- a/dowsing/poetry.py +++ b/dowsing/poetry.py @@ -1,3 +1,4 @@ +import posixpath from pathlib import Path from typing import Sequence @@ -36,29 +37,43 @@ def get_metadata(self) -> Distribution: d = Distribution() d.metadata_version = "2.1" d.project_urls = {} + d.entry_points = {} d.requires_dist = [] d.packages = [] + d.packages_dict = {} for k, v in doc["tool"]["poetry"].items(): if k in ("homepage", "repository", "documentation"): d.project_urls[k] = v elif k == "packages": + # TODO improve and add tests; this works for tf2_utils and + # poetry itself but include can be a glob and there are excludes for x in v: - d.packages.extend( - p - for p in find_packages(self.path.as_posix()) - if p == x["include"] or p.startswith(f"{x['include']}.") - ) + f = x.get("from", ".") + for p in find_packages((self.path / f).as_posix()): + if p == x["include"] or p.startswith(f"{x['include']}."): + d.packages_dict[p] = posixpath.normpath( + posixpath.join(f, p.replace(".", "/")) + ) + d.packages.append(p) elif k in METADATA_MAPPING: setattr(d, METADATA_MAPPING[k], v) + if not d.packages: + for p in find_packages(self.path.as_posix()): + d.packages_dict[p] = p.replace(".", "/") + d.packages.append(p) + for k, v in doc["tool"]["poetry"].get("dependencies", {}).items(): if k == "python": pass # TODO translate to requires_python else: - d.requires_dist.append(k) + d.requires_dist.append(k) # TODO something with version for k, v in doc["tool"]["poetry"].get("urls", {}).items(): d.project_urls[k] = v + for k, v in doc["tool"]["poetry"].get("scripts", {}).items(): + d.entry_points[k] = v + return d From 7dcda671dbfdae8232b18d7a3659bf2d41ced4d8 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Mon, 19 Oct 2020 20:14:26 -0700 Subject: [PATCH 11/71] Flit packages_dict improvements --- dowsing/flit.py | 1 + dowsing/tests/flit.py | 1 + 2 files changed, 2 insertions(+) diff --git a/dowsing/flit.py b/dowsing/flit.py index 83350f8..b41ed9e 100644 --- a/dowsing/flit.py +++ b/dowsing/flit.py @@ -36,6 +36,7 @@ def get_metadata(self) -> Distribution: elif k == "module": k = "packages" v = find_packages(self.path.as_posix(), include=(f"{v}.*")) + d.packages_dict = {i: i.replace(".", "/") for i in v} elif k == "description-file": k = "description" v = f"file: {v}" diff --git a/dowsing/tests/flit.py b/dowsing/tests/flit.py index 281bc29..ae15ad6 100644 --- a/dowsing/tests/flit.py +++ b/dowsing/tests/flit.py @@ -63,6 +63,7 @@ def test_normal(self) -> None: "metadata_version": "2.1", "name": "Name", "packages": ["foo", "foo.tests"], + "packages_dict": {"foo": "foo", "foo.tests": "foo/tests"}, "requires_dist": ["abc", "def"], "project_urls": {"Foo": "https://"}, }, From 4e716c0d56e267dfe8dd22e0ca6119b5bd5f815b Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Mon, 19 Oct 2020 20:21:29 -0700 Subject: [PATCH 12/71] Small Windows bug in setuptools find_packages --- dowsing/setuptools/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dowsing/setuptools/__init__.py b/dowsing/setuptools/__init__.py index deef847..b8b6104 100644 --- a/dowsing/setuptools/__init__.py +++ b/dowsing/setuptools/__init__.py @@ -60,18 +60,21 @@ def mangle(package: str) -> str: raise Exception("Should have stopped by now") d1.packages_dict = {} # Break shared class-level dict + + # The following as_posix calls are necessary for Windows, but don't + # hurt elsewhere. if isinstance(d1.packages, FindPackages): # This encodes a lot of sketchy logic, and deserves more test cases, # plus some around py_modules for p in find_packages( - self.path / d1.packages.where, + (self.path / d1.packages.where).as_posix(), d1.packages.exclude, d1.packages.include, ): d1.packages_dict[p] = mangle(p) elif d1.packages == ["find:"]: for p in find_packages( - self.path / d1.find_packages_where, + (self.path / d1.find_packages_where).as_posix(), d1.find_packages_exclude, d1.find_packages_include, ): From 87860e009ce4f045ab12dd272ccd6952986a6331 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Tue, 20 Oct 2020 06:56:49 -0700 Subject: [PATCH 13/71] Use safer default for empty dict in Distribution --- dowsing/types.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/dowsing/types.py b/dowsing/types.py index b596a1c..b802913 100644 --- a/dowsing/types.py +++ b/dowsing/types.py @@ -1,4 +1,5 @@ from pathlib import Path +from types import MappingProxyType from typing import Any, Dict, Mapping, Optional, Sequence, Tuple import pkginfo.distribution @@ -34,24 +35,26 @@ def get_metadata(self) -> "Distribution": raise NotImplementedError +DEFAULT_EMPTY_DICT: Mapping[str, Any] = MappingProxyType({}) + # TODO: pkginfo isn't typed, and is doing to require a yak-shave to send a PR # since it's on launchpad. class Distribution(pkginfo.distribution.Distribution): # type: ignore # These are not actually part of the metadata, see PEP 566 setup_requires: Sequence[str] = () tests_require: Sequence[str] = () - extras_require: Dict[str, Sequence[str]] = {} + extras_require: Mapping[str, Sequence[str]] = DEFAULT_EMPTY_DICT use_scm_version: Optional[bool] = None zip_safe: Optional[bool] = None include_package_data: Optional[bool] = None test_suite: str = "" test_loader: str = "" namespace_packages: Sequence[str] = () - package_data: Dict[str, Sequence[str]] = {} + package_data: Mapping[str, Sequence[str]] = DEFAULT_EMPTY_DICT packages: Sequence[str] = () - package_dir: Mapping[str, str] = {} - packages_dict: Mapping[str, str] = {} - entry_points: Dict[str, Sequence[str]] = {} + package_dir: Mapping[str, str] = DEFAULT_EMPTY_DICT + packages_dict: Mapping[str, str] = DEFAULT_EMPTY_DICT + entry_points: Mapping[str, Sequence[str]] = DEFAULT_EMPTY_DICT find_packages_where: str = "." find_packages_exclude: Sequence[str] = () find_packages_include: Sequence[str] = ("*",) From 0e98b21bd2fb92e606403e239dfb54de783352b2 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Tue, 20 Oct 2020 08:36:53 -0700 Subject: [PATCH 14/71] Initial implementation for maturin This backend is only used for 4 popular projects so far, but one of those is orjson. --- dowsing/maturin.py | 52 +++++++++++++++++++++++++++++++++++++++ dowsing/pep517.py | 1 + dowsing/tests/__init__.py | 2 ++ dowsing/tests/maturin.py | 51 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 106 insertions(+) create mode 100644 dowsing/maturin.py create mode 100644 dowsing/tests/maturin.py diff --git a/dowsing/maturin.py b/dowsing/maturin.py new file mode 100644 index 0000000..4576b34 --- /dev/null +++ b/dowsing/maturin.py @@ -0,0 +1,52 @@ +from pathlib import Path +from typing import Sequence + +import tomlkit + +from .types import BaseReader, Distribution + + +class MaturinReader(BaseReader): + def __init__(self, path: Path): + self.path = path + + def get_requires_for_build_sdist(self) -> Sequence[str]: + return [] # TODO + + def get_requires_for_build_wheel(self) -> Sequence[str]: + return [] # TODO + + def get_metadata(self) -> Distribution: + pyproject = self.path / "pyproject.toml" + doc = tomlkit.parse(pyproject.read_text()) + + d = Distribution() + d.metadata_version = "2.1" + + cargo = self.path / "Cargo.toml" + doc = tomlkit.parse(cargo.read_text()) + for k, v in doc["package"].items(): + if k == "name": + d.name = v + elif k == "version": + d.version = v + elif k == "license": + d.license = v + elif k == "description": + d.summary = v + # authors ["foo "] + # repository + # homepage + # readme (filename) + + for k, v in doc["package"]["metadata"]["maturin"].items(): + if k == "requires-python": + d.requires_python = v + elif k == "classifier": + d.classifiers = v + elif k == "requires-dist": + d.requires_dist = v + # Many others, see https://docs.rs/maturin/0.8.3/maturin/struct.Metadata21.html + # but these do not seem to be that popular. + + return d diff --git a/dowsing/pep517.py b/dowsing/pep517.py index b13be01..ac697d2 100644 --- a/dowsing/pep517.py +++ b/dowsing/pep517.py @@ -12,6 +12,7 @@ "setuptools.build_meta:__legacy__": "dowsing.setuptools:SetuptoolsReader", "setuptools.build_meta": "dowsing.setuptools:SetuptoolsReader", "flit_core.buildapi": "dowsing.flit:FlitReader", + "maturin": "dowsing.maturin:MaturinReader", "poetry.core.masonry.api": "dowsing.poetry:PoetryReader", "poetry.masonry.api": "dowsing.poetry:PoetryReader", } diff --git a/dowsing/tests/__init__.py b/dowsing/tests/__init__.py index 9e24cfe..7eb139d 100644 --- a/dowsing/tests/__init__.py +++ b/dowsing/tests/__init__.py @@ -1,5 +1,6 @@ from .api import ApiTest from .flit import FlitReaderTest +from .maturin import MaturinReaderTest from .pep517 import Pep517Test from .poetry import PoetryReaderTest from .setuptools import SetuptoolsReaderTest @@ -9,6 +10,7 @@ __all__ = [ "ApiTest", "FlitReaderTest", + "MaturinReaderTest", "Pep517Test", "PoetryReaderTest", "SetuptoolsReaderTest", diff --git a/dowsing/tests/maturin.py b/dowsing/tests/maturin.py new file mode 100644 index 0000000..38409af --- /dev/null +++ b/dowsing/tests/maturin.py @@ -0,0 +1,51 @@ +import unittest +from pathlib import Path + +import volatile + +from dowsing.maturin import MaturinReader + + +class MaturinReaderTest(unittest.TestCase): + def test_orjson(self) -> None: + # This is a simplified version of orjson 3.4.0 + with volatile.dir() as d: + dp = Path(d) + (dp / "pyproject.toml").write_text( + """\ +[project] +name = "orjson" +repository = "https://example.com/" + +[build-system] +build-backend = "maturin" +requires = ["maturin>=0.8.1,<0.9"] +""" + ) + + (dp / "Cargo.toml").write_text( + """\ +[package] +name = "orjson" +version = "3.4.0" +authors = ["foo "] +description = "Summary here" +license = "Apache-2.0 OR MIT" +repository = "https://example.com/repo" +homepage = "https://example.com/home" +readme = "README.md" +keywords = ["foo", "bar", "baz"] + +[package.metadata.maturin] +requires-python = ">=3.6" +classifer = [ + "License :: OSI Approved :: Apache Software License", + "License :: OSI Approved :: MIT License", +] +""" + ) + r = MaturinReader(dp) + md = r.get_metadata() + self.assertEqual("orjson", md.name) + self.assertEqual("3.4.0", md.version) + # TODO more tests From 57cc3dd2d8240452e0c14faf78dd72d4c97047f5 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Tue, 20 Oct 2020 08:37:31 -0700 Subject: [PATCH 15/71] Support flit scripts --- dowsing/flit.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dowsing/flit.py b/dowsing/flit.py index b41ed9e..c795e6b 100644 --- a/dowsing/flit.py +++ b/dowsing/flit.py @@ -24,6 +24,7 @@ def get_metadata(self) -> Distribution: d = Distribution() d.metadata_version = "2.1" d.project_urls = {} + d.entry_points = {} for k, v in doc["tool"]["flit"]["metadata"].items(): # TODO description-file -> long_description @@ -50,7 +51,11 @@ def get_metadata(self) -> Distribution: for k, v in doc["tool"]["flit"]["metadata"].get("urls", {}).items(): d.project_urls[k] = v + for k, v in doc["tool"]["flit"].get("scripts", {}).items(): + d.entry_points[k] = v + # TODO extras-require + # TODO distutils commands (e.g. pex 2.1.19) return d From 08160dad356f696f15cacda6c92f3023d3e71e84 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Tue, 20 Oct 2020 08:37:42 -0700 Subject: [PATCH 16/71] Support setup.cfg fields with dashes instead --- dowsing/setuptools/setup_and_metadata.py | 2 +- dowsing/setuptools/setup_cfg_parsing.py | 7 ++++++- dowsing/tests/setuptools.py | 18 ++++++++++++++++++ 3 files changed, 25 insertions(+), 2 deletions(-) diff --git a/dowsing/setuptools/setup_and_metadata.py b/dowsing/setuptools/setup_and_metadata.py index 49e47d6..e9c1cea 100644 --- a/dowsing/setuptools/setup_and_metadata.py +++ b/dowsing/setuptools/setup_and_metadata.py @@ -94,7 +94,7 @@ # Metadata 1.2, not at all supported by distutils ConfigField( "python_requires", - SetupCfg("options", "python_requires"), + SetupCfg("options", "python_requires"), # also requires_python :/ Metadata("Requires-Python"), sample_value="<4.0", ), diff --git a/dowsing/setuptools/setup_cfg_parsing.py b/dowsing/setuptools/setup_cfg_parsing.py index ab38653..aa93448 100644 --- a/dowsing/setuptools/setup_cfg_parsing.py +++ b/dowsing/setuptools/setup_cfg_parsing.py @@ -30,7 +30,12 @@ def from_setup_cfg(path: Path, markers: Dict[str, Any]) -> Distribution: parsed = SectionWriter().from_ini_section(raw_section_data) # type: ignore else: try: - raw_data = cfg[field.cfg.section][field.cfg.key] + # All fields are defined as underscore, but it appears + # setuptools normalizes so dashes are ok too. + key = field.cfg.key + if key not in cfg[field.cfg.section]: + key = key.replace("_", "-") + raw_data = cfg[field.cfg.section][key] except KeyError: continue parsed = cls().from_ini(raw_data) diff --git a/dowsing/tests/setuptools.py b/dowsing/tests/setuptools.py index f3ac136..61bb9ad 100644 --- a/dowsing/tests/setuptools.py +++ b/dowsing/tests/setuptools.py @@ -28,6 +28,24 @@ def test_setup_cfg(self) -> None: ("setuptools", "wheel", "def"), r.get_requires_for_build_wheel() ) + def test_setup_cfg_dash_normalization(self) -> None: + # I can't find documentation for this, but e.g. auditwheel 3.2.0 uses + # dashes instead of underscores and it works. + with volatile.dir() as d: + dp = Path(d) + (dp / "setup.cfg").write_text( + """\ +[metadata] +name = foo +author = Foo +author-email = foo@example.com +""" + ) + + r = SetuptoolsReader(dp) + md = r.get_metadata() + self.assertEqual("foo@example.com", md.author_email) + def test_setup_py(self) -> None: with volatile.dir() as d: dp = Path(d) From 5f441aca287bb682361c206cc00fc287b2706891 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Mon, 19 Oct 2020 20:15:57 -0700 Subject: [PATCH 17/71] Update CHANGELOG --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index b6e2801..5d401c4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.7.0 + +* Adds Poetry support +* Addd Maturin support +* Adds `packages_dict` and better `packages` support across supported backends +* Allows `setup.cfg` fields to use dashes + ## v0.6.0 * Fix many bugs in Flit and Setuptools support, better test coverage. From fe091110b71c46c6a11418fde883638457775944 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Mon, 9 Nov 2020 20:19:28 -0800 Subject: [PATCH 18/71] Provide source_mapping This emulates much of the "install" procedure, to let you know the site-packages-relative-path that each pure python source file is installed in. --- dowsing/flit.py | 1 + dowsing/poetry.py | 1 + dowsing/setuptools/__init__.py | 1 + dowsing/tests/setuptools.py | 37 ++++++++++++++++++++++++++++ dowsing/tests/setuptools_metadata.py | 7 +++++- dowsing/types.py | 26 +++++++++++++++++++ 6 files changed, 72 insertions(+), 1 deletion(-) diff --git a/dowsing/flit.py b/dowsing/flit.py index c795e6b..45c6a93 100644 --- a/dowsing/flit.py +++ b/dowsing/flit.py @@ -57,6 +57,7 @@ def get_metadata(self) -> Distribution: # TODO extras-require # TODO distutils commands (e.g. pex 2.1.19) + d.source_mapping = d._source_mapping(self.path) return d def _get_requires(self) -> Sequence[str]: diff --git a/dowsing/poetry.py b/dowsing/poetry.py index 7f164cc..37b6157 100644 --- a/dowsing/poetry.py +++ b/dowsing/poetry.py @@ -76,4 +76,5 @@ def get_metadata(self) -> Distribution: for k, v in doc["tool"]["poetry"].get("scripts", {}).items(): d.entry_points[k] = v + d.source_mapping = d._source_mapping(self.path) return d diff --git a/dowsing/setuptools/__init__.py b/dowsing/setuptools/__init__.py index b8b6104..59941f1 100644 --- a/dowsing/setuptools/__init__.py +++ b/dowsing/setuptools/__init__.py @@ -84,6 +84,7 @@ def mangle(package: str) -> str: for p in d1.packages: d1.packages_dict[p] = mangle(p) + d1.source_mapping = d1._source_mapping(self.path) return d1 def _get_requires(self) -> Tuple[str, ...]: diff --git a/dowsing/tests/setuptools.py b/dowsing/tests/setuptools.py index 61bb9ad..5f4ad65 100644 --- a/dowsing/tests/setuptools.py +++ b/dowsing/tests/setuptools.py @@ -157,6 +157,13 @@ def test_packages_find_packages_call_package_dir2(self) -> None: ) self.assertEqual(d.packages, FindPackages(".", ("pkg.sub",), ("*",))) self.assertEqual(d.packages_dict, {"pkg": "pkg", "pkg.tests": "pkg/tests"}) + self.assertEqual( + d.source_mapping, + { + "pkg/__init__.py": "pkg/__init__.py", + "pkg/tests/__init__.py": "pkg/tests/__init__.py", + }, + ) def test_packages_find_packages_call_package_dir3(self) -> None: d = self._read( @@ -170,6 +177,13 @@ def test_packages_find_packages_call_package_dir3(self) -> None: ) self.assertEqual(d.packages, FindPackages("pkg", (), ("*",))) self.assertEqual(d.packages_dict, {"sub": "pkg/sub", "tests": "pkg/tests"}) + self.assertEqual( + d.source_mapping, + { + "sub/__init__.py": "pkg/sub/__init__.py", + "tests/__init__.py": "pkg/tests/__init__.py", + }, + ) def test_packages_find_packages_include(self) -> None: # This is weird behavior but documented. @@ -183,3 +197,26 @@ def test_packages_find_packages_include(self) -> None: ) self.assertEqual(d.packages, FindPackages(".", (), ("pkg",))) self.assertEqual(d.packages_dict, {"pkg": "pkg"}) + self.assertEqual(d.source_mapping, {"pkg/__init__.py": "pkg/__init__.py"}) + + def test_py_modules(self) -> None: + d = self._read( + """\ +from setuptools import setup, find_packages +setup( + py_modules=["a", "b"], +) + """ + ) + self.assertEqual(d.source_mapping, {"a.py": "a.py", "b.py": "b.py"}) + + def test_invalid_packages(self) -> None: + d = self._read( + """\ +from setuptools import setup, find_packages +setup( + packages = ["zzz"], +) + """ + ) + self.assertEqual(d.source_mapping, None) diff --git a/dowsing/tests/setuptools_metadata.py b/dowsing/tests/setuptools_metadata.py index cdef42b..d1ac9b5 100644 --- a/dowsing/tests/setuptools_metadata.py +++ b/dowsing/tests/setuptools_metadata.py @@ -22,7 +22,8 @@ def egg_info(files: Dict[str, str]) -> Tuple[Message, Distribution]: # and whether that gives a Distribution that knows setuptools-only options with tempfile.TemporaryDirectory() as d: for relname, contents in files.items(): - (Path(d) / relname).write_text(contents) + Path(d, relname).parent.mkdir(exist_ok=True, parents=True) + Path(d, relname).write_text(contents) try: cwd = os.getcwd() @@ -61,6 +62,8 @@ def test_arg_mapping(self) -> None: { "setup.py": "from setuptools import setup\n" f"setup({field.keyword}={foo!r})\n", + "a/__init__.py": "", + "b/__init__.py": "", } ) @@ -70,6 +73,8 @@ def test_arg_mapping(self) -> None: "setup.cfg": f"[{field.cfg.section}]\n" f"{field.cfg.key} = {cfg_format_foo}\n", "setup.py": "from setuptools import setup\n" "setup()\n", + "a/__init__.py": "", + "b/__init__.py": "", } ) diff --git a/dowsing/types.py b/dowsing/types.py index b802913..c170129 100644 --- a/dowsing/types.py +++ b/dowsing/types.py @@ -54,10 +54,12 @@ class Distribution(pkginfo.distribution.Distribution): # type: ignore packages: Sequence[str] = () package_dir: Mapping[str, str] = DEFAULT_EMPTY_DICT packages_dict: Mapping[str, str] = DEFAULT_EMPTY_DICT + py_modules: Sequence[str] = () entry_points: Mapping[str, Sequence[str]] = DEFAULT_EMPTY_DICT find_packages_where: str = "." find_packages_exclude: Sequence[str] = () find_packages_include: Sequence[str] = ("*",) + source_mapping: Optional[Mapping[str, str]] = None def _getHeaderAttrs(self) -> Sequence[Tuple[str, str, bool]]: # Until I invent a metadata version to include this, do so @@ -76,6 +78,7 @@ def _getHeaderAttrs(self) -> Sequence[Tuple[str, str, bool]]: ("X-Packages", "packages", True), ("X-Package-Dir", "package_dir", False), ("X-Packages-Dict", "packages_dict", False), + ("X-Py-Modules", "py_modules", True), ("X-Entry-Points", "entry_points", False), ) @@ -85,3 +88,26 @@ def asdict(self) -> Dict[str, Any]: if getattr(self, x): d[x] = getattr(self, x) return d + + def _source_mapping(self, root: Path) -> Optional[Dict[str, str]]: + """ + Returns install path -> src path + + If an exception like FileNotFound is encountered, returns None. + """ + d: Dict[str, str] = {} + + for m in self.py_modules: + d[f"{m}.py"] = f"{m}.py" + + try: + # k = foo.bar, v = src/foo/bar + for k, v in self.packages_dict.items(): + kp = k.replace(".", "/") + for item in (root / v).iterdir(): + if item.is_file(): + d[f"{kp}/{item.name}"] = f"{v}/{item.name}" + except IOError: + return None + + return d From f281fcdb46a1617fa14f91ec5e945805b280709c Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Mon, 9 Nov 2020 20:25:06 -0800 Subject: [PATCH 19/71] Update skel 2020-11-09 --- .github/workflows/build.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index adf7550..9191bc6 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,6 +3,8 @@ on: push: branches: - master + - main + - tmp-* tags: - v* pull_request: @@ -13,14 +15,14 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.6, 3.7, 3.8] + python-version: ["3.6", "3.7", "3.8", "3.9"] os: [macOS-latest, ubuntu-latest, windows-latest] steps: - name: Checkout uses: actions/checkout@v1 - name: Set Up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install @@ -32,6 +34,7 @@ jobs: run: make test - name: Lint run: make lint + if: ${{ matrix.python-version != '3.9' }} - name: Coverage run: codecov --token ${{ secrets.CODECOV_TOKEN }} --branch ${{ github.ref }} continue-on-error: true From 0c4a2332deb98db1119f908a676f2e296a010695 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Mon, 9 Nov 2020 20:28:13 -0800 Subject: [PATCH 20/71] Update changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d401c4..d878dbd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +## v0.8.0 + +* Adds `Distribution.source_mapping` +* Enable gh actions on 3.9 + ## v0.7.0 * Adds Poetry support From 52fae381ee39c0ff9fb0fb8ad9235133dbabde3c Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Mon, 9 Nov 2020 20:42:32 -0800 Subject: [PATCH 21/71] Bump min dep versions --- setup.cfg | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 19d0687..e6b7df7 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,10 +19,10 @@ setup_requires = python_requires = >=3.6 install_requires = highlighter>=0.1.1 - imperfect - LibCST>=0.3.1 + imperfect>=0.1.0 + LibCST>=0.3.7 tomlkit>=0.2.0 - pkginfo>=0.6 + pkginfo>=1.4.2 [check] metadata = true From 0b99ddec39f3341260babc7811ec483544c918d8 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sun, 22 Nov 2020 09:06:41 -0800 Subject: [PATCH 22/71] requires-dist is a repeated entry --- dowsing/setuptools/setup_and_metadata.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dowsing/setuptools/setup_and_metadata.py b/dowsing/setuptools/setup_and_metadata.py index e9c1cea..9c44087 100644 --- a/dowsing/setuptools/setup_and_metadata.py +++ b/dowsing/setuptools/setup_and_metadata.py @@ -133,7 +133,7 @@ ConfigField( "install_requires", SetupCfg("options", "install_requires", writer_cls=ListCommaWriter), - Metadata("Requires-Dist"), + Metadata("Requires-Dist", repeated=True), sample_value=["a", "b ; python_version < '3'"], distribution_key="requires_dist", ), From 05443fa03d3960bff9088d5a08212e434b88a305 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Wed, 9 Dec 2020 09:49:19 -0800 Subject: [PATCH 23/71] By default include all package data. This more closely matches the flit/poetry behavior, but does not yet handle any excludes. With this changes, on the top 5000 packages: ``` 2100 dowsing computes the same* source mapping as the wheel 579 dowsing computes a different* source mapping than the wheel 327 dowsing thinks there are no sources 145 dowsing raises an exception (some of these are 404) 1849 do not have wheel+sdist to easily check ``` `*` -- this is after some simple normalization, but many of these are either harmless differences, or exposing other problems. For example: `pytz` includes the in-package tests w/ dowsing `boto3` wheel cannot be recreated from the published sdist `inflection` wheel includes both a module and a package, and cannot be recreated from the published sdist `bleach` includes the dist-info for a vendored dep `regex` produces filenames with `.` instead of `/` because of an unusual use of `py_modules` containing dots. --- dowsing/pep517.py | 1 + dowsing/setuptools/__init__.py | 5 ++++- dowsing/tests/setuptools.py | 16 +++++++++++++-- dowsing/types.py | 37 +++++++++++++++++++++++++++++----- 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/dowsing/pep517.py b/dowsing/pep517.py index ac697d2..26a7e99 100644 --- a/dowsing/pep517.py +++ b/dowsing/pep517.py @@ -12,6 +12,7 @@ "setuptools.build_meta:__legacy__": "dowsing.setuptools:SetuptoolsReader", "setuptools.build_meta": "dowsing.setuptools:SetuptoolsReader", "flit_core.buildapi": "dowsing.flit:FlitReader", + "flit.buildapi": "dowsing.flit:FlitReader", "maturin": "dowsing.maturin:MaturinReader", "poetry.core.masonry.api": "dowsing.poetry:PoetryReader", "poetry.masonry.api": "dowsing.poetry:PoetryReader", diff --git a/dowsing/setuptools/__init__.py b/dowsing/setuptools/__init__.py index 59941f1..57bb7cb 100644 --- a/dowsing/setuptools/__init__.py +++ b/dowsing/setuptools/__init__.py @@ -57,7 +57,10 @@ def mangle(package: str) -> str: for x, rest in _prefixes(package): if x in package_dir: return posixpath.normpath(posixpath.join(package_dir[x], rest)) - raise Exception("Should have stopped by now") + + # Some projects seem to set only a partial package_dir, but then + # use find_packages which wants to include some outside. + return package d1.packages_dict = {} # Break shared class-level dict diff --git a/dowsing/tests/setuptools.py b/dowsing/tests/setuptools.py index 5f4ad65..5295975 100644 --- a/dowsing/tests/setuptools.py +++ b/dowsing/tests/setuptools.py @@ -161,6 +161,8 @@ def test_packages_find_packages_call_package_dir2(self) -> None: d.source_mapping, { "pkg/__init__.py": "pkg/__init__.py", + # TODO this line should not be here as it's excluded + "pkg/sub/__init__.py": "pkg/sub/__init__.py", "pkg/tests/__init__.py": "pkg/tests/__init__.py", }, ) @@ -197,7 +199,16 @@ def test_packages_find_packages_include(self) -> None: ) self.assertEqual(d.packages, FindPackages(".", (), ("pkg",))) self.assertEqual(d.packages_dict, {"pkg": "pkg"}) - self.assertEqual(d.source_mapping, {"pkg/__init__.py": "pkg/__init__.py"}) + # TODO strict interpretation should be this commented line + # self.assertEqual(d.source_mapping, {"pkg/__init__.py": "pkg/__init__.py"}) + self.assertEqual( + d.source_mapping, + { + "pkg/__init__.py": "pkg/__init__.py", + "pkg/sub/__init__.py": "pkg/sub/__init__.py", + "pkg/tests/__init__.py": "pkg/tests/__init__.py", + }, + ) def test_py_modules(self) -> None: d = self._read( @@ -219,4 +230,5 @@ def test_invalid_packages(self) -> None: ) """ ) - self.assertEqual(d.source_mapping, None) + # TODO wish this were None + self.assertEqual(d.source_mapping, {}) diff --git a/dowsing/types.py b/dowsing/types.py index c170129..349c456 100644 --- a/dowsing/types.py +++ b/dowsing/types.py @@ -1,6 +1,6 @@ from pathlib import Path from types import MappingProxyType -from typing import Any, Dict, Mapping, Optional, Sequence, Tuple +from typing import Any, Dict, Mapping, Optional, Sequence, Set, Tuple import pkginfo.distribution @@ -98,15 +98,42 @@ def _source_mapping(self, root: Path) -> Optional[Dict[str, str]]: d: Dict[str, str] = {} for m in self.py_modules: + if m == "?": + return None d[f"{m}.py"] = f"{m}.py" try: - # k = foo.bar, v = src/foo/bar - for k, v in self.packages_dict.items(): + # This commented block is approximately correct for setuptools, but + # does not understand package_data. + # # k = foo.bar, v = src/foo/bar + # for k, v in self.packages_dict.items(): + # kp = k.replace(".", "/") + # for item in (root / v).iterdir(): + # if item.is_file(): + # d[f"{kp}/{item.name}"] = f"{v}/{item.name}" + + # Instead, this behavior is more like flit/poetry by including all + # files under package dirs, in a way that's mostly compatible with + # setuptools setting package_dir dicts. This tends to include + # in-package tests, which is a behavior I like, but I'm sure some + # people won't. + + seen_paths: Set[Path] = set() + + # Longest source path first, will "own" the item + for k, v in sorted( + self.packages_dict.items(), key=lambda x: len(x[1]), reverse=True + ): kp = k.replace(".", "/") - for item in (root / v).iterdir(): + vp = root / v + for item in vp.rglob("*"): + if item in seen_paths: + continue + seen_paths.add(item) if item.is_file(): - d[f"{kp}/{item.name}"] = f"{v}/{item.name}" + rel = item.relative_to(vp) + d[(kp / rel).as_posix()] = (v / rel).as_posix() + except IOError: return None From 276e83ef833637365550b8eb0c53727006f25194 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Wed, 9 Dec 2020 10:05:40 -0800 Subject: [PATCH 24/71] Script to check source mapping --- .github/workflows/build.yml | 2 +- dowsing/check_source_mapping.py | 84 +++++++++++++++++++++++++++++++++ 2 files changed, 85 insertions(+), 1 deletion(-) create mode 100644 dowsing/check_source_mapping.py diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 12bc235..b8394ac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -58,4 +58,4 @@ jobs: echo 'importall>=0.2.1' > importall.txt - name: Check Deps run: | - python -m pessimist --requirements=importall.txt --fast -c 'importall --root=. --exclude=tests,_demo_pep517.py dowsing' . + python -m pessimist --requirements=importall.txt --fast -c 'importall --root=. --exclude=tests,_demo_pep517.py,check_source_mapping.py dowsing' . diff --git a/dowsing/check_source_mapping.py b/dowsing/check_source_mapping.py new file mode 100644 index 0000000..9d0c292 --- /dev/null +++ b/dowsing/check_source_mapping.py @@ -0,0 +1,84 @@ +import sys +from pathlib import Path +from typing import List + +import click +from honesty.archive import extract_and_get_names +from honesty.cache import Cache +from honesty.cmdline import select_versions, wrap_async +from honesty.releases import FileType, async_parse_index +from moreorless.click import echo_color_unified_diff + +from dowsing.pep517 import get_metadata + + +@click.command() +@click.argument("packages", nargs=-1) +@wrap_async +async def main(packages: List[str]) -> None: + # Much of this code mirrors the methods in honesty/cmdline.py + async with Cache(fresh_index=True) as cache: + for package_name in packages: + package_name, operator, version = package_name.partition("==") + try: + package = await async_parse_index(package_name, cache, use_json=True) + except Exception as e: + print(package_name, repr(e), file=sys.stderr) + continue + + selected_versions = select_versions(package, operator, version) + rel = package.releases[selected_versions[0]] + + sdists = [f for f in rel.files if f.file_type == FileType.SDIST] + wheels = [f for f in rel.files if f.file_type == FileType.BDIST_WHEEL] + + if not sdists or not wheels: + print(f"{package_name}: insufficient artifacts") + continue + + sdist_path = await cache.async_fetch(pkg=package_name, url=sdists[0].url) + wheel_path = await cache.async_fetch(pkg=package_name, url=wheels[0].url) + + sdist_root, sdist_filenames = extract_and_get_names( + sdist_path, strip_top_level=True, patterns=("*.*") + ) + wheel_root, wheel_filenames = extract_and_get_names( + wheel_path, strip_top_level=True, patterns=("*.*") + ) + + try: + subdirs = tuple(Path(sdist_root).iterdir()) + metadata = get_metadata(Path(sdist_root, subdirs[0])) + assert metadata.source_mapping is not None, "no source_mapping" + except Exception as e: + print(package_name, repr(e), file=sys.stderr) + continue + + skip_patterns = [ + ".so", + ".pyc", + "nspkg", + ".dist-info", + ".data/scripts", + ] + wheel_blob = "".join( + sorted( + f"{f[0]}\n" + for f in wheel_filenames + if not any(s in f[0] for s in skip_patterns) + ) + ) + md_blob = "".join(sorted(f"{f}\n" for f in metadata.source_mapping.keys())) + + if md_blob == wheel_blob: + print(f"{package_name}: ok") + elif md_blob in ("", "?.py\n"): + print(f"{package_name}: COMPLETELY MISSING") + else: + echo_color_unified_diff( + wheel_blob, md_blob, f"{package_name}/files.txt" + ) + + +if __name__ == "__main__": + main() From f0634b0ddd9a7be78309cae6450961a4a398d9a7 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Wed, 9 Dec 2020 12:26:14 -0800 Subject: [PATCH 25/71] Switch to usort --- Makefile | 4 ++-- requirements-dev.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index dee08c4..cfd7927 100644 --- a/Makefile +++ b/Makefile @@ -21,12 +21,12 @@ test: .PHONY: format format: - python -m isort --recursive -y $(SOURCES) + python -m usort format $(SOURCES) python -m black $(SOURCES) .PHONY: lint lint: - python -m isort --recursive --diff $(SOURCES) + python -m usort check $(SOURCES) python -m black --check $(SOURCES) python -m flake8 $(SOURCES) mypy --strict dowsing diff --git a/requirements-dev.txt b/requirements-dev.txt index 0479c95..add1b04 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,9 @@ black==19.10b0 coverage==4.5.4 flake8==3.7.9 -isort==4.3.21 mypy==0.750 tox==3.14.1 twine==3.1.1 +usort==0.6.2 volatile==2.1.0 wheel==0.33.6 From e4f444b1b6c7ebbc5ae6b62d3726b85c58a559b9 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Wed, 9 Dec 2020 12:39:39 -0800 Subject: [PATCH 26/71] Dep on honesty for check_source_mapping debug --- requirements-dev.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index add1b04..0b2bf04 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ black==19.10b0 +click==7.1.2 coverage==4.5.4 flake8==3.7.9 mypy==0.750 @@ -7,3 +8,4 @@ twine==3.1.1 usort==0.6.2 volatile==2.1.0 wheel==0.33.6 +honesty==0.3.0a1 From ee81fb26ea4ac67bb951652706107d7f40a546bd Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Wed, 9 Dec 2020 10:06:41 -0800 Subject: [PATCH 27/71] Get ready for 0.9.0b1 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d878dbd..1b7ccc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,9 @@ +## v0.9.0b1 + +* Includes package data in `source_mapping` all the time. +* Support `flit.buildapi` as alternate flit build-backend +* Switch to usort for import sorting + ## v0.8.0 * Adds `Distribution.source_mapping` From c62b6fdadedd9c6a6c223f17f46621da741ed4ce Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sun, 13 Dec 2020 07:31:03 -0800 Subject: [PATCH 28/71] List sources for most pbr projects. Refs #7 --- dowsing/check_source_mapping.py | 4 +- dowsing/setuptools/__init__.py | 14 +++++ dowsing/setuptools/setup_and_metadata.py | 11 ++++ dowsing/tests/setuptools.py | 73 +++++++++++++++++++++++- dowsing/types.py | 6 ++ 5 files changed, 106 insertions(+), 2 deletions(-) diff --git a/dowsing/check_source_mapping.py b/dowsing/check_source_mapping.py index 9d0c292..789732f 100644 --- a/dowsing/check_source_mapping.py +++ b/dowsing/check_source_mapping.py @@ -70,7 +70,9 @@ async def main(packages: List[str]) -> None: ) md_blob = "".join(sorted(f"{f}\n" for f in metadata.source_mapping.keys())) - if md_blob == wheel_blob: + if metadata.source_mapping == {}: + print(f"{package_name}: empty dict") + elif md_blob == wheel_blob: print(f"{package_name}: ok") elif md_blob in ("", "?.py\n"): print(f"{package_name}: COMPLETELY MISSING") diff --git a/dowsing/setuptools/__init__.py b/dowsing/setuptools/__init__.py index 57bb7cb..aac7890 100644 --- a/dowsing/setuptools/__init__.py +++ b/dowsing/setuptools/__init__.py @@ -43,6 +43,20 @@ def get_metadata(self) -> Distribution: if getattr(d2, k): setattr(d1, k, getattr(d2, k)) + # This is the bare minimum to get pbr projects to show as having any + # sources. I don't want to use pbr.util.cfg_to_args because it appears + # to import and run arbitrary code. + if d1.pbr: + where = "." + if d1.pbr__files__packages_root: + d1.package_dir = {"": d1.pbr__files__packages_root} + where = d1.pbr__files__packages_root + + if d1.pbr__files__packages: + d1.packages = d1.pbr__files__packages + else: + d1.packages = FindPackages(where, (), ("*",)) # type: ignore + # package_dir can both add and remove components, see docs # https://docs.python.org/2/distutils/setupscript.html#listing-whole-packages package_dir: Mapping[str, str] = d1.package_dir diff --git a/dowsing/setuptools/setup_and_metadata.py b/dowsing/setuptools/setup_and_metadata.py index 9c44087..dea96f4 100644 --- a/dowsing/setuptools/setup_and_metadata.py +++ b/dowsing/setuptools/setup_and_metadata.py @@ -222,4 +222,15 @@ SetupCfg("options.packages.find", "include", writer_cls=ListCommaWriter), sample_value=None, ), + ConfigField("pbr", SetupCfg("--unused--", "--unused--"), sample_value=None,), + ConfigField( + "pbr__files__packages_root", + SetupCfg("files", "packages_root"), + sample_value=None, + ), + ConfigField( + "pbr__files__packages", + SetupCfg("files", "packages", writer_cls=ListCommaWriter), + sample_value=None, + ), ] diff --git a/dowsing/tests/setuptools.py b/dowsing/tests/setuptools.py index 5295975..4d3f548 100644 --- a/dowsing/tests/setuptools.py +++ b/dowsing/tests/setuptools.py @@ -1,5 +1,6 @@ import unittest from pathlib import Path +from typing import Dict, Optional import volatile @@ -63,10 +64,18 @@ def test_setup_py(self) -> None: ("setuptools", "wheel", "def"), r.get_requires_for_build_wheel() ) - def _read(self, data: str, src_dir: str = ".") -> Distribution: + def _read( + self, + data: str, + src_dir: str = ".", + extra_files: Optional[Dict[str, str]] = None, + ) -> Distribution: with volatile.dir() as d: sp = Path(d, "setup.py") sp.write_text(data) + if extra_files: + for k, v in extra_files.items(): + Path(d, k).write_text(v) Path(d, src_dir, "pkg").mkdir(parents=True) Path(d, src_dir, "pkg", "__init__.py").touch() Path(d, src_dir, "pkg", "sub").mkdir() @@ -232,3 +241,65 @@ def test_invalid_packages(self) -> None: ) # TODO wish this were None self.assertEqual(d.source_mapping, {}) + + def test_pbr_properly_enabled(self) -> None: + d = self._read( + """\ +from setuptools import setup + +setup( + setup_requires=['pbr>=1.9', 'setuptools>=17.1'], + pbr=True, +)""", + extra_files={ + "setup.cfg": """\ +[metadata] +name = pbr +author = OpenStack Foundation + +[files] +packages = + pkg +""" + }, + ) + self.assertEqual( + d.source_mapping, + { + "pkg/__init__.py": "pkg/__init__.py", + "pkg/sub/__init__.py": "pkg/sub/__init__.py", + "pkg/tests/__init__.py": "pkg/tests/__init__.py", + }, + ) + + def test_pbr_properly_enabled_src(self) -> None: + d = self._read( + """\ +from setuptools import setup + +setup( + setup_requires=['pbr>=1.9', 'setuptools>=17.1'], + pbr=True, +)""", + src_dir="src", + extra_files={ + "setup.cfg": """\ +[metadata] +name = pbr +author = OpenStack Foundation + +[files] +packages = + pkg +packages_root = src +""" + }, + ) + self.assertEqual( + d.source_mapping, + { + "pkg/__init__.py": "src/pkg/__init__.py", + "pkg/sub/__init__.py": "src/pkg/sub/__init__.py", + "pkg/tests/__init__.py": "src/pkg/tests/__init__.py", + }, + ) diff --git a/dowsing/types.py b/dowsing/types.py index 349c456..045cf77 100644 --- a/dowsing/types.py +++ b/dowsing/types.py @@ -60,6 +60,9 @@ class Distribution(pkginfo.distribution.Distribution): # type: ignore find_packages_exclude: Sequence[str] = () find_packages_include: Sequence[str] = ("*",) source_mapping: Optional[Mapping[str, str]] = None + pbr: Optional[bool] = None + pbr__files__packages_root: Optional[str] = None + pbr__files__packages: Optional[str] = None def _getHeaderAttrs(self) -> Sequence[Tuple[str, str, bool]]: # Until I invent a metadata version to include this, do so @@ -80,6 +83,9 @@ def _getHeaderAttrs(self) -> Sequence[Tuple[str, str, bool]]: ("X-Packages-Dict", "packages_dict", False), ("X-Py-Modules", "py_modules", True), ("X-Entry-Points", "entry_points", False), + ("X-Pbr", "pbr", False), + ("X-pbr__files__packages_root", "pbr__files__packages_root", False), + ("X-pbr__files__packages", "pbr__files__packages", True), ) def asdict(self) -> Dict[str, Any]: From bf7299209fc29708f578fbef6d68fefd04384b31 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sun, 13 Dec 2020 10:15:43 -0800 Subject: [PATCH 29/71] Also consider pbr-looking projects. This simple heuristic works for pbr itself without complex setup.py-matching. --- dowsing/setuptools/__init__.py | 2 +- dowsing/tests/setuptools.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/dowsing/setuptools/__init__.py b/dowsing/setuptools/__init__.py index aac7890..4de386b 100644 --- a/dowsing/setuptools/__init__.py +++ b/dowsing/setuptools/__init__.py @@ -46,7 +46,7 @@ def get_metadata(self) -> Distribution: # This is the bare minimum to get pbr projects to show as having any # sources. I don't want to use pbr.util.cfg_to_args because it appears # to import and run arbitrary code. - if d1.pbr: + if d1.pbr or (d1.pbr__files__packages and not d1.packages): where = "." if d1.pbr__files__packages_root: d1.package_dir = {"": d1.pbr__files__packages_root} diff --git a/dowsing/tests/setuptools.py b/dowsing/tests/setuptools.py index 4d3f548..31d7f14 100644 --- a/dowsing/tests/setuptools.py +++ b/dowsing/tests/setuptools.py @@ -303,3 +303,31 @@ def test_pbr_properly_enabled_src(self) -> None: "pkg/tests/__init__.py": "src/pkg/tests/__init__.py", }, ) + + def test_pbr_improperly_enabled(self) -> None: + # pbr itself is something like this. + d = self._read( + """\ +from setuptools import setup + +setup()""", + extra_files={ + "setup.cfg": """\ +[metadata] +name = pbr +author = OpenStack Foundation + +[files] +packages = + pkg +""" + }, + ) + self.assertEqual( + d.source_mapping, + { + "pkg/__init__.py": "pkg/__init__.py", + "pkg/sub/__init__.py": "pkg/sub/__init__.py", + "pkg/tests/__init__.py": "pkg/tests/__init__.py", + }, + ) From 01fffc8d42c21e503a80e59c7193cf18fcf7b784 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 13 Mar 2021 07:37:23 -0800 Subject: [PATCH 30/71] Change dowsing.pep517 to also show source_mapping --- dowsing/pep517.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dowsing/pep517.py b/dowsing/pep517.py index 26a7e99..73d30e2 100644 --- a/dowsing/pep517.py +++ b/dowsing/pep517.py @@ -74,10 +74,12 @@ def _default(obj: Any) -> Any: def main(path: Path) -> None: + metadata = get_metadata(path) d = { "get_requires_for_build_sdist": get_requires_for_build_sdist(path), "get_requires_for_build_wheel": get_requires_for_build_wheel(path), - "get_metadata": get_metadata(path).asdict(), + "get_metadata": metadata.asdict(), + "source_mapping": metadata.source_mapping, } print(json.dumps(d, default=_default)) From c5ca8ec55f040fa29c1d151831efd9827f8229ac Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 13 Mar 2021 07:37:44 -0800 Subject: [PATCH 31/71] Support flit modules, not just packages Fixes #24 --- dowsing/flit.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dowsing/flit.py b/dowsing/flit.py index 45c6a93..76b1514 100644 --- a/dowsing/flit.py +++ b/dowsing/flit.py @@ -35,9 +35,13 @@ def get_metadata(self) -> Distribution: d.project_urls["Homepage"] = v continue elif k == "module": - k = "packages" - v = find_packages(self.path.as_posix(), include=(f"{v}.*")) - d.packages_dict = {i: i.replace(".", "/") for i in v} + if (self.path / f"{v}.py").exists(): + k = "py_modules" + v = [v] + else: + k = "packages" + v = find_packages(self.path.as_posix(), include=(f"{v}.*")) + d.packages_dict = {i: i.replace(".", "/") for i in v} elif k == "description-file": k = "description" v = f"file: {v}" From 968e2de2ab543d95d0ab3e1c727051a486745024 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 13 Mar 2021 07:47:51 -0800 Subject: [PATCH 32/71] setup.py parsing: support list/str addition Fixes #25 --- dowsing/setuptools/setup_py_parsing.py | 10 ++++++++++ dowsing/tests/setuptools.py | 13 +++++++++++++ 2 files changed, 23 insertions(+) diff --git a/dowsing/setuptools/setup_py_parsing.py b/dowsing/setuptools/setup_py_parsing.py index 8d9819b..9efcde2 100644 --- a/dowsing/setuptools/setup_py_parsing.py +++ b/dowsing/setuptools/setup_py_parsing.py @@ -281,6 +281,16 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: else: # LOG.warning(f"Omit2 {type(item.slice[0].slice)!r}") return "??" + elif isinstance(item, cst.BinaryOperation): + lhs = self.evaluate_in_scope(item.left, scope) + rhs = self.evaluate_in_scope(item.right, scope) + if isinstance(item.operator, cst.Add): + try: + return lhs + rhs + except Exception: + return "??" + else: + return "??" else: # LOG.warning(f"Omit1 {type(item)!r}") return "??" diff --git a/dowsing/tests/setuptools.py b/dowsing/tests/setuptools.py index 31d7f14..0b13207 100644 --- a/dowsing/tests/setuptools.py +++ b/dowsing/tests/setuptools.py @@ -331,3 +331,16 @@ def test_pbr_improperly_enabled(self) -> None: "pkg/tests/__init__.py": "pkg/tests/__init__.py", }, ) + + def test_add_items(self) -> None: + d = self._read( + """\ +from setuptools import setup +a = "aaaa" +p = ["a", "b", "c"] +setup(name = a + "1111", packages=[] + p, classifiers=a + p) + """ + ) + self.assertEqual(d.name, "aaaa1111") + self.assertEqual(d.packages, ["a", "b", "c"]) + self.assertEqual(d.classifiers, "??") From 5cfa546a69545186e99b8e21f58fa6e8eb36ab12 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 13 Mar 2021 07:53:16 -0800 Subject: [PATCH 33/71] Support py_modules with dots in them Fixes #22 --- dowsing/types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dowsing/types.py b/dowsing/types.py index 045cf77..ffc89b7 100644 --- a/dowsing/types.py +++ b/dowsing/types.py @@ -106,6 +106,7 @@ def _source_mapping(self, root: Path) -> Optional[Dict[str, str]]: for m in self.py_modules: if m == "?": return None + m = m.replace(".", "/") d[f"{m}.py"] = f"{m}.py" try: From 8fa493d225fd5908fa84064262a4acf736c68683 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 13 Mar 2021 07:58:41 -0800 Subject: [PATCH 34/71] Ignore falsy items in packages list Fixes #20 --- dowsing/setuptools/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dowsing/setuptools/__init__.py b/dowsing/setuptools/__init__.py index 4de386b..b3ffcaf 100644 --- a/dowsing/setuptools/__init__.py +++ b/dowsing/setuptools/__init__.py @@ -99,7 +99,8 @@ def mangle(package: str) -> str: elif d1.packages != "??": assert isinstance(d1.packages, (list, tuple)) for p in d1.packages: - d1.packages_dict[p] = mangle(p) + if p: + d1.packages_dict[p] = mangle(p) d1.source_mapping = d1._source_mapping(self.path) return d1 From 7aad2e47c6bd71eb5dcd400a9121f04d3a80bd86 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 13 Mar 2021 07:50:04 -0800 Subject: [PATCH 35/71] Update changelog --- CHANGELOG.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1b7ccc0..c566f50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +## v0.9.0b2 + +* `source_mapping` bugfixes + * `packages` being an empty string (#20) + * `py_modules` containing dots (#22) + * Flit modules instead of packages (#24) + * `setup.py` parsing addition operator (#25) + ## v0.9.0b1 * Includes package data in `source_mapping` all the time. From cbede183e43a08474648dabb0b92f40c9c941698 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 1 Apr 2021 07:25:56 -0700 Subject: [PATCH 36/71] Update skel 2021-04-01 --- Makefile | 6 ++---- requirements-dev.txt | 5 +++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 4c166f6..1050aed 100644 --- a/Makefile +++ b/Makefile @@ -21,13 +21,11 @@ test: .PHONY: format format: - python -m isort --recursive -y $(SOURCES) - python -m black $(SOURCES) + python -m ufmt format $(SOURCES) .PHONY: lint lint: - python -m isort --recursive --diff $(SOURCES) - python -m black --check $(SOURCES) + python -m ufmt check $(SOURCES) python -m flake8 $(SOURCES) mypy --strict dowsing diff --git a/requirements-dev.txt b/requirements-dev.txt index 0479c95..b5f7505 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,9 +1,10 @@ -black==19.10b0 +black==20.8b1 coverage==4.5.4 flake8==3.7.9 -isort==4.3.21 mypy==0.750 tox==3.14.1 twine==3.1.1 +ufmt==1.1 +usort==0.6.3 volatile==2.1.0 wheel==0.33.6 From 72f6af54070bf295db2fe5ff7048f06bd5150e93 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Thu, 1 Apr 2021 07:27:24 -0700 Subject: [PATCH 37/71] `make format` with new black --- dowsing/setuptools/setup_and_metadata.py | 20 ++++++++++++++++---- dowsing/tests/setuptools_metadata.py | 9 ++++++--- dowsing/tests/setuptools_types.py | 10 ++++++++-- 3 files changed, 30 insertions(+), 9 deletions(-) diff --git a/dowsing/setuptools/setup_and_metadata.py b/dowsing/setuptools/setup_and_metadata.py index dea96f4..bd7a598 100644 --- a/dowsing/setuptools/setup_and_metadata.py +++ b/dowsing/setuptools/setup_and_metadata.py @@ -41,14 +41,22 @@ ), ConfigField("author", SetupCfg("metadata", "author"), Metadata("Author")), ConfigField( - "author_email", SetupCfg("metadata", "author_email"), Metadata("Author-email"), + "author_email", + SetupCfg("metadata", "author_email"), + Metadata("Author-email"), + ), + ConfigField( + "license", + SetupCfg("metadata", "license"), + Metadata("License"), ), - ConfigField("license", SetupCfg("metadata", "license"), Metadata("License"),), # TODO licence (alternate spelling) # TODO license_file, license_files (setuptools-specific) ConfigField("url", SetupCfg("metadata", "url"), Metadata("Home-page")), ConfigField( - "description", SetupCfg("metadata", "description"), Metadata("Summary"), + "description", + SetupCfg("metadata", "description"), + Metadata("Summary"), ), ConfigField( "long_description", @@ -222,7 +230,11 @@ SetupCfg("options.packages.find", "include", writer_cls=ListCommaWriter), sample_value=None, ), - ConfigField("pbr", SetupCfg("--unused--", "--unused--"), sample_value=None,), + ConfigField( + "pbr", + SetupCfg("--unused--", "--unused--"), + sample_value=None, + ), ConfigField( "pbr__files__packages_root", SetupCfg("files", "packages_root"), diff --git a/dowsing/tests/setuptools_metadata.py b/dowsing/tests/setuptools_metadata.py index d1ac9b5..6c226b5 100644 --- a/dowsing/tests/setuptools_metadata.py +++ b/dowsing/tests/setuptools_metadata.py @@ -80,13 +80,16 @@ def test_arg_mapping(self) -> None: name = field.get_distribution_key() self.assertNotEqual( - getattr(setup_py_dist, name), None, + getattr(setup_py_dist, name), + None, ) self.assertEqual( - foo, getattr(setup_py_dist, name), + foo, + getattr(setup_py_dist, name), ) self.assertEqual( - foo, getattr(setup_cfg_dist, name), + foo, + getattr(setup_cfg_dist, name), ) if field.metadata: diff --git a/dowsing/tests/setuptools_types.py b/dowsing/tests/setuptools_types.py index b73cf8d..dc51813 100644 --- a/dowsing/tests/setuptools_types.py +++ b/dowsing/tests/setuptools_types.py @@ -19,7 +19,10 @@ class WriterTest(unittest.TestCase): @parameterized.expand( # type: ignore - [(False,), (True,),] + [ + (False,), + (True,), + ] ) def test_bool_writer(self, arg: bool) -> None: c = ConfigFile() @@ -32,7 +35,10 @@ def test_bool_writer(self, arg: bool) -> None: self.assertEqual(str(arg).lower(), rcp["a"]["b"]) @parameterized.expand( # type: ignore - [("hello",), ("a\nb\nc",),] + [ + ("hello",), + ("a\nb\nc",), + ] ) def test_str_writer(self, arg: str) -> None: c = ConfigFile() From b7af6977b9b29169aec958a2d956d8c6a26aa570 Mon Sep 17 00:00:00 2001 From: John Reese Date: Mon, 6 Dec 2021 16:52:32 -0800 Subject: [PATCH 38/71] Add provides_extra metadata to types --- dowsing/types.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dowsing/types.py b/dowsing/types.py index ffc89b7..cefbdae 100644 --- a/dowsing/types.py +++ b/dowsing/types.py @@ -63,6 +63,7 @@ class Distribution(pkginfo.distribution.Distribution): # type: ignore pbr: Optional[bool] = None pbr__files__packages_root: Optional[str] = None pbr__files__packages: Optional[str] = None + provides_extra: Optional[Sequence[str]] = () def _getHeaderAttrs(self) -> Sequence[Tuple[str, str, bool]]: # Until I invent a metadata version to include this, do so From aebd317869c9bae824d4d504ec68b8b939795333 Mon Sep 17 00:00:00 2001 From: John Reese Date: Mon, 6 Dec 2021 17:14:00 -0800 Subject: [PATCH 39/71] Enhance package compatibility Handle more edge cases, adding support for parsing setup.py from fastai/fastprogress, dirsync, and plotly. --- dowsing/setuptools/__init__.py | 6 ++++-- dowsing/setuptools/setup_py_parsing.py | 20 +++++++++++++++++--- 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/dowsing/setuptools/__init__.py b/dowsing/setuptools/__init__.py index b3ffcaf..3864fee 100644 --- a/dowsing/setuptools/__init__.py +++ b/dowsing/setuptools/__init__.py @@ -96,8 +96,10 @@ def mangle(package: str) -> str: d1.find_packages_include, ): d1.packages_dict[p] = mangle(p) - elif d1.packages != "??": - assert isinstance(d1.packages, (list, tuple)) + elif d1.packages not in ("??", "????"): + assert isinstance( + d1.packages, (list, tuple) + ), f"{d1.packages!r} is not a list/tuple" for p in d1.packages: if p: d1.packages_dict[p] = mangle(p) diff --git a/dowsing/setuptools/setup_py_parsing.py b/dowsing/setuptools/setup_py_parsing.py index 9efcde2..8b0766a 100644 --- a/dowsing/setuptools/setup_py_parsing.py +++ b/dowsing/setuptools/setup_py_parsing.py @@ -141,7 +141,13 @@ def visit_Call(self, node: cst.Call) -> Optional[bool]: # TODO sometimes there is more than one setup call, we might # prioritize/merge... if any( - q.name in ("setuptools.setup", "distutils.core.setup", "setup3lib") + q.name + in ( + "setuptools.setup", + "distutils.core.setup", + "setup3lib", + "skbuild.setup", + ) for q in names ): self.found_setup = True @@ -177,7 +183,8 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: if isinstance(item, cst.SimpleString): return item.evaluated_value - # TODO int/float/etc + elif isinstance(item, (cst.Integer, cst.Float)): + return int(item.value) elif isinstance(item, cst.Name) and item.value in self.BOOL_NAMES: return self.BOOL_NAMES[item.value] elif isinstance(item, cst.Name): @@ -277,7 +284,14 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: # TODO: Figure out why this is Sequence if isinstance(item.slice[0].slice, cst.Index): rhs = self.evaluate_in_scope(item.slice[0].slice.value, scope) - return lhs.get(rhs, "??") + try: + if isinstance(lhs, dict): + return lhs.get(rhs, "??") + else: + return lhs[rhs] + except Exception: + return "??" + else: # LOG.warning(f"Omit2 {type(item.slice[0].slice)!r}") return "??" From 2b57fcb99bb9dc997444c7897ea01eee5ca1a346 Mon Sep 17 00:00:00 2001 From: John Reese Date: Mon, 6 Dec 2021 18:08:06 -0800 Subject: [PATCH 40/71] Simple dependabot config --- .github/dependabot.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .github/dependabot.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..55dbe85 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,6 @@ +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "monthly" From 7fcd5f2534080babfe0935cd5ff35c064b829d73 Mon Sep 17 00:00:00 2001 From: John Reese Date: Mon, 6 Dec 2021 18:51:41 -0800 Subject: [PATCH 41/71] Fix #33: Try reading long_description from payload body Updates the test_arg_mapping case to handle setuptools >= 57, which writes the long_description field as the payload/body of PKG-INFO, skipping the previous `Description:` field entirely. This results in the missing key from the Message object, so the test then expects to read long_description from the payload. --- dowsing/tests/setuptools_metadata.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/dowsing/tests/setuptools_metadata.py b/dowsing/tests/setuptools_metadata.py index 6c226b5..10893e8 100644 --- a/dowsing/tests/setuptools_metadata.py +++ b/dowsing/tests/setuptools_metadata.py @@ -96,6 +96,12 @@ def test_arg_mapping(self) -> None: a = setup_py_info.get_all(field.metadata.key) b = setup_cfg_info.get_all(field.metadata.key) + # setuptools>=57 writes long_description to the body/payload + # of PKG-INFO, and skips the description field entirely. + if field.keyword == "long_description" and a is None: + a = setup_py_info.get_payload() + b = setup_cfg_info.get_payload() + # install_requires gets written out to *.egg-info/requires.txt # instead if field.keyword != "install_requires": From f072c484a3181580b0bf0c5c5e9da5fe4c7a0c39 Mon Sep 17 00:00:00 2001 From: John Reese Date: Thu, 20 Jan 2022 20:52:13 -0800 Subject: [PATCH 42/71] Initial support for PEP 621 metadata with Flit This adds a `Pep621Reader` class that pulls metadata from the [project] table of pyproject.toml. The `FlitReader` class uses this as a basis for reading metadata, and is also updated to not fail if the flit table is missing. --- dowsing/flit.py | 24 ++++++++--------- dowsing/pep621.py | 49 +++++++++++++++++++++++++++++++++ dowsing/tests/__init__.py | 2 ++ dowsing/tests/flit.py | 45 +++++++++++++++++++++++++++++-- dowsing/tests/pep621.py | 57 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 163 insertions(+), 14 deletions(-) create mode 100644 dowsing/pep621.py create mode 100644 dowsing/tests/pep621.py diff --git a/dowsing/flit.py b/dowsing/flit.py index 76b1514..a61c632 100644 --- a/dowsing/flit.py +++ b/dowsing/flit.py @@ -4,10 +4,11 @@ import tomlkit from setuptools import find_packages -from .types import BaseReader, Distribution +from .pep621 import Pep621Reader +from .types import Distribution -class FlitReader(BaseReader): +class FlitReader(Pep621Reader): def __init__(self, path: Path): self.path = path @@ -21,12 +22,12 @@ def get_metadata(self) -> Distribution: pyproject = self.path / "pyproject.toml" doc = tomlkit.parse(pyproject.read_text()) - d = Distribution() - d.metadata_version = "2.1" - d.project_urls = {} - d.entry_points = {} + d = self.get_pep621_metadata() + d.entry_points = dict(d.entry_points) or {} - for k, v in doc["tool"]["flit"]["metadata"].items(): + flit = doc.get("tool", {}).get("flit", {}) + metadata = flit.get("metadata", {}) + for k, v in metadata.items(): # TODO description-file -> long_description # TODO home-page -> urls # TODO requires -> requires_dist @@ -52,10 +53,10 @@ def get_metadata(self) -> Distribution: if k2 in d: setattr(d, k2, v) - for k, v in doc["tool"]["flit"]["metadata"].get("urls", {}).items(): + for k, v in metadata.get("urls", {}).items(): d.project_urls[k] = v - for k, v in doc["tool"]["flit"].get("scripts", {}).items(): + for k, v in flit.get("scripts", {}).items(): d.entry_points[k] = v # TODO extras-require @@ -72,8 +73,7 @@ def _get_requires(self) -> Sequence[str]: https://github.com/takluyver/flit/issues/141 """ - pyproject = self.path / "pyproject.toml" - doc = tomlkit.parse(pyproject.read_text()) - seq = doc["tool"]["flit"]["metadata"].get("requires", ()) + dist = self.get_metadata() + seq = dist.requires_dist assert isinstance(seq, (list, tuple)) return seq diff --git a/dowsing/pep621.py b/dowsing/pep621.py new file mode 100644 index 0000000..b5e9082 --- /dev/null +++ b/dowsing/pep621.py @@ -0,0 +1,49 @@ +import tomlkit +from setuptools import find_packages + +from .types import BaseReader, Distribution + + +class Pep621Reader(BaseReader): + def get_pep621_metadata(self) -> Distribution: + pyproject = self.path / "pyproject.toml" + doc = tomlkit.parse(pyproject.read_text()) + + d = Distribution() + d.metadata_version = "2.1" + d.project_urls = {} + d.entry_points = {} + d.requires_dist = [] + d.packages = [] + d.packages_dict = {} + + table = doc.get("project", None) + if table: + for k, v in table.items(): + if k == "name": + if (self.path / f"{v}.py").exists(): + d.py_modules = [v] + else: + d.packages = find_packages( + self.path.as_posix(), include=(f"{v}.*") + ) + d.packages_dict = {i: i.replace(".", "/") for i in d.packages} + elif k == "license": + if "text" in v: + v = v["text"] + elif "file" in v: + v = f"file: {v['file']}" + else: + raise ValueError("no known license field values") + elif k == "dependencies": + k = "requires_dist" + elif k == "optional-dependencies": + pass + elif k == "urls": + d.project_urls.update(v) + + k2 = k.replace("-", "_") + if k2 in d: + setattr(d, k2, v) + + return d diff --git a/dowsing/tests/__init__.py b/dowsing/tests/__init__.py index 7eb139d..80335e6 100644 --- a/dowsing/tests/__init__.py +++ b/dowsing/tests/__init__.py @@ -2,6 +2,7 @@ from .flit import FlitReaderTest from .maturin import MaturinReaderTest from .pep517 import Pep517Test +from .pep621 import Pep621ReaderTest from .poetry import PoetryReaderTest from .setuptools import SetuptoolsReaderTest from .setuptools_metadata import SetupArgsTest @@ -12,6 +13,7 @@ "FlitReaderTest", "MaturinReaderTest", "Pep517Test", + "Pep621ReaderTest", "PoetryReaderTest", "SetuptoolsReaderTest", "WriterTest", diff --git a/dowsing/tests/flit.py b/dowsing/tests/flit.py index ae15ad6..a36448d 100644 --- a/dowsing/tests/flit.py +++ b/dowsing/tests/flit.py @@ -23,8 +23,8 @@ def test_simplest(self) -> None: # handle missing metadata appropriately. r = FlitReader(dp) - self.assertEqual((), r.get_requires_for_build_sdist()) - self.assertEqual((), r.get_requires_for_build_wheel()) + self.assertEqual([], r.get_requires_for_build_sdist()) + self.assertEqual([], r.get_requires_for_build_wheel()) md = r.get_metadata() self.assertEqual("Name", md.name) @@ -69,3 +69,44 @@ def test_normal(self) -> None: }, md.asdict(), ) + + def test_pep621(self) -> None: + with volatile.dir() as d: + dp = Path(d) + (dp / "pyproject.toml").write_text( + """\ +[build-system] +requires = ["flit_core >=2,<4"] +build-backend = "flit_core.buildapi" + +[project] +name = "foo" +dependencies = ["abc", "def"] + +[project.urls] +Foo = "https://" +""" + ) + (dp / "foo").mkdir() + (dp / "foo" / "tests").mkdir() + (dp / "foo" / "__init__.py").write_text("") + (dp / "foo" / "tests" / "__init__.py").write_text("") + + r = FlitReader(dp) + # Notably these do not include flit itself; that's handled by + # dowsing.pep517 + self.assertEqual(["abc", "def"], r.get_requires_for_build_sdist()) + self.assertEqual(["abc", "def"], r.get_requires_for_build_wheel()) + md = r.get_metadata() + self.assertEqual("foo", md.name) + self.assertEqual( + { + "metadata_version": "2.1", + "name": "foo", + "packages": ["foo", "foo.tests"], + "packages_dict": {"foo": "foo", "foo.tests": "foo/tests"}, + "requires_dist": ["abc", "def"], + "project_urls": {"Foo": "https://"}, + }, + md.asdict(), + ) diff --git a/dowsing/tests/pep621.py b/dowsing/tests/pep621.py new file mode 100644 index 0000000..3b3ab8b --- /dev/null +++ b/dowsing/tests/pep621.py @@ -0,0 +1,57 @@ +import unittest +from pathlib import Path + +import volatile + +from ..pep621 import Pep621Reader + + +class Pep621ReaderTest(unittest.TestCase): + def test_simplest(self) -> None: + with volatile.dir() as d: + dp = Path(d) + (dp / "pyproject.toml").write_text( + """\ +[project] +name = "Name" +""" + ) + + r = Pep621Reader(dp) + md = r.get_pep621_metadata() + self.assertEqual("Name", md.name) + + def test_normal(self) -> None: + with volatile.dir() as d: + dp = Path(d) + (dp / "pyproject.toml").write_text( + """\ +[project] +name = "foo" +dependencies = ["abc", "def"] +license = {text = "MIT"} + +[project.urls] +Foo = "https://" +""" + ) + (dp / "foo").mkdir() + (dp / "foo" / "tests").mkdir() + (dp / "foo" / "__init__.py").write_text("") + (dp / "foo" / "tests" / "__init__.py").write_text("") + + r = Pep621Reader(dp) + md = r.get_pep621_metadata() + self.assertEqual("foo", md.name) + self.assertEqual( + { + "metadata_version": "2.1", + "name": "foo", + "license": "MIT", + "packages": ["foo", "foo.tests"], + "packages_dict": {"foo": "foo", "foo.tests": "foo/tests"}, + "requires_dist": ["abc", "def"], + "project_urls": {"Foo": "https://"}, + }, + md.asdict(), + ) From ceb4048da1c31bdfdf107d2d29f6331f61b0e86b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 00:10:05 -0800 Subject: [PATCH 43/71] Bump wheel from 0.33.6 to 0.37.1 (#40) --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 19d9292..8abc6bd 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -8,5 +8,5 @@ twine==3.1.1 ufmt==1.1 usort==0.6.3 volatile==2.1.0 -wheel==0.33.6 +wheel==0.37.1 honesty==0.3.0a1 From 323d7c054342cd7fae0c50bec2c51c0e8f7e2e05 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 00:10:50 -0800 Subject: [PATCH 44/71] Bump click from 7.1.2 to 8.0.3 (#38) --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 8abc6bd..33d6001 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ black==20.8b1 -click==7.1.2 +click==8.0.3 coverage==4.5.4 flake8==3.7.9 mypy==0.750 From f863a431ac4de83ccde37d304690415dc6ca0102 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 00:29:07 -0800 Subject: [PATCH 45/71] Bump tox from 3.14.1 to 3.24.5 (#44) --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 33d6001..b6ebc27 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -3,7 +3,7 @@ click==8.0.3 coverage==4.5.4 flake8==3.7.9 mypy==0.750 -tox==3.14.1 +tox==3.24.5 twine==3.1.1 ufmt==1.1 usort==0.6.3 From d5cddeff6d0118ed210d7dedb91c89c8acc89012 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 26 Jan 2022 00:38:26 -0800 Subject: [PATCH 46/71] Bump black from 20.8b1 to 21.12b0 (#35) Bumps [black](https://github.com/psf/black) from 20.8b1 to 21.12b0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits) --- updated-dependencies: - dependency-name: black dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index b6ebc27..b6d9817 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -black==20.8b1 +black==21.12b0 click==8.0.3 coverage==4.5.4 flake8==3.7.9 From a2b87845e5ff9a042cbd7d23fbcae6be1a9e42c4 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Wed, 26 Jan 2022 10:13:06 -0800 Subject: [PATCH 47/71] mypy can pass on 3.9 now --- .github/workflows/build.yml | 1 - dowsing/setuptools/__init__.py | 2 +- dowsing/setuptools/setup_py_parsing.py | 4 ++-- requirements-dev.txt | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b8394ac..8209f9e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -34,7 +34,6 @@ jobs: run: make test - name: Lint run: make lint - if: ${{ matrix.python-version != '3.9' }} check-deps: runs-on: ${{ matrix.os }} diff --git a/dowsing/setuptools/__init__.py b/dowsing/setuptools/__init__.py index 3864fee..ce8452e 100644 --- a/dowsing/setuptools/__init__.py +++ b/dowsing/setuptools/__init__.py @@ -61,7 +61,7 @@ def get_metadata(self) -> Distribution: # https://docs.python.org/2/distutils/setupscript.html#listing-whole-packages package_dir: Mapping[str, str] = d1.package_dir # If there was an error, we might have written "??" - if package_dir != "??": + if package_dir != "??": # type: ignore if not package_dir: package_dir = {"": "."} diff --git a/dowsing/setuptools/setup_py_parsing.py b/dowsing/setuptools/setup_py_parsing.py index 8b0766a..2a61bba 100644 --- a/dowsing/setuptools/setup_py_parsing.py +++ b/dowsing/setuptools/setup_py_parsing.py @@ -92,7 +92,7 @@ def __init__(self, filename: str) -> None: class SetupCallTransformer(cst.CSTTransformer): - METADATA_DEPENDENCIES = (ScopeProvider, ParentNodeProvider, QualifiedNameProvider) # type: ignore + METADATA_DEPENDENCIES = (ScopeProvider, ParentNodeProvider, QualifiedNameProvider) def __init__( self, @@ -124,7 +124,7 @@ def leave_Call( class SetupCallAnalyzer(cst.CSTVisitor): - METADATA_DEPENDENCIES = (ScopeProvider, ParentNodeProvider, QualifiedNameProvider) # type: ignore + METADATA_DEPENDENCIES = (ScopeProvider, ParentNodeProvider, QualifiedNameProvider) # TODO names resulting from other than 'from setuptools import setup' # TODO wrapper funcs that modify args diff --git a/requirements-dev.txt b/requirements-dev.txt index b6d9817..074221c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,7 @@ black==21.12b0 click==8.0.3 coverage==4.5.4 flake8==3.7.9 -mypy==0.750 +mypy==0.931 tox==3.24.5 twine==3.1.1 ufmt==1.1 From 0a318d430d685a67c0563f8296ef4cb7008d78ad Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Wed, 26 Jan 2022 10:13:41 -0800 Subject: [PATCH 48/71] Bump some deps, including usort with minor format change --- dowsing/check_source_mapping.py | 2 +- requirements-dev.txt | 4 ++-- requirements.txt | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dowsing/check_source_mapping.py b/dowsing/check_source_mapping.py index 789732f..efe38f3 100644 --- a/dowsing/check_source_mapping.py +++ b/dowsing/check_source_mapping.py @@ -6,7 +6,7 @@ from honesty.archive import extract_and_get_names from honesty.cache import Cache from honesty.cmdline import select_versions, wrap_async -from honesty.releases import FileType, async_parse_index +from honesty.releases import async_parse_index, FileType from moreorless.click import echo_color_unified_diff from dowsing.pep517 import get_metadata diff --git a/requirements-dev.txt b/requirements-dev.txt index 074221c..c6faaca 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,8 +5,8 @@ flake8==3.7.9 mypy==0.931 tox==3.24.5 twine==3.1.1 -ufmt==1.1 -usort==0.6.3 +ufmt==1.3.1.post1 +usort==1.0.0 volatile==2.1.0 wheel==0.37.1 honesty==0.3.0a1 diff --git a/requirements.txt b/requirements.txt index 578b979..74c5df2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ highlighter==0.1.1 imperfect==0.3.0 LibCST==0.3.12 -tomlkit==0.7.0 -pkginfo==1.5.0.1 +tomlkit==0.7.2 +pkginfo==1.8.1 From 0980cadc3ee1409d736a54394379bedd9bb1f852 Mon Sep 17 00:00:00 2001 From: John Reese Date: Wed, 26 Jan 2022 21:03:04 -0800 Subject: [PATCH 49/71] Upgrade to newest tomlkit, fix type issues --- dowsing/maturin.py | 6 ++++-- dowsing/pep517.py | 15 ++++++++------- dowsing/poetry.py | 9 +++++---- requirements.txt | 2 +- 4 files changed, 18 insertions(+), 14 deletions(-) diff --git a/dowsing/maturin.py b/dowsing/maturin.py index 4576b34..299ea51 100644 --- a/dowsing/maturin.py +++ b/dowsing/maturin.py @@ -25,7 +25,8 @@ def get_metadata(self) -> Distribution: cargo = self.path / "Cargo.toml" doc = tomlkit.parse(cargo.read_text()) - for k, v in doc["package"].items(): + package = doc.get("package", {}) + for k, v in package.items(): if k == "name": d.name = v elif k == "version": @@ -39,7 +40,8 @@ def get_metadata(self) -> Distribution: # homepage # readme (filename) - for k, v in doc["package"]["metadata"]["maturin"].items(): + maturin = package.get("metadata", {}).get("maturin", {}) + for k, v in maturin.items(): if k == "requires-python": d.requires_python = v elif k == "classifier": diff --git a/dowsing/pep517.py b/dowsing/pep517.py index 73d30e2..ef17b11 100644 --- a/dowsing/pep517.py +++ b/dowsing/pep517.py @@ -26,13 +26,14 @@ def get_backend(path: Path) -> Tuple[List[str], BaseReader]: requires: List[str] = [] if pyproject.exists(): doc = tomlkit.parse(pyproject.read_text()) - if "build-system" in doc: - # 1b. include any build-system requires - if "requires" in doc["build-system"]: - requires.extend(doc["build-system"]["requires"]) - if "build-backend" in doc["build-system"]: - backend = doc["build-system"]["build-backend"] - # TODO backend-path + table = doc.get("build-system", {}) + + # 1b. include any build-system requires + if "requires" in table: + requires.extend(table["requires"]) + if "build-backend" in table: + backend = table["build-backend"] + # TODO backend-path try: backend_path = KNOWN_BACKENDS[backend] diff --git a/dowsing/poetry.py b/dowsing/poetry.py index 37b6157..ebb5a72 100644 --- a/dowsing/poetry.py +++ b/dowsing/poetry.py @@ -42,7 +42,8 @@ def get_metadata(self) -> Distribution: d.packages = [] d.packages_dict = {} - for k, v in doc["tool"]["poetry"].items(): + poetry = doc.get("tool", {}).get("poetry", {}) + for k, v in poetry.items(): if k in ("homepage", "repository", "documentation"): d.project_urls[k] = v elif k == "packages": @@ -64,16 +65,16 @@ def get_metadata(self) -> Distribution: d.packages_dict[p] = p.replace(".", "/") d.packages.append(p) - for k, v in doc["tool"]["poetry"].get("dependencies", {}).items(): + for k, v in poetry.get("dependencies", {}).items(): if k == "python": pass # TODO translate to requires_python else: d.requires_dist.append(k) # TODO something with version - for k, v in doc["tool"]["poetry"].get("urls", {}).items(): + for k, v in poetry.get("urls", {}).items(): d.project_urls[k] = v - for k, v in doc["tool"]["poetry"].get("scripts", {}).items(): + for k, v in poetry.get("scripts", {}).items(): d.entry_points[k] = v d.source_mapping = d._source_mapping(self.path) diff --git a/requirements.txt b/requirements.txt index 74c5df2..bccf3e8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ highlighter==0.1.1 imperfect==0.3.0 LibCST==0.3.12 -tomlkit==0.7.2 +tomlkit==0.8.0 pkginfo==1.8.1 From e49c4a27a6b7d3941fef35eb1f7cc17c37456a22 Mon Sep 17 00:00:00 2001 From: John Reese Date: Wed, 26 Jan 2022 21:07:33 -0800 Subject: [PATCH 50/71] Add dataclasses for 3.6 --- requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements.txt b/requirements.txt index bccf3e8..09e6376 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ imperfect==0.3.0 LibCST==0.3.12 tomlkit==0.8.0 pkginfo==1.8.1 +dataclasses==0.8; python_version<"3.7" From 86884a044002e7347b1d19d48aec888a4764357f Mon Sep 17 00:00:00 2001 From: John Reese Date: Thu, 27 Jan 2022 05:39:48 +0000 Subject: [PATCH 51/71] Run mypy against python 3.7 --- setup.cfg | 2 ++ 1 file changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index e6b7df7..0bda0bf 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,8 @@ use_parentheses = True [mypy] ignore_missing_imports = True +python_version = 3.7 +strict = True [tox:tox] envlist = py36, py37, py38 From 74e6599943c700845067039836fdf555e18dbd1b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 3 Feb 2022 11:57:56 -0800 Subject: [PATCH 52/71] Bump tomlkit from 0.8.0 to 0.9.0 (#48) Bumps [tomlkit](https://github.com/sdispater/tomlkit) from 0.8.0 to 0.9.0. - [Release notes](https://github.com/sdispater/tomlkit/releases) - [Changelog](https://github.com/sdispater/tomlkit/blob/master/CHANGELOG.md) - [Commits](https://github.com/sdispater/tomlkit/compare/0.8.0...0.9.0) --- updated-dependencies: - dependency-name: tomlkit dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 09e6376..d972937 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ highlighter==0.1.1 imperfect==0.3.0 LibCST==0.3.12 -tomlkit==0.8.0 +tomlkit==0.9.0 pkginfo==1.8.1 dataclasses==0.8; python_version<"3.7" From 2b658be5e64294a2e29b5483eab70ce6b71d60cb Mon Sep 17 00:00:00 2001 From: Brendan Gerrity Date: Fri, 4 Feb 2022 10:36:32 -0500 Subject: [PATCH 53/71] set python variable to use 3 --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9103317..adb21fc 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -PYTHON?=python +PYTHON?=python3 SOURCES=dowsing setup.py .PHONY: venv From a97cd507d64599a84fe9a944ee373fd9bb89d9cb Mon Sep 17 00:00:00 2001 From: Brendan Gerrity Date: Fri, 4 Feb 2022 11:05:37 -0500 Subject: [PATCH 54/71] add jupyter packaging as backend --- dowsing/pep517.py | 1 + 1 file changed, 1 insertion(+) diff --git a/dowsing/pep517.py b/dowsing/pep517.py index ef17b11..57ee59a 100644 --- a/dowsing/pep517.py +++ b/dowsing/pep517.py @@ -11,6 +11,7 @@ KNOWN_BACKENDS: Dict[str, str] = { "setuptools.build_meta:__legacy__": "dowsing.setuptools:SetuptoolsReader", "setuptools.build_meta": "dowsing.setuptools:SetuptoolsReader", + "jupyter_packaging.build_api": "dowsing.setuptools:SetuptoolsReader", "flit_core.buildapi": "dowsing.flit:FlitReader", "flit.buildapi": "dowsing.flit:FlitReader", "maturin": "dowsing.maturin:MaturinReader", From 4c6fa30d419bdea3aaf6fb8936ad40888d4641f2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Feb 2022 20:27:43 -0800 Subject: [PATCH 55/71] Bump black from 21.12b0 to 22.1.0 (#51) Bumps [black](https://github.com/psf/black) from 21.12b0 to 22.1.0. - [Release notes](https://github.com/psf/black/releases) - [Changelog](https://github.com/psf/black/blob/main/CHANGES.md) - [Commits](https://github.com/psf/black/commits/22.1.0) --- updated-dependencies: - dependency-name: black dependency-type: direct:development ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index c6faaca..b3e704c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,4 @@ -black==21.12b0 +black==22.1.0 click==8.0.3 coverage==4.5.4 flake8==3.7.9 From 36441b60a2070dd0dea157a91253aabe1c7e207e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Feb 2022 20:27:54 -0800 Subject: [PATCH 56/71] Bump honesty from 0.3.0a1 to 0.3.0a2 (#50) Bumps [honesty](https://github.com/python-packaging/honesty) from 0.3.0a1 to 0.3.0a2. - [Release notes](https://github.com/python-packaging/honesty/releases) - [Changelog](https://github.com/python-packaging/honesty/blob/main/CHANGELOG.md) - [Commits](https://github.com/python-packaging/honesty/compare/v0.3.0a1...v0.3.0a2) --- updated-dependencies: - dependency-name: honesty dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index b3e704c..6df737c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -9,4 +9,4 @@ ufmt==1.3.1.post1 usort==1.0.0 volatile==2.1.0 wheel==0.37.1 -honesty==0.3.0a1 +honesty==0.3.0a2 From f2ad1becb10fab631dc5143ba9547eb478cbf90e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 4 Feb 2022 20:28:14 -0800 Subject: [PATCH 57/71] Bump usort from 1.0.0 to 1.0.1 (#47) Bumps [usort](https://github.com/facebookexperimental/usort) from 1.0.0 to 1.0.1. - [Release notes](https://github.com/facebookexperimental/usort/releases) - [Changelog](https://github.com/facebookexperimental/usort/blob/main/CHANGELOG.md) - [Commits](https://github.com/facebookexperimental/usort/compare/v1.0.0...v1.0.1) --- updated-dependencies: - dependency-name: usort dependency-type: direct:development update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 6df737c..0d90b9b 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,7 +6,7 @@ mypy==0.931 tox==3.24.5 twine==3.1.1 ufmt==1.3.1.post1 -usort==1.0.0 +usort==1.0.1 volatile==2.1.0 wheel==0.37.1 honesty==0.3.0a2 From ac4db2f7cb1c04c87ea539a8c3106ea9c8942219 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 23 Nov 2024 18:33:40 -0800 Subject: [PATCH 58/71] Make tests work with the current version of setuptools Previously was failing because of multiple top-level packages. --- dowsing/setuptools/setup_and_metadata.py | 2 +- dowsing/tests/setuptools_metadata.py | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/dowsing/setuptools/setup_and_metadata.py b/dowsing/setuptools/setup_and_metadata.py index bd7a598..ff64c4c 100644 --- a/dowsing/setuptools/setup_and_metadata.py +++ b/dowsing/setuptools/setup_and_metadata.py @@ -175,7 +175,7 @@ ConfigField( "packages", SetupCfg("options", "packages", writer_cls=ListCommaWriter), - sample_value=["a", "b"], + sample_value=["a"], ), ConfigField( "package_dir", diff --git a/dowsing/tests/setuptools_metadata.py b/dowsing/tests/setuptools_metadata.py index 10893e8..9d36509 100644 --- a/dowsing/tests/setuptools_metadata.py +++ b/dowsing/tests/setuptools_metadata.py @@ -31,13 +31,13 @@ def egg_info(files: Dict[str, str]) -> Tuple[Message, Distribution]: os.chdir(d) sys.stdout = io.StringIO() - dist = run_setup(f"setup.py", ["egg_info"]) + dist = run_setup("setup.py", ["egg_info"]) finally: os.chdir(cwd) sys.stdout = stdout sources = list(Path(d).rglob("PKG-INFO")) - assert len(sources) == 1 + assert len(sources) == 1, sources with open(sources[0]) as f: parser = email.parser.Parser() @@ -63,7 +63,6 @@ def test_arg_mapping(self) -> None: "setup.py": "from setuptools import setup\n" f"setup({field.keyword}={foo!r})\n", "a/__init__.py": "", - "b/__init__.py": "", } ) @@ -74,7 +73,6 @@ def test_arg_mapping(self) -> None: f"{field.cfg.key} = {cfg_format_foo}\n", "setup.py": "from setuptools import setup\n" "setup()\n", "a/__init__.py": "", - "b/__init__.py": "", } ) From 1057a840afc4bddea4939491aa59eed40f68f35a Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 23 Nov 2024 18:59:22 -0800 Subject: [PATCH 59/71] Upgrade testing dependencies, fix types --- dowsing/_demo_pep517.py | 13 +++++++------ dowsing/flit.py | 5 ++++- dowsing/pep621.py | 6 ++++-- dowsing/poetry.py | 8 +++++--- dowsing/tests/setuptools_metadata.py | 8 ++++---- dowsing/types.py | 8 ++++---- requirements-dev.txt | 22 +++++++++++----------- requirements.txt | 9 ++++----- setup.cfg | 4 ++-- 9 files changed, 45 insertions(+), 38 deletions(-) diff --git a/dowsing/_demo_pep517.py b/dowsing/_demo_pep517.py index 967d9c7..e66cb8b 100644 --- a/dowsing/_demo_pep517.py +++ b/dowsing/_demo_pep517.py @@ -1,6 +1,7 @@ """ For testing, dump the requirements that we find using the pep517 project. """ + import json import sys @@ -19,12 +20,12 @@ def main(path: str) -> None: d = {} with BuildEnvironment() as env: env.pip_install(requires) - d[ - "get_requires_for_build_sdist" - ] = requires + hooks.get_requires_for_build_sdist(None) - d[ - "get_requires_for_build_wheel" - ] = requires + hooks.get_requires_for_build_wheel(None) + d["get_requires_for_build_sdist"] = ( + requires + hooks.get_requires_for_build_sdist(None) + ) + d["get_requires_for_build_wheel"] = ( + requires + hooks.get_requires_for_build_wheel(None) + ) print(json.dumps(d)) diff --git a/dowsing/flit.py b/dowsing/flit.py index a61c632..095aa29 100644 --- a/dowsing/flit.py +++ b/dowsing/flit.py @@ -24,6 +24,9 @@ def get_metadata(self) -> Distribution: d = self.get_pep621_metadata() d.entry_points = dict(d.entry_points) or {} + d.project_urls = list(d.project_urls) + + assert isinstance(d.project_urls, list) flit = doc.get("tool", {}).get("flit", {}) metadata = flit.get("metadata", {}) @@ -33,7 +36,7 @@ def get_metadata(self) -> Distribution: # TODO requires -> requires_dist # TODO tool.flit.metadata.urls if k == "home-page": - d.project_urls["Homepage"] = v + d.project_urls.append("Homepage={v}") continue elif k == "module": if (self.path / f"{v}.py").exists(): diff --git a/dowsing/pep621.py b/dowsing/pep621.py index b5e9082..8915e2e 100644 --- a/dowsing/pep621.py +++ b/dowsing/pep621.py @@ -11,12 +11,14 @@ def get_pep621_metadata(self) -> Distribution: d = Distribution() d.metadata_version = "2.1" - d.project_urls = {} + d.project_urls = [] d.entry_points = {} d.requires_dist = [] d.packages = [] d.packages_dict = {} + assert isinstance(d.project_urls, list) + table = doc.get("project", None) if table: for k, v in table.items(): @@ -40,7 +42,7 @@ def get_pep621_metadata(self) -> Distribution: elif k == "optional-dependencies": pass elif k == "urls": - d.project_urls.update(v) + d.project_urls.extend(v) k2 = k.replace("-", "_") if k2 in d: diff --git a/dowsing/poetry.py b/dowsing/poetry.py index ebb5a72..f6dc977 100644 --- a/dowsing/poetry.py +++ b/dowsing/poetry.py @@ -36,16 +36,18 @@ def get_metadata(self) -> Distribution: d = Distribution() d.metadata_version = "2.1" - d.project_urls = {} + d.project_urls = [] d.entry_points = {} d.requires_dist = [] d.packages = [] d.packages_dict = {} + assert isinstance(d.project_urls, list) + poetry = doc.get("tool", {}).get("poetry", {}) for k, v in poetry.items(): if k in ("homepage", "repository", "documentation"): - d.project_urls[k] = v + d.project_urls.append(f"{k}={v}") elif k == "packages": # TODO improve and add tests; this works for tf2_utils and # poetry itself but include can be a glob and there are excludes @@ -72,7 +74,7 @@ def get_metadata(self) -> Distribution: d.requires_dist.append(k) # TODO something with version for k, v in poetry.get("urls", {}).items(): - d.project_urls[k] = v + d.project_urls.append(f"{k}={v}") for k, v in poetry.get("scripts", {}).items(): d.entry_points[k] = v diff --git a/dowsing/tests/setuptools_metadata.py b/dowsing/tests/setuptools_metadata.py index 9d36509..3e3c0ac 100644 --- a/dowsing/tests/setuptools_metadata.py +++ b/dowsing/tests/setuptools_metadata.py @@ -43,8 +43,8 @@ def egg_info(files: Dict[str, str]) -> Tuple[Message, Distribution]: parser = email.parser.Parser() info = parser.parse(f) reader = SetuptoolsReader(Path(d)) - dist = reader.get_metadata() - return info, dist + dist = reader.get_metadata() # type: ignore[assignment] + return info, dist # type: ignore[return-value] # These tests do not increase coverage, and just verify that we have the right @@ -97,8 +97,8 @@ def test_arg_mapping(self) -> None: # setuptools>=57 writes long_description to the body/payload # of PKG-INFO, and skips the description field entirely. if field.keyword == "long_description" and a is None: - a = setup_py_info.get_payload() - b = setup_cfg_info.get_payload() + a = setup_py_info.get_payload() # type: ignore[assignment] + b = setup_cfg_info.get_payload() # type: ignore[assignment] # install_requires gets written out to *.egg-info/requires.txt # instead diff --git a/dowsing/types.py b/dowsing/types.py index cefbdae..7c93096 100644 --- a/dowsing/types.py +++ b/dowsing/types.py @@ -37,9 +37,8 @@ def get_metadata(self) -> "Distribution": DEFAULT_EMPTY_DICT: Mapping[str, Any] = MappingProxyType({}) -# TODO: pkginfo isn't typed, and is doing to require a yak-shave to send a PR -# since it's on launchpad. -class Distribution(pkginfo.distribution.Distribution): # type: ignore + +class Distribution(pkginfo.distribution.Distribution): # These are not actually part of the metadata, see PEP 566 setup_requires: Sequence[str] = () tests_require: Sequence[str] = () @@ -68,7 +67,8 @@ class Distribution(pkginfo.distribution.Distribution): # type: ignore def _getHeaderAttrs(self) -> Sequence[Tuple[str, str, bool]]: # Until I invent a metadata version to include this, do so # unconditionally. - return tuple(super()._getHeaderAttrs()) + ( + # Stubs are wrong, this does too exist. + return tuple(super()._getHeaderAttrs()) + ( # type: ignore[misc] ("X-Setup-Requires", "setup_requires", True), ("X-Tests-Require", "tests_require", True), ("???", "extras_require", False), diff --git a/requirements-dev.txt b/requirements-dev.txt index 0d90b9b..fc8019c 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,12 +1,12 @@ -black==22.1.0 -click==8.0.3 -coverage==4.5.4 -flake8==3.7.9 -mypy==0.931 -tox==3.24.5 -twine==3.1.1 -ufmt==1.3.1.post1 -usort==1.0.1 +black==24.10.0 +click==8.1.7 +coverage==7.6.8 +flake8==7.1.1 +mypy==1.13.0 +tox==4.23.2 +twine==5.1.1 +ufmt==2.8.0 +usort==1.0.8.post1 volatile==2.1.0 -wheel==0.37.1 -honesty==0.3.0a2 +wheel==0.45.1 +honesty==0.3.0b1 diff --git a/requirements.txt b/requirements.txt index d972937..7afca69 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ -highlighter==0.1.1 +highlighter==0.2.0 imperfect==0.3.0 -LibCST==0.3.12 -tomlkit==0.9.0 -pkginfo==1.8.1 -dataclasses==0.8; python_version<"3.7" +LibCST==1.5.1 +tomlkit==0.13.2 +pkginfo==1.11.2 diff --git a/setup.cfg b/setup.cfg index 0bda0bf..d123a00 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,7 @@ packages = setup_requires = setuptools_scm setuptools >= 38.3.0 -python_requires = >=3.6 +python_requires = >=3.7 install_requires = highlighter>=0.1.1 imperfect>=0.1.0 @@ -48,7 +48,7 @@ use_parentheses = True [mypy] ignore_missing_imports = True -python_version = 3.7 +python_version = 3.8 strict = True [tox:tox] From 1a562021a1119a2ea1c04455474a65a85e10d521 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 23 Nov 2024 19:02:38 -0800 Subject: [PATCH 60/71] Modernize GH Actions --- .github/workflows/build.yml | 69 ++++++++++++++++++++++--------------- Makefile | 2 +- requirements-dev.txt | 12 ------- setup.cfg | 16 +++++++++ 4 files changed, 58 insertions(+), 41 deletions(-) delete mode 100644 requirements-dev.txt diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8209f9e..5d02cb9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,52 +9,65 @@ on: - v* pull_request: +env: + UV_SYSTEM_PYTHON: 1 + jobs: - dowsing: + test: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: - python-version: ["3.6", "3.7", "3.8", "3.9"] + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] os: [macOS-latest, ubuntu-latest, windows-latest] steps: - name: Checkout - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: Set Up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - uses: astral-sh/setup-uv@v3 - name: Install run: | - python -m pip install --upgrade pip - make setup - pip install -U . + uv pip install -e .[test] - name: Test run: make test - name: Lint - run: make lint - - check-deps: - runs-on: ${{ matrix.os }} - strategy: - fail-fast: false - matrix: - python-version: ["3.6", "3.7", "3.8", "3.9"] - os: [ubuntu-latest] + run: | + uv pip install -e .[test,dev] + make lint + if: ${{ matrix.python-version != '3.9' && matrix.python-version != '3.8' }} + build: + needs: test + runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v1 - - name: Set Up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.12" + - uses: astral-sh/setup-uv@v3 - name: Install - run: | - python -m pip install --upgrade pip - pip install 'pessimist>=0.8.0' - echo 'importall>=0.2.1' > importall.txt - - name: Check Deps - run: | - python -m pessimist --requirements=importall.txt --fast -c 'importall --root=. --exclude=tests,_demo_pep517.py,check_source_mapping.py dowsing' . + run: uv pip install build + - name: Build + run: python -m build + - name: Upload + uses: actions/upload-artifact@v3 + with: + name: sdist + path: dist + + publish: + needs: build + runs-on: ubuntu-latest + if: startsWith(github.ref, 'refs/tags/v') + permissions: + id-token: write + steps: + - uses: actions/download-artifact@v3 + with: + name: sdist + path: dist + - uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/Makefile b/Makefile index adb21fc..da69f1f 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ venv: .PHONY: setup setup: - python -m pip install -U -r requirements.txt -r requirements-dev.txt + python -m pip install -Ue .[dev,test] .PHONY: test test: diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index fc8019c..0000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,12 +0,0 @@ -black==24.10.0 -click==8.1.7 -coverage==7.6.8 -flake8==7.1.1 -mypy==1.13.0 -tox==4.23.2 -twine==5.1.1 -ufmt==2.8.0 -usort==1.0.8.post1 -volatile==2.1.0 -wheel==0.45.1 -honesty==0.3.0b1 diff --git a/setup.cfg b/setup.cfg index d123a00..8f00d86 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,22 @@ install_requires = tomlkit>=0.2.0 pkginfo>=1.4.2 +[options.extras_require] +dev = + black==24.10.0 + click==8.1.7 + flake8==7.1.1 + mypy==1.13.0 + tox==4.23.2 + twine==5.1.1 + ufmt==2.8.0 + usort==1.0.8.post1 + volatile==2.1.0 + wheel==0.45.1 + honesty==0.3.0b1 +test = + coverage >= 6 + [check] metadata = true strict = true From f50cb665ada67e07b5f035a1714e0a11a931b7a7 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 23 Nov 2024 19:15:17 -0800 Subject: [PATCH 61/71] Change tests to match --- dowsing/flit.py | 2 +- dowsing/pep621.py | 2 +- dowsing/tests/flit.py | 4 ++-- dowsing/tests/pep621.py | 2 +- dowsing/tests/poetry.py | 8 ++++---- setup.cfg | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/dowsing/flit.py b/dowsing/flit.py index 095aa29..60907d0 100644 --- a/dowsing/flit.py +++ b/dowsing/flit.py @@ -57,7 +57,7 @@ def get_metadata(self) -> Distribution: setattr(d, k2, v) for k, v in metadata.get("urls", {}).items(): - d.project_urls[k] = v + d.project_urls.append(f"{k}={v}") for k, v in flit.get("scripts", {}).items(): d.entry_points[k] = v diff --git a/dowsing/pep621.py b/dowsing/pep621.py index 8915e2e..4a956b2 100644 --- a/dowsing/pep621.py +++ b/dowsing/pep621.py @@ -42,7 +42,7 @@ def get_pep621_metadata(self) -> Distribution: elif k == "optional-dependencies": pass elif k == "urls": - d.project_urls.extend(v) + d.project_urls.extend([f"{x}={y}" for x, y in v.items()]) k2 = k.replace("-", "_") if k2 in d: diff --git a/dowsing/tests/flit.py b/dowsing/tests/flit.py index a36448d..98a15b0 100644 --- a/dowsing/tests/flit.py +++ b/dowsing/tests/flit.py @@ -65,7 +65,7 @@ def test_normal(self) -> None: "packages": ["foo", "foo.tests"], "packages_dict": {"foo": "foo", "foo.tests": "foo/tests"}, "requires_dist": ["abc", "def"], - "project_urls": {"Foo": "https://"}, + "project_urls": ["Foo=https://"], }, md.asdict(), ) @@ -106,7 +106,7 @@ def test_pep621(self) -> None: "packages": ["foo", "foo.tests"], "packages_dict": {"foo": "foo", "foo.tests": "foo/tests"}, "requires_dist": ["abc", "def"], - "project_urls": {"Foo": "https://"}, + "project_urls": ["Foo=https://"], }, md.asdict(), ) diff --git a/dowsing/tests/pep621.py b/dowsing/tests/pep621.py index 3b3ab8b..99069b9 100644 --- a/dowsing/tests/pep621.py +++ b/dowsing/tests/pep621.py @@ -51,7 +51,7 @@ def test_normal(self) -> None: "packages": ["foo", "foo.tests"], "packages_dict": {"foo": "foo", "foo.tests": "foo/tests"}, "requires_dist": ["abc", "def"], - "project_urls": {"Foo": "https://"}, + "project_urls": ["Foo=https://"], }, md.asdict(), ) diff --git a/dowsing/tests/poetry.py b/dowsing/tests/poetry.py index ae8b2d4..f8a1f9e 100644 --- a/dowsing/tests/poetry.py +++ b/dowsing/tests/poetry.py @@ -41,10 +41,10 @@ def test_basic(self) -> None: self.assertEqual("1.5.2", md.version) self.assertEqual("BSD-3-Clause", md.license) self.assertEqual( - { - "homepage": "http://example.com", - "Bug Tracker": "https://github.com/python-poetry/poetry/issues", - }, + [ + "homepage=http://example.com", + "Bug Tracker=https://github.com/python-poetry/poetry/issues", + ], md.project_urls, ) self.assertEqual(["Not a real classifier"], md.classifiers) diff --git a/setup.cfg b/setup.cfg index 8f00d86..cf0b25f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -34,11 +34,11 @@ dev = twine==5.1.1 ufmt==2.8.0 usort==1.0.8.post1 - volatile==2.1.0 wheel==0.45.1 honesty==0.3.0b1 test = coverage >= 6 + volatile==2.1.0 [check] metadata = true From 2b08a16c85f1218d0d6459681ad21bc195adefe0 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 23 Nov 2024 19:17:32 -0800 Subject: [PATCH 62/71] Bring in setuptools in runtime deps --- setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.cfg b/setup.cfg index cf0b25f..622ebc9 100644 --- a/setup.cfg +++ b/setup.cfg @@ -23,6 +23,7 @@ install_requires = LibCST>=0.3.7 tomlkit>=0.2.0 pkginfo>=1.4.2 + setuptools >= 38.3.0 [options.extras_require] dev = From 77a421449b8db787cb420f76347e340379562545 Mon Sep 17 00:00:00 2001 From: Amethyst Reese Date: Mon, 21 Oct 2024 17:24:49 -0700 Subject: [PATCH 63/71] Support PEP 639 style license metadata --- dowsing/pep621.py | 4 +++- dowsing/tests/pep621.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/dowsing/pep621.py b/dowsing/pep621.py index 4a956b2..47697f8 100644 --- a/dowsing/pep621.py +++ b/dowsing/pep621.py @@ -31,7 +31,9 @@ def get_pep621_metadata(self) -> Distribution: ) d.packages_dict = {i: i.replace(".", "/") for i in d.packages} elif k == "license": - if "text" in v: + if isinstance(v, str): + pass # PEP 639 proposes `license = "MIT"` style metadata + elif "text" in v: v = v["text"] elif "file" in v: v = f"file: {v['file']}" diff --git a/dowsing/tests/pep621.py b/dowsing/tests/pep621.py index 99069b9..408f3ae 100644 --- a/dowsing/tests/pep621.py +++ b/dowsing/tests/pep621.py @@ -55,3 +55,19 @@ def test_normal(self) -> None: }, md.asdict(), ) + + def test_pep639(self) -> None: + with volatile.dir() as d: + dp = Path(d) + (dp / "pyproject.toml").write_text( + """\ +[project] +name = "Name" +license = "MIT" +""" + ) + + r = Pep621Reader(dp) + md = r.get_pep621_metadata() + self.assertEqual("Name", md.name) + self.assertEqual("MIT", md.license) From 6927be9a814cc6061838374b438d32ee3d4b3e2b Mon Sep 17 00:00:00 2001 From: John Reese Date: Mon, 7 Feb 2022 19:49:29 -0800 Subject: [PATCH 64/71] More robust evaluation for self-referential names Tracks the `target_name` that we are recursively evaluating, and short-circuits evaluation if we attempt to evaluate that name again. Also allows the assignment evaluation to try multiple assignments until either a real value is found, or all assignments are exhausted. Also prevents combining lhs and rhs of binary addition if either side is the `"??"` sentinel value. Fixes #56 --- dowsing/setuptools/setup_py_parsing.py | 40 +++++++++++++++++++------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/dowsing/setuptools/setup_py_parsing.py b/dowsing/setuptools/setup_py_parsing.py index 2a61bba..00e2f7f 100644 --- a/dowsing/setuptools/setup_py_parsing.py +++ b/dowsing/setuptools/setup_py_parsing.py @@ -178,7 +178,9 @@ def visit_Call(self, node: cst.Call) -> Optional[bool]: BOOL_NAMES = {"True": True, "False": False, "None": None} PRETEND_ARGV = ["setup.py", "bdist_wheel"] - def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: + def evaluate_in_scope( + self, item: cst.CSTNode, scope: Any, target_name: str = "" + ) -> Any: qnames = self.get_metadata(QualifiedNameProvider, item) if isinstance(item, cst.SimpleString): @@ -224,13 +226,23 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: # module scope isn't in the dict return "??" - return self.evaluate_in_scope(gp.value, scope) + # we have recursed, likey due to `x = x + y` assignment or similar + # self-referential evaluation + if target_name and target_name == name: + return "??" + + # keep trying assignments until we get something other than ?? + result = self.evaluate_in_scope(gp.value, scope, name) + if result and result != "??": + return result + # give up + return "??" elif isinstance(item, (cst.Tuple, cst.List)): lst = [] for el in item.elements: lst.append( self.evaluate_in_scope( - el.value, self.get_metadata(ScopeProvider, el) + el.value, self.get_metadata(ScopeProvider, el), target_name ) ) if isinstance(item, cst.Tuple): @@ -248,10 +260,10 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: for arg in item.args: if isinstance(arg.keyword, cst.Name): args[names.index(arg.keyword.value)] = self.evaluate_in_scope( - arg.value, scope + arg.value, scope, target_name ) else: - args[i] = self.evaluate_in_scope(arg.value, scope) + args[i] = self.evaluate_in_scope(arg.value, scope, target_name) i += 1 # TODO clear ones that are still default @@ -264,7 +276,9 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: d = {} for arg in item.args: if isinstance(arg.keyword, cst.Name): - d[arg.keyword.value] = self.evaluate_in_scope(arg.value, scope) + d[arg.keyword.value] = self.evaluate_in_scope( + arg.value, scope, target_name + ) # TODO something with **kwargs return d elif isinstance(item, cst.Dict): @@ -272,18 +286,20 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: for el2 in item.elements: if isinstance(el2, cst.DictElement): d[self.evaluate_in_scope(el2.key, scope)] = self.evaluate_in_scope( - el2.value, scope + el2.value, scope, target_name ) return d elif isinstance(item, cst.Subscript): - lhs = self.evaluate_in_scope(item.value, scope) + lhs = self.evaluate_in_scope(item.value, scope, target_name) if isinstance(lhs, str): # A "??" entry, propagate return "??" # TODO: Figure out why this is Sequence if isinstance(item.slice[0].slice, cst.Index): - rhs = self.evaluate_in_scope(item.slice[0].slice.value, scope) + rhs = self.evaluate_in_scope( + item.slice[0].slice.value, scope, target_name + ) try: if isinstance(lhs, dict): return lhs.get(rhs, "??") @@ -296,8 +312,10 @@ def evaluate_in_scope(self, item: cst.CSTNode, scope: Any) -> Any: # LOG.warning(f"Omit2 {type(item.slice[0].slice)!r}") return "??" elif isinstance(item, cst.BinaryOperation): - lhs = self.evaluate_in_scope(item.left, scope) - rhs = self.evaluate_in_scope(item.right, scope) + lhs = self.evaluate_in_scope(item.left, scope, target_name) + rhs = self.evaluate_in_scope(item.right, scope, target_name) + if lhs == "??" or rhs == "??": + return "??" if isinstance(item.operator, cst.Add): try: return lhs + rhs From 9ae3ac377e10d325f419ba82e1f6bce2523eefc7 Mon Sep 17 00:00:00 2001 From: John Reese Date: Tue, 8 Feb 2022 23:26:48 -0800 Subject: [PATCH 65/71] Improved assignment handling with sorting --- dowsing/setuptools/setup_py_parsing.py | 98 ++++++++++++++++++-------- dowsing/tests/setuptools.py | 29 ++++++++ 2 files changed, 97 insertions(+), 30 deletions(-) diff --git a/dowsing/setuptools/setup_py_parsing.py b/dowsing/setuptools/setup_py_parsing.py index 00e2f7f..ebf7b61 100644 --- a/dowsing/setuptools/setup_py_parsing.py +++ b/dowsing/setuptools/setup_py_parsing.py @@ -8,7 +8,12 @@ from typing import Any, Dict, Optional import libcst as cst -from libcst.metadata import ParentNodeProvider, QualifiedNameProvider, ScopeProvider +from libcst.metadata import ( + ParentNodeProvider, + PositionProvider, + QualifiedNameProvider, + ScopeProvider, +) from ..types import Distribution from .setup_and_metadata import SETUP_ARGS @@ -124,7 +129,12 @@ def leave_Call( class SetupCallAnalyzer(cst.CSTVisitor): - METADATA_DEPENDENCIES = (ScopeProvider, ParentNodeProvider, QualifiedNameProvider) + METADATA_DEPENDENCIES = ( + ScopeProvider, + ParentNodeProvider, + QualifiedNameProvider, + PositionProvider, + ) # TODO names resulting from other than 'from setuptools import setup' # TODO wrapper funcs that modify args @@ -179,7 +189,7 @@ def visit_Call(self, node: cst.Call) -> Optional[bool]: PRETEND_ARGV = ["setup.py", "bdist_wheel"] def evaluate_in_scope( - self, item: cst.CSTNode, scope: Any, target_name: str = "" + self, item: cst.CSTNode, scope: Any, target_name: str = "", target_line: int = 0 ) -> Any: qnames = self.get_metadata(QualifiedNameProvider, item) @@ -192,19 +202,30 @@ def evaluate_in_scope( elif isinstance(item, cst.Name): name = item.value assignments = scope[name] - for a in assignments: - # TODO: Only assignments "before" this node matter if in the - # same scope; really if we had a call graph and walked the other - # way, we could have a better idea of what has already happened. - + assignment_nodes = sorted( + ( + (self.get_metadata(PositionProvider, a.node).start.line, a.node) + for a in assignments + if a.node + ), + reverse=True, + ) + # Walk assignments from bottom to top, evaluating them recursively. + # When recursing, only look at assignments above the "target line". + for lineno, node in assignment_nodes: # Assign( # targets=[AssignTarget(target=Name(value="v"))], # value=SimpleString(value="'x'"), # ) # TODO or an import... # TODO builtins have BuiltinAssignment + + # we have recursed, likey due to `x = x + y` assignment or similar + # self-referential evaluation, and can't + if target_name and target_name == name and lineno >= target_line: + continue + try: - node = a.node if node: parent = self.get_metadata(ParentNodeProvider, node) if parent: @@ -214,27 +235,27 @@ def evaluate_in_scope( else: raise KeyError except (KeyError, AttributeError): - return "??" - - # This presumes a single assignment - if not isinstance(gp, cst.Assign) or len(gp.targets) != 1: - return "??" # TooComplicated(repr(gp)) + continue try: scope = self.get_metadata(ScopeProvider, gp) except KeyError: # module scope isn't in the dict - return "??" + continue - # we have recursed, likey due to `x = x + y` assignment or similar - # self-referential evaluation - if target_name and target_name == name: - return "??" + # This presumes a single assignment + if isinstance(gp, cst.Assign) and len(gp.targets) == 1: + result = self.evaluate_in_scope(gp.value, scope, name, lineno) + elif isinstance(parent, cst.AugAssign): + result = self.evaluate_in_scope(parent, scope, name, lineno) + else: + # too complicated? + continue # keep trying assignments until we get something other than ?? - result = self.evaluate_in_scope(gp.value, scope, name) - if result and result != "??": + if result != "??": return result + # give up return "??" elif isinstance(item, (cst.Tuple, cst.List)): @@ -242,7 +263,10 @@ def evaluate_in_scope( for el in item.elements: lst.append( self.evaluate_in_scope( - el.value, self.get_metadata(ScopeProvider, el), target_name + el.value, + self.get_metadata(ScopeProvider, el), + target_name, + target_line, ) ) if isinstance(item, cst.Tuple): @@ -260,10 +284,12 @@ def evaluate_in_scope( for arg in item.args: if isinstance(arg.keyword, cst.Name): args[names.index(arg.keyword.value)] = self.evaluate_in_scope( - arg.value, scope, target_name + arg.value, scope, target_name, target_line ) else: - args[i] = self.evaluate_in_scope(arg.value, scope, target_name) + args[i] = self.evaluate_in_scope( + arg.value, scope, target_name, target_line + ) i += 1 # TODO clear ones that are still default @@ -277,7 +303,7 @@ def evaluate_in_scope( for arg in item.args: if isinstance(arg.keyword, cst.Name): d[arg.keyword.value] = self.evaluate_in_scope( - arg.value, scope, target_name + arg.value, scope, target_name, target_line ) # TODO something with **kwargs return d @@ -286,11 +312,11 @@ def evaluate_in_scope( for el2 in item.elements: if isinstance(el2, cst.DictElement): d[self.evaluate_in_scope(el2.key, scope)] = self.evaluate_in_scope( - el2.value, scope, target_name + el2.value, scope, target_name, target_line ) return d elif isinstance(item, cst.Subscript): - lhs = self.evaluate_in_scope(item.value, scope, target_name) + lhs = self.evaluate_in_scope(item.value, scope, target_name, target_line) if isinstance(lhs, str): # A "??" entry, propagate return "??" @@ -298,7 +324,7 @@ def evaluate_in_scope( # TODO: Figure out why this is Sequence if isinstance(item.slice[0].slice, cst.Index): rhs = self.evaluate_in_scope( - item.slice[0].slice.value, scope, target_name + item.slice[0].slice.value, scope, target_name, target_line ) try: if isinstance(lhs, dict): @@ -312,8 +338,8 @@ def evaluate_in_scope( # LOG.warning(f"Omit2 {type(item.slice[0].slice)!r}") return "??" elif isinstance(item, cst.BinaryOperation): - lhs = self.evaluate_in_scope(item.left, scope, target_name) - rhs = self.evaluate_in_scope(item.right, scope, target_name) + lhs = self.evaluate_in_scope(item.left, scope, target_name, target_line) + rhs = self.evaluate_in_scope(item.right, scope, target_name, target_line) if lhs == "??" or rhs == "??": return "??" if isinstance(item.operator, cst.Add): @@ -323,6 +349,18 @@ def evaluate_in_scope( return "??" else: return "??" + elif isinstance(item, cst.AugAssign): + lhs = self.evaluate_in_scope(item.target, scope, target_name, target_line) + rhs = self.evaluate_in_scope(item.value, scope, target_name, target_line) + if lhs == "??" or rhs == "??": + return "??" + if isinstance(item.operator, cst.AddAssign): + try: + return lhs + rhs + except Exception: + return "??" + else: + return "??" else: # LOG.warning(f"Omit1 {type(item)!r}") return "??" diff --git a/dowsing/tests/setuptools.py b/dowsing/tests/setuptools.py index 0b13207..320c5cc 100644 --- a/dowsing/tests/setuptools.py +++ b/dowsing/tests/setuptools.py @@ -344,3 +344,32 @@ def test_add_items(self) -> None: self.assertEqual(d.name, "aaaa1111") self.assertEqual(d.packages, ["a", "b", "c"]) self.assertEqual(d.classifiers, "??") + + def test_self_reference_assignments(self) -> None: + d = self._read( + """\ +from setuptools import setup + +version = "base" +name = "foo" +name += "bar" +version = version + ".suffix" + +classifiers = [ + "123", + "abc", +] + +if True: + classifiers = classifiers + ["xyz"] + +setup( + name=name, + version=version, + classifiers=classifiers, +) + """ + ) + self.assertEqual(d.name, "foobar") + self.assertEqual(d.version, "base.suffix") + self.assertListEqual(d.classifiers, ["123", "abc", "xyz"]) From 1a80f10487a3f8ffec93cb060d756ea63d1df0c0 Mon Sep 17 00:00:00 2001 From: John Reese Date: Tue, 8 Feb 2022 23:53:58 -0800 Subject: [PATCH 66/71] Don't track names, just line numbers --- dowsing/setuptools/setup_py_parsing.py | 47 ++++++++++++++------------ dowsing/tests/setuptools.py | 20 +++++++++++ 2 files changed, 45 insertions(+), 22 deletions(-) diff --git a/dowsing/setuptools/setup_py_parsing.py b/dowsing/setuptools/setup_py_parsing.py index ebf7b61..8774e6d 100644 --- a/dowsing/setuptools/setup_py_parsing.py +++ b/dowsing/setuptools/setup_py_parsing.py @@ -189,7 +189,7 @@ def visit_Call(self, node: cst.Call) -> Optional[bool]: PRETEND_ARGV = ["setup.py", "bdist_wheel"] def evaluate_in_scope( - self, item: cst.CSTNode, scope: Any, target_name: str = "", target_line: int = 0 + self, item: cst.CSTNode, scope: Any, target_line: int = 0 ) -> Any: qnames = self.get_metadata(QualifiedNameProvider, item) @@ -211,20 +211,26 @@ def evaluate_in_scope( reverse=True, ) # Walk assignments from bottom to top, evaluating them recursively. - # When recursing, only look at assignments above the "target line". for lineno, node in assignment_nodes: + + # When recursing, only look at assignments above the "target line". + if target_line and lineno >= target_line: + continue + # Assign( # targets=[AssignTarget(target=Name(value="v"))], # value=SimpleString(value="'x'"), # ) + # + # AugAssign( + # target=Name(value="v"), + # operator=AddAssign(...), + # value=SimpleString(value="'x'"), + # ) + # # TODO or an import... # TODO builtins have BuiltinAssignment - # we have recursed, likey due to `x = x + y` assignment or similar - # self-referential evaluation, and can't - if target_name and target_name == name and lineno >= target_line: - continue - try: if node: parent = self.get_metadata(ParentNodeProvider, node) @@ -245,9 +251,9 @@ def evaluate_in_scope( # This presumes a single assignment if isinstance(gp, cst.Assign) and len(gp.targets) == 1: - result = self.evaluate_in_scope(gp.value, scope, name, lineno) + result = self.evaluate_in_scope(gp.value, scope, lineno) elif isinstance(parent, cst.AugAssign): - result = self.evaluate_in_scope(parent, scope, name, lineno) + result = self.evaluate_in_scope(parent, scope, lineno) else: # too complicated? continue @@ -265,7 +271,6 @@ def evaluate_in_scope( self.evaluate_in_scope( el.value, self.get_metadata(ScopeProvider, el), - target_name, target_line, ) ) @@ -284,12 +289,10 @@ def evaluate_in_scope( for arg in item.args: if isinstance(arg.keyword, cst.Name): args[names.index(arg.keyword.value)] = self.evaluate_in_scope( - arg.value, scope, target_name, target_line + arg.value, scope, target_line ) else: - args[i] = self.evaluate_in_scope( - arg.value, scope, target_name, target_line - ) + args[i] = self.evaluate_in_scope(arg.value, scope, target_line) i += 1 # TODO clear ones that are still default @@ -303,7 +306,7 @@ def evaluate_in_scope( for arg in item.args: if isinstance(arg.keyword, cst.Name): d[arg.keyword.value] = self.evaluate_in_scope( - arg.value, scope, target_name, target_line + arg.value, scope, target_line ) # TODO something with **kwargs return d @@ -312,11 +315,11 @@ def evaluate_in_scope( for el2 in item.elements: if isinstance(el2, cst.DictElement): d[self.evaluate_in_scope(el2.key, scope)] = self.evaluate_in_scope( - el2.value, scope, target_name, target_line + el2.value, scope, target_line ) return d elif isinstance(item, cst.Subscript): - lhs = self.evaluate_in_scope(item.value, scope, target_name, target_line) + lhs = self.evaluate_in_scope(item.value, scope, target_line) if isinstance(lhs, str): # A "??" entry, propagate return "??" @@ -324,7 +327,7 @@ def evaluate_in_scope( # TODO: Figure out why this is Sequence if isinstance(item.slice[0].slice, cst.Index): rhs = self.evaluate_in_scope( - item.slice[0].slice.value, scope, target_name, target_line + item.slice[0].slice.value, scope, target_line ) try: if isinstance(lhs, dict): @@ -338,8 +341,8 @@ def evaluate_in_scope( # LOG.warning(f"Omit2 {type(item.slice[0].slice)!r}") return "??" elif isinstance(item, cst.BinaryOperation): - lhs = self.evaluate_in_scope(item.left, scope, target_name, target_line) - rhs = self.evaluate_in_scope(item.right, scope, target_name, target_line) + lhs = self.evaluate_in_scope(item.left, scope, target_line) + rhs = self.evaluate_in_scope(item.right, scope, target_line) if lhs == "??" or rhs == "??": return "??" if isinstance(item.operator, cst.Add): @@ -350,8 +353,8 @@ def evaluate_in_scope( else: return "??" elif isinstance(item, cst.AugAssign): - lhs = self.evaluate_in_scope(item.target, scope, target_name, target_line) - rhs = self.evaluate_in_scope(item.value, scope, target_name, target_line) + lhs = self.evaluate_in_scope(item.target, scope, target_line) + rhs = self.evaluate_in_scope(item.value, scope, target_line) if lhs == "??" or rhs == "??": return "??" if isinstance(item.operator, cst.AddAssign): diff --git a/dowsing/tests/setuptools.py b/dowsing/tests/setuptools.py index 320c5cc..5e99c7b 100644 --- a/dowsing/tests/setuptools.py +++ b/dowsing/tests/setuptools.py @@ -373,3 +373,23 @@ def test_self_reference_assignments(self) -> None: self.assertEqual(d.name, "foobar") self.assertEqual(d.version, "base.suffix") self.assertListEqual(d.classifiers, ["123", "abc", "xyz"]) + + def test_circular_references(self) -> None: + d = self._read( + """\ +from setuptools import setup + +name = "foo" + +foo = bar +bar = version +version = foo + +setup( + name=name, + version=version, +) + """ + ) + self.assertEqual(d.name, "foo") + self.assertEqual(d.version, "??") From 1b7b694e88fd483914acf20ba8273b56840e15ef Mon Sep 17 00:00:00 2001 From: John Reese Date: Tue, 8 Feb 2022 23:59:50 -0800 Subject: [PATCH 67/71] x=x test case --- dowsing/tests/setuptools.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/dowsing/tests/setuptools.py b/dowsing/tests/setuptools.py index 5e99c7b..3bf1119 100644 --- a/dowsing/tests/setuptools.py +++ b/dowsing/tests/setuptools.py @@ -385,6 +385,8 @@ def test_circular_references(self) -> None: bar = version version = foo +classifiers = classifiers + setup( name=name, version=version, @@ -393,3 +395,4 @@ def test_circular_references(self) -> None: ) self.assertEqual(d.name, "foo") self.assertEqual(d.version, "??") + self.assertEqual(d.classifiers, ()) From e42a849b0602768408b52474d0af6525506a4f8f Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 23 Nov 2024 19:31:30 -0800 Subject: [PATCH 68/71] Add test confirming builtin redefinition is ok (see pr comment) --- dowsing/tests/setuptools.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/dowsing/tests/setuptools.py b/dowsing/tests/setuptools.py index 3bf1119..5ba2a8c 100644 --- a/dowsing/tests/setuptools.py +++ b/dowsing/tests/setuptools.py @@ -396,3 +396,24 @@ def test_circular_references(self) -> None: self.assertEqual(d.name, "foo") self.assertEqual(d.version, "??") self.assertEqual(d.classifiers, ()) + + def test_redefines_builtin(self) -> None: + d = self._read( + """\ +import setuptools +with open("CREDITS.txt", "r", encoding="utf-8") as fp: + credits = fp.read() + +long_desc = "a" + credits + "b" +name = "foo" + +kwargs = dict( + long_description = long_desc, + name = name, +) + +setuptools.setup(**kwargs) +""" + ) + self.assertEqual(d.name, "foo") + self.assertEqual(d.description, "??") From 10a837a691e4093ad0160fd99a3571ce1aac64d9 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 23 Nov 2024 19:37:48 -0800 Subject: [PATCH 69/71] Fix typing glitch --- dowsing/tests/setuptools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dowsing/tests/setuptools.py b/dowsing/tests/setuptools.py index 5ba2a8c..88840a7 100644 --- a/dowsing/tests/setuptools.py +++ b/dowsing/tests/setuptools.py @@ -372,7 +372,7 @@ def test_self_reference_assignments(self) -> None: ) self.assertEqual(d.name, "foobar") self.assertEqual(d.version, "base.suffix") - self.assertListEqual(d.classifiers, ["123", "abc", "xyz"]) + self.assertSequenceEqual(d.classifiers, ["123", "abc", "xyz"]) def test_circular_references(self) -> None: d = self._read( From be7dee696fcecb7778820974e801f3345e9115c7 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 23 Nov 2024 19:36:10 -0800 Subject: [PATCH 70/71] Update tox config --- setup.cfg | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/setup.cfg b/setup.cfg index 622ebc9..db662a1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -69,15 +69,15 @@ python_version = 3.8 strict = True [tox:tox] -envlist = py36, py37, py38 +envlist = py{38,39,310,311,312,313}-tests [testenv] -deps = -rrequirements-dev.txt -whitelist_externals = make +deps = .[test] +allowlist_externals = make commands = make test setenv = - py{36,37,38}: COVERAGE_FILE={envdir}/.coverage + tests: COVERAGE_FILE={envdir}/.coverage [flake8] ignore = E203, E231, E266, E302, E501, W503 From 71a65988f29cb143c399f986ae56f4195fb9e860 Mon Sep 17 00:00:00 2001 From: Tim Hatch Date: Sat, 23 Nov 2024 19:47:56 -0800 Subject: [PATCH 71/71] Update changelog to get ready for release --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c566f50..8ff4ed4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## v0.9.0b3 + +* Support PEP 639 style metadata (#76) +* Support more `setup.py` assignments (#57) +* 3.12 compat (depends on setuptools) +* Fix tests to work on modern Python + ## v0.9.0b2 * `source_mapping` bugfixes