diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 8a0c736726..2ed4548cee 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -29,12 +29,13 @@ jobs: python-version: ${{ matrix.python-version }} cache: poetry - - name: Install PyGobject dependencies on Ubuntu + - name: Install PyGobject and release script dependencies on Ubuntu if: matrix.platform == 'ubuntu-latest' run: | sudo apt update - sudo apt install ffmpeg gobject-introspection libgirepository1.0-dev - poetry install --extras=replaygain --extras=reflink + sudo apt install ffmpeg gobject-introspection libgirepository1.0-dev pandoc + poetry install --with=release --extras=docs --extras=replaygain --extras=reflink + poe docs - name: Install Python dependencies run: poetry install --only=main,test --extras=autobpm diff --git a/.github/workflows/make_release.yaml b/.github/workflows/make_release.yaml index cf1b12fe24..248755703b 100644 --- a/.github/workflows/make_release.yaml +++ b/.github/workflows/make_release.yaml @@ -65,9 +65,10 @@ jobs: - name: Obtain the changelog id: generate_changelog run: | + poe docs { echo 'changelog<> "$GITHUB_OUTPUT" diff --git a/extra/release.py b/extra/release.py index 329fbb208f..e1c036b286 100755 --- a/extra/release.py +++ b/extra/release.py @@ -6,22 +6,141 @@ import re import subprocess +from contextlib import redirect_stdout from datetime import datetime, timezone +from functools import partial +from io import StringIO from pathlib import Path -from typing import Callable +from typing import Callable, NamedTuple import click import tomli from packaging.version import Version, parse +from sphinx.ext import intersphinx +from typing_extensions import TypeAlias BASE = Path(__file__).parent.parent.absolute() PYPROJECT = BASE / "pyproject.toml" CHANGELOG = BASE / "docs" / "changelog.rst" +DOCS = "https://beets.readthedocs.io/en/stable" -MD_CHANGELOG_SECTION_LIST = re.compile(r"- .+?(?=\n\n###|$)", re.DOTALL) -version_header = r"\d+\.\d+\.\d+ \([^)]+\)" +VERSION_HEADER = r"\d+\.\d+\.\d+ \([^)]+\)" RST_LATEST_CHANGES = re.compile( - rf"{version_header}\n--+\s+(.+?)\n\n+{version_header}", re.DOTALL + rf"{VERSION_HEADER}\n--+\s+(.+?)\n\n+{VERSION_HEADER}", re.DOTALL +) + +Replacement: TypeAlias = "tuple[str, str | Callable[[re.Match[str]], str]]" + + +class Ref(NamedTuple): + """A reference to documentation with ID, path, and optional title.""" + + id: str + path: str | None + title: str | None + + @classmethod + def from_line(cls, line: str) -> Ref: + """Create Ref from a Sphinx objects.inv line. + + Each line has the following structure: + [optional title : ] + + """ + if len(line_parts := line.split(" ", 1)) == 1: + return cls(line, None, None) + + id, path_with_name = line_parts + parts = [p.strip() for p in path_with_name.split(":", 1)] + + if len(parts) == 1: + path, name = parts[0], None + else: + name, path = parts + + return cls(id, path, name) + + @property + def url(self) -> str: + """Full documentation URL.""" + return f"{DOCS}/{self.path}" + + @property + def name(self) -> str: + """Display name (title if available, otherwise ID).""" + return self.title or self.id + + +def get_refs() -> dict[str, Ref]: + """Parse Sphinx objects.inv and return dict of documentation references.""" + objects_filepath = Path("docs/_build/html/objects.inv") + if not objects_filepath.exists(): + raise ValueError("Documentation does not exist. Run 'poe docs' first.") + + captured_output = StringIO() + + with redirect_stdout(captured_output): + intersphinx.inspect_main([str(objects_filepath)]) + + return { + r.id: r + for ln in captured_output.getvalue().split("\n") + if ln.startswith("\t") and (r := Ref.from_line(ln.strip())) + } + + +def create_rst_replacements() -> list[Replacement]: + """Generate list of pattern replacements for RST changelog.""" + refs = get_refs() + + def make_ref_link(ref_id: str, name: str | None = None) -> str: + ref = refs[ref_id] + return rf"`{name or ref.name} <{ref.url}>`_" + + commands = "|".join(r.split("-")[0] for r in refs if r.endswith("-cmd")) + plugins = "|".join( + r.split("/")[-1] for r in refs if r.startswith("plugins/") + ) + return [ + # Fix nested bullet points indent: use 2 spaces consistently + (r"(?<=\n) {3,4}(?=\*)", " "), + # Fix nested text indent: use 4 spaces consistently + (r"(?<=\n) {5,6}(?=[\w:`])", " "), + # Replace Sphinx :ref: and :doc: directives by documentation URLs + # :ref:`/plugins/autobpm` -> [AutoBPM Plugin](DOCS/plugins/autobpm.html) + ( + r":(?:ref|doc):`+(?:([^`<]+)<)?/?([\w./_-]+)>?`+", + lambda m: make_ref_link(m[2], m[1]), + ), + # Convert command references to documentation URLs + # `beet move` or `move` command -> [import](DOCS/reference/cli.html#import) + ( + rf"`+beet ({commands})`+|`+({commands})`+(?= command)", + lambda m: make_ref_link(f"{m[1] or m[2]}-cmd"), + ), + # Convert plugin references to documentation URLs + # `fetchart` plugin -> [fetchart](DOCS/plugins/fetchart.html) + (rf"`+({plugins})`+", lambda m: make_ref_link(f"plugins/{m[1]}")), + # Add additional backticks around existing backticked text to ensure it + # is rendered as inline code in Markdown + (r"(?<=[\s])(`[^`]+`)(?!_)", r"`\1`"), + # Convert bug references to GitHub issue links + (r":bug:`(\d+)`", r":bug: (#\1)"), + # Convert user references to GitHub @mentions + (r":user:`(\w+)`", r"\@\1"), + ] + + +MD_REPLACEMENTS: list[Replacement] = [ + (r"^ (- )", r"\1"), # remove indent from top-level bullet points + (r"^ +( - )", r"\1"), # adjust nested bullet points indent + (r"^(\w[^\n]{,80}):(?=\n\n[^ ])", r"### \1"), # format section headers + (r"^(\w[^\n]{81,}):(?=\n\n[^ ])", r"**\1**"), # and bolden too long ones + (r"### [^\n]+\n+(?=### )", ""), # remove empty sections +] +order_bullet_points = partial( + re.compile("(\n- .*?(?=\n(?! *- )|$))", flags=re.DOTALL).sub, + lambda m: "\n- ".join(sorted(m.group().split("\n- "))), ) @@ -41,8 +160,11 @@ def update_changelog(text: str, new: Version) -> str: ---------- New features: + Bug fixes: + For packagers: + Other changes: {new_header} @@ -95,50 +217,36 @@ def bump_version(new: Version) -> None: def rst2md(text: str) -> str: """Use Pandoc to convert text from ReST to Markdown.""" - # Other backslashes with verbatim ranges. - rst = re.sub(r"(?<=[\s(])`([^`]+)`(?=[^_])", r"``\1``", text) - - # Bug numbers. - rst = re.sub(r":bug:`(\d+)`", r":bug: (#\1)", rst) - - # Users. - rst = re.sub(r":user:`(\w+)`", r"@\1", rst) return ( subprocess.check_output( - ["/usr/bin/pandoc", "--from=rst", "--to=gfm", "--wrap=none"], - input=rst.encode(), + ["pandoc", "--from=rst", "--to=gfm+hard_line_breaks"], + input=text.encode(), ) .decode() .strip() ) -def changelog_as_markdown() -> str: - """Get the latest changelog entry as hacked up Markdown.""" - with CHANGELOG.open() as f: - contents = f.read() +def get_changelog_contents() -> str | None: + if m := RST_LATEST_CHANGES.search(CHANGELOG.read_text()): + return m.group(1) - m = RST_LATEST_CHANGES.search(contents) - rst = m.group(1) if m else "" + return None - # Convert with Pandoc. - md = rst2md(rst) - # Make sections stand out - md = re.sub(r"^(\w.+?):$", r"### \1", md, flags=re.M) +def changelog_as_markdown(rst: str) -> str: + """Get the latest changelog entry as hacked up Markdown.""" + for pattern, repl in create_rst_replacements(): + rst = re.sub(pattern, repl, rst, flags=re.M | re.DOTALL) - # Highlight plugin names - md = re.sub( - r"^- `/?plugins/(\w+)`:?", r"- Plugin **`\1`**:", md, flags=re.M - ) + md = rst2md(rst) - # Highlights command names. - md = re.sub(r"^- `(\w+)-cmd`:?", r"- Command **`\1`**:", md, flags=re.M) + for pattern, repl in MD_REPLACEMENTS: + md = re.sub(pattern, repl, md, flags=re.M | re.DOTALL) - # sort list items alphabetically for each of the sections - return MD_CHANGELOG_SECTION_LIST.sub( - lambda m: "\n".join(sorted(m.group().splitlines())), md - ) + # order bullet points in each of the lists alphabetically to + # improve readability + return order_bullet_points(md) @click.group() @@ -156,7 +264,11 @@ def bump(version: Version) -> None: @cli.command() def changelog(): """Get the most recent version's changelog as Markdown.""" - print(changelog_as_markdown()) + if changelog := get_changelog_contents(): + try: + print(changelog_as_markdown(changelog)) + except ValueError as e: + raise click.exceptions.UsageError(str(e)) if __name__ == "__main__": diff --git a/test/test_release.py b/test/test_release.py new file mode 100644 index 0000000000..4b3f37113b --- /dev/null +++ b/test/test_release.py @@ -0,0 +1,108 @@ +"""Tests for the release utils.""" + +import os +import shutil +import sys + +import pytest + +release = pytest.importorskip("extra.release") + + +pytestmark = pytest.mark.skipif( + not ( + (os.environ.get("GITHUB_ACTIONS") == "true" and sys.platform != "win32") + or bool(shutil.which("pandoc")) + ), + reason="pandoc isn't available", +) + + +@pytest.fixture +def rst_changelog(): + return """New features: + +* :doc:`/plugins/substitute`: Some substitute + multi-line change. + :bug:`5467` +* :ref:`list-cmd` Update. + +You can do something with this command:: + + $ do-something + +Bug fixes: + +* Some fix that refers to an issue. + :bug:`5467` +* Some fix that mentions user :user:`username`. +* Some fix thanks to + :user:`username`. :bug:`5467` +* Some fix with its own bullet points using incorrect indentation: + * First nested bullet point + with some text that wraps to the next line + * Second nested bullet point +* Another fix with its own bullet points using correct indentation: + * First + * Second + +Section naaaaaaaaaaaaaaaaaaaaaaaammmmmmmmmmmmmmmmeeeeeeeeeeeeeee with over 80 +characters: + +Empty section: + +Other changes: + +* Changed `bitesize` label to `good first issue`. Our `contribute`_ page is now + automatically populated with these issues. :bug:`4855` + +.. _contribute: https://github.com/beetbox/beets/contribute + +2.1.0 (November 22, 2024) +------------------------- + +Bug fixes: + +* Fixed something.""" + + +@pytest.fixture +def md_changelog(): + return r"""### New features + +- [Substitute Plugin](https://beets.readthedocs.io/en/stable/plugins/substitute.html): Some substitute multi-line change. :bug: (\#5467) +- [list](https://beets.readthedocs.io/en/stable/reference/cli.html#list-cmd) Update. + +You can do something with this command: + + $ do-something + +### Bug fixes + +- Another fix with its own bullet points using correct indentation: + - First + - Second +- Some fix thanks to @username. :bug: (\#5467) +- Some fix that mentions user @username. +- Some fix that refers to an issue. :bug: (\#5467) +- Some fix with its own bullet points using incorrect indentation: + - First nested bullet point with some text that wraps to the next line + - Second nested bullet point + +**Section naaaaaaaaaaaaaaaaaaaaaaaammmmmmmmmmmmmmmmeeeeeeeeeeeeeee with over 80 characters** + +### Other changes + +- Changed `bitesize` label to `good first issue`. Our [contribute](https://github.com/beetbox/beets/contribute) page is now automatically populated with these issues. :bug: (\#4855) + +# 2.1.0 (November 22, 2024) + +### Bug fixes + +- Fixed something.""" # noqa: E501 + + +def test_convert_rst_to_md(rst_changelog, md_changelog): + actual = release.changelog_as_markdown(rst_changelog) + + assert actual == md_changelog