diff --git a/.gitattributes b/.gitattributes index 12a49d1bda..647009ed89 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,9 @@ # Make Github ignore the designs folder when determining repo language flow/designs/src/* linguist-vendored flow/scripts/variables.json linguist-generated=true + +# Byte-for-byte copies of OpenROAD files, regenerated by +# docs/scripts/refresh_openroad_docs.py. Upstream's trailing whitespace must +# survive verbatim, or the refresh is no longer idempotent and the vendored +# copies stop matching the commit SOURCE.json pins. +docs/_vendor/openroad/** -whitespace diff --git a/.github/workflows/github-actions-cron-update-OR.yml b/.github/workflows/github-actions-cron-update-OR.yml index 7b80699270..ec6faa2b1a 100644 --- a/.github/workflows/github-actions-cron-update-OR.yml +++ b/.github/workflows/github-actions-cron-update-OR.yml @@ -22,6 +22,13 @@ jobs: echo "::set-output name=has_update::$(git --no-pager diff --name-only origin/master..HEAD)" git checkout master git pull + - name: Refresh vendored OpenROAD docs + # docs/conf.py reads docs/_vendor/openroad/ instead of downloading from + # GitHub while Sphinx runs, so the vendored copies must be re-taken at + # the new submodule SHA. Running it here puts the docs refresh in the + # same commit as the submodule bump, which is what + # docs/scripts/check_vendored_docs.py asserts. + run: python3 docs/scripts/refresh_openroad_docs.py - if: "steps.remote-update.outputs.has_update != ''" name: Create Draft PR uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1 diff --git a/.gitignore b/.gitignore index 84d1e39569..8d5142778c 100644 --- a/.gitignore +++ b/.gitignore @@ -85,12 +85,15 @@ perf.data perf.data.old # documentation specific +# Everything below is generated by docs/conf.py during `make html`; the sources +# are docs/index.md, ../README.md and docs/_vendor/openroad/ (see +# docs/_vendor/openroad/README.md). docs/main docs/build -SupportedOS.md index2.md Manpage.md mainREADME.md +docs/contrib/GitGuide.md build .scala-build/ .bsp/ @@ -107,4 +110,5 @@ MODULE.bazel.lock # python venv venv/ +.venv/ tmp/ diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 9c99c03866..5edb8e8edb 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -8,7 +8,7 @@ version: 2 build: os: ubuntu-22.04 tools: - python: "3.10" + python: "3.12" # Build documentation in the "docs/" directory with Sphinx sphinx: diff --git a/docs/Makefile b/docs/Makefile index 486af0ac24..558768fa14 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -7,12 +7,20 @@ SPHINXOPTS ?= SPHINXBUILD ?= sphinx-build SOURCEDIR = BUILDDIR = build +# Read the Docs builds with 3.12 (see .readthedocs.yaml); override if you want +# a specific interpreter locally. +PYTHON ?= python3 # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -.PHONY: help Makefile +# Create a local virtualenv and install the hash-pinned doc-build dependencies. +venv: + $(PYTHON) -m venv .venv + .venv/bin/pip install -r requirements_lock.txt + +.PHONY: help Makefile venv # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). diff --git a/docs/README.md b/docs/README.md index 81cddb3edd..08c99af809 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,28 +11,33 @@ repository and is published at ### Requires -- Python 3.10 — the version Read the Docs builds with (see `.readthedocs.yaml`). - The pinned Sphinx release does not run on Python 3.13 or newer. +- Python 3.10+ (tested through 3.14) — Read the Docs builds with 3.12 (see + `.readthedocs.yaml`); any recent `python3` on PATH works locally. - `pip` - `virtualenv` (or the standard-library `venv`) ### Install prerequisites -Run from the repository root: +Run from `docs/`: ``` shell -virtualenv .venv +make venv source .venv/bin/activate -pip install -r docs/requirements_lock.txt ``` +`make venv` just runs `python3 -m venv .venv && .venv/bin/pip install -r +requirements_lock.txt`; pass `PYTHON=/path/to/python3` to use a specific +interpreter instead of whatever `python3` resolves to on your PATH. + `docs/requirements_lock.txt` is the hash-pinned lock file that Read the Docs installs, so a local build matches the published one. The direct dependencies are listed in `docs/requirements.in`; after editing that file, regenerate the -lock with: +lock with [`uv`](https://docs.astral.sh/uv/) (`pip-compile` from `pip-tools` +is a documented alternative, but at time of writing has a compatibility bug +with recent `pip` releases): ``` shell -pip-compile --generate-hashes --output-file=docs/requirements_lock.txt docs/requirements.in +uv pip compile --upgrade --generate-hashes --output-file=docs/requirements_lock.txt docs/requirements.in ``` ### Build @@ -62,7 +67,7 @@ It exits non-zero when any link is unreachable. `SupportedOS.md`, `Manpage.md`, `mainREADME.md`, and — because it is fetched rather than authored here — the tracked file `contrib/GitGuide.md`. Do not commit those changes; `git checkout docs/contrib/GitGuide.md` after a build. -- Always `rm -rf docs/build` before rebuilding. The output directory is - `build/`, but `conf.py`'s `exclude_patterns` only excludes `_build`, so a - second run picks up the Markdown files written under `build/html/` as if they - were source pages. +- `conf.py`'s `exclude_patterns` excludes `build` (the actual output directory + — Sphinx's own default is `_build`) and any local `.venv`/`venv`, so re-running + a build, or building from a checkout with a local virtualenv already in + `docs/`, does not pick that up as source content. diff --git a/docs/_vendor/openroad/GitGuide.md b/docs/_vendor/openroad/GitGuide.md new file mode 100644 index 0000000000..da22ff87d3 --- /dev/null +++ b/docs/_vendor/openroad/GitGuide.md @@ -0,0 +1,207 @@ +# Git Quickstart + +This tutorial serves as a quickstart to Git and contributing to our repository. If you have not already set up OpenROAD, please follow the instructions [here](../user/Build.md). + +```{tip} This basic tutorial gives instruction for basic password Git authentication. +If you would like to setup SSH authentication, please follow this [guide](https://help.github.com/set-up-git-redirect). +``` + +## Forking + +You will need your own fork to work on the code. Go to the `OpenROAD` project +[page](https://github.com/The-OpenROAD-Project/OpenROAD) and hit the `Fork` button. You will +want to clone your fork to your machine: + +```shell +git clone https://github.com/your-user-name/OpenROAD.git +cd OpenROAD +git remote add upstream https://github.com/The-OpenROAD-Project/OpenROAD.git +git fetch upstream +``` + +This creates the directory `OpenROAD` and connects your repository to +the upstream (master project) *OpenROAD* repository. + +## Creating a branch + +You want your master branch to reflect only production-ready code, so create a +feature branch for making your changes. For example: + +```shell +git checkout master && git branch shiny-new-feature +git checkout shiny-new-feature +# Or equivalently, +git checkout master && git checkout -b shiny-new-feature +``` + +This changes your working directory to the shiny-new-feature branch. Keep any +changes in this branch specific to one bug or feature so it is clear +what the branch brings to OpenROAD. You can have many shiny-new-features +and switch in between them using the git checkout command. + +When creating this branch, make sure your master branch is up to date with +the latest upstream master version. To update your local master branch, you +can do: + +```shell +git checkout master +git pull upstream master +``` + +When you want to update the feature branch with changes in master after +you created the branch, check the section on +[updating a PR](#updating-your-pull-request). + +## Committing your code +Keep style fixes to a separate commit to make your pull request more readable. Once you've made changes, you can see them by typing: + +```shell +git status +``` + +If you have created a new file, it is not being tracked by git. Add it by typing: +```shell +git add path/to/file-to-be-added.py +``` + +Doing `git status` again should give something like: +```shell +# On branch shiny-new-feature +# +# modified: /relative/path/to/file-you-added.py +# +``` + +Finally, commit your changes to your local repository with an explanatory commit +message. Do note the `-s` option is needed for developer signoff. +```shell +git commit -s -m "your commit message goes here" +``` + +## Pushing your changes + +When you want your changes to appear publicly on your GitHub page, push your +forked feature branch's commits: + +```shell +git push origin shiny-new-feature +``` + +Here `origin` is the default name given to your remote repository on GitHub. +You can see the remote repositories: + +```shell +git remote -v +``` + +If you added the upstream repository as described above you will see something +like: + +```shell +origin https://github.com/your-user-name/OpenROAD.git (fetch) +origin https://github.com/your-user-name/OpenROAD.git (push) +upstream https://github.com/The-OpenROAD-Project/OpenROAD.git (fetch) +upstream https://github.com/The-OpenROAD-Project/OpenROAD.git (push) +``` + +Now your code is on GitHub, but it is not yet a part of the OpenROAD project. For that to +happen, a pull request needs to be submitted on GitHub. + +## Review your code + +When you're ready to ask for a code review, file a pull request. Before you do, once +again make sure that you have followed all the guidelines outlined in the [Developer's Guide](./DeveloperGuide.md) +regarding code style, tests, performance tests, and documentation. You should also +double check your branch changes against the branch it was based on: + +1. Navigate to your repository on GitHub -- https://github.com/your-user-name/OpenROAD +1. Click on `Branches` +1. Click on the `Compare` button for your feature branch +1. Select the `base` and `compare` branches, if necessary. This will be `master` and + `shiny-new-feature`, respectively. + +## Submitting the pull request + +If everything looks good, you are ready to make a pull request. A pull request is how +code from a local repository becomes available to the GitHub community and can be looked +at and eventually merged into the master version. This pull request and its associated +changes will eventually be committed to the master branch and available in the next +release. To submit a pull request: + +1. Navigate to your repository on GitHub +1. Click on the ``Compare & pull request`` button +1. You can then click on ``Commits`` and ``Files Changed`` to make sure everything looks + okay one last time +1. Write a description of your changes in the ``Preview Discussion`` tab +1. Click ``Send Pull Request``. + +This request then goes to the repository maintainers, and they will review +the code. + +## Updating your pull request + +Based on the review you get on your pull request, you will probably need to make +some changes to the code. In that case, you can make them in your branch, +add a new commit to that branch, push it to GitHub, and the pull request will be +automatically updated. Pushing them to GitHub again is done by: + +```shell +git push origin shiny-new-feature +``` + +This will automatically update your pull request with the latest code and restart the +[Continuous Integration](./CI.md) tests. + +Another reason you might need to update your pull request is to solve conflicts +with changes that have been merged into the master branch since you opened your +pull request. + +To do this, you need to `merge upstream master` in your branch: + +```shell +git checkout shiny-new-feature +git fetch upstream +git merge upstream/master +``` + +If there are no conflicts (or they could be fixed automatically), a file with a +default commit message will open, and you can simply save and quit this file. + +If there are merge conflicts, you need to solve those conflicts. See +this [article](https://help.github.com/articles/resolving-a-merge-conflict-using-the-command-line/) +for an explanation on how to do this. +Once the conflicts are merged and the files where the conflicts were solved are +added, you can run ``git commit`` to save those fixes. + +If you have uncommitted changes at the moment you want to update the branch with +master, you will need to ``stash`` them prior to updating. + +```{seealso} +See the stash [docs](https://git-scm.com/book/en/v2/Git-Tools-Stashing-and-Cleaning). +``` +This will effectively store your changes and they can be reapplied after updating. + +After the feature branch has been updated locally, you can now update your pull +request by pushing to the branch on GitHub: + +```shell +git push origin shiny-new-feature +``` + +## Tips for a successful pull request + +If you have made it to the `Review your code` phase, one of the core contributors may +take a look. Please note however that a handful of people are responsible for reviewing +all of the contributions, which can often lead to bottlenecks. + +To improve the chances of your pull request being reviewed, you should: + +- **Reference an open issue** for non-trivial changes to clarify the PR's purpose +- **Ensure you have appropriate tests**. These should be the first part of any PR +- **Keep your pull requests as simple as possible**. Larger PRs take longer to review +- **Ensure that CI is in a green state**. Reviewers may not even look otherwise +- **Keep updating your pull request**, either by request or every few days + +## Acknowledgements + +This page has been adapted from [pandas Developer Guide](https://pandas.pydata.org/docs/development/contributing.html). diff --git a/docs/_vendor/openroad/README.md b/docs/_vendor/openroad/README.md new file mode 100644 index 0000000000..6282b1bf72 --- /dev/null +++ b/docs/_vendor/openroad/README.md @@ -0,0 +1,57 @@ +# Vendored OpenROAD documentation sources + +**Do not edit these files by hand.** They are verbatim copies of files from the +[OpenROAD](https://github.com/The-OpenROAD-Project/OpenROAD) repository, taken at +the exact commit that the `tools/OpenROAD` submodule pins. `SOURCE.json` records +the repository, the commit SHA, and the upstream path of every file here. + +| File | Upstream path | Used by `docs/conf.py` to build | +| --- | --- | --- | +| `GitGuide.md` | `docs/contrib/GitGuide.md` | `docs/contrib/GitGuide.md` (renamed for ORFS, with `contrib/GitGuideAdapter.md` spliced in) | +| `index-SupportedOS.md` | `docs/index.md`, `## Supported Operating Systems` section only | that section, demoted to `####`, spliced into `docs/index2.md` under `### Setup` | +| `utl-README.md` | `src/utl/README.md` | `docs/Manpage.md` | + +Only what `docs/conf.py` consumes is stored. OpenROAD's `docs/index.md` is a +landing page and ORFS renders exactly one section of it, so the refresh script +extracts that section and vendors it alone — copying the whole page in would +bring unrelated project prose into this repository to render one table. + +The repo-relative links in `utl-README.md` are rewritten to absolute GitHub URLs +at the vendored SHA, because they would otherwise resolve against this +repository and 404. + +These files are exempt from the whitespace checks in `.gitattributes`: they are +byte-for-byte upstream copies, and the refresh script is only idempotent if +nothing rewrites them. + +## Why these are vendored + +`docs/conf.py` used to `requests.get()` all three files from +`raw.githubusercontent.com/.../OpenROAD/master/` while Sphinx was running. That +made the documentation build require network access, produced non-reproducible +output, tracked OpenROAD `master` rather than the version ORFS actually ships, +and overwrote files in the working tree on every local `make html`. Vendoring +gives the documentation the same hash-pinned reproducibility as +`docs/requirements_lock.txt`. + +## Refreshing + +```shell +python3 docs/scripts/refresh_openroad_docs.py +``` + +This is the only thing that downloads from GitHub, and it is wired into +`.github/workflows/github-actions-cron-update-OR.yml` so the nightly +submodule-bump PR carries the refreshed docs in the same commit. Run it by hand +whenever you bump `tools/OpenROAD` yourself. + +## Checking + +```shell +python3 docs/scripts/check_vendored_docs.py +``` + +Asserts, offline, that `SOURCE.json`'s SHA still matches +`git rev-parse HEAD:tools/OpenROAD`, that every heading `docs/conf.py` splices +on is still present, and that `index-SupportedOS.md` still holds exactly one +`##` section. Run it on any pull request that bumps `tools/OpenROAD`. diff --git a/docs/_vendor/openroad/SOURCE.json b/docs/_vendor/openroad/SOURCE.json new file mode 100644 index 0000000000..805b162380 --- /dev/null +++ b/docs/_vendor/openroad/SOURCE.json @@ -0,0 +1,21 @@ +{ + "_comment": "Generated by docs/scripts/refresh_openroad_docs.py -- do not edit by hand. Run that script to re-vendor after bumping the tools/OpenROAD submodule.", + "repo": "The-OpenROAD-Project/OpenROAD", + "sha": "ab6fd26351dc449e69059684dc6aa9ae9046eb36", + "files": { + "docs/contrib/GitGuide.md": { + "vendored": "GitGuide.md" + }, + "docs/index.md": { + "vendored": "index-SupportedOS.md", + "section": { + "start": "## Supported Operating Systems", + "end": "## Code of conduct" + } + }, + "src/utl/README.md": { + "vendored": "utl-README.md", + "links_rewritten": true + } + } +} diff --git a/docs/_vendor/openroad/index-SupportedOS.md b/docs/_vendor/openroad/index-SupportedOS.md new file mode 100644 index 0000000000..d289271d17 --- /dev/null +++ b/docs/_vendor/openroad/index-SupportedOS.md @@ -0,0 +1,23 @@ +## Supported Operating Systems + +Note that depending on the installation method, we have varying levels of +support for various operating systems. + +Legend: +- `Y` for supported. +- `-` for unsupported. + +| Operating System | Local Installation | Prebuilt Binaries | Docker Installation | Windows Subsystem for Linux | +| --- | --- | --- | --- | --- | +| Debian 12 | `Y` | `Y` | `Y` | `-` | +| Debian 13 | `Y` | `Y` | `Y` | `-` | +| Ubuntu 22.04 | `Y` | `Y` | `Y` | `-` | +| Ubuntu 24.04 | `Y` | `Y` | `Y` | `-` | +| Ubuntu 26.04 | `Y` | `Y` | `Y` | `-` | +| RHEL 8/9 | `Y` | `-` | `Y` | `-` | +| Rocky 8 | `Y` | `-` | `Y` | `-` | +| Rocky 9 | `Y` | `-` | `Y` | `-` | +| macOS | `Y` | `-` | `Y` | `-` | +| Windows 10 and above | `-` | `-` | `Y` | `Y` | + + diff --git a/docs/_vendor/openroad/utl-README.md b/docs/_vendor/openroad/utl-README.md new file mode 100644 index 0000000000..6f7c1d32e0 --- /dev/null +++ b/docs/_vendor/openroad/utl-README.md @@ -0,0 +1,135 @@ +# Utilities + +The utility module contains the `man` command. + +## Commands + +```{note} +- Parameters in square brackets `[-param param]` are optional. +- Parameters without square brackets `-param2 param2` are required. +``` + +### Prometheus Metrics + +OpenROAD includes a metrics endpoint server that can track internal tool metrics over time. + +![page](https://raw.githubusercontent.com/The-OpenROAD-Project/OpenROAD/ab6fd26351dc449e69059684dc6aa9ae9046eb36/docs/images/grafana.png) + +To use this feature you need to do the following start the prometheus and grafana collectors + +[Detailed instructions](https://github.com/The-OpenROAD-Project/OpenROAD/blob/ab6fd26351dc449e69059684dc6aa9ae9046eb36/etc/monitoring/README.md): +```shell +$ cd etc/monitoring +$ docker compose up -d +``` + +This will start a grafana endpoint ready to collect from the OpenROAD application you would +like to track. By default it's looking for an http server running on port 8080 on your localhost. + +To start the metrics endpoint in OpenROAD, run: +```tcl +utl::startPrometheusEndpoint 8080 +``` + +This is all configurable in the docker compose file, and you should be able to access grafana by going to +http://localhost:3000 username: admin, password: grafana. Go to the dashboard tab and click service, +then OpenROAD to see the pre-made dashboard. + +## Man installation + +The `man` command can be installed optionally as part of the OpenROAD +binary. For more instructions, please refer to [here](https://github.com/The-OpenROAD-Project/OpenROAD/blob/ab6fd26351dc449e69059684dc6aa9ae9046eb36/src/utl/manREADME.md). + +### Man + +The `man` command in OpenROAD is similar in functionality to Unix +(and Unix-like operating systems such as Linux) . It is used to +display the manual pages for various applications, tools and error +messages. These manual pages provide detailed information about how +to use a particular command or function, along with its syntax and options. + +This can be used for a range of commands in different levels as follows: +- Level 1: Top-level openroad command (e.g. `man openroad`) +- Level 2: Individual module commands (e.g. `man clock_tree_synthesis`) +- Level 3: Info, error, warning messages (e.g. `man CTS-0001`) + +```tcl +man + name + [-manpath manpath] + [-no_pager] +``` + +#### Options + +| Switch Name | Description | +| ----- | ----- | +| `name` | Name of the command/message to query. | +| `-manpath` | Include optional path to man pages (e.g. ~/OpenROAD/docs/cat). | +| `-no_pager` | This flag determines whether you wish to see all of the man output at once. Default value is `False`, which shows a buffered output. | + +## Example scripts + +You may run various commands or message IDs for man pages. +``` +man openroad +man clock_tree_synthesis +man CTS-0005 +``` + +### tee + +Redirect a commands output to a file and standard out. + +```tcl +tee (-file filename | -variable name) + [-append] + [-quiet] + command +``` + +#### Options + +| Switch Name | Description | +| ----- | ----- | +| `-file filename` | File to redirect output into. | +| `-variable name` | Direct output into a variable. | +| `-append` | Append to file. | +| `-quiet` | Do not send output to standard out. | +| `command` | Command to execute. | + +## Example scripts + +``` +tee -file output.rpt { report_design_area } +tee -quiet -file output.rpt { report_floating_nets } +``` + +## Regression tests + +There are a set of regression tests in `./test`. For more information, refer to this [section](https://github.com/The-OpenROAD-Project/OpenROAD/blob/ab6fd26351dc449e69059684dc6aa9ae9046eb36/README.md#regression-tests). + +For information regarding the Man page test framework, refer to this +[link](https://github.com/The-OpenROAD-Project/OpenROAD/blob/ab6fd26351dc449e69059684dc6aa9ae9046eb36/docs/src/test/README.md). + +You should also be aware of the [README](https://github.com/The-OpenROAD-Project/OpenROAD/blob/ab6fd26351dc449e69059684dc6aa9ae9046eb36/docs/contrib/ReadmeFormat.md) and [Tcl](https://github.com/The-OpenROAD-Project/OpenROAD/blob/ab6fd26351dc449e69059684dc6aa9ae9046eb36/docs/contrib/TclFormat.md) format enforced to ensure +accurate parsing of man pages. + +Simply run the following script: + +```shell +./test/regression +``` + +## Limitations + +## References + +## Authors + +MAN command is written by Jack Luar with guidance from members of the OpenROAD team, +including: Cho Moon, Matt Liberty. + +## License + +BSD 3-Clause License. See [LICENSE](https://github.com/The-OpenROAD-Project/OpenROAD/blob/ab6fd26351dc449e69059684dc6aa9ae9046eb36/LICENSE) file. diff --git a/docs/conf.py b/docs/conf.py index 96e0c84cf4..0806ef78e1 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -13,7 +13,6 @@ import docutils import os import re -import requests # -- Project information ----------------------------------------------------- @@ -71,6 +70,15 @@ # This pattern also affects html_static_path and html_extra_path. exclude_patterns = [ "_build", + # The Makefile's actual BUILDDIR ("build", not the Sphinx-default "_build") + # and any local virtualenv -- without these, a second build treats its own + # previous output and installed packages' bundled images as source pages. + "build", + ".venv*", + "venv", + # Verbatim copies of OpenROAD files; conf.py splices them into ORFS pages + # rather than publishing them directly. See docs/_vendor/openroad/README.md. + "_vendor", "Thumbs.db", ".DS_Store", "**/LICENSE", @@ -135,17 +143,38 @@ # so a file named "default.css" will overwrite the builtin "default.css". -def get_file_from_url(url, fname): - r = requests.get(url) - with open(fname, "wb") as f: - f.write(r.content) +# Files copied verbatim from the OpenROAD repository at the SHA that the +# tools/OpenROAD submodule pins. They are checked in, so this build performs no +# network access -- matching the intent of the hash-pinned requirements_lock.txt. +# Refresh them with REFRESH_CMD; docs/scripts/check_vendored_docs.py asserts +# they are still in sync with the submodule pointer. +VENDOR_DIR = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "_vendor", "openroad" +) +REFRESH_CMD = "python3 docs/scripts/refresh_openroad_docs.py" + +# Holds one extracted section of OpenROAD's docs/index.md, not the whole page. +SUPPORTED_OS_FILE = "index-SupportedOS.md" + + +def read_vendored(fname): + """Read a file vendored from the OpenROAD repository.""" + path = os.path.join(VENDOR_DIR, fname) + try: + with open(path, "r", encoding="utf-8") as f: + return f.read() + except OSError as error: + raise RuntimeError( + "Cannot read the vendored OpenROAD file '%s': %s. " + "Regenerate the vendored files with: %s" % (path, error, REFRESH_CMD) + ) from error def swap_prefix(file, old, new): - with open(file, "r") as f: + with open(file, "r", encoding="utf-8") as f: lines = f.read() lines = lines.replace(old, new) - with open(file, "wt") as f: + with open(file, "wt", encoding="utf-8") as f: f.write(lines) @@ -156,57 +185,78 @@ def setup(app): shutil.copy("../README.md", "mainREADME.md") swap_prefix("mainREADME.md", "```mermaid", "```{mermaid}\n:align: center\n") - # Grab the reference file from OR - url = "https://raw.githubusercontent.com/The-OpenROAD-Project/OpenROAD/master/docs/contrib/GitGuide.md" - get_file_from_url(url, "contrib/GitGuide.md") - - # Temporarily using commit number, will change once OR commit merged. - url = "https://raw.githubusercontent.com/The-OpenROAD-Project/OpenROAD/master/docs/index.md" - get_file_from_url(url, "SupportedOS.md") - - # Adapt the downloaded GitGuide for ORFS: rename references and inject - # ORFS-specific submodule forking instructions from GitGuideAdapter.md. - with open("contrib/GitGuide.md", "r") as f: - content = f.read() - content = ( - content.replace( - "user/Build.md", "../index.md#build-or-installing-orfs-dependencies" - ) - .replace("OpenROAD", "OpenROAD-flow-scripts") - .replace("The-OpenROAD-flow-scripts", "The-OpenROAD") - ) - with open("contrib/GitGuideAdapter.md", "r") as f: - adapter_content = f.read() + # Adapt the vendored OpenROAD GitGuide for ORFS: rename references and + # inject ORFS-specific submodule forking instructions from + # GitGuideAdapter.md. + content = read_vendored("GitGuide.md") content = content.replace( - "## Creating a branch", adapter_content + "\n## Creating a branch" + "user/Build.md", "../index.md#build-or-installing-orfs-dependencies" + ) + # Rename OpenROAD -> OpenROAD-flow-scripts, but leave the GitHub + # organisation name ("The-OpenROAD-Project") and anything that already + # carries the "-flow-scripts" suffix alone. Equivalent to the previous + # blanket replace + "The-OpenROAD-flow-scripts" self-patch, without + # corrupting those two forms. + content = re.sub( + r"(? str | None: + """Return the SHA the committed ``tools/OpenROAD`` gitlink points at.""" + result = subprocess.run( + ["git", "rev-parse", "HEAD:%s" % SUBMODULE_PATH], + cwd=str(REPO_ROOT), + capture_output=True, + text=True, + ) + if result.returncode != 0: + problems.append( + "could not read the pinned submodule commit: " + "`git rev-parse HEAD:%s` failed: %s" + % (SUBMODULE_PATH, result.stderr.strip()) + ) + return None + return result.stdout.strip() + + +def load_source(problems: list[str]) -> dict | None: + if not SOURCE_JSON.exists(): + problems.append( + "%s is missing. Create it with: %s" + % (SOURCE_JSON.relative_to(REPO_ROOT), REFRESH_CMD) + ) + return None + try: + return json.loads(SOURCE_JSON.read_text(encoding="utf-8")) + except json.JSONDecodeError as error: + problems.append( + "%s is not valid JSON (%s). Regenerate it with: %s" + % (SOURCE_JSON.relative_to(REPO_ROOT), error, REFRESH_CMD) + ) + return None + + +def check_sha(source: dict, problems: list[str]) -> None: + pinned = committed_submodule_sha(problems) + if pinned is None: + return + + vendored = source.get("sha") + if not vendored: + problems.append( + '%s has no "sha" field. Regenerate it with: %s' + % (SOURCE_JSON.relative_to(REPO_ROOT), REFRESH_CMD) + ) + return + + if vendored != pinned: + problems.append( + "the vendored OpenROAD docs are stale:\n" + " %s says sha = %s\n" + " git rev-parse HEAD:%s = %s\n" + " The tools/OpenROAD submodule was bumped without re-vendoring " + "the documentation it ships. Fix with: %s" + % ( + SOURCE_JSON.relative_to(REPO_ROOT), + vendored, + SUBMODULE_PATH, + pinned, + REFRESH_CMD, + ) + ) + + +def check_files_present(source: dict, problems: list[str]) -> None: + files = source.get("files") + if not isinstance(files, dict) or not files: + problems.append( + '%s has no usable "files" mapping. Regenerate it with: %s' + % (SOURCE_JSON.relative_to(REPO_ROOT), REFRESH_CMD) + ) + return + + for upstream_path, record in sorted(files.items()): + if not isinstance(record, dict) or not record.get("vendored"): + problems.append( + '%s has a malformed entry for "%s". Regenerate it with: %s' + % (SOURCE_JSON.relative_to(REPO_ROOT), upstream_path, REFRESH_CMD) + ) + continue + name = record["vendored"] + path = VENDOR_DIR / name + if not path.exists(): + problems.append( + "%s is missing (vendored from %s:%s). Fix with: %s" + % ( + path.relative_to(REPO_ROOT), + source.get("repo", "OpenROAD"), + upstream_path, + REFRESH_CMD, + ) + ) + elif not path.read_text(encoding="utf-8").strip(): + problems.append( + "%s is empty. Fix with: %s" % (path.relative_to(REPO_ROOT), REFRESH_CMD) + ) + + +def check_anchors(source: dict, problems: list[str]) -> None: + cache: dict[str, str | None] = {} + + def read(rel_path: str) -> str | None: + if rel_path not in cache: + path = REPO_ROOT / rel_path + cache[rel_path] = ( + path.read_text(encoding="utf-8") if path.exists() else None + ) + return cache[rel_path] + + for rel_path, anchor in REQUIRED_ANCHORS: + text = read(rel_path) + if text is None: + # check_files_present already reported missing vendored files. + if not rel_path.startswith("docs/_vendor/"): + problems.append( + "%s is missing; it must contain %r" % (rel_path, anchor) + ) + continue + if anchor not in text: + if rel_path.startswith("docs/_vendor/"): + remedy = ( + "OpenROAD most likely renamed it. Re-vendor with `%s`; if " + "the heading really changed, update it in docs/conf.py and " + "in docs/scripts/check_vendored_docs.py." % REFRESH_CMD + ) + else: + remedy = ( + "This is an ORFS-owned page. Restore the heading, or update " + "the anchor in docs/conf.py and in " + "docs/scripts/check_vendored_docs.py." + ) + problems.append( + "%s does not contain the heading %r, which docs/conf.py splices " + "on.\n %s" % (rel_path, anchor, remedy) + ) + + files = source.get("files") if isinstance(source.get("files"), dict) else {} + + for rel_path, upstream_path, heading in EXTRACTED_SECTIONS: + remedy = ( + "Re-vendor with `%s`. If OpenROAD really renamed or reordered the " + "heading, update the section bounds in " + "docs/scripts/refresh_openroad_docs.py, the anchor in " + "docs/conf.py, and EXTRACTED_SECTIONS here to match." % REFRESH_CMD + ) + text = read(rel_path) + if text is None: + # check_files_present already reported missing vendored files. + continue + + if not text.startswith(heading): + problems.append( + "%s does not start with %r. It must hold that one section of " + "%s and nothing else, because docs/conf.py strips the heading " + "and splices the rest into docs/index2.md.\n %s" + % (rel_path, heading, upstream_path, remedy) + ) + elif not text[len(heading) :].strip(): + problems.append( + "%s contains %r with no body under it, so docs/index2.md would " + "publish an empty supported-OS section.\n %s" + % (rel_path, heading, remedy) + ) + + extra = [h for h in _H2_RE.findall(text) if h != heading] + if extra: + problems.append( + "%s contains %d unexpected `## ` heading(s) besides %r: %s.\n" + " The extraction from %s ran past the end of the section, so " + "docs/index2.md would publish unrelated content.\n %s" + % ( + rel_path, + len(extra), + heading, + ", ".join(repr(h) for h in extra), + upstream_path, + remedy, + ) + ) + + # The declarations above must agree with what the refresh script + # recorded; otherwise conf.py and the refresh script have drifted and + # the checks above are guarding the wrong heading. + recorded = (files.get(upstream_path) or {}).get("section") or {} + if recorded.get("start") not in (None, heading): + problems.append( + "%s records that %s was cut at %r, but docs/conf.py and this " + "script expect %r.\n Reconcile " + "docs/scripts/refresh_openroad_docs.py, docs/conf.py and " + "EXTRACTED_SECTIONS here, then re-vendor with `%s`." + % ( + SOURCE_JSON.relative_to(REPO_ROOT), + upstream_path, + recorded.get("start"), + heading, + REFRESH_CMD, + ) + ) + + +def main() -> int: + problems: list[str] = [] + + source = load_source(problems) + if source is not None: + check_sha(source, problems) + check_files_present(source, problems) + check_anchors(source or {}, problems) + + if problems: + print("check_vendored_docs: FAIL (%d problem(s))" % len(problems)) + for problem in problems: + print(" - %s" % problem) + return 1 + + print( + "check_vendored_docs: OK (vendored at %s @ %s)" + % (source.get("repo"), source.get("sha")) + ) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/docs/scripts/refresh_openroad_docs.py b/docs/scripts/refresh_openroad_docs.py new file mode 100755 index 0000000000..3fe1026f84 --- /dev/null +++ b/docs/scripts/refresh_openroad_docs.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +"""Refresh ``docs/_vendor/openroad/`` from the pinned ``tools/OpenROAD`` submodule. + +The ORFS documentation splices three files that live in the OpenROAD repository +into its own pages (see ``docs/conf.py``). Those files used to be downloaded +from ``raw.githubusercontent.com/.../OpenROAD/master/`` while Sphinx was +running, which made the docs build depend on the network, non-reproducible, and +liable to document a version of OpenROAD that ORFS does not ship. + +They are vendored instead. This script is the *only* thing that talks to the +network: it downloads the three files **at the SHA that ``tools/OpenROAD`` is +pinned to**, rewrites the repo-relative links inside them so they still resolve +when rendered from this repository, and records the SHA in ``SOURCE.json``. + +Only what ``docs/conf.py`` actually consumes is stored. For OpenROAD's +``docs/index.md`` that is a single section -- the supported-OS table -- so this +script extracts it and vendors the section alone rather than the whole landing +page, which is mostly prose ORFS never renders. + +Usage:: + + python3 docs/scripts/refresh_openroad_docs.py [--sha SHA] + +It is idempotent: running it twice in a row leaves the working tree unchanged. + +``docs/scripts/check_vendored_docs.py`` is the offline counterpart: run it to +assert the vendored tree is still in sync with the submodule pointer. +""" + +from __future__ import annotations + +import argparse +import json +import posixpath +import re +import subprocess +import sys +import urllib.error +import urllib.request +from pathlib import Path +from typing import NamedTuple + +REPO = "The-OpenROAD-Project/OpenROAD" +SUBMODULE_PATH = "tools/OpenROAD" + +REPO_ROOT = Path(__file__).resolve().parents[2] +VENDOR_DIR = REPO_ROOT / "docs" / "_vendor" / "openroad" +SOURCE_JSON = VENDOR_DIR / "SOURCE.json" + + +class Vendored(NamedTuple): + """One file to vendor out of the OpenROAD repository. + + ``section`` restricts what is stored to a single ``## `` section of the + upstream file, delimited by ``(start, end)`` headings. The vendored file + then holds ``start`` followed by everything up to -- but not including -- + ``end``; ``docs/conf.py`` strips the leading heading and splices the rest. + + ``rewrite_links`` turns repo-relative markdown links into absolute GitHub + URLs at the vendored SHA. Without it a link such as ``../../LICENSE`` + would resolve against *this* repository and 404. + """ + + upstream: str + name: str + section: tuple[str, str] | None = None + rewrite_links: bool = False + + +# Keep this in sync with the readers in docs/conf.py. +# +# GitGuide.md is deliberately not link-rewritten: conf.py adapts it for ORFS +# and its remaining relative links (./DeveloperGuide.md, ./CI.md) resolve +# against ORFS's own docs/contrib/ pages. +# +# Only the supported-OS table is taken from docs/index.md. Vendoring that page +# whole would drag OpenROAD's entire landing page -- partner and programme +# prose included -- into this repository to render one table. +VENDORED_FILES = ( + Vendored("docs/contrib/GitGuide.md", "GitGuide.md"), + Vendored( + "docs/index.md", + "index-SupportedOS.md", + section=("## Supported Operating Systems", "## Code of conduct"), + ), + Vendored("src/utl/README.md", "utl-README.md", rewrite_links=True), +) + +_LINK_RE = re.compile(r"(!?)\[([^\]]*)\]\(\s*([^)\s]+)\s*\)") +_ABSOLUTE_PREFIXES = ("http://", "https://", "mailto:", "ftp://", "//") +_IMAGE_SUFFIXES = (".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp", ".ico") + +_SHA_RE = re.compile(r"^[0-9a-f]{40}$") + + +def _git(*args: str, cwd: Path = REPO_ROOT) -> str: + """Run git and return stripped stdout, or raise with a readable message.""" + result = subprocess.run( + ["git", *args], + cwd=str(cwd), + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise RuntimeError( + "command failed: git %s (in %s)\n%s" + % (" ".join(args), cwd, result.stderr.strip()) + ) + return result.stdout.strip() + + +def resolve_pinned_sha() -> tuple[str, str]: + """Return ``(sha, how)`` for the OpenROAD commit ORFS pins. + + The nightly bot updates the ``tools/OpenROAD`` working tree first and only + then commits, so when the submodule is checked out its ``HEAD`` is what the + resulting commit will pin -- and that is the SHA the docs must match. When + the submodule is not initialised (a plain ``git clone`` without + ``--recursive``) fall back to the committed gitlink. + """ + submodule_dir = REPO_ROOT / SUBMODULE_PATH + if (submodule_dir / ".git").exists(): + sha = _git("rev-parse", "HEAD", cwd=submodule_dir) + how = "%s working tree (git -C %s rev-parse HEAD)" % ( + SUBMODULE_PATH, + SUBMODULE_PATH, + ) + else: + sha = _git("rev-parse", "HEAD:%s" % SUBMODULE_PATH) + how = "committed submodule pointer (git rev-parse HEAD:%s)" % SUBMODULE_PATH + + if not _SHA_RE.match(sha): + raise RuntimeError( + "resolved OpenROAD SHA %r from the %s is not a 40-character " + "commit id" % (sha, how) + ) + return sha, how + + +def download(url: str) -> str: + """Fetch ``url`` as text, failing loudly on any non-200 response. + + The old ``conf.py`` helper wrote ``r.content`` unconditionally, so a 404 or + an outage silently produced a doc page containing GitHub's error body. + """ + request = urllib.request.Request( + url, headers={"User-Agent": "orfs-refresh-openroad-docs"} + ) + try: + with urllib.request.urlopen(request, timeout=60) as response: + status = response.status + body = response.read() + except urllib.error.HTTPError as exc: + raise RuntimeError( + "GET %s failed with HTTP %s %s" % (url, exc.code, exc.reason) + ) from exc + except urllib.error.URLError as exc: + raise RuntimeError("GET %s failed: %s" % (url, exc.reason)) from exc + + if status != 200: + raise RuntimeError("GET %s returned HTTP %s, expected 200" % (url, status)) + if not body.strip(): + raise RuntimeError("GET %s returned an empty body" % url) + return body.decode("utf-8") + + +def _to_github_url(target: str, src_dir: str, sha: str) -> str | None: + """Map a repo-relative markdown target to an absolute GitHub URL. + + Returns ``None`` when the target should be left alone (already absolute, or + a pure in-page anchor). + """ + if target.startswith(_ABSOLUTE_PREFIXES) or target.startswith("#"): + return None + + path, sep, fragment = target.partition("#") + if not path: + return None + + if path.startswith("/"): + # Repo-root-relative, e.g. /docs/images/grafana.png + repo_path = path.lstrip("/") + else: + repo_path = posixpath.normpath(posixpath.join(src_dir, path)) + + if repo_path.startswith(".."): + raise RuntimeError( + "link target %r in %s/ escapes the OpenROAD repository root" + % (target, src_dir) + ) + + if repo_path.lower().endswith(_IMAGE_SUFFIXES): + # Images must point at the raw blob so they actually render. + return "https://raw.githubusercontent.com/%s/%s/%s" % (REPO, sha, repo_path) + return "https://github.com/%s/blob/%s/%s%s%s" % ( + REPO, + sha, + repo_path, + sep, + fragment, + ) + + +def rewrite_links(text: str, upstream_path: str, sha: str) -> str: + """Rewrite repo-relative links in a vendored file to absolute GitHub URLs.""" + src_dir = posixpath.dirname(upstream_path) + + def replace(match: re.Match) -> str: + bang, label, target = match.group(1), match.group(2), match.group(3) + url = _to_github_url(target, src_dir, sha) + if url is None: + return match.group(0) + return "%s[%s](%s)" % (bang, label, url) + + return _LINK_RE.sub(replace, text) + + +def extract_section(text: str, upstream_path: str, start: str, end: str) -> str: + """Return the ``start`` heading plus everything up to (not including) ``end``. + + This is the extraction ``docs/conf.py`` used to perform on the whole + downloaded file. Doing it here means an upstream heading rename fails + *now*, naming both headings, instead of vendoring a page the documentation + build cannot splice. + """ + match = re.search("%s(.*?)%s" % (re.escape(start), re.escape(end)), text, re.DOTALL) + if match is None: + raise RuntimeError( + "cannot find a section starting at '%s' and ending at '%s' in " + "%s:%s.\n" + " OpenROAD most likely renamed or reordered one of those " + "headings. Look at the upstream file, then update the section " + "bounds in this script, the anchor in docs/conf.py, and the " + "anchor in docs/scripts/check_vendored_docs.py to match." + % (start, end, REPO, upstream_path) + ) + body = match.group(1) + if not body.strip(): + raise RuntimeError( + "the section '%s' in %s:%s is empty. Vendoring it would silently " + "publish a supported-OS heading with nothing under it." + % (start, REPO, upstream_path) + ) + return start + body + + +def _write_if_changed(path: Path, content: str) -> bool: + """Write ``content`` to ``path``; return True if the file changed on disk.""" + if path.exists() and path.read_text(encoding="utf-8") == content: + return False + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + return True + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument( + "--sha", + help="OpenROAD commit to vendor from (default: the pinned " + "tools/OpenROAD submodule commit)", + ) + args = parser.parse_args(argv) + + if args.sha: + sha, how = args.sha, "--sha command-line override" + if not _SHA_RE.match(sha): + parser.error("--sha must be a full 40-character commit id") + else: + sha, how = resolve_pinned_sha() + + print("OpenROAD repo: %s" % REPO) + print("OpenROAD sha: %s" % sha) + print("resolved from: %s" % how) + + changed = [] + files = {} + for entry in VENDORED_FILES: + url = "https://raw.githubusercontent.com/%s/%s/%s" % (REPO, sha, entry.upstream) + text = download(url) + record = {"vendored": entry.name} + if entry.section is not None: + start, end = entry.section + text = extract_section(text, entry.upstream, start, end) + record["section"] = {"start": start, "end": end} + if entry.rewrite_links: + text = rewrite_links(text, entry.upstream, sha) + record["links_rewritten"] = True + files[entry.upstream] = record + if _write_if_changed(VENDOR_DIR / entry.name, text): + changed.append("docs/_vendor/openroad/%s" % entry.name) + print( + " %-24s <- %s%s" + % ( + entry.name, + entry.upstream, + " ['%s' section only]" % entry.section[0] if entry.section else "", + ) + ) + + source = { + "_comment": ( + "Generated by docs/scripts/refresh_openroad_docs.py -- do not edit " + "by hand. Run that script to re-vendor after bumping the " + "tools/OpenROAD submodule." + ), + "repo": REPO, + "sha": sha, + "files": files, + } + if _write_if_changed(SOURCE_JSON, json.dumps(source, indent=2) + "\n"): + changed.append("docs/_vendor/openroad/SOURCE.json") + + if changed: + print("updated:") + for name in changed: + print(" %s" % name) + else: + print("already up to date; nothing written.") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except RuntimeError as error: + print("error: %s" % error, file=sys.stderr) + sys.exit(1)