diff --git a/README.md b/README.md index e866e72e..3d8b4eea 100644 --- a/README.md +++ b/README.md @@ -102,8 +102,14 @@ advertised or used anywhere): - `.htaccess`: additional redirect rules; copy of `etc/docs.htaccess` in the `GapWWW` repository -The directories `v4.X.Y` can be (re)generated from a GAP installation and the -corresponding `.json` file in `_data/package-infos/`. For example: +A new GAP release is picked up automatically: `etc/mirror-manuals.py`, run +hourly by a systemd timer on that host, builds the manuals of any release it +does not have yet and moves the `latest` symlink. See +[`etc/README.gap-docs.md`](etc/README.gap-docs.md). + +The directories `v4.X.Y` can also be (re)generated by hand from a GAP +installation and the corresponding `.json` file in `_data/package-infos/`. For +example: etc/extract_manuals.py /usr/local/gap-4.27.3 _data/package-infos/4-27-3.json @@ -116,5 +122,10 @@ symlink suitably. Various archives for GAP and packages are served from a separate subdomain, namely . The files served there -are from the directory `/srv/www/www-gap-docs/files/http` on +are from the directory `/srv/www/www-gap-files/data/http` on `www-admin13.rz.rptu.de`, username `www-gap-files`. + +Nothing there needs updating by hand: both the package archives and the +archives of GAP releases are mirrored automatically. The scripts and systemd +units that do this live in their own repository, +, which also documents that host. diff --git a/etc/README.gap-docs.md b/etc/README.gap-docs.md new file mode 100644 index 00000000..a76d316d --- /dev/null +++ b/etc/README.gap-docs.md @@ -0,0 +1,120 @@ +# docs.gap-system.org + +The GAP and GAP package manuals, one directory per GAP release. See +[`README.server.md`](README.server.md) for the conventions shared with the other +two sites. + + ssh gap-docs # www-gap-docs@www-admin13.rz.rptu.de + +Note that this site is *not* the Jekyll website: nothing here is generated by +Jekyll, and the webhook that rebuilds has no effect +on it. The only thing the two have in common is that the scripts for both live +in this repository. + +## What is where + +``` +/srv/www/www-gap-docs/data/ (== ~/data; ~/http is the document root) +├── http/ document root +│ ├── v4.16.0/{doc,pkg}/ the manuals of one release, ~310 MB each +│ ├── v4.15.1/... one directory per release, back to v4.11.1 +│ ├── latest -> v4.16.0 always the newest installed release +│ ├── doc -> latest/doc stable URL for the GAP manual +│ ├── pkg -> latest/pkg stable URL for the package manuals +│ ├── index.html placeholder +│ ├── manual.js +│ └── .htaccess hand-made copy of etc/docs.htaccess +├── GapWWW/ git clone of this repository +└── .mirror-manuals/ scratch space, only while a build runs +~/.config/systemd/user/gap-docs-manuals.{service,timer} +``` + +Nothing in the document root is version specific except the `vX.Y.Z` directories +themselves and the `latest` symlink: `.htaccess` and `manual.js` contain no +version numbers, and `doc` and `pkg` resolve through `latest`. Installing a +release therefore only means creating one directory and moving one symlink. + +## How an update happens + +`gap-docs-manuals.timer` runs hourly. It first pulls this repository — both the +job and `extract_manuals.py` live in it — and then runs +[`mirror-manuals.py`](mirror-manuals.py), which: + +1. asks GitHub for the releases of `gap-system/gap`, ignoring pre-releases; +2. skips every release for which `~/http/vX.Y.Z` already exists, so the usual + run does nothing and costs one API call; +3. for anything left, downloads `gap-X.Y.Z.tar.gz` and `package-infos.json.gz`, + verifying both against the `.sha256` files published beside them, unpacks + them, and runs [`extract_manuals.py`](extract_manuals.py); +4. moves the finished tree into `~/http/vX.Y.Z` in a single rename, so a + half-extracted set of manuals is never served; +5. repoints `latest` by renaming a new symlink over the old one, so that + `latest` — and `doc` and `pkg` with it — is never even briefly missing; +6. deletes its scratch directory. + +Because pre-releases are ignored, the manuals of a new release appear within an +hour of it being switched from "pre-release" to "latest release" on GitHub, and +not before. A build takes roughly ten minutes and needs about 2 GB of scratch +space: the release tarball is ~570 MB and unpacks to ~1.3 GB. + +Releases older than 4.11.1 are ignored, because they predate the +`package-infos.json` file that `extract_manuals.py` needs. + +This replaces the manual `./download_manuals.sh X.Y.Z` step that used to be part +of the GAP release process; that script still exists in the home directory on +this host and can be used if the automation is broken. + +## Troubleshooting + + systemctl --user list-timers + systemctl --user status gap-docs-manuals.service + journalctl --user -u gap-docs-manuals.service -n 100 + +To see what it would do, or to run it immediately rather than waiting: + + ~/data/GapWWW/etc/mirror-manuals.py --dry-run + systemctl --user start gap-docs-manuals.service + +A failed run leaves the scratch directory behind; it is cleared at the start of +the next run, and it is always safe to delete by hand. Since the check is +"does `~/http/vX.Y.Z` exist", the way to force a rebuild of one release is to +move that directory aside and start the service. + +Rebuilding the manuals of an arbitrary GAP installation by hand, without going +through a release, is also possible: + + etc/extract_manuals.py /usr/local/gap-4.27.3 _data/package-infos/4-27-3.json + +which produces a `Manuals` directory to be renamed to `~/http/v4.27.3`. Remember +to update `latest` too. + +## `.htaccess` + +The document root's `.htaccess` is a hand-made copy of +[`docs.htaccess`](docs.htaccess) in this directory; nothing deploys it +automatically, so the two can drift apart, and have. If you change one, change +the other. Most of it maps the old mixed-case package directory names to the +lowercase ones used now. + +## Setting this up from scratch + +Beyond the shared requirements in [`README.server.md`](README.server.md), this +site needs only Python 3 with `requests`. As `www-gap-docs`: + + ln -s /srv/www/www-gap-docs/data ~/data + git clone https://github.com/gap-system/GapWWW ~/data/GapWWW + + mkdir -p ~/.config/systemd/user + cp ~/data/GapWWW/etc/gap-docs-manuals.* ~/.config/systemd/user/ + systemctl --user daemon-reload + systemctl --user enable --now gap-docs-manuals.timer + +Then copy `etc/docs.htaccess` to `~/http/.htaccess`, and create the `doc` and +`pkg` symlinks: + + ln -s latest/doc ~/http/doc + ln -s latest/pkg ~/http/pkg + +On an empty document root the first run will build the manuals of *every* +release since 4.11.1, which takes hours and a lot of bandwidth. Use `--since` to +limit that to the releases you actually want. diff --git a/etc/README.gap-www.md b/etc/README.gap-www.md new file mode 100644 index 00000000..7d1fb00f --- /dev/null +++ b/etc/README.gap-www.md @@ -0,0 +1,119 @@ +# www.gap-system.org + +The GAP website itself: a Jekyll site built from this repository. See +[`README.server.md`](README.server.md) for the conventions shared with the other +two sites. + + ssh gap-www # www-gap-systems@www-admin13.rz.rptu.de + +## What is where + +``` +/srv/www/www-gap-systems/data/ (== ~/data; ~/http is the document root) +├── http/ document root; the Jekyll build output +├── GapWWW/ git clone of this repository, branch master +├── gap-website.trigger touched by webhook.php to request an update +├── webhook.secret the shared secret, see below; not in git +├── ForumArchive/ symlinked into http/ by etc/update.sh +└── ForumArchive2/ +~/.config/systemd/user/gap-website.{path,service} +``` + +The clone is owned by `www-gap-systems:www-gap-systems`. + +## How an update happens + +1. Something is pushed to `master` of this repository. +2. GitHub calls the webhook at , which + is `webhook.php` from the top of this repository, served from the document + root. +3. `webhook.php` checks the signature and, for a push event, `touch`es + `~/data/gap-website.trigger`. That is all it does — it deliberately runs no + code itself. +4. `gap-website.path`, a systemd user unit watching that file, notices and + starts `gap-website.service`. +5. That runs [`update.sh`](update.sh): it resets the clone to `origin/master`, + re-appends the webhook secret to `.htaccess`, runs `bundle install`, builds + the site with Jekyll into `~/http`, and restores the `ForumArchive` symlinks. + +Note step 5 resets hard and rebuilds unconditionally; the webhook payload is not +inspected beyond checking that it is a push. + +## The webhook secret + +The secret lives only on the server, in `~/data/webhook.secret`, as a single +line of Apache configuration: + + SetEnv GITHUB_WEBHOOK_SECRET "MY_SECRET" + +`update.sh` appends this file to `.htaccess` after every `git reset --hard`, +which is how the setting survives without the secret ever being committed. The +side effect is that the working tree is permanently dirty in `.htaccess`, which +is why the script resets rather than pulls. + +The same value must be set at +, where the webhook should +be configured as: + +- Payload URL: +- Content type: `application/x-www-form-urlencoded` (TODO: switch to JSON?) +- Secret: matching `GITHUB_WEBHOOK_SECRET` +- SSL verification enabled +- Trigger: just the push event + +If `GITHUB_WEBHOOK_SECRET` is not set at all, `webhook.php` skips the signature +check entirely rather than failing, so a lost secret does not break updates — it +silently makes the endpoint unauthenticated. Worth remembering when debugging. + +## Troubleshooting + + systemctl --user status gap-website.service gap-website.path + journalctl --user -f -u 'gap-website.*' + +If systemd reports that the units do not exist, reinstall them: + + cp ~/data/GapWWW/etc/gap-website.* ~/.config/systemd/user/ + systemctl --user daemon-reload + systemctl --user enable --now gap-website.service gap-website.path + +Broken file ownership — typically after poking at the clone as another user — +stops git or Jekyll from writing. As root: + + chown -R www-gap-systems:www-gap-systems ~/data/GapWWW ~/data/http + touch ~/data/gap-website.trigger + chown www-gap-systems:www-gap-systems ~/data/gap-website.trigger + chmod 0664 ~/data/gap-website.trigger + +The trigger file needs to be writable by the Apache/PHP user, which is a +different uid from `www-gap-systems`, while being watched by the +`www-gap-systems` systemd session — hence the group-writable mode. This is the +fiddliest part of the whole setup. + +## Setting this up from scratch + +Requirements beyond the shared ones in [`README.server.md`](README.server.md): +Ruby 2.7 or newer including development headers, and bundler +(`apt install bundler`); PHP, needed only for the webhook. + +As root: create the account and virtual host with document root `~/http`, set +`GITHUB_WEBHOOK_SECRET` in the vhost configuration, enable PHP, and +`loginctl enable-linger www-gap-systems`. + +Then, as `www-gap-systems` (`sudo -u www-gap-systems -g www-gap-systems bash`): + + ln -s /srv/www/www-gap-systems/data ~/data + git clone https://github.com/gap-system/GapWWW ~/data/GapWWW + # otherwise adjust the path in gap-website.service + + touch ~/data/gap-website.trigger + chmod 0664 ~/data/gap-website.trigger + + # create ~/data/webhook.secret with the SetEnv line described above + + mkdir -p ~/.config/systemd/user/ + cp ~/data/GapWWW/etc/gap-website.* ~/.config/systemd/user/ + systemctl --user daemon-reload + systemctl --user enable --now gap-website.service gap-website.path + +Finally configure the webhook on GitHub as described above, and check that a +push to `master` really does rebuild the site. diff --git a/etc/README.server.md b/etc/README.server.md index fe9c72a2..27f618fc 100644 --- a/etc/README.server.md +++ b/etc/README.server.md @@ -1,142 +1,114 @@ -# Technical info about the web server setup +# Technical info about the GAP web server setup -This document describes how the GAP website hosting is set up, to help -people who need to troubleshoot it or migrate it to a new host. +This document is the starting point for anyone who has to troubleshoot the GAP +web hosting, or rebuild it from scratch after it has been lost. It gives the +overview; the details of each individual site are in a separate document, listed +below. -## Where it is hosted - -The server can be reached via SSH: - - ssh www-gap-systems@www-admin13.rz.rptu.de - -The website is update from a git clone of the website repository at - - ~/data/GapWWW - -This clone is owned by user `www-gap-systems` and group `www-gap-systems`. If anything goes -wrong with these permissions, they can be fixed via - - chown -R www-gap-systems:www-gap-systems ~/data/GapWWW - -## Automatic updates via webhook - -Whenever a change is pushed to the `master` branch of the website -repository, GitHub activates a webhook we provide via `webhook.php` at -. - -The crucial bit is at the end of this .php file, where an empty file -`~/data/gap-website.trigger` is created. This is detected by a -systemd unit `~/.config/systemd/user/gap-website.path` (a copy of this file is -in the `etc` directory of the website repository). - -This then triggers `~/.config/systemd/user/gap-website.service` -(a copy of this file is in the `etc` directory of the website repository). - -This finally executes `etc/update.sh`, which runs jekyll. - - -For authentication, we set a secret token in `~/data/webhook.secret` -which looks like this: - - SetEnv GITHUB_WEBHOOK_SECRET "MY_SECRET" - -with the actual secret key taking the place of `MY_SECRET`. The same value -must be entered in the GitHub settings at -. - - -## Troubleshooting - -The following assumes you are logged in as root (resp. used `sudo` to become root) -on the webserver. - -If updates stop working, a good first place to look at is this output of this: - - systemctl --user status gap-website.service gap-website.path - -This prints a log with extra info. However, it might also say "service not -found". In that case, make sure that `gap-website.service` and -`gap-website.path` are installed and enabled: - - cp ~/data/GapWWW/etc/gap-website.* ~/.config/systemd/user - systemctl --user enable gap-website.service gap-website.path - -Also helpful is to study the log for the relevant systemd units - - journalctl --user -f -u "gap-website.*" - -A problem that sometimes happens (e.g. if one directly pokes into the git -clone) are broken file permissions which can impede further operations, such -as git pulling updates or jekyll updating the website. To fix these, run the -following as root: - - chown -R www-gap-systems:www-gap-systems ~/data/GapWWW - chown -R www-gap-systems:www-gap-systems ~/data/http - - touch ~/data/gap-website.trigger - chown www-gap-systems:www-gap-systems ~/data/gap-website.trigger - chmod 0664 ~/data/gap-website.trigger - - -## Initial setup / what if the server VM is upgraded - -### Requirements - -- Ubuntu or Debian VM -- Apache 2 (`apt install apache2`) -- Ruby 2.7 or newer, including development headers, and bundler (`apt-get install bundler`) -- PHP (only for the webhook) (`apt install libapache2-mod-php ; a2enmod php7.4`) - - -## Further steps as `root` - -1. Set up a user `www-gap-systems` in group `www-gap-systems` - -2. Set up an Apache2 site with data in `~/data/http/` (or modify the units - here for alternate locations); ensure `www-gap-systems` owns it, i.e. - - chown -R www-gap-systems:www-gap-systems ~/data/http - - In the config for that site, make sure to set `GITHUB_WEBHOOK_SECRET` as described - elsewhere in this file, and enable PHP. - Of course also set up SSL/TLS and a scheme to update the certificates. - -3. Activate systemd user units: - - loginctl enable-linger www-gap-systems - -## Further steps as `www-gap-systems` - -As `www-gap-systems:www-gap-systems` (`sudo -u www-gap-systems -g www-gap-systems bash`): - -In the `www-gap-systems` home directory add symlinks pointing to the web -server directories: - - ln -s /srv/www/www-gap-systems/data ~/data - ln -s /srv/www/www-gap-systems/data/http ~/data/http - -Then clone the `GapWWW` git repository inside the data directory, i.e., -as `~/data/GapWWW` (otherwise adjust `gap-website.service`). Also do - - touch ~/data/gap-website.trigger - chown www-gap-systems:www-gap-systems ~/data/gap-website.trigger - chmod 0644 ~/data/gap-website.trigger - -Next install and activate the systemd units: - - mkdir -p ~/.config/systemd/user/ - cp ~/data/GapWWW/etc/gap-website.* ~/.config/systemd/user/ - systemctl --user enable gap-website.service gap-website.path - systemctl --user start gap-website.service gap-website.path - - -## On GitHub - -Go to and -make sure the webhook there is setup right: - - - Payload URL: - - Content-type: `application/x-www-form-urlencoded` (TODO: switch to JSON at some point?) - - Secret should of course match `GITHUB_WEBHOOK_SECRET` used elsewhere - - enable SSL validation - - trigger: "just the push event" +## The big picture + +Three separate websites are hosted, each as its own Apache virtual host with its +own Unix account, all on the same machine `www-admin13.rz.rptu.de`. That machine +is run by the central IT department of RPTU Kaiserslautern-Landau, not by us. In +German they are the *Rechenzentrum*, usually abbreviated RZ, which is where the +`rz` in the hostname comes from; below they are simply "the IT department". The +administrative contact there is Max Horn . + +| Site | Unix account | SSH alias | Driven by | Updated by | +| --- | --- | --- | --- | --- | +| | `www-gap-systems` | `gap-www` | this repository | GitHub webhook, on every push to `master` | +| | `www-gap-docs` | `gap-docs` | this repository | systemd timer, hourly | +| | `www-gap-files` | `gap-files` | [gap-files](https://github.com/gap-system/gap-files) | systemd timers, every 15 min and hourly | + +The SSH aliases are a local convention; they assume entries such as the +following in your `~/.ssh/config`: + + Host gap-www + Hostname www-admin13.rz.rptu.de + User www-gap-systems + +Per-site documentation: + +- [`README.gap-www.md`](README.gap-www.md) — the website itself +- [`README.gap-docs.md`](README.gap-docs.md) — the manuals +- the archive server is documented in the `README.md` of the + [gap-files](https://github.com/gap-system/gap-files) repository, which is + where its scripts and units live + +## What the three accounts have in common + +Knowing these conventions explains most of what you will find on any of the +three accounts. + +**Directory layout.** Each account has the same two symlinks in its home +directory: + + ~/data -> /srv/www//data # created by us + ~/http -> /srv/www//data/http # created for us, root owned + +`~/http` is the document root of that site. Everything else we keep — git +clones, work directories, state files — lives next to it in `~/data`, outside +the document root, so that it is not served. + +**Storage** is an NFS mount from `nas.rhrk.uni-kl.de`, shared by all accounts on +the machine, about 1.7 TB in total. It is not fast: reading a few gigabytes back +is noticeably expensive, which is why the update jobs are written to avoid +re-reading data they have already checked. + +**Apache** is installed and configured centrally by the IT department, one +instance per virtual host. The vhost configuration is *not* readable or writable +by our accounts. However, we are not boxed in by that: `.htaccess` files are +honoured, and PHP is enabled (8.4 as of this writing), which is enough for +redirect rules and for the webhook endpoint. Certificates and DNS are handled by +them too. + +**systemd user units** are how all automatic updating is driven. This works +without an active login session only because lingering is enabled for each +account, which requires root: + + loginctl enable-linger + +If that is ever lost, the units silently stop running when nobody is logged in — +which looks exactly like "the website stopped updating for no reason". Units are +installed as *copies* in `~/.config/systemd/user/`, not symlinks into the git +clone, so after changing a unit in the repository it must be copied again. + +Unit names are prefixed with the site they belong to, so that `systemctl --user +list-timers` on any account is self-explanatory. + +## If you have to rebuild all of this + +Roughly in order: + +1. Ask the IT department for the virtual hosts and the accounts, with PHP + enabled and `AllowOverride` sufficient for `.htaccess`, and get + `loginctl enable-linger` set for each account. Everything after this can be + done without root. +2. Recreate the `~/data` symlink on each account (`~/http` is created for us). +3. Set up each site following its own document, in this order: the website + first (it is the only one with a secret to configure), then the manuals, then + the archive server. +4. For the archive server, note that the package archives cannot be recovered + from anywhere else: is itself the archive of + record for old package releases. If it is ever lost, the only copies are + whatever backups the IT department holds and whatever developers happen to + have locally. Everything else — the website, the manuals, the GAP releases — + can be rebuilt from GitHub. + +## Troubleshooting anywhere + +The same handful of commands apply on all three accounts: + + systemctl --user list-timers # what is scheduled, and when it last ran + systemctl --user list-units --failed # anything broken + journalctl --user -u '' -n 100 # what it said + journalctl --user -f -u '' # follow a running job + +There is no email notification anywhere in this setup: a job that starts failing +will keep failing quietly until somebody looks. If updates have stopped, check in +this order: is lingering still enabled (`loginctl show-user `), is the +timer or path unit still enabled, and does the journal show the job failing. + +A recurring cause of trouble on the website account is broken file ownership in +the git clone or document root, usually after someone has poked at it as the +wrong user. See [`README.gap-www.md`](README.gap-www.md) for how to repair it. diff --git a/etc/docs.htaccess b/etc/docs.htaccess index 9d2cb282..739be68e 100644 --- a/etc/docs.htaccess +++ b/etc/docs.htaccess @@ -1,3 +1,5 @@ +RedirectMatch temp ^/$ https://www.gap-system.org/doc/ + RedirectMatch permanent ^(.*)/pkg/4ti2Interface(-[^/]+)?/(.*)$ $1/pkg/4ti2interface/$3 RedirectMatch permanent ^(.*)/pkg/AGT(-[^/]+)?/(.*)$ $1/pkg/agt/$3 RedirectMatch permanent ^(.*)/pkg/AutoDoc(-[^/]+)?/(.*)$ $1/pkg/autodoc/$3 diff --git a/etc/gap-docs-manuals.service b/etc/gap-docs-manuals.service new file mode 100644 index 00000000..64de7496 --- /dev/null +++ b/etc/gap-docs-manuals.service @@ -0,0 +1,10 @@ +[Unit] +Description=Mirror the GAP manuals to docs.gap-system.org +Documentation=https://github.com/gap-system/GapWWW +After=network-online.target + +[Service] +Type=oneshot +# Update the clone first: extract_manuals.py, and this job itself, live in it. +ExecStartPre=/usr/bin/git -C %h/data/GapWWW pull --ff-only --quiet +ExecStart=%h/data/GapWWW/etc/mirror-manuals.py diff --git a/etc/gap-docs-manuals.timer b/etc/gap-docs-manuals.timer new file mode 100644 index 00000000..c02f246b --- /dev/null +++ b/etc/gap-docs-manuals.timer @@ -0,0 +1,13 @@ +[Unit] +Description=Mirror the GAP manuals hourly +Documentation=https://github.com/gap-system/GapWWW + +[Timer] +# Releases are rare, so hourly is ample; the manuals of a new release appear +# within an hour of it ceasing to be a pre-release. +OnCalendar=hourly +Persistent=true +RandomizedDelaySec=10min + +[Install] +WantedBy=timers.target diff --git a/etc/mirror-manuals.py b/etc/mirror-manuals.py new file mode 100755 index 00000000..20f1fbf5 --- /dev/null +++ b/etc/mirror-manuals.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python3 +# +# This script mirrors the GAP manuals onto https://docs.gap-system.org. +# +# It automates what used to be the manual step `./download_manuals.sh X.Y.Z` on +# that host, described in dev/releases/README.md in the GAP repository. For +# every published GAP release it does not have yet, it downloads the release +# tarball and the accompanying package metadata, extracts the GAP and package +# manuals with extract_manuals.py, installs them as `/vX.Y.Z`, and points +# the `latest` symlink at the newest release. +# +# Releases that are already installed are skipped, so running this repeatedly +# is cheap; it is meant to be run hourly from a systemd timer. + +import argparse +import gzip +import os +import re +import shutil +import subprocess +import sys +import tarfile +from typing import List, Optional, Tuple + +import requests +from utils import download_with_sha256, error, notice, warning + +REPO = "gap-system/gap" +API_URL = f"https://api.github.com/repos/{REPO}/releases" + +TAG_RE = re.compile(r"^v(\d+)\.(\d+)\.(\d+)$") + +# Releases before this one do not ship the package-infos.json file that +# extract_manuals.py needs, so their manuals cannot be built this way. +MIN_VERSION = (4, 11, 1) + +HERE = os.path.dirname(os.path.abspath(__file__)) +EXTRACT_MANUALS = os.path.join(HERE, "extract_manuals.py") + + +def stable_releases() -> List[Tuple[int, int, int]]: + """Return the versions of all published, non-prerelease GAP releases.""" + versions = [] + page = 1 + while True: + response = requests.get( + API_URL, params={"per_page": 100, "page": page}, timeout=(30, 60) + ) + response.raise_for_status() + batch = response.json() + if not batch: + break + for release in batch: + if release["draft"] or release["prerelease"]: + continue + match = TAG_RE.match(release["tag_name"]) + if match: + versions.append( + (int(match.group(1)), int(match.group(2)), int(match.group(3))) + ) + page += 1 + return versions + + +def installed_versions(dest: str) -> List[Tuple[int, int, int]]: + """Return the versions for which manuals are already installed in `dest`.""" + versions = [] + for name in os.listdir(dest): + match = TAG_RE.match(name) + if match and os.path.isdir(os.path.join(dest, name)): + versions.append( + (int(match.group(1)), int(match.group(2)), int(match.group(3))) + ) + return versions + + +def version_str(version: Tuple[int, int, int]) -> str: + return ".".join(str(n) for n in version) + + +def build_manuals(version: Tuple[int, int, int], workdir: str, dest: str) -> None: + """Build the manuals for `version` and install them into `dest`.""" + ver = version_str(version) + base = f"https://github.com/{REPO}/releases/download/v{ver}" + + os.makedirs(workdir, exist_ok=True) + + # Both downloads are checked against the .sha256 file published beside them; + # the old download_manuals.sh fetched them without verifying anything. + tarball = os.path.join(workdir, f"gap-{ver}.tar.gz") + infos_gz = os.path.join(workdir, "package-infos.json.gz") + download_with_sha256(f"{base}/gap-{ver}.tar.gz", tarball) + download_with_sha256(f"{base}/package-infos.json.gz", infos_gz) + + notice(f"unpacking package-infos.json for {ver}") + infos = os.path.join(workdir, "package-infos.json") + with gzip.open(infos_gz, "rb") as src, open(infos, "wb") as dst: + shutil.copyfileobj(src, dst) + + notice(f"extracting gap-{ver}.tar.gz") + gaproot = os.path.join(workdir, f"gap-{ver}") + shutil.rmtree(gaproot, ignore_errors=True) + with tarfile.open(tarball) as archive: + # Extract as `tar x` would, symlinks and all: GAP tarballs contain + # package symlinks that the stricter filters reject. The tarball is an + # official release archive whose checksum was just verified. + archive.extractall(workdir, filter="fully_trusted") + + # extract_manuals.py writes into a directory called `Manuals` in the current + # working directory, so run it inside the work directory. + manuals = os.path.join(workdir, "Manuals") + shutil.rmtree(manuals, ignore_errors=True) + notice(f"extracting the manuals for {ver}") + subprocess.run( + [sys.executable, EXTRACT_MANUALS, gaproot, infos], cwd=workdir, check=True + ) + + # Move the finished tree into place in one step, so that a half-built set of + # manuals is never visible on the website. This is a rename within the same + # filesystem, since the work directory sits next to the document root. + target = os.path.join(dest, f"v{ver}") + if os.path.exists(target): + shutil.rmtree(target) + os.replace(manuals, target) + notice(f"installed the manuals for {ver} in {target}") + + +def update_latest(dest: str) -> None: + """Point the `latest` symlink at the newest installed release.""" + versions = installed_versions(dest) + if not versions: + return + newest = f"v{version_str(max(versions))}" + link = os.path.join(dest, "latest") + + if os.path.islink(link) and os.readlink(link) == newest: + return + + # Replace the symlink by renaming a new one over it, so that `latest` -- and + # with it the `doc` and `pkg` symlinks pointing through it -- is never + # missing, not even briefly. + tmp = os.path.join(dest, ".latest.new") + if os.path.lexists(tmp): + os.remove(tmp) + os.symlink(newest, tmp) + os.replace(tmp, link) + notice(f"pointed latest at {newest}") + + +def main(argv: Optional[List[str]] = None) -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--dest", + default=os.path.expanduser("~/http"), + help="directory served by the web server (default: ~/http)", + ) + parser.add_argument( + "--workdir", + default=os.path.expanduser("~/data/.mirror-manuals"), + help="scratch directory for downloads and unpacking", + ) + parser.add_argument( + "--since", + default=version_str(MIN_VERSION), + help="oldest release to consider, as X.Y.Z (default: %(default)s)", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="report what would be built, without downloading", + ) + args = parser.parse_args(argv) + + try: + since = tuple(int(n) for n in args.since.split(".")) + if len(since) != 3: + raise ValueError + except ValueError: + parser.error(f"--since must be of the form X.Y.Z, not {args.since!r}") + + if not os.path.isdir(args.dest): + error(f"{args.dest} is not a directory") + + # Manuals are served over HTTP, so they must be world readable. + os.umask(0o022) + + try: + available = stable_releases() + except requests.RequestException as e: + error(f"could not list releases: {e}") + + have = set(installed_versions(args.dest)) + todo = sorted(v for v in available if v >= since and v not in have) + + notice( + f"{len(available)} releases published, {len(have)} installed, " + f"{len(todo)} to build" + ) + + if args.dry_run: + for version in todo: + notice(f"would build manuals for {version_str(version)}") + return 0 + + # Start from a clean slate: the work directory holds a release tarball and + # its unpacked copy, well over a gigabyte, which there is no reason to keep + # between runs. + shutil.rmtree(args.workdir, ignore_errors=True) + + failures = [] + for version in todo: + try: + build_manuals(version, args.workdir, args.dest) + # utils.error() reports a problem by exiting, so catch that too and + # carry on with the remaining releases. + except ( + requests.RequestException, + subprocess.CalledProcessError, + OSError, + tarfile.TarError, + SystemExit, + ) as e: + warning(f"{version_str(version)}: {e}") + failures.append(version_str(version)) + + update_latest(args.dest) + shutil.rmtree(args.workdir, ignore_errors=True) + + if failures: + error(f"failed to build manuals for: {' '.join(failures)}") + + return 0 + + +if __name__ == "__main__": + sys.exit(main())