Resolve library repos from infra's libraries.yaml

.gitignore

@@ -102,3 +102,6 @@ dist
 
 # TernJS port file
 .tern-port
+
+# Local build log database
+buildslogs.db*

cpp-build-results.js

@@ -10,17 +10,19 @@ const {DynamoDBClient, ScanCommand, QueryCommand} = require ('@aws-sdk/client-dy
 class CppBuildResultsView {
     /**
      * 
-     * @param {BuildLogging} logging 
+     * @param {BuildLogging} logging
      * @param {BuildAnnotations} annotations
+     * @param {LibraryRepos} libraryrepos
      */
-    constructor(logging, annotations, conanfunc, compilernames, compilersemvers) {
+    constructor(logging, annotations, conanfunc, compilernames, compilersemvers, libraryrepos) {
         this.results_view = pug.compileFile('views/library_build_results.pug');
 
         this.logging = logging;
         this.annotations = annotations;
         this.conanfunc = conanfunc;
         this.compilernames = compilernames;
         this.compilersemvers = compilersemvers;
+        this.libraryrepos = libraryrepos;
 
         this.ddbClient = new DynamoDBClient({region: 'us-east-1'});
     }
@@ -107,25 +109,15 @@ class CppBuildResultsView {
         const clang_semvers = succeeded_compilers.filter(result => result.compiler === 'clang').map(result => semver.parse(result.compiler_semver)).filter(Boolean);
         const min_clang_ver = _.first(semver.sort(clang_semvers));
 
-        let repo_owner = "";
-        let repo_name = "";
-        if (library === "fmt") {
-          repo_owner = "fmtlib";
-          repo_name = "fmt"
-        } else if (library === "beman_iterator_interface") {
-          repo_owner = "bemanproject";
-          repo_name = "iterator_interface"
-        } else {
-          repo_owner = "notsupported"
-          repo_name = "notsupported"
-        }
-
+        const repo = this.libraryrepos ? this.libraryrepos.get('c++', library, library_version) : null;
+
         return await this.results_view({
             lib: {
-                commit_url: `https://github.com/${repo_owner}/${repo_name}/commit/${commit_hash}`,
+                commit_url: repo ? `${repo.commit_url_prefix}${commit_hash}` : '',
                 commit_hash: commit_hash,
-                repo_owner,
-                repo_name,
+                repo_owner: repo ? repo.owner : '',
+                repo_name: repo ? repo.name : '',
+                repo_host: repo ? repo.host : '',
                 name: `${library}`,
                 version: `${library_version}`
             },

docs/api.md

@@ -3,12 +3,22 @@
 
 The proxy forwards all /v1 and /v2 API calls to Conan, everything there should be as usual.
 
+Note that the Conan passthrough only ever lists packages that *exist*. If a library/compiler
+combination is absent from `/v1` (or from `/binaries`), that says nothing about why: it may never
+have been attempted, or it may have been attempted and failed. The endpoints under
+[Build failures and logs](#build-failures-and-logs) are what tell those two cases apart.
+
 ## Listings
 
 ### GET `/libraries`
 
 To get a list of available libraries and versions, this includes library and version information from Compiler-Explorer.
 
+### GET `/libraries/cpp`, `/libraries/rust`, `/libraries/fortran`, `/libraries/go`
+
+The same document as `/libraries`, filtered down to the libraries of one language. Anything not
+tagged with that language is left out; the entries themselves are unchanged.
+
 ### GET `/binaries/:libraryid/:version`
 
 To get a list of available binaries and for what compiler and configuration they were built.
@@ -81,6 +91,25 @@ Returns:
 }
 ```
 
+### GET `/annotations/:libraryid/:version`
+
+Every annotation held for a library version, in one call, rather than one build hash at a time.
+
+Returns an array with one entry per build hash found on disk, each holding the `annotation` (the
+same document the call below returns) and the `buildinfo` read from Conan's own `conaninfo.txt`:
+
+```
+[
+    {
+        "buildhash": "16ee6b51398740e37d34e117ee3defa26bd4104e",
+        "annotation": { "commithash": "7eddbfed", "cxx11": true, ... },
+        "buildinfo": { ... }
+    }
+]
+```
+
+Returns an empty array if the library version has no packages at all.
+
 ### GET `/annotations/:libraryid/:version/:buildhash`
 
 Annotations are json documents that are saved next to a certain library version and build.
@@ -102,6 +131,144 @@ Returns:
 ```
 
 
+## Downloads
+
+These redirect (302) to the package tarball on the Conan server rather than serving it themselves.
+
+Packages for proprietary compilers (Intel ICC, MSVC, QNX, EDG) are never handed out. The
+five-argument form below rejects them with a 403; the three-argument form usually returns 404
+instead, because those compilers have more than one binary per library version and so fail its
+single-binary check before the restriction is even consulted.
+
+### GET `/downloadcshared/:libraryid/:version`
+
+Redirects to the `cshared` build of a library version — the one built for use from C. Only works
+when exactly one `cshared` binary exists for that version; returns 404 otherwise.
+
+### GET `/downloadpkg/:libraryid/:version/:compiler`
+
+Redirects to the package built by a given compiler, e.g. `/downloadpkg/fmt/10.0.0/g131`.
+
+This only resolves when that compiler has exactly one binary for the version. A library built for
+several architectures or standard libraries will have more than one, so use the five-argument form
+below for those. Returns 404 when nothing matches.
+
+### GET `/downloadpkg/:library/:library_version/:compiler_version/:arch/:libcxx`
+
+Redirects to the package for one fully specified build combination, e.g.
+`/downloadpkg/fmt/10.0.0/g131/x86_64/libstdc%2B%2B`.
+
+`libcxx` needs url-encoding: `libstdc++` becomes `libstdc%2B%2B`. Combinations with no architecture
+or no libcxx are addressed with a literal space (`%20`) in that position, since an empty path
+segment would not route.
+
+Returns 404 when nothing matches.
+
+
+## Build failures and logs
+
+A build that is missing from Conan was either never attempted or attempted and failed. These are
+the endpoints that say which.
+
+### GET `/failedbuilds/:library/:library_version`
+
+The build combinations that were attempted for a library version and failed, one record per
+combination:
+
+```
+[
+    {
+        "compiler": "gcc",
+        "compiler_version": "g161",
+        "arch": "x86_64",
+        "libcxx": "libstdc++",
+        "compiler_flags": "",
+        "commithash": "6.7.0"
+    }
+]
+```
+
+The absence of a combination here means it was never attempted; its presence means it was attempted
+and failed. That distinction is the point of this endpoint — Conan itself cannot tell you the
+difference, because it only ever lists packages that exist.
+
+Only failures are returned; successful builds are not listed. Use the `commithash` from a record
+here as the `:commithash` for `/getlogging_forcommit` below.
+
+### GET `/getlogging/:library/:library_version/:arch/:dt`
+
+The stored build log for a library version, architecture and build timestamp, served as a text
+attachment.
+
+`:dt` is the `build_dt` of the build, in `YYYYMMDDHHMMSS` form, as returned by `/allfailedbuilds`.
+An architecture that is empty is addressed with a literal space (`%20`). If several builds share
+that timestamp and architecture, their logs are concatenated, separated by
+`--- More entries ---`. Returns 404 when no log is stored, and logs trimmed by the age-based
+cleanup come back as the single line `Log eliminated due to age`.
+
+Both logging endpoints send `Content-Type: text/html` regardless of content; it is the
+`Content-Disposition` header that makes browsers save it as a `.txt` file.
+
+### GET `/getlogging_forcommit/:library/:library_version/:commithash/:compiler_version/:arch/:libcxx`
+
+The stored build log for one exact build combination, served as a text attachment. This is usually
+the one you want after `/failedbuilds` has told you which combination failed.
+
+```
+/getlogging_forcommit/qt/6.7.0/6.7.0/g161/x86_64/libstdc%2B%2B
+```
+
+Two things to watch:
+
+* `libcxx` needs url-encoding — `libstdc++` is `libstdc%2B%2B` — and an empty `arch` or `libcxx` is
+  addressed with a literal space (`%20`).
+* `:commithash` is whatever was recorded with `/buildfailed`, which for a released library version
+  is frequently just the version string (`6.7.0`) rather than a git hash. Take it from the
+  `/failedbuilds` record or from `/whathasfailedbefore`, both of which return it, instead of
+  guessing.
+
+What comes back is the whole captured build output, which begins with the generated `#!/bin/sh`
+build script — compiler paths, flags, the `cmake` invocation — and only then the compiler output.
+The first screenful is therefore never the failure; the logs run to hundreds of kilobytes, so skip
+to the end.
+
+Returns 404 when no log is stored for that combination.
+
+### GET `/cpp_library_build_results/:library/:library_version/:commit_hash`
+
+An **HTML** page (not JSON) summarising how one commit of a library built across every compiler,
+with per-compiler links to the log or to the package. Build history comes from the DynamoDB
+`library-build-history` table, so this covers more than the recent failures in the local database.
+
+```
+/cpp_library_build_results/fmt/trunk/7eddbfed
+```
+
+A `?allcompilers=1` query parameter is accepted and currently has no effect — the compressed view
+it used to select is commented out in the template.
+
+### GET `/cpp_compiler_build_results/:compiler_id`
+
+An **HTML** page (not JSON), the same data from the other direction: how every library fared for
+one compiler, e.g. `/cpp_compiler_build_results/g161`. Also sourced from DynamoDB, and covers all
+architecture/libcxx variants of that compiler id.
+
+
+## Service
+
+### GET `/healthcheck`
+
+Returns `OK`. No authentication.
+
+### GET `/reinitialize`
+
+Re-reads everything the proxy caches at startup: the compiler and library lists from
+Compiler-Explorer, the packages on the Conan server, the library repository mapping and the
+compiler variants. Returns `done` once that has finished.
+
+No authentication, and it does real work on every call, so do not put it behind anything that
+retries.
+
 ## Secure API
 
 The following calls require authentication. This is done by posting to `/login` and using the resulting token in the `Authentication` header.

html/gh-commits.js

@@ -1,6 +1,6 @@
-import { Octokit, App } from "https://esm.sh/octokit";
+import { Octokit } from "https://esm.sh/octokit";
 
-async function list_last_10(owner, repo, lib_name, lib_version) {
+async function list_last_10(morediv, owner, repo, lib_name, lib_version) {
     const octokit = new Octokit({
         // auth: 'YOUR-TOKEN'
     });
@@ -13,7 +13,6 @@ async function list_last_10(owner, repo, lib_name, lib_version) {
         }
     });
 
-    const morediv = $(".more-commits");
     for (const item of list.data) {
         const shorthash = item.sha.substring(0, 8);
 
@@ -31,4 +30,8 @@ async function list_last_10(owner, repo, lib_name, lib_version) {
     }
 }
 
-list_last_10(window.repo_owner, window.repo_name, window.lib_name, window.lib_version);
+const morediv = $(".more-commits");
+const config = morediv.length ? morediv[0].dataset : null;
+if (config && config.repoOwner && config.repoName) {
+    list_last_10(morediv, config.repoOwner, config.repoName, config.libName, config.libVersion);
+}

index.js

@@ -6,6 +6,7 @@ const
     { CppBuildResultsView } = require('./cpp-build-results'),
     { CppCompilerBuildResultsView, CompilerVariantsCache } = require('./cpp-compiler-build-results'),
     { is_restricted_compiler } = require('./compiler-restrictions'),
+    { LibraryRepos } = require('./library-repos'),
     path = require('path'),
     express = require('express'),
     { expressjwt } = require('express-jwt'),
@@ -39,6 +40,7 @@ let modifiedDt = null;
 
 const annotations = new BuildAnnotations(conanserverroot);
 const buildlogging = new BuildLogging(buildlogspath);
+const libraryRepos = new LibraryRepos();
 const compilerVariantsCache = new CompilerVariantsCache([
     'fmt#10.0.0#10.0.0',
     'zlib#1.3.1#1.3.1',
@@ -194,6 +196,15 @@ async function refreshCELibraries() {
     allGoLibrariesAndVersions = go;
 }
 
+// A stale repo map only costs us a missing link, so never let this take the server down.
+async function refreshLibraryRepos() {
+    try {
+        await libraryRepos.refresh();
+    } catch (e) {
+        console.error('libraryRepos refresh failed:', e);
+    }
+}
+
 function newProxy() {
     const proxy = httpProxy.createProxyServer({});
 
@@ -397,6 +408,7 @@ function main() {
             await refreshCECompilers();
             await refreshCELibraries();
             await refreshConanLibraries(true);
+            await refreshLibraryRepos();
             await compilerVariantsCache.refresh();
             modifiedDt = new Date();
             res.send('done');
@@ -557,7 +569,7 @@ function main() {
                 }
 
                 if (!found)
-                    res.send(404);
+                    res.sendStatus(404);
             } catch (e) {
                 console.error(e);
                 res.send(e);
@@ -744,7 +756,8 @@ function main() {
                 new RemoteAnnotations(conanserverurl),
                 getConanBinaries,
                 compilernames,
-                compilersemvers
+                compilersemvers,
+                libraryRepos
             );
             res.send(await view.get(req.params.library, req.params.library_version, req.params.commit_hash, req.query.allcompilers === '1'));
         })
@@ -774,7 +787,7 @@ function main() {
 
 buildlogging.connect().then(refreshCECompilers).then(refreshCELibraries).then(() => {
     return refreshConanLibraries(true);
-}).then(() => {
+}).then(refreshLibraryRepos).then(() => {
     compilerVariantsCache.refresh().catch(e => console.error('compilerVariantsCache initial refresh failed:', e));
     return main();
 });