use vendor script to generate build file sources

simplicity-sys/build.rs

@@ -1,61 +1,117 @@
 extern crate cc;
 
 use std::env;
-use std::path::Path;
+use std::path::{Path, PathBuf};
+
+// What we compile, from the lists vendor-simplicity.sh takes from libsimplicity's Makefile:
+// - library sources (CORE, BITCOIN, ELEMENTS): always, except TEST_UTILS_ONLY (test-utils only);
+// - test sources (TEST): with test-utils only;
+// - never anything in EXCLUDED.
+// Each source goes in the library named by its directory: bitcoin/, elements/, or common.
+include!("depend/simplicity-sources.rs");
+
+const SIMPLICITY_DIR: &str = "depend/simplicity";
+
+/// Rust has its own decoder, type inference and evaluator, so the C versions are only used to
+/// cross-check them in tests.
+const TEST_UTILS_ONLY: &[&str] = &[
+    "bitstream.c",
+    "dag.c",
+    "deserialize.c",
+    "eval.c",
+    "type.c",
+    "typeInference.c",
+    "elements/exec.c",
+    "elements/primitive.c",
+    // #385: uncomment along with the Bitcoin library below.
+    // "bitcoin/exec.c",
+    // "bitcoin/primitive.c",
+];
+
+/// Sources we never compile, and why.
+const EXCLUDED: &[(&str, &str)] = &[
+    ("elements/cmr.c", "Rust computes CMRs itself"),
+    // #385: uncomment when vendoring a libsimplicity that has the Bitcoin application (#324).
+    // Otherwise these get compiled into the libraries, and MSRV links fail on their undefined
+    // decoder symbols.
+    // ("cmr.c", "Rust computes CMRs itself"),
+    // ("bitcoin/cmr.c", "Rust computes CMRs itself"),
+    ("test.c", "libsimplicity's test runner; defines main()"),
+    ("regression4.c", "only used by test.c"),
+    ("typeSkipTest.c", "only used by test.c"),
+];
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum Library {
+    Common,
+    Bitcoin,
+    Elements,
+}
+
+/// The library a source belongs in, by its directory in libsimplicity.
+fn library(source: &str) -> Library {
+    if source.starts_with("bitcoin/") {
+        Library::Bitcoin
+    } else if source.starts_with("elements/") {
+        Library::Elements
+    } else {
+        Library::Common
+    }
+}
+
+/// The sources to compile into `lib`, following the rules at the top of this file.
+fn sources(lib: Library) -> Vec<PathBuf> {
+    let test_utils = cfg!(feature = "test-utils");
+    let library_sources = CORE_SOURCES
+        .iter()
+        .chain(BITCOIN_SOURCES)
+        .chain(ELEMENTS_SOURCES)
+        .copied()
+        .filter(|source| test_utils || !TEST_UTILS_ONLY.contains(source));
+    let test_sources = TEST_SOURCES.iter().copied().filter(|_| test_utils);
+
+    library_sources
+        .chain(test_sources)
+        .filter(|source| !EXCLUDED.iter().any(|(excluded, _)| excluded == source))
+        .filter(|source| library(source) == lib)
+        .map(|source| Path::new(SIMPLICITY_DIR).join(source))
+        .collect()
+}
+
+/// Fail the build if the tables above name a source that libsimplicity's Makefile doesn't build,
+/// e.g. because of a typo or because upstream renamed or removed it. Otherwise the entry would
+/// silently do nothing. (Whether the vendored tree matches upstream is checked in CI, by re-running
+/// vendor-simplicity.sh.)
+fn check_tables() {
+    let upstream = || {
+        CORE_SOURCES
+            .iter()
+            .chain(BITCOIN_SOURCES)
+            .chain(ELEMENTS_SOURCES)
+            .chain(TEST_SOURCES)
+    };
+    let stale: Vec<_> = TEST_UTILS_ONLY
+        .iter()
+        .chain(EXCLUDED.iter().map(|(source, _)| source))
+        .filter(|source| !upstream().any(|upstream| upstream == *source))
+        .collect();
+    assert!(
+        stale.is_empty(),
+        "build.rs mentions {stale:?}, but libsimplicity's Makefile doesn't build them.",
+    );
+}
 
 fn main() {
     // rerun if changes to the C code
     println!("cargo:rerun-if-changed=depend");
-    let simplicity_path = Path::new("depend/simplicity");
-    let jet_files: Vec<_> = vec![
-        "frame.c",
-        "jets.c",
-        "jets-secp256k1.c",
-        "rsort.c",
-        "sha256.c",
-        "elements/env.c",
-        "elements/ops.c",
-        "elements/elementsJets.c",
-        "elements/txEnv.c",
-    ]
-    .into_iter()
-    .map(|x| simplicity_path.join(x))
-    .collect();
+    check_tables();
 
     let mut build = cc::Build::new();
     build
         .std("c11")
         .flag_if_supported("-fno-inline-functions")
         .opt_level(2)
-        .files(jet_files)
-        .file(Path::new("depend/wrapper.c"))
-        .file(Path::new("depend/env.c"))
-        .file(Path::new("depend/jets_wrapper.c"))
-        .include(simplicity_path.join("include"));
-
-    if cfg!(feature = "test-utils") {
-        let test_files: Vec<_> = vec![
-            "bitstream.c",
-            "dag.c",
-            "deserialize.c",
-            "eval.c",
-            "type.c",
-            "typeInference.c",
-            "elements/exec.c",
-            "elements/primitive.c",
-            "ctx8Pruned.c",
-            "ctx8Unpruned.c",
-            "hashBlock.c",
-            "schnorr0.c",
-            "schnorr6.c",
-            "elements/checkSigHashAllTx1.c",
-        ]
-        .into_iter()
-        .map(|x| simplicity_path.join(x))
-        .collect();
-
-        build.files(test_files);
-    }
+        .include(Path::new(SIMPLICITY_DIR).join("include"));
 
     if cfg!(not(fuzzing)) {
         build.define("PRODUCTION", None);
@@ -66,5 +122,25 @@ fn main() {
         build.include("wasm-sysroot");
     }
 
-    build.compile("ElementsSimplicity");
+    // Chain-independent code goes in its own library, which each chain's library links against.
+    // Compiling it into every chain's library instead breaks the MSRV build: Rust 1.74 links native
+    // libraries with --whole-archive, so the duplicate objects collide.
+    build
+        .clone()
+        .files(sources(Library::Common))
+        .file("depend/wrapper.c")
+        .file("depend/jets_wrapper.c")
+        .compile("RustSimplicityCommon");
+
+    // #385: uncomment once the Bitcoin transaction environment lands.
+    // build
+    //     .clone()
+    //     .files(sources(Library::Bitcoin))
+    //     .file("depend/bitcoin_env.c")
+    //     .compile("BitcoinSimplicity");
+
+    build
+        .files(sources(Library::Elements))
+        .file("depend/env.c")
+        .compile("ElementsSimplicity");
 }

simplicity-sys/depend/simplicity-sources.rs

@@ -0,0 +1,50 @@
+// This file has been automatically generated by vendor-simplicity.sh from libsimplicity's
+// C/Makefile. Do not edit it by hand; re-run vendor-simplicity.sh instead.
+
+/// `CORE_OBJS` in libsimplicity's Makefile.
+const CORE_SOURCES: &[&str] = &[
+    "bitstream.c",
+    "dag.c",
+    "deserialize.c",
+    "eval.c",
+    "frame.c",
+    "jets.c",
+    "jets-secp256k1.c",
+    "rsort.c",
+    "sha256.c",
+    "type.c",
+    "typeInference.c",
+];
+
+/// `BITCOIN_OBJS` in libsimplicity's Makefile.
+const BITCOIN_SOURCES: &[&str] = &[
+    "bitcoin/env.c",
+    "bitcoin/ops.c",
+    "bitcoin/bitcoinJets.c",
+    "bitcoin/primitive.c",
+    "bitcoin/txEnv.c",
+];
+
+/// `ELEMENTS_OBJS` in libsimplicity's Makefile.
+const ELEMENTS_SOURCES: &[&str] = &[
+    "elements/env.c",
+    "elements/exec.c",
+    "elements/ops.c",
+    "elements/elementsJets.c",
+    "elements/primitive.c",
+    "elements/cmr.c",
+    "elements/txEnv.c",
+];
+
+/// `TEST_OBJS` in libsimplicity's Makefile.
+const TEST_SOURCES: &[&str] = &[
+    "test.c",
+    "ctx8Pruned.c",
+    "ctx8Unpruned.c",
+    "hashBlock.c",
+    "regression4.c",
+    "schnorr0.c",
+    "schnorr6.c",
+    "typeSkipTest.c",
+    "elements/checkSigHashAllTx1.c",
+];

simplicity-sys/vendor-simplicity.sh

@@ -71,7 +71,34 @@ git ls-files | rsync -av --files-from=- . "$VENDORED_SIM_DIR"
 
 popd
 
-## 3. Patch things to include versions
+## 3. Record which sources libsimplicity's Makefile builds, for build.rs
+
+SOURCES_FILE="$DEPEND_PATH/simplicity-sources.rs"
+{
+    echo "// This file has been automatically generated by vendor-simplicity.sh from libsimplicity's"
+    echo "// C/Makefile. Do not edit it by hand; re-run vendor-simplicity.sh instead."
+    for GROUP in CORE BITCOIN ELEMENTS TEST; do
+        # We only understand a list on a single line, so refuse continuations and appends.
+        if grep -qE "^${GROUP}_OBJS"' *(\+=|:=.*\\$)' "$VENDORED_SIM_DIR/Makefile"; then
+            echo "${GROUP}_OBJS spans several lines in $VENDORED_SIM_DIR/Makefile; update vendor-simplicity.sh." >&2
+            exit 1
+        fi
+        # e.g. "CORE_OBJS := bitstream.o dag.o ..." becomes one quoted "bitstream.c" per line.
+        SOURCES=$(sed -n "s/^${GROUP}_OBJS *:= *//p" "$VENDORED_SIM_DIR/Makefile" \
+            | tr ' ' '\n' | sed -n 's/^\(.*\)\.o$/    "\1.c",/p')
+        if [ -z "$SOURCES" ]; then
+            echo "${GROUP}_OBJS not found in $VENDORED_SIM_DIR/Makefile; update vendor-simplicity.sh." >&2
+            exit 1
+        fi
+        echo
+        echo "/// \`${GROUP}_OBJS\` in libsimplicity's Makefile."
+        echo "const ${GROUP}_SOURCES: &[&str] = &["
+        echo "$SOURCES"
+        echo "];"
+    done
+} > "$SOURCES_FILE"
+
+## 4. Patch things to include versions
 
 # a. patch our own drop/new functions for C structures.
 find "$DEPEND_PATH/.." -name "*.rs" -type f -exec sed -i "s/rust_[0-9_]*_free/rust_${SIMPLICITY_ALLOC_VERSION_CODE}_free/" {} \;