test: run the test suite on WASI

.github/workflows/main.yml

@@ -75,6 +75,49 @@ jobs:
       run: rustup component add clippy
     - run: cargo clippy --all-targets --all-features -- -Dwarnings
 
+  wasi:
+    name: WASI
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        build: [stable, beta, nightly, MSRV]
+        include:
+          - build: stable
+            rust: stable
+          - build: beta
+            rust: beta
+          - build: nightly
+            rust: nightly
+          - build: MSRV
+            rust: "1.87"
+    env:
+      WASI_SDK_VERSION: 34
+      WASMTIME_VERSION: 49.0.2
+    steps:
+    - uses: actions/checkout@master
+      with:
+        submodules: true
+    - name: Install Rust (rustup)
+      run: rustup update ${{ matrix.rust }} --no-self-update && rustup default ${{ matrix.rust }} && rustup target add wasm32-wasip1
+    - name: Install wasi-sdk
+      run: |
+        curl -fsSL https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-${WASI_SDK_VERSION}/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux.tar.gz | tar xz
+        echo "CC_wasm32_wasip1=$PWD/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux/bin/clang" >> $GITHUB_ENV
+        echo "AR_wasm32_wasip1=$PWD/wasi-sdk-${WASI_SDK_VERSION}.0-x86_64-linux/bin/llvm-ar" >> $GITHUB_ENV
+    - name: Install wasmtime
+      run: |
+        curl -fsSL https://github.com/bytecodealliance/wasmtime/releases/download/v${WASMTIME_VERSION}/wasmtime-v${WASMTIME_VERSION}-x86_64-linux.tar.xz | tar xJ
+        echo "$PWD/wasmtime-v${WASMTIME_VERSION}-x86_64-linux" >> $GITHUB_PATH
+    # Before Rust 1.94, std on WASI can't canonicalize paths and its
+    # remove_dir_all fails on the directories the tests create, so MSRV only
+    # builds.
+    - run: cargo build --locked --target wasm32-wasip1
+      if: matrix.build == 'MSRV'
+    - run: cargo test --locked --target wasm32-wasip1
+      if: matrix.build != 'MSRV'
+      env:
+        CARGO_TARGET_WASM32_WASIP1_RUNNER: wasmtime run --dir /tmp
+
   rustfmt:
     name: Rustfmt
     runs-on: ubuntu-latest
@@ -94,6 +137,7 @@ jobs:
     if: always()
     needs:
       - test
+      - wasi
       - rustfmt
     runs-on: ubuntu-latest
     steps:

Cargo.toml

@@ -64,6 +64,11 @@ zlib-ng-compat = ["libgit2-sys/zlib-ng-compat"]
 [workspace]
 members = ["systest", "git2-curl"]
 
+# DO NOT MERGE: wasm32-wasip1 needs rust-lang/libz-sys#273, which is not
+# released yet. Drop this once rust-lang/libz-sys#276 ships 1.1.30.
+[patch.crates-io]
+libz-sys = { git = "https://github.com/rust-lang/libz-sys", rev = "86e55f804bc117885acdde4b4d663d5e20bc293f" }
+
 [package.metadata.docs.rs]
 features = ["https", "ssh"]
 

README.md

@@ -62,6 +62,20 @@ If the `ssh` feature is enabled then this library depends
 on libssh2 which depends on OpenSSL. To get OpenSSL working follow the
 [`openssl` crate's instructions](https://github.com/sfackler/rust-openssl/blob/master/openssl/src/lib.rs#L31).
 
+## Building for WASI
+
+libgit2 can be built for `wasm32-wasip1`, without the `https` and `ssh`
+features since WASI preview 1 cannot open network connections. libgit2 is
+always built from source for this target, which needs a C compiler for WASI
+such as [wasi-sdk](https://github.com/WebAssembly/wasi-sdk); point the
+[`cc` crate](https://docs.rs/cc) at it:
+
+```sh
+$ export CC_wasm32_wasip1=/path/to/wasi-sdk/bin/clang
+$ export AR_wasm32_wasip1=/path/to/wasi-sdk/bin/llvm-ar
+$ cargo build --target wasm32-wasip1
+```
+
 # License
 
 This project is licensed under either of

libgit2-sys/build.rs

@@ -109,8 +109,18 @@ The build is now aborting. To disable, unset the variable or use `LIBGIT2_NO_VEN
         return;
     }
 
+    let target = env::var("TARGET").unwrap();
+    let wasi = env::var("CARGO_CFG_TARGET_OS").unwrap() == "wasi";
+    if wasi && target.ends_with("-threads") {
+        panic!("libgit2 is built without thread support on WASI, so the `{target}` target is not supported");
+    }
+    if wasi && (https || ssh) {
+        panic!("the `https` and `ssh` features are not supported on WASI, which cannot open network connections");
+    }
+
     // To use zlib-ng in zlib-compat mode, we have to build libgit2 ourselves.
-    let try_to_use_system_libgit2 = !vendored && !zlib_ng_compat;
+    // There is no system libgit2 to link against when targeting WASI.
+    let try_to_use_system_libgit2 = !vendored && !zlib_ng_compat && !wasi;
     if try_to_use_system_libgit2 && try_system_libgit2(unstable_sha256).is_ok() {
         // using system libgit2 has worked
         return;
@@ -124,7 +134,6 @@ The build is now aborting. To disable, unset the variable or use `LIBGIT2_NO_VEN
             .status();
     }
 
-    let target = env::var("TARGET").unwrap();
     let windows = target.contains("windows");
     let dst = PathBuf::from(env::var_os("OUT_DIR").unwrap());
     let include = dst.join("include");
@@ -153,7 +162,13 @@ The build is now aborting. To disable, unset the variable or use `LIBGIT2_NO_VEN
     // compiled apparently and have internal #define's to make sure they're
     // compiled correctly.
     add_c_files(&mut cfg, "libgit2/src/libgit2/transports");
-    add_c_files(&mut cfg, "libgit2/src/libgit2/streams");
+    if wasi {
+        // WASI preview 1 has no sockets or name resolution; `wasi/compat.c`
+        // provides a socket stream that always fails instead.
+        add_c_files_except(&mut cfg, "libgit2/src/libgit2/streams", &["socket.c"]);
+    } else {
+        add_c_files(&mut cfg, "libgit2/src/libgit2/streams");
+    }
 
     // Always use bundled HTTP parser (llhttp) for now
     cfg.include("libgit2/deps/llhttp");
@@ -204,6 +219,27 @@ The build is now aborting. To disable, unset the variable or use `LIBGIT2_NO_VEN
         if target.contains("gnu") {
             cfg.define("__USE_MINGW_ANSI_STDIO", "1");
         }
+    } else if wasi {
+        // Of the unix sources only realpath.c applies: WASI has no processes,
+        // and no mmap (NO_MMAP makes libgit2 read files into memory, see
+        // util/posix.c).
+        cfg.file("libgit2/src/util/unix/realpath.c");
+        cfg.define("NO_MMAP", None);
+        // Declarations and stubs for the POSIX bits wasi-libc lacks
+        // (user/process identity, pwd.h, sockets).
+        cfg.include("wasi/include");
+        cfg.flag("-include").flag("wasi/compat.h");
+        cfg.file("wasi/compat.c");
+        // Declare getpid without wasi-libc's deprecation warning. Rust's
+        // bundled wasi-libc has no libwasi-emulated-getpid, so wasi/compat.c
+        // defines it.
+        cfg.define("_WASI_EMULATED_GETPID", None);
+        // On wasm32 size_t is unsigned long while SIZE_MAX == UINT_MAX, so
+        // libgit2's integer.h passes size_t * to __builtin_uadd_overflow. Same
+        // width, so downgrade clang's error until libgit2 uses the type-generic
+        // builtin: https://github.com/libgit2/libgit2/pull/7406
+        cfg.flag("-Wno-incompatible-pointer-types");
+        cfg.flag("-fvisibility=hidden");
     } else {
         add_c_files(&mut cfg, "libgit2/src/util/unix");
         cfg.flag("-fvisibility=hidden");
@@ -217,7 +253,10 @@ The build is now aborting. To disable, unset the variable or use `LIBGIT2_NO_VEN
 
     features.push_str("#ifndef INCLUDE_features_h\n");
     features.push_str("#define INCLUDE_features_h\n");
-    features.push_str("#define GIT_THREADS 1\n");
+    // WASI is built without threads; the -threads targets are rejected above.
+    if !wasi {
+        features.push_str("#define GIT_THREADS 1\n");
+    }
     features.push_str("#define GIT_TRACE 1\n");
     features.push_str("#define GIT_HTTPPARSER_BUILTIN 1\n");
 
@@ -227,6 +266,11 @@ The build is now aborting. To disable, unset the variable or use `LIBGIT2_NO_VEN
 
     if windows {
         features.push_str("#define GIT_IO_WSAPOLL 1\n");
+    } else if wasi {
+        // posix.c requires a poll backend even though only socket.c polls.
+        features.push_str("#define GIT_IO_POLL 1\n");
+        // wasi-libc has getentropy (WASI random_get) and no /dev/urandom.
+        features.push_str("#define GIT_RAND_GETENTROPY 1\n");
     } else {
         // Should we fallback to `select` as more systems have that?
         features.push_str("#define GIT_IO_POLL 1\n");
@@ -327,6 +371,7 @@ The build is now aborting. To disable, unset the variable or use `LIBGIT2_NO_VEN
     println!("cargo:rerun-if-changed=libgit2/include");
     println!("cargo:rerun-if-changed=libgit2/src");
     println!("cargo:rerun-if-changed=libgit2/deps");
+    println!("cargo:rerun-if-changed=wasi");
 }
 
 fn cp_r(from: impl AsRef<Path>, to: impl AsRef<Path>) {
@@ -345,6 +390,10 @@ fn cp_r(from: impl AsRef<Path>, to: impl AsRef<Path>) {
 }
 
 fn add_c_files(build: &mut cc::Build, path: impl AsRef<Path>) {
+    add_c_files_except(build, path, &[]);
+}
+
+fn add_c_files_except(build: &mut cc::Build, path: impl AsRef<Path>, skip: &[&str]) {
     let path = path.as_ref();
     if !path.exists() {
         panic!("Path {} does not exist", path.display());
@@ -358,7 +407,9 @@ fn add_c_files(build: &mut cc::Build, path: impl AsRef<Path>) {
         let path = e.path();
         if e.file_type().unwrap().is_dir() {
             // skip dirs for now
-        } else if path.extension().and_then(|s| s.to_str()) == Some("c") {
+        } else if path.extension().and_then(|s| s.to_str()) == Some("c")
+            && !skip.iter().any(|&s| e.file_name() == s)
+        {
             build.file(&path);
         }
     }

libgit2-sys/wasi/compat.c

@@ -0,0 +1,64 @@
+/*
+ * Definitions for symbols libgit2 references that have no WASI preview 1
+ * equivalent. There is a single user (uid 0) and a single process, and no
+ * sockets.
+ */
+#include <sys/types.h>
+
+#include "git2_util.h"
+#include "streams/socket.h"
+
+uid_t getuid(void)
+{
+	return 0;
+}
+
+uid_t geteuid(void)
+{
+	return 0;
+}
+
+gid_t getgid(void)
+{
+	return 0;
+}
+
+pid_t getpid(void)
+{
+	return 1;
+}
+
+pid_t getppid(void)
+{
+	return 0;
+}
+
+pid_t getpgid(pid_t pid)
+{
+	GIT_UNUSED(pid);
+	return 1;
+}
+
+pid_t getsid(pid_t pid)
+{
+	GIT_UNUSED(pid);
+	return 1;
+}
+
+/* Replaces streams/socket.c: settable through git_libgit2_opts, unused. */
+int git_socket_stream__connect_timeout = 0;
+int git_socket_stream__timeout = 0;
+
+int git_socket_stream_global_init(void)
+{
+	return 0;
+}
+
+int git_socket_stream_new(git_stream **out, const char *host, const char *port)
+{
+	GIT_UNUSED(host);
+	GIT_UNUSED(port);
+	*out = NULL;
+	git_error_set(GIT_ERROR_NET, "sockets are not supported on WASI");
+	return -1;
+}

libgit2-sys/wasi/compat.h

@@ -0,0 +1,19 @@
+/*
+ * Force-included into every libgit2 translation unit when targeting WASI.
+ * wasi-libc omits these POSIX functions (libgit2 calls them to seed its PRNG,
+ * check file ownership and find the home directory); compat.c defines them.
+ * getpid is declared by wasi-libc itself under _WASI_EMULATED_GETPID.
+ */
+#ifndef LIBGIT2_SYS_WASI_COMPAT_H
+#define LIBGIT2_SYS_WASI_COMPAT_H
+
+#include <sys/types.h>
+
+uid_t getuid(void);
+uid_t geteuid(void);
+gid_t getgid(void);
+pid_t getppid(void);
+pid_t getpgid(pid_t pid);
+pid_t getsid(pid_t pid);
+
+#endif

libgit2-sys/wasi/include/pwd.h

@@ -0,0 +1,31 @@
+/*
+ * Minimal pwd.h for WASI, whose libc has none. libgit2 includes it from
+ * sysdir.c and only calls getpwuid_r when running setuid or in a macOS
+ * sandbox, neither of which happens with compat.c's identity stubs, so this
+ * just needs to compile; it reports no entry if ever reached.
+ */
+#ifndef LIBGIT2_SYS_WASI_PWD_H
+#define LIBGIT2_SYS_WASI_PWD_H
+
+#include <stddef.h>
+#include <sys/types.h>
+
+struct passwd {
+	char *pw_name;
+	char *pw_passwd;
+	uid_t pw_uid;
+	gid_t pw_gid;
+	char *pw_gecos;
+	char *pw_dir;
+	char *pw_shell;
+};
+
+static inline int getpwuid_r(uid_t uid, struct passwd *pwd, char *buf,
+	size_t buflen, struct passwd **result)
+{
+	(void)uid; (void)pwd; (void)buf; (void)buflen;
+	*result = NULL;
+	return 0;
+}
+
+#endif

src/blame.rs

@@ -475,7 +475,7 @@ mod tests {
     #[test]
     fn buffer_signatures() {
         // Regression tests for #1253
-        let td = tempfile::TempDir::new().unwrap();
+        let td = crate::test::tempdir().unwrap();
         let path = td.path();
 
         let repo = crate::Repository::init(path).unwrap();
@@ -560,7 +560,7 @@ mod tests {
     #[test]
     fn buffer_empty() {
         // Regression tests for #1288
-        let td = tempfile::TempDir::new().unwrap();
+        let td = crate::test::tempdir().unwrap();
         let path = td.path();
 
         let repo = crate::Repository::init(path).unwrap();

src/blob.rs

@@ -156,11 +156,10 @@ mod tests {
     use std::fs::File;
     use std::io::prelude::*;
     use std::path::Path;
-    use tempfile::TempDir;
 
     #[test]
     fn buffer() {
-        let td = TempDir::new().unwrap();
+        let td = crate::test::tempdir().unwrap();
         let repo = Repository::init(td.path()).unwrap();
         let id = repo.blob(&[5, 4, 6]).unwrap();
         let blob = repo.find_blob(id).unwrap();
@@ -180,7 +179,7 @@ mod tests {
 
     #[test]
     fn path() {
-        let td = TempDir::new().unwrap();
+        let td = crate::test::tempdir().unwrap();
         let path = td.path().join("foo");
         File::create(&path).unwrap().write_all(&[7, 8, 9]).unwrap();
         let repo = Repository::init(td.path()).unwrap();
@@ -192,7 +191,7 @@ mod tests {
 
     #[test]
     fn stream() {
-        let td = TempDir::new().unwrap();
+        let td = crate::test::tempdir().unwrap();
         let repo = Repository::init(td.path()).unwrap();
         let mut ws = repo.blob_writer(Some(Path::new("foo"))).unwrap();
         let wl = ws.write(&[10, 11, 12]).unwrap();