From c1bd40155693575e33feec683954bf546f31f843 Mon Sep 17 00:00:00 2001 From: Zed Spencer-Milnes Date: Tue, 29 Sep 2026 18:19:26 +0100 Subject: [PATCH] Build and select glibc and musl natives in a concurrent platform matrix Add explicit Linux libc classifiers and detect the libc used by the running JVM. Build all eight desktop targets concurrently on matching native runners, using containers only for Alpine, and verify packaged native loading on each target. Assisted-by: OpenAI Codex (gpt-6-astra high) --- .github/workflows/native-builds.yml | 204 ++++++++++++++++++ .github/workflows/publish-opencollab.yml | 113 +--------- README.md | 32 ++- build.gradle.kts | 135 +++++++++--- ...ibdatachannel.convention.common.gradle.kts | 5 +- jni/build-alpine.sh | 7 + jni/build-linux.sh | 18 ++ jni/build-windows.sh | 34 +++ native-test/NativeLibrarySmoke.java | 34 +++ native-test/README.md | 2 +- scripts/package-development.sh | 13 +- .../tel/schich/libdatachannel/LinuxLibc.java | 57 +++++ .../tel/schich/libdatachannel/Platform.java | 47 ++-- .../schich/libdatachannel/LinuxLibcTest.java | 50 +++++ 14 files changed, 595 insertions(+), 156 deletions(-) create mode 100644 .github/workflows/native-builds.yml create mode 100644 jni/build-alpine.sh create mode 100644 jni/build-linux.sh create mode 100644 jni/build-windows.sh create mode 100644 native-test/NativeLibrarySmoke.java create mode 100644 src/main/java/tel/schich/libdatachannel/LinuxLibc.java create mode 100644 src/test/java/tel/schich/libdatachannel/LinuxLibcTest.java diff --git a/.github/workflows/native-builds.yml b/.github/workflows/native-builds.yml new file mode 100644 index 0000000..ea855ae --- /dev/null +++ b/.github/workflows/native-builds.yml @@ -0,0 +1,204 @@ +name: Native build matrix + +on: + pull_request: + branches: [ main ] + workflow_call: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: native-builds-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + native: + name: ${{ matrix.classifier }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - classifier: linux-glibc-x86_64 + family: linux + runner: ubuntu-22.04 + task: compileNativeForLinuxGlibcX8664 + directory: linux-glibc-x86_64 + library: libdatachannel-java.so + platform: amd64 + libc: glibc + - classifier: linux-glibc-aarch64 + family: linux + runner: ubuntu-22.04-arm + task: compileNativeForLinuxGlibcAarch64 + directory: linux-glibc-aarch64 + library: libdatachannel-java.so + platform: arm64 + libc: glibc + - classifier: linux-musl-x86_64 + family: linux + runner: ubuntu-latest + task: compileNativeForLinuxMuslX8664 + directory: linux-musl-x86_64 + library: libdatachannel-java.so + platform: amd64 + libc: musl + runtime: alpine:3.22 + - classifier: linux-musl-aarch64 + family: linux + runner: ubuntu-24.04-arm + task: compileNativeForLinuxMuslAarch64 + directory: linux-musl-aarch64 + library: libdatachannel-java.so + platform: arm64 + libc: musl + runtime: alpine:3.22 + - classifier: windows-x86_64 + family: windows + runner: windows-2022 + msystem: MINGW64 + package-prefix: mingw-w64-x86_64 + compiler: gcc + task: compileNativeForWindowsX8664 + directory: windows-x86_64 + library: libdatachannel-java.dll + - classifier: windows-aarch64 + family: windows + runner: windows-11-arm + msystem: CLANGARM64 + package-prefix: mingw-w64-clang-aarch64 + compiler: clang + task: compileNativeForWindowsAarch64 + directory: windows-aarch64 + library: libdatachannel-java.dll + - classifier: macos-arm64 + family: macos + runner: macos-26 + task: compileNativeForHost + directory: host + library: libdatachannel-java.dylib + - classifier: macos-x86_64 + family: macos + runner: macos-15-intel + task: compileNativeForHost + directory: host + library: libdatachannel-java.dylib + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: recursive + - uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: ${{ matrix.classifier == 'windows-aarch64' && 'microsoft' || 'temurin' }} + - uses: gradle/actions/setup-gradle@v5 + - name: Install glibc build dependencies + if: matrix.libc == 'glibc' + run: | + sudo apt-get update + sudo apt-get install -y gcc-12 g++-12 libssl-dev + echo "CC=gcc-12" >> "$GITHUB_ENV" + echo "CXX=g++-12" >> "$GITHUB_ENV" + - uses: msys2/setup-msys2@v2 + if: matrix.family == 'windows' + with: + msystem: ${{ matrix.msystem }} + path-type: inherit + update: true + install: >- + ${{ matrix.package-prefix }}-${{ matrix.compiler }} + ${{ matrix.package-prefix }}-cmake + ${{ matrix.package-prefix }}-ninja + ${{ matrix.package-prefix }}-openssl + ${{ matrix.package-prefix }}-winpthreads + - name: Install macOS build dependencies + if: matrix.family == 'macos' + env: + HOMEBREW_NO_AUTO_UPDATE: '1' + HOMEBREW_NO_INSTALL_CLEANUP: '1' + run: | + brew uninstall --ignore-dependencies cmake || true + brew install cmake + pipx install conan || brew install conan + cmake --version + conan --version + - name: Build release native + if: matrix.family != 'windows' + env: + CLASSIFIER: ${{ matrix.classifier }} + TASK: ${{ matrix.task }} + run: | + ./gradlew "$TASK" --max-workers=2 --no-configuration-cache \ + -Plibdatachannel.targets="$CLASSIFIER" \ + -Plibdatachannel.build-release-binaries=true + - name: Build Windows native + if: matrix.family == 'windows' + shell: msys2 {0} + env: + CLASSIFIER: ${{ matrix.classifier }} + TASK: ${{ matrix.task }} + run: | + export JAVA_HOME="$(cygpath -u "$JAVA_HOME")" + export MSYS2_BASH="$(cygpath -w /usr/bin/bash)" + ./gradlew "$TASK" --max-workers=2 --no-configuration-cache \ + -Plibdatachannel.targets="$CLASSIFIER" \ + -Plibdatachannel.build-release-binaries=true + - name: Package bundle for runtime checks + shell: bash + env: + CLASSIFIER: ${{ matrix.classifier }} + NATIVE: build/dockcross/${{ matrix.directory }}/native/${{ matrix.library }} + run: | + if command -v cygpath >/dev/null; then + export JAVA_HOME="$(cygpath -u "$JAVA_HOME")" + fi + ./gradlew prepareNativeSmoke --max-workers=2 --no-configuration-cache \ + -Plibdatachannel.targets="$CLASSIFIER" \ + "-Plibdatachannel.$CLASSIFIER.prebuilt-path=$NATIVE" + - name: Load Windows bundle without toolchain DLLs on PATH + if: matrix.family == 'windows' + shell: pwsh + env: + CLASSIFIER: ${{ matrix.classifier }} + run: | + $env:PATH = "$env:JAVA_HOME\bin;$env:SystemRoot\System32;$env:SystemRoot" + foreach ($mode in @("lazy", "eager")) { + & "$env:JAVA_HOME\bin\java.exe" -Xcheck:jni -cp "build/native-smoke/*;build/native-smoke/classes" ` + tel.schich.libdatachannel.NativeLibrarySmoke $env:CLASSIFIER $mode + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + } + - name: Load bundle in the host JVM + if: matrix.family == 'macos' || matrix.libc == 'glibc' + env: + CLASSIFIER: ${{ matrix.classifier }} + run: | + for mode in lazy eager; do + java -Xcheck:jni -cp "build/native-smoke/*:build/native-smoke/classes" \ + tel.schich.libdatachannel.NativeLibrarySmoke "$CLASSIFIER" "$mode" + done + - name: Load musl bundle in an Alpine JVM + if: matrix.libc == 'musl' + env: + CLASSIFIER: ${{ matrix.classifier }} + PLATFORM: linux/${{ matrix.platform }} + RUNTIME: ${{ matrix.runtime }} + run: | + docker run --rm --platform "$PLATFORM" -v "$PWD:/work:ro" -w /work \ + -e CLASSIFIER "$RUNTIME" sh -ec ' + apk add --no-cache openjdk17-jre-headless libstdc++ libssl3 + for mode in lazy eager; do + java -Xcheck:jni -cp "build/native-smoke/*:build/native-smoke/classes" \ + tel.schich.libdatachannel.NativeLibrarySmoke "$CLASSIFIER" "$mode" + done + ' + - uses: actions/upload-artifact@v6 + with: + name: libdatachannel-${{ matrix.classifier }} + path: build/dockcross/${{ matrix.directory }}/native/${{ matrix.library }} + retention-days: 30 + if-no-files-found: error diff --git a/.github/workflows/publish-opencollab.yml b/.github/workflows/publish-opencollab.yml index aff9d2f..49d53db 100644 --- a/.github/workflows/publish-opencollab.yml +++ b/.github/workflows/publish-opencollab.yml @@ -13,113 +13,16 @@ permissions: contents: read env: - # linux and windows are cross compiled on the publishing job, macOS is built on its own runners - TARGETS: linux-x86_64,linux-aarch64,windows-x86_64,windows-aarch64,macos-arm64,macos-x86_64 + # Linux builds and JVM smoke tests cover both libc implementations. + TARGETS: linux-glibc-x86_64,linux-glibc-aarch64,linux-musl-x86_64,linux-musl-aarch64,windows-x86_64,windows-aarch64,macos-arm64,macos-x86_64 jobs: - build-macos: - name: Build macOS Native Libraries - strategy: - matrix: - include: - - os: macos-26 # Apple Silicon runner - classifier: macos-arm64 - - os: macos-15-intel # Intel runner - classifier: macos-x86_64 - runs-on: ${{ matrix.os }} - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - fetch-tags: 'true' - fetch-depth: '0' - submodules: recursive - - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - java-version: '17' - distribution: 'temurin' - cache: gradle - - - name: Install dependencies - env: - HOMEBREW_NO_AUTO_UPDATE: '1' - HOMEBREW_NO_INSTALL_CLEANUP: '1' - run: | - brew uninstall --ignore-dependencies cmake || true - brew install cmake - pipx install conan || brew install conan - cmake --version - conan --version - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v5 - - - name: Build native library for host - run: ./gradlew compileNativeForHost -Plibdatachannel.build-release-binaries=true - - - name: Upload artifacts - uses: actions/upload-artifact@v6 - with: - name: libdatachannel-${{ matrix.classifier }} - path: | - build/dockcross/host/native/libdatachannel-java.dylib - retention-days: 30 - if-no-files-found: error - - build-cross: - name: Cross Compile Native Libraries - strategy: - matrix: - include: - - classifier: linux-x86_64 - task: compileNativeForLinuxX8664 - library: libdatachannel-java.so - - classifier: linux-aarch64 - task: compileNativeForLinuxAarch64 - library: libdatachannel-java.so - - classifier: windows-x86_64 - task: compileNativeForWindowsX8664 - library: libdatachannel-java.dll - - classifier: windows-aarch64 - task: compileNativeForWindowsAarch64 - library: libdatachannel-java.dll - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - fetch-tags: 'true' - fetch-depth: '0' - submodules: recursive - - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - java-version: '17' - distribution: 'temurin' - cache: gradle - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v5 - - - name: Build native library - run: | - ./gradlew ${{ matrix.task }} \ - -Plibdatachannel.build-release-binaries=true \ - -Plibdatachannel.targets=${{ matrix.classifier }} - - - name: Upload artifacts - uses: actions/upload-artifact@v6 - with: - name: libdatachannel-${{ matrix.classifier }} - path: build/dockcross/${{ matrix.classifier }}/native/${{ matrix.library }} - retention-days: 30 - if-no-files-found: error + build-natives: + uses: ./.github/workflows/native-builds.yml publish: name: Publish to OpenCollab - needs: [ build-macos, build-cross ] + needs: [ build-natives ] runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -148,8 +51,10 @@ jobs: run: | ./gradlew --no-configuration-cache openCollabDeploy \ -Plibdatachannel.targets="$TARGETS" \ - -Plibdatachannel.linux-x86_64.prebuilt-path=artifacts/libdatachannel-linux-x86_64/libdatachannel-java.so \ - -Plibdatachannel.linux-aarch64.prebuilt-path=artifacts/libdatachannel-linux-aarch64/libdatachannel-java.so \ + -Plibdatachannel.linux-glibc-x86_64.prebuilt-path=artifacts/libdatachannel-linux-glibc-x86_64/libdatachannel-java.so \ + -Plibdatachannel.linux-glibc-aarch64.prebuilt-path=artifacts/libdatachannel-linux-glibc-aarch64/libdatachannel-java.so \ + -Plibdatachannel.linux-musl-x86_64.prebuilt-path=artifacts/libdatachannel-linux-musl-x86_64/libdatachannel-java.so \ + -Plibdatachannel.linux-musl-aarch64.prebuilt-path=artifacts/libdatachannel-linux-musl-aarch64/libdatachannel-java.so \ -Plibdatachannel.windows-x86_64.prebuilt-path=artifacts/libdatachannel-windows-x86_64/libdatachannel-java.dll \ -Plibdatachannel.windows-aarch64.prebuilt-path=artifacts/libdatachannel-windows-aarch64/libdatachannel-java.dll \ -Plibdatachannel.macos-arm64.prebuilt-path=artifacts/libdatachannel-macos-arm64/libdatachannel-java.dylib \ diff --git a/README.md b/README.md index 44d5f43..42cfc72 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,37 @@ implementation("dev.opencollab:libdatachannel-java:0.24.1.1") Additionally, pull the architecture-specific native components using their the architecture-specific classifier. Alternatively, use the `libdatachannel-java-arch-detect` module, which includes common architectures and has -code to detect which one to apply. +code to detect which one to apply, including the libc loaded by a Linux JVM. + +Linux classifiers are `linux-glibc-x86_64`, `linux-glibc-aarch64`, +`linux-musl-x86_64`, and `linux-musl-aarch64`. Alpine uses the musl variants. +These replace the ambiguous `linux-x86_64` and `linux-aarch64` classifiers; +update explicit classifier dependencies when upgrading. Other platforms keep +`windows-x86_64`, `windows-aarch64`, `macos-x86_64`, and `macos-arm64`. + +Linux requires OpenSSL 3 and the target distribution's C++ runtime (on Alpine, +`apk add libssl3 libstdc++`). Installing `gcompat` is unnecessary. The bundle +selects libc from `/proc/self/maps`; if procfs is unavailable, explicitly set +`-Dlibdatachannel.libc=glibc` or `-Dlibdatachannel.libc=musl` to match the JVM. + +Build one Linux native with, for example: + +```sh +./gradlew compileNativeForLinuxMuslAarch64 \ + -Plibdatachannel.targets=linux-musl-aarch64 \ + -Plibdatachannel.build-release-binaries=true +``` + +The CI matrix builds all eight Linux, Windows, and macOS targets concurrently +on matching x86_64 or ARM64 runners. Only musl uses a container (Alpine 3.22); +glibc builds use GCC 12 and OpenSSL from Ubuntu 22.04, Windows builds use native +MSYS2 MINGW64/CLANGARM64 with static OpenSSL, and macOS builds use the host +toolchain. Install the corresponding build dependencies for local builds. + +Every job checks lazy and eager loading of the packaged bundle in a matching +JVM. Windows checks exclude toolchain DLLs from PATH. Publishing waits for every +matrix job to pass. Docker is required only for the musl targets; local builds +of those targets need a matching host architecture or Docker/QEMU emulation. ### Offerer example diff --git a/build.gradle.kts b/build.gradle.kts index 463a457..5ea5039 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -183,7 +183,7 @@ fun DockcrossRunTask.baseConfigure(outputTo: Directory, target: BuildTarget) { configureSshRemoteBuild(target) } -fun Jar.baseConfigure(compileTask: TaskProvider, buildOutputDir: Directory) { +fun Jar.baseConfigure(compileTask: TaskProvider, buildOutputDir: Directory) { group = nativeGroup dependsOn(compileTask) @@ -212,6 +212,7 @@ data class BuildTarget( val image: String?, val family: String, val classifier: String, + val nativeLinuxContainer: Boolean = false, val env: Map = emptyMap(), val args: List = emptyList(), val outputTo: NamedDomainObjectProvider = archDetectConfiguration, @@ -234,9 +235,9 @@ fun macosTarget(classifier: String, arch: String) = BuildTarget( val allTargets = listOf( BuildTarget( - image = "linux-x64", + image = null, family = "linux", - classifier = "${Constants.LINUX_CLASSIFIER_PREFIX}x86_64", + classifier = "${Constants.LINUX_CLASSIFIER_PREFIX}glibc-x86_64", ), // BuildTarget( // image = "linux-x86", @@ -244,9 +245,21 @@ val allTargets = listOf( // classifier = "${Constants.LINUX_CLASSIFIER_PREFIX}x86_32", // ), BuildTarget( - image = "linux-arm64", + image = null, family = "linux", - classifier = "${Constants.LINUX_CLASSIFIER_PREFIX}aarch64", + classifier = "${Constants.LINUX_CLASSIFIER_PREFIX}glibc-aarch64", + ), + BuildTarget( + image = "alpine:3.22", + family = "linux", + classifier = "linux-musl-x86_64", + nativeLinuxContainer = true, + ), + BuildTarget( + image = "alpine:3.22", + family = "linux", + classifier = "linux-musl-aarch64", + nativeLinuxContainer = true, ), BuildTarget( image = "windows-static-x64", @@ -304,36 +317,90 @@ for (target in targets) { val packageTaskName = "packageNativeFor$taskSuffix" val packageNative = if (prebuiltPath == null) { - val compileNative = tasks.register("compileNativeFor$taskSuffix", DockcrossRunTask::class) { - baseConfigure(outputDir, target) - unsafeWritableMountSource = true - containerName = "dockcross-${project.name}-${target.classifier}" - } - - - if (ci) { - val previous = previousCompileNative - compileNative { - if (target.image == null) { - runner(NonContainerRunner) + val compileNative: TaskProvider + if (target.nativeLinuxContainer) { + compileNative = tasks.register("compileNativeFor$taskSuffix") { + group = nativeGroup + dependsOn(tasks.compileJava) + inputs.dir(jniPath) + inputs.property("version", project.version.toString()) + inputs.property("release", buildReleaseBinaries) + inputs.property("image", target.image!!) + outputs.file(outputDir.file("native/libdatachannel-java.so")) + val architecture = if (target.classifier.endsWith("aarch64")) { + "arm64" } else { - runner(DockerRunner()) - } - if (previous != null) { - mustRunAfter(previous) + "amd64" } + commandLine("docker", "run", "--rm", "--platform", "linux/$architecture", + "-v", "${project.projectDir}:/work:z", "-w", "/work", + "-e", "TARGET_CLASSIFIER=${target.classifier}", + "-e", "PROJECT_VERSION=${project.version}", + "-e", "PROJECT_BUILD_TYPE=${if (buildReleaseBinaries) "Release" else "Debug"}", + "-e", "JOBS=${project.gradle.startParameter.maxWorkerCount}", + target.image!!, "sh", "jni/build-alpine.sh") + } + } else if (target.classifier.startsWith("linux-glibc-")) { + compileNative = tasks.register("compileNativeFor$taskSuffix") { + group = nativeGroup + dependsOn(tasks.compileJava) + inputs.dir(jniPath) + inputs.property("version", project.version.toString()) + inputs.property("release", buildReleaseBinaries) + outputs.file(outputDir.file("native/libdatachannel-java.so")) + environment("TARGET_CLASSIFIER", target.classifier) + environment("PROJECT_VERSION", project.version.toString()) + environment("PROJECT_BUILD_TYPE", if (buildReleaseBinaries) "Release" else "Debug") + environment("JOBS", project.gradle.startParameter.maxWorkerCount.toString()) + commandLine("sh", "jni/build-linux.sh") + } + } else if (target.family == "windows" && System.getProperty("os.name").startsWith("Windows")) { + compileNative = tasks.register("compileNativeFor$taskSuffix") { + group = nativeGroup + dependsOn(tasks.compileJava) + inputs.dir(jniPath) + inputs.property("version", project.version.toString()) + inputs.property("release", buildReleaseBinaries) + outputs.file(outputDir.file("native/libdatachannel-java.dll")) + environment("TARGET_CLASSIFIER", target.classifier) + environment("PROJECT_VERSION", project.version.toString()) + environment("PROJECT_BUILD_TYPE", if (buildReleaseBinaries) "Release" else "Debug") + environment("JOBS", project.gradle.startParameter.maxWorkerCount.toString()) + commandLine(providers.environmentVariable("MSYS2_BASH").getOrElse("bash"), "jni/build-windows.sh") + } + } else { + val dockcrossCompile = tasks.register("compileNativeFor$taskSuffix", DockcrossRunTask::class) { + baseConfigure(outputDir, target) + unsafeWritableMountSource = true + containerName = "dockcross-${project.name}-${target.classifier}" + } + + + if (ci) { + val previous = previousCompileNative + dockcrossCompile { + if (target.image == null) { + runner(NonContainerRunner) + } else { + runner(DockerRunner()) + } + if (previous != null) { + mustRunAfter(previous) + } - if (target.image != null) { - val execOps = project.serviceOf() - doLast { - execOps.exec { - commandLine("docker", "image", "prune", "-af") + if (target.image != null) { + val execOps = project.serviceOf() + doLast { + execOps.exec { + commandLine("docker", "image", "prune", "-af") + } } } } - } - previousCompileNative = compileNative + previousCompileNative = dockcrossCompile + } + compileNative = dockcrossCompile } tasks.register(packageTaskName, Jar::class) { @@ -543,3 +610,15 @@ tasks.register("nativeStunMonitorProbe") { mainClass = "tel.schich.libdatachannel.NativeStunMonitorProbe" systemProperty("libdatachannel.native.datachannel-java.path", layout.buildDirectory.file("native-probe/libdatachannel-java.so").get().asFile.absolutePath) } + +// A portable classpath for running the shipped bundle inside target JVMs. +tasks.register("prepareNativeSmoke") { + dependsOn(tasks.named(probeSourceSet.classesTaskName)) + from(tasks.jar) + from(project(":libdatachannel-java-arch-detect").tasks.named("jar")) + from(configurations[probeSourceSet.runtimeClasspathConfigurationName]) + from(probeSourceSet.output) { + into("classes") + } + into(layout.buildDirectory.dir("native-smoke")) +} diff --git a/conventions/src/main/kotlin/tel.schich.libdatachannel.convention.common.gradle.kts b/conventions/src/main/kotlin/tel.schich.libdatachannel.convention.common.gradle.kts index 15150d7..d636ca6 100644 --- a/conventions/src/main/kotlin/tel.schich.libdatachannel.convention.common.gradle.kts +++ b/conventions/src/main/kotlin/tel.schich.libdatachannel.convention.common.gradle.kts @@ -20,10 +20,9 @@ tasks.test { } tasks.compileJava { + // Keep Java 11 bytecode/API compatibility while using the JDK 17 toolchain + // available natively on Windows ARM64 as well as the other build hosts. options.release = 11 - javaCompiler = javaToolchains.compilerFor { - languageVersion = JavaLanguageVersion.of(11) - } } tasks.withType().configureEach { diff --git a/jni/build-alpine.sh b/jni/build-alpine.sh new file mode 100644 index 0000000..d9605de --- /dev/null +++ b/jni/build-alpine.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu + +# Only musl needs a container: keep the native compiler and libc in Alpine. +apk add --no-cache build-base cmake openssl-dev linux-headers su-exec +build_owner=$(stat -c '%u:%g' jni/generated) +exec su-exec "$build_owner" sh jni/build-linux.sh diff --git a/jni/build-linux.sh b/jni/build-linux.sh new file mode 100644 index 0000000..be3681a --- /dev/null +++ b/jni/build-linux.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu + +architecture=$(uname -m) +case "$TARGET_CLASSIFIER" in + linux-glibc-"$architecture"|linux-musl-"$architecture") + ;; + *) + echo "Build $TARGET_CLASSIFIER on a matching Linux host (found $architecture)" >&2 + exit 1 + ;; +esac +cmake -S jni -B "build/dockcross/$TARGET_CLASSIFIER/native" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_BUILD_TYPE="$PROJECT_BUILD_TYPE" \ + -DPROJECT_VERSION="$PROJECT_VERSION" +cmake --build "build/dockcross/$TARGET_CLASSIFIER/native" \ + --target datachannel-java --parallel "$JOBS" diff --git a/jni/build-windows.sh b/jni/build-windows.sh new file mode 100644 index 0000000..6dafb65 --- /dev/null +++ b/jni/build-windows.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +case "$TARGET_CLASSIFIER:$MSYSTEM" in + windows-aarch64:CLANGARM64) + compiler=clang + cxx_compiler=clang++ + expected_target=aarch64-w64-windows-gnu + ;; + windows-x86_64:MINGW64) + compiler=gcc + cxx_compiler=g++ + expected_target=x86_64-w64-mingw32 + ;; + *) + echo "Use CLANGARM64 on Windows ARM64 or MINGW64 on Windows x86_64" >&2 + exit 1 + ;; +esac +compiler_target=$("$compiler" -dumpmachine) +if [ "$compiler_target" != "$expected_target" ]; then + echo "Expected $expected_target, found $compiler_target" >&2 + exit 1 +fi +cmake -S jni -B "build/dockcross/$TARGET_CLASSIFIER/native" -G Ninja \ + -DCMAKE_C_COMPILER="$compiler" \ + -DCMAKE_CXX_COMPILER="$cxx_compiler" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_BUILD_TYPE="$PROJECT_BUILD_TYPE" \ + -DPROJECT_VERSION="$PROJECT_VERSION" \ + -DOPENSSL_ROOT_DIR="$MINGW_PREFIX" \ + -DOPENSSL_USE_STATIC_LIBS=ON +cmake --build "build/dockcross/$TARGET_CLASSIFIER/native" \ + --target datachannel-java --parallel "$JOBS" diff --git a/native-test/NativeLibrarySmoke.java b/native-test/NativeLibrarySmoke.java new file mode 100644 index 0000000..bbfc333 --- /dev/null +++ b/native-test/NativeLibrarySmoke.java @@ -0,0 +1,34 @@ +package tel.schich.libdatachannel; + +import java.time.Duration; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.TimeUnit; + +/** Runs against packaged release natives in a JVM for the target platform. */ +public final class NativeLibrarySmoke { + public static void main(String[] args) throws Exception { + String classifier = Platform.detectArch(); + if (!classifier.equals(args[0])) { + throw new AssertionError("Expected " + args[0] + ", detected " + classifier); + } + if (args.length > 1 && args[1].equals("eager")) { + Class.forName("tel.schich.libdatachannel.LibDataChannelArchDetect") + .getMethod("initialize").invoke(null); + } + ArrayBlockingQueue descriptions = new ArrayBlockingQueue<>(1); + try (PeerConnection peer = PeerConnection.createPeer( + PeerConnectionConfiguration.DEFAULT.withDisableAutoNegotiation(true))) { + peer.onLocalDescription.register((connection, sdp, type) -> descriptions.offer(sdp)); + peer.createDataChannel("libc-smoke"); + peer.setLocalDescription("offer"); + String description = descriptions.poll(10, TimeUnit.SECONDS); + if (description == null || !description.contains("a=fingerprint:sha-256")) { + throw new AssertionError("Native offer/certificate callback did not complete"); + } + if (!peer.closeAndAwait(Duration.ofSeconds(10))) { + throw new AssertionError("Native peer cleanup did not complete"); + } + } + System.out.println("Native loading, certificate, callback and cleanup passed: " + classifier); + } +} diff --git a/native-test/README.md b/native-test/README.md index 30ecd82..951e483 100644 --- a/native-test/README.md +++ b/native-test/README.md @@ -114,7 +114,7 @@ use native-generated certificates. From a clean committed tree, `scripts/package-development.sh [maven-directory]` runs native and JVM regressions and writes an immutable local artifact under `io.github.teamziax:libdatachannel-java:.0-dev.`. -The classifier is `linux-x86_64`. `provenance.json` records all three source +The classifier identifies the host libc and architecture, for example `linux-glibc-x86_64`. `provenance.json` records all three source SHAs and artifact hashes. This binary targets the current host's system OpenSSL/ABI; it is not a portable dockcross release. Nothing is uploaded. Rebuild headers and JNI together after changing the pinned native version. diff --git a/scripts/package-development.sh b/scripts/package-development.sh index 9b42676..c9302fe 100755 --- a/scripts/package-development.sh +++ b/scripts/package-development.sh @@ -15,10 +15,17 @@ version="${native_version}.0-dev.${revision}" -Plibdatachannel.development-version="$version" \ -Plibdatachannel.test-native-path="$PWD/build/native-probe/libdatachannel-java.so" python3 - "$output" "$version" "$revision" <<'PY' -import hashlib, json, pathlib, subprocess, sys, zipfile +import hashlib, json, pathlib, platform, subprocess, sys, zipfile output, version, revision = pathlib.Path(sys.argv[1]), sys.argv[2], sys.argv[3] root = output / 'dev/opencollab/libdatachannel-java' / version root.mkdir(parents=True, exist_ok=True) +architecture = platform.machine() +if architecture not in {"x86_64", "aarch64"}: + raise SystemExit(f"Unsupported development architecture: {architecture}") +mappings = pathlib.Path('/proc/self/maps').read_text() +libc = 'musl' if 'ld-musl-' in mappings or 'libc.musl-' in mappings else 'glibc' +classifier = f'linux-{libc}-{architecture}' + def add(jar, path, name): info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) info.compress_type = zipfile.ZIP_DEFLATED @@ -29,7 +36,7 @@ with zipfile.ZipFile(root / f'libdatachannel-java-{version}.jar', 'w') as jar: if folder.exists(): for path in sorted(folder.rglob('*')): if path.is_file(): add(jar, path, path.relative_to(folder).as_posix()) -with zipfile.ZipFile(root / f'libdatachannel-java-{version}-linux-x86_64.jar', 'w') as jar: +with zipfile.ZipFile(root / f'libdatachannel-java-{version}-{classifier}.jar', 'w') as jar: add(jar, pathlib.Path('build/native-probe/libdatachannel-java.so'), 'native/libdatachannel-java.so') (root / f'libdatachannel-java-{version}.pom').write_text(f'''4.0.0 dev.opencollablibdatachannel-java{version} @@ -43,7 +50,7 @@ provenance = { 'bindingRevision': revision, 'libdatachannelRevision': head('jni/libdatachannel'), 'libjuiceRevision': head('jni/libdatachannel/deps/libjuice'), - 'platform': 'linux-x86_64', + 'platform': classifier, 'nativeBuild': 'system OpenSSL, Debug, current host ABI; not a portable release', 'checks': ['nativeTransportProbe', 'nativeCallbackCleanupProbe', 'nativeLoggingProbe', 'test'], 'sha256': {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(root.iterdir()) if path.is_file()}, diff --git a/src/main/java/tel/schich/libdatachannel/LinuxLibc.java b/src/main/java/tel/schich/libdatachannel/LinuxLibc.java new file mode 100644 index 0000000..c34714c --- /dev/null +++ b/src/main/java/tel/schich/libdatachannel/LinuxLibc.java @@ -0,0 +1,57 @@ +package tel.schich.libdatachannel; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +/** Detects the libc loaded by this JVM, even when both libcs are installed. */ +final class LinuxLibc { + static final String PROPERTY = "libdatachannel.libc"; + + private LinuxLibc() { + } + + static String detect() { + String override = System.getProperty(PROPERTY); + if (override != null) { + if (!override.equals("glibc") && !override.equals("musl")) { + throw new IllegalArgumentException(PROPERTY + " must be glibc or musl"); + } + return override; + } + try { + return fromMappings(Files.readAllLines(Path.of("/proc/self/maps"))); + } catch (IOException | SecurityException e) { + throw new IllegalStateException("Cannot identify the JVM's libc; set -D" + PROPERTY + + "=glibc or -D" + PROPERTY + "=musl", e); + } + } + + static String fromMappings(List mappings) { + boolean glibc = false; + boolean musl = false; + for (String mapping : mappings) { + int pathStart = mapping.indexOf('/'); + if (pathStart < 0) { + continue; + } + String path = mapping.substring(pathStart).replace(" (deleted)", ""); + String name = path.substring(path.lastIndexOf('/') + 1); + if (name.startsWith("ld-musl-") || name.startsWith("libc.musl-")) { + musl = true; + } else if (name.equals("libc.so.6") || name.matches("libc-[0-9]+\\.[0-9]+\\.so")) { + glibc = true; + } + } + // gcompat can map libc.so.6 into a musl JVM, so the musl loader takes precedence. + if (musl) { + return "musl"; + } + if (glibc) { + return "glibc"; + } + throw new IllegalStateException("Cannot identify the JVM's libc; set -D" + PROPERTY + + "=glibc or -D" + PROPERTY + "=musl"); + } +} diff --git a/src/main/java/tel/schich/libdatachannel/Platform.java b/src/main/java/tel/schich/libdatachannel/Platform.java index c4ea67c..21f179e 100644 --- a/src/main/java/tel/schich/libdatachannel/Platform.java +++ b/src/main/java/tel/schich/libdatachannel/Platform.java @@ -46,10 +46,10 @@ public static boolean isMacOS() { } public static OS getOS() { - if (isLinux()) { - return OS.LINUX; - } else if (isAndroid()) { + if (isAndroid()) { return OS.ANDROID; + } else if (isLinux()) { + return OS.LINUX; } else if (isMacOS()) { return OS.MACOS; } else if (isWindows()) { @@ -75,22 +75,36 @@ public static String classPathPropertyNameForLibrary(String name) { private static String archPrefixForOs() { switch (getOS()) { - case LINUX: - return "linux-"; - case WINDOWS: + case LINUX: { + return "linux-" + LinuxLibc.detect() + "-"; + } + case WINDOWS: { return "windows-"; - case ANDROID: + } + case ANDROID: { return "android-"; - case MACOS: + } + case MACOS: { return "macos-"; - default: + } + default: { return ""; + } } } private static String detectCpuArch() { String arch = System.getProperty("os.arch").toLowerCase(); - if (arch.contains("arm")) { + return detectCpuArch(arch, getOS()); + } + + static String detectCpuArch(String arch, OS os) { + if (arch.equals("aarch64") || arch.equals("arm64")) { + if (os == OS.MACOS) { + return "arm64"; + } + return "aarch64"; + } else if (arch.contains("arm")) { return "armv7"; } else if (arch.contains("86") || arch.contains("amd")) { if (arch.contains("64")) { @@ -102,11 +116,6 @@ private static String detectCpuArch() { return "riscv64"; } return "riscv32"; - } else if (arch.contains("aarch64") || arch.contains("arm64")) { - if (getOS() == OS.MACOS) { - return "arm64"; // macOS uses arm64 instead of aarch64 - } - return "aarch64"; } return arch; } @@ -133,7 +142,13 @@ private static void loadExplicitLibrary(String name, Class base) { return; } - final String sourceLibPath = classPathLocation(name, base, detectArch()); + String explicitClassPath = System.getProperty(classPathPropertyNameForLibrary(name)); + final String sourceLibPath; + if (explicitClassPath != null) { + sourceLibPath = explicitClassPath; + } else { + sourceLibPath = classPathLocation(name, base, detectArch()); + } LOGGER.trace("Loading native library {} from {}", name, sourceLibPath); try { final Path tempDirectory = Files.createTempDirectory(name + "-"); diff --git a/src/test/java/tel/schich/libdatachannel/LinuxLibcTest.java b/src/test/java/tel/schich/libdatachannel/LinuxLibcTest.java new file mode 100644 index 0000000..649e62b --- /dev/null +++ b/src/test/java/tel/schich/libdatachannel/LinuxLibcTest.java @@ -0,0 +1,50 @@ +package tel.schich.libdatachannel; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class LinuxLibcTest { + @AfterEach + void clearOverride() { + System.clearProperty(LinuxLibc.PROPERTY); + } + + @Test + void identifiesGlibcWithBothLibraryNamingSchemes() { + assertEquals("glibc", LinuxLibc.fromMappings(List.of("0000 r-xp /usr/lib64/libc.so.6"))); + assertEquals("glibc", LinuxLibc.fromMappings(List.of("0000 r-xp /lib/libc-2.31.so (deleted)"))); + } + + @Test + void identifiesMuslEvenWhenGcompatIsLoaded() { + assertEquals("musl", LinuxLibc.fromMappings(List.of( + "0000 r-xp /lib/libc.so.6", "0000 r-xp /lib/ld-musl-x86_64.so.1"))); + assertEquals("musl", LinuxLibc.fromMappings(List.of("0000 r-xp /lib/libc.musl-aarch64.so.1"))); + } + + @Test + void unknownMappingsRequireAnExplicitOverride() { + assertThrows(IllegalStateException.class, + () -> LinuxLibc.fromMappings(List.of("0000 r-xp /home/musl/libjvm.so"))); + System.setProperty(LinuxLibc.PROPERTY, "musl"); + assertEquals("musl", LinuxLibc.detect()); + System.setProperty(LinuxLibc.PROPERTY, "glibc"); + assertEquals("glibc", LinuxLibc.detect()); + System.setProperty(LinuxLibc.PROPERTY, "unknown"); + assertThrows(IllegalArgumentException.class, LinuxLibc::detect); + } + + @Test + void arm64AliasesDoNotSelectArmv7() { + assertEquals("aarch64", Platform.detectCpuArch("arm64", Platform.OS.LINUX)); + assertEquals("aarch64", Platform.detectCpuArch("aarch64", Platform.OS.LINUX)); + assertEquals("arm64", Platform.detectCpuArch("aarch64", Platform.OS.MACOS)); + assertEquals("armv7", Platform.detectCpuArch("armv7l", Platform.OS.LINUX)); + assertEquals("x86_64", Platform.detectCpuArch("amd64", Platform.OS.LINUX)); + } +}