diff --git a/.github/workflows/native-builds.yml b/.github/workflows/native-builds.yml new file mode 100644 index 0000000..ea855ae --- /dev/null +++ b/.github/workflows/native-builds.yml @@ -0,0 +1,204 @@ +name: Native build matrix + +on: + pull_request: + branches: [ main ] + workflow_call: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: native-builds-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + native: + name: ${{ matrix.classifier }} + runs-on: ${{ matrix.runner }} + strategy: + fail-fast: false + matrix: + include: + - classifier: linux-glibc-x86_64 + family: linux + runner: ubuntu-22.04 + task: compileNativeForLinuxGlibcX8664 + directory: linux-glibc-x86_64 + library: libdatachannel-java.so + platform: amd64 + libc: glibc + - classifier: linux-glibc-aarch64 + family: linux + runner: ubuntu-22.04-arm + task: compileNativeForLinuxGlibcAarch64 + directory: linux-glibc-aarch64 + library: libdatachannel-java.so + platform: arm64 + libc: glibc + - classifier: linux-musl-x86_64 + family: linux + runner: ubuntu-latest + task: compileNativeForLinuxMuslX8664 + directory: linux-musl-x86_64 + library: libdatachannel-java.so + platform: amd64 + libc: musl + runtime: alpine:3.22 + - classifier: linux-musl-aarch64 + family: linux + runner: ubuntu-24.04-arm + task: compileNativeForLinuxMuslAarch64 + directory: linux-musl-aarch64 + library: libdatachannel-java.so + platform: arm64 + libc: musl + runtime: alpine:3.22 + - classifier: windows-x86_64 + family: windows + runner: windows-2022 + msystem: MINGW64 + package-prefix: mingw-w64-x86_64 + compiler: gcc + task: compileNativeForWindowsX8664 + directory: windows-x86_64 + library: libdatachannel-java.dll + - classifier: windows-aarch64 + family: windows + runner: windows-11-arm + msystem: CLANGARM64 + package-prefix: mingw-w64-clang-aarch64 + compiler: clang + task: compileNativeForWindowsAarch64 + directory: windows-aarch64 + library: libdatachannel-java.dll + - classifier: macos-arm64 + family: macos + runner: macos-26 + task: compileNativeForHost + directory: host + library: libdatachannel-java.dylib + - classifier: macos-x86_64 + family: macos + runner: macos-15-intel + task: compileNativeForHost + directory: host + library: libdatachannel-java.dylib + steps: + - uses: actions/checkout@v6 + with: + fetch-depth: 0 + submodules: recursive + - uses: actions/setup-java@v5 + with: + java-version: '17' + distribution: ${{ matrix.classifier == 'windows-aarch64' && 'microsoft' || 'temurin' }} + - uses: gradle/actions/setup-gradle@v5 + - name: Install glibc build dependencies + if: matrix.libc == 'glibc' + run: | + sudo apt-get update + sudo apt-get install -y gcc-12 g++-12 libssl-dev + echo "CC=gcc-12" >> "$GITHUB_ENV" + echo "CXX=g++-12" >> "$GITHUB_ENV" + - uses: msys2/setup-msys2@v2 + if: matrix.family == 'windows' + with: + msystem: ${{ matrix.msystem }} + path-type: inherit + update: true + install: >- + ${{ matrix.package-prefix }}-${{ matrix.compiler }} + ${{ matrix.package-prefix }}-cmake + ${{ matrix.package-prefix }}-ninja + ${{ matrix.package-prefix }}-openssl + ${{ matrix.package-prefix }}-winpthreads + - name: Install macOS build dependencies + if: matrix.family == 'macos' + env: + HOMEBREW_NO_AUTO_UPDATE: '1' + HOMEBREW_NO_INSTALL_CLEANUP: '1' + run: | + brew uninstall --ignore-dependencies cmake || true + brew install cmake + pipx install conan || brew install conan + cmake --version + conan --version + - name: Build release native + if: matrix.family != 'windows' + env: + CLASSIFIER: ${{ matrix.classifier }} + TASK: ${{ matrix.task }} + run: | + ./gradlew "$TASK" --max-workers=2 --no-configuration-cache \ + -Plibdatachannel.targets="$CLASSIFIER" \ + -Plibdatachannel.build-release-binaries=true + - name: Build Windows native + if: matrix.family == 'windows' + shell: msys2 {0} + env: + CLASSIFIER: ${{ matrix.classifier }} + TASK: ${{ matrix.task }} + run: | + export JAVA_HOME="$(cygpath -u "$JAVA_HOME")" + export MSYS2_BASH="$(cygpath -w /usr/bin/bash)" + ./gradlew "$TASK" --max-workers=2 --no-configuration-cache \ + -Plibdatachannel.targets="$CLASSIFIER" \ + -Plibdatachannel.build-release-binaries=true + - name: Package bundle for runtime checks + shell: bash + env: + CLASSIFIER: ${{ matrix.classifier }} + NATIVE: build/dockcross/${{ matrix.directory }}/native/${{ matrix.library }} + run: | + if command -v cygpath >/dev/null; then + export JAVA_HOME="$(cygpath -u "$JAVA_HOME")" + fi + ./gradlew prepareNativeSmoke --max-workers=2 --no-configuration-cache \ + -Plibdatachannel.targets="$CLASSIFIER" \ + "-Plibdatachannel.$CLASSIFIER.prebuilt-path=$NATIVE" + - name: Load Windows bundle without toolchain DLLs on PATH + if: matrix.family == 'windows' + shell: pwsh + env: + CLASSIFIER: ${{ matrix.classifier }} + run: | + $env:PATH = "$env:JAVA_HOME\bin;$env:SystemRoot\System32;$env:SystemRoot" + foreach ($mode in @("lazy", "eager")) { + & "$env:JAVA_HOME\bin\java.exe" -Xcheck:jni -cp "build/native-smoke/*;build/native-smoke/classes" ` + tel.schich.libdatachannel.NativeLibrarySmoke $env:CLASSIFIER $mode + if ($LASTEXITCODE -ne 0) { + exit $LASTEXITCODE + } + } + - name: Load bundle in the host JVM + if: matrix.family == 'macos' || matrix.libc == 'glibc' + env: + CLASSIFIER: ${{ matrix.classifier }} + run: | + for mode in lazy eager; do + java -Xcheck:jni -cp "build/native-smoke/*:build/native-smoke/classes" \ + tel.schich.libdatachannel.NativeLibrarySmoke "$CLASSIFIER" "$mode" + done + - name: Load musl bundle in an Alpine JVM + if: matrix.libc == 'musl' + env: + CLASSIFIER: ${{ matrix.classifier }} + PLATFORM: linux/${{ matrix.platform }} + RUNTIME: ${{ matrix.runtime }} + run: | + docker run --rm --platform "$PLATFORM" -v "$PWD:/work:ro" -w /work \ + -e CLASSIFIER "$RUNTIME" sh -ec ' + apk add --no-cache openjdk17-jre-headless libstdc++ libssl3 + for mode in lazy eager; do + java -Xcheck:jni -cp "build/native-smoke/*:build/native-smoke/classes" \ + tel.schich.libdatachannel.NativeLibrarySmoke "$CLASSIFIER" "$mode" + done + ' + - uses: actions/upload-artifact@v6 + with: + name: libdatachannel-${{ matrix.classifier }} + path: build/dockcross/${{ matrix.directory }}/native/${{ matrix.library }} + retention-days: 30 + if-no-files-found: error diff --git a/.github/workflows/publish-opencollab.yml b/.github/workflows/publish-opencollab.yml index aff9d2f..49d53db 100644 --- a/.github/workflows/publish-opencollab.yml +++ b/.github/workflows/publish-opencollab.yml @@ -13,113 +13,16 @@ permissions: contents: read env: - # linux and windows are cross compiled on the publishing job, macOS is built on its own runners - TARGETS: linux-x86_64,linux-aarch64,windows-x86_64,windows-aarch64,macos-arm64,macos-x86_64 + # Linux builds and JVM smoke tests cover both libc implementations. + TARGETS: linux-glibc-x86_64,linux-glibc-aarch64,linux-musl-x86_64,linux-musl-aarch64,windows-x86_64,windows-aarch64,macos-arm64,macos-x86_64 jobs: - build-macos: - name: Build macOS Native Libraries - strategy: - matrix: - include: - - os: macos-26 # Apple Silicon runner - classifier: macos-arm64 - - os: macos-15-intel # Intel runner - classifier: macos-x86_64 - runs-on: ${{ matrix.os }} - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - fetch-tags: 'true' - fetch-depth: '0' - submodules: recursive - - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - java-version: '17' - distribution: 'temurin' - cache: gradle - - - name: Install dependencies - env: - HOMEBREW_NO_AUTO_UPDATE: '1' - HOMEBREW_NO_INSTALL_CLEANUP: '1' - run: | - brew uninstall --ignore-dependencies cmake || true - brew install cmake - pipx install conan || brew install conan - cmake --version - conan --version - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v5 - - - name: Build native library for host - run: ./gradlew compileNativeForHost -Plibdatachannel.build-release-binaries=true - - - name: Upload artifacts - uses: actions/upload-artifact@v6 - with: - name: libdatachannel-${{ matrix.classifier }} - path: | - build/dockcross/host/native/libdatachannel-java.dylib - retention-days: 30 - if-no-files-found: error - - build-cross: - name: Cross Compile Native Libraries - strategy: - matrix: - include: - - classifier: linux-x86_64 - task: compileNativeForLinuxX8664 - library: libdatachannel-java.so - - classifier: linux-aarch64 - task: compileNativeForLinuxAarch64 - library: libdatachannel-java.so - - classifier: windows-x86_64 - task: compileNativeForWindowsX8664 - library: libdatachannel-java.dll - - classifier: windows-aarch64 - task: compileNativeForWindowsAarch64 - library: libdatachannel-java.dll - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v6 - with: - fetch-tags: 'true' - fetch-depth: '0' - submodules: recursive - - - name: Set up JDK 17 - uses: actions/setup-java@v5 - with: - java-version: '17' - distribution: 'temurin' - cache: gradle - - - name: Setup Gradle - uses: gradle/actions/setup-gradle@v5 - - - name: Build native library - run: | - ./gradlew ${{ matrix.task }} \ - -Plibdatachannel.build-release-binaries=true \ - -Plibdatachannel.targets=${{ matrix.classifier }} - - - name: Upload artifacts - uses: actions/upload-artifact@v6 - with: - name: libdatachannel-${{ matrix.classifier }} - path: build/dockcross/${{ matrix.classifier }}/native/${{ matrix.library }} - retention-days: 30 - if-no-files-found: error + build-natives: + uses: ./.github/workflows/native-builds.yml publish: name: Publish to OpenCollab - needs: [ build-macos, build-cross ] + needs: [ build-natives ] runs-on: ubuntu-latest steps: - uses: actions/checkout@v6 @@ -148,8 +51,10 @@ jobs: run: | ./gradlew --no-configuration-cache openCollabDeploy \ -Plibdatachannel.targets="$TARGETS" \ - -Plibdatachannel.linux-x86_64.prebuilt-path=artifacts/libdatachannel-linux-x86_64/libdatachannel-java.so \ - -Plibdatachannel.linux-aarch64.prebuilt-path=artifacts/libdatachannel-linux-aarch64/libdatachannel-java.so \ + -Plibdatachannel.linux-glibc-x86_64.prebuilt-path=artifacts/libdatachannel-linux-glibc-x86_64/libdatachannel-java.so \ + -Plibdatachannel.linux-glibc-aarch64.prebuilt-path=artifacts/libdatachannel-linux-glibc-aarch64/libdatachannel-java.so \ + -Plibdatachannel.linux-musl-x86_64.prebuilt-path=artifacts/libdatachannel-linux-musl-x86_64/libdatachannel-java.so \ + -Plibdatachannel.linux-musl-aarch64.prebuilt-path=artifacts/libdatachannel-linux-musl-aarch64/libdatachannel-java.so \ -Plibdatachannel.windows-x86_64.prebuilt-path=artifacts/libdatachannel-windows-x86_64/libdatachannel-java.dll \ -Plibdatachannel.windows-aarch64.prebuilt-path=artifacts/libdatachannel-windows-aarch64/libdatachannel-java.dll \ -Plibdatachannel.macos-arm64.prebuilt-path=artifacts/libdatachannel-macos-arm64/libdatachannel-java.dylib \ diff --git a/README.md b/README.md index 44d5f43..42cfc72 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,37 @@ implementation("dev.opencollab:libdatachannel-java:0.24.1.1") Additionally, pull the architecture-specific native components using their the architecture-specific classifier. Alternatively, use the `libdatachannel-java-arch-detect` module, which includes common architectures and has -code to detect which one to apply. +code to detect which one to apply, including the libc loaded by a Linux JVM. + +Linux classifiers are `linux-glibc-x86_64`, `linux-glibc-aarch64`, +`linux-musl-x86_64`, and `linux-musl-aarch64`. Alpine uses the musl variants. +These replace the ambiguous `linux-x86_64` and `linux-aarch64` classifiers; +update explicit classifier dependencies when upgrading. Other platforms keep +`windows-x86_64`, `windows-aarch64`, `macos-x86_64`, and `macos-arm64`. + +Linux requires OpenSSL 3 and the target distribution's C++ runtime (on Alpine, +`apk add libssl3 libstdc++`). Installing `gcompat` is unnecessary. The bundle +selects libc from `/proc/self/maps`; if procfs is unavailable, explicitly set +`-Dlibdatachannel.libc=glibc` or `-Dlibdatachannel.libc=musl` to match the JVM. + +Build one Linux native with, for example: + +```sh +./gradlew compileNativeForLinuxMuslAarch64 \ + -Plibdatachannel.targets=linux-musl-aarch64 \ + -Plibdatachannel.build-release-binaries=true +``` + +The CI matrix builds all eight Linux, Windows, and macOS targets concurrently +on matching x86_64 or ARM64 runners. Only musl uses a container (Alpine 3.22); +glibc builds use GCC 12 and OpenSSL from Ubuntu 22.04, Windows builds use native +MSYS2 MINGW64/CLANGARM64 with static OpenSSL, and macOS builds use the host +toolchain. Install the corresponding build dependencies for local builds. + +Every job checks lazy and eager loading of the packaged bundle in a matching +JVM. Windows checks exclude toolchain DLLs from PATH. Publishing waits for every +matrix job to pass. Docker is required only for the musl targets; local builds +of those targets need a matching host architecture or Docker/QEMU emulation. ### Offerer example diff --git a/build.gradle.kts b/build.gradle.kts index 463a457..5ea5039 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -183,7 +183,7 @@ fun DockcrossRunTask.baseConfigure(outputTo: Directory, target: BuildTarget) { configureSshRemoteBuild(target) } -fun Jar.baseConfigure(compileTask: TaskProvider, buildOutputDir: Directory) { +fun Jar.baseConfigure(compileTask: TaskProvider, buildOutputDir: Directory) { group = nativeGroup dependsOn(compileTask) @@ -212,6 +212,7 @@ data class BuildTarget( val image: String?, val family: String, val classifier: String, + val nativeLinuxContainer: Boolean = false, val env: Map = emptyMap(), val args: List = emptyList(), val outputTo: NamedDomainObjectProvider = archDetectConfiguration, @@ -234,9 +235,9 @@ fun macosTarget(classifier: String, arch: String) = BuildTarget( val allTargets = listOf( BuildTarget( - image = "linux-x64", + image = null, family = "linux", - classifier = "${Constants.LINUX_CLASSIFIER_PREFIX}x86_64", + classifier = "${Constants.LINUX_CLASSIFIER_PREFIX}glibc-x86_64", ), // BuildTarget( // image = "linux-x86", @@ -244,9 +245,21 @@ val allTargets = listOf( // classifier = "${Constants.LINUX_CLASSIFIER_PREFIX}x86_32", // ), BuildTarget( - image = "linux-arm64", + image = null, family = "linux", - classifier = "${Constants.LINUX_CLASSIFIER_PREFIX}aarch64", + classifier = "${Constants.LINUX_CLASSIFIER_PREFIX}glibc-aarch64", + ), + BuildTarget( + image = "alpine:3.22", + family = "linux", + classifier = "linux-musl-x86_64", + nativeLinuxContainer = true, + ), + BuildTarget( + image = "alpine:3.22", + family = "linux", + classifier = "linux-musl-aarch64", + nativeLinuxContainer = true, ), BuildTarget( image = "windows-static-x64", @@ -304,36 +317,90 @@ for (target in targets) { val packageTaskName = "packageNativeFor$taskSuffix" val packageNative = if (prebuiltPath == null) { - val compileNative = tasks.register("compileNativeFor$taskSuffix", DockcrossRunTask::class) { - baseConfigure(outputDir, target) - unsafeWritableMountSource = true - containerName = "dockcross-${project.name}-${target.classifier}" - } - - - if (ci) { - val previous = previousCompileNative - compileNative { - if (target.image == null) { - runner(NonContainerRunner) + val compileNative: TaskProvider + if (target.nativeLinuxContainer) { + compileNative = tasks.register("compileNativeFor$taskSuffix") { + group = nativeGroup + dependsOn(tasks.compileJava) + inputs.dir(jniPath) + inputs.property("version", project.version.toString()) + inputs.property("release", buildReleaseBinaries) + inputs.property("image", target.image!!) + outputs.file(outputDir.file("native/libdatachannel-java.so")) + val architecture = if (target.classifier.endsWith("aarch64")) { + "arm64" } else { - runner(DockerRunner()) - } - if (previous != null) { - mustRunAfter(previous) + "amd64" } + commandLine("docker", "run", "--rm", "--platform", "linux/$architecture", + "-v", "${project.projectDir}:/work:z", "-w", "/work", + "-e", "TARGET_CLASSIFIER=${target.classifier}", + "-e", "PROJECT_VERSION=${project.version}", + "-e", "PROJECT_BUILD_TYPE=${if (buildReleaseBinaries) "Release" else "Debug"}", + "-e", "JOBS=${project.gradle.startParameter.maxWorkerCount}", + target.image!!, "sh", "jni/build-alpine.sh") + } + } else if (target.classifier.startsWith("linux-glibc-")) { + compileNative = tasks.register("compileNativeFor$taskSuffix") { + group = nativeGroup + dependsOn(tasks.compileJava) + inputs.dir(jniPath) + inputs.property("version", project.version.toString()) + inputs.property("release", buildReleaseBinaries) + outputs.file(outputDir.file("native/libdatachannel-java.so")) + environment("TARGET_CLASSIFIER", target.classifier) + environment("PROJECT_VERSION", project.version.toString()) + environment("PROJECT_BUILD_TYPE", if (buildReleaseBinaries) "Release" else "Debug") + environment("JOBS", project.gradle.startParameter.maxWorkerCount.toString()) + commandLine("sh", "jni/build-linux.sh") + } + } else if (target.family == "windows" && System.getProperty("os.name").startsWith("Windows")) { + compileNative = tasks.register("compileNativeFor$taskSuffix") { + group = nativeGroup + dependsOn(tasks.compileJava) + inputs.dir(jniPath) + inputs.property("version", project.version.toString()) + inputs.property("release", buildReleaseBinaries) + outputs.file(outputDir.file("native/libdatachannel-java.dll")) + environment("TARGET_CLASSIFIER", target.classifier) + environment("PROJECT_VERSION", project.version.toString()) + environment("PROJECT_BUILD_TYPE", if (buildReleaseBinaries) "Release" else "Debug") + environment("JOBS", project.gradle.startParameter.maxWorkerCount.toString()) + commandLine(providers.environmentVariable("MSYS2_BASH").getOrElse("bash"), "jni/build-windows.sh") + } + } else { + val dockcrossCompile = tasks.register("compileNativeFor$taskSuffix", DockcrossRunTask::class) { + baseConfigure(outputDir, target) + unsafeWritableMountSource = true + containerName = "dockcross-${project.name}-${target.classifier}" + } + + + if (ci) { + val previous = previousCompileNative + dockcrossCompile { + if (target.image == null) { + runner(NonContainerRunner) + } else { + runner(DockerRunner()) + } + if (previous != null) { + mustRunAfter(previous) + } - if (target.image != null) { - val execOps = project.serviceOf() - doLast { - execOps.exec { - commandLine("docker", "image", "prune", "-af") + if (target.image != null) { + val execOps = project.serviceOf() + doLast { + execOps.exec { + commandLine("docker", "image", "prune", "-af") + } } } } - } - previousCompileNative = compileNative + previousCompileNative = dockcrossCompile + } + compileNative = dockcrossCompile } tasks.register(packageTaskName, Jar::class) { @@ -543,3 +610,15 @@ tasks.register("nativeStunMonitorProbe") { mainClass = "tel.schich.libdatachannel.NativeStunMonitorProbe" systemProperty("libdatachannel.native.datachannel-java.path", layout.buildDirectory.file("native-probe/libdatachannel-java.so").get().asFile.absolutePath) } + +// A portable classpath for running the shipped bundle inside target JVMs. +tasks.register("prepareNativeSmoke") { + dependsOn(tasks.named(probeSourceSet.classesTaskName)) + from(tasks.jar) + from(project(":libdatachannel-java-arch-detect").tasks.named("jar")) + from(configurations[probeSourceSet.runtimeClasspathConfigurationName]) + from(probeSourceSet.output) { + into("classes") + } + into(layout.buildDirectory.dir("native-smoke")) +} diff --git a/conventions/src/main/kotlin/tel.schich.libdatachannel.convention.common.gradle.kts b/conventions/src/main/kotlin/tel.schich.libdatachannel.convention.common.gradle.kts index 15150d7..d636ca6 100644 --- a/conventions/src/main/kotlin/tel.schich.libdatachannel.convention.common.gradle.kts +++ b/conventions/src/main/kotlin/tel.schich.libdatachannel.convention.common.gradle.kts @@ -20,10 +20,9 @@ tasks.test { } tasks.compileJava { + // Keep Java 11 bytecode/API compatibility while using the JDK 17 toolchain + // available natively on Windows ARM64 as well as the other build hosts. options.release = 11 - javaCompiler = javaToolchains.compilerFor { - languageVersion = JavaLanguageVersion.of(11) - } } tasks.withType().configureEach { diff --git a/jni/build-alpine.sh b/jni/build-alpine.sh new file mode 100644 index 0000000..d9605de --- /dev/null +++ b/jni/build-alpine.sh @@ -0,0 +1,7 @@ +#!/bin/sh +set -eu + +# Only musl needs a container: keep the native compiler and libc in Alpine. +apk add --no-cache build-base cmake openssl-dev linux-headers su-exec +build_owner=$(stat -c '%u:%g' jni/generated) +exec su-exec "$build_owner" sh jni/build-linux.sh diff --git a/jni/build-linux.sh b/jni/build-linux.sh new file mode 100644 index 0000000..be3681a --- /dev/null +++ b/jni/build-linux.sh @@ -0,0 +1,18 @@ +#!/bin/sh +set -eu + +architecture=$(uname -m) +case "$TARGET_CLASSIFIER" in + linux-glibc-"$architecture"|linux-musl-"$architecture") + ;; + *) + echo "Build $TARGET_CLASSIFIER on a matching Linux host (found $architecture)" >&2 + exit 1 + ;; +esac +cmake -S jni -B "build/dockcross/$TARGET_CLASSIFIER/native" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_BUILD_TYPE="$PROJECT_BUILD_TYPE" \ + -DPROJECT_VERSION="$PROJECT_VERSION" +cmake --build "build/dockcross/$TARGET_CLASSIFIER/native" \ + --target datachannel-java --parallel "$JOBS" diff --git a/jni/build-windows.sh b/jni/build-windows.sh new file mode 100644 index 0000000..6dafb65 --- /dev/null +++ b/jni/build-windows.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +set -euo pipefail + +case "$TARGET_CLASSIFIER:$MSYSTEM" in + windows-aarch64:CLANGARM64) + compiler=clang + cxx_compiler=clang++ + expected_target=aarch64-w64-windows-gnu + ;; + windows-x86_64:MINGW64) + compiler=gcc + cxx_compiler=g++ + expected_target=x86_64-w64-mingw32 + ;; + *) + echo "Use CLANGARM64 on Windows ARM64 or MINGW64 on Windows x86_64" >&2 + exit 1 + ;; +esac +compiler_target=$("$compiler" -dumpmachine) +if [ "$compiler_target" != "$expected_target" ]; then + echo "Expected $expected_target, found $compiler_target" >&2 + exit 1 +fi +cmake -S jni -B "build/dockcross/$TARGET_CLASSIFIER/native" -G Ninja \ + -DCMAKE_C_COMPILER="$compiler" \ + -DCMAKE_CXX_COMPILER="$cxx_compiler" \ + -DCMAKE_POLICY_VERSION_MINIMUM=3.5 \ + -DCMAKE_BUILD_TYPE="$PROJECT_BUILD_TYPE" \ + -DPROJECT_VERSION="$PROJECT_VERSION" \ + -DOPENSSL_ROOT_DIR="$MINGW_PREFIX" \ + -DOPENSSL_USE_STATIC_LIBS=ON +cmake --build "build/dockcross/$TARGET_CLASSIFIER/native" \ + --target datachannel-java --parallel "$JOBS" diff --git a/native-test/NativeLibrarySmoke.java b/native-test/NativeLibrarySmoke.java new file mode 100644 index 0000000..bbfc333 --- /dev/null +++ b/native-test/NativeLibrarySmoke.java @@ -0,0 +1,34 @@ +package tel.schich.libdatachannel; + +import java.time.Duration; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.TimeUnit; + +/** Runs against packaged release natives in a JVM for the target platform. */ +public final class NativeLibrarySmoke { + public static void main(String[] args) throws Exception { + String classifier = Platform.detectArch(); + if (!classifier.equals(args[0])) { + throw new AssertionError("Expected " + args[0] + ", detected " + classifier); + } + if (args.length > 1 && args[1].equals("eager")) { + Class.forName("tel.schich.libdatachannel.LibDataChannelArchDetect") + .getMethod("initialize").invoke(null); + } + ArrayBlockingQueue descriptions = new ArrayBlockingQueue<>(1); + try (PeerConnection peer = PeerConnection.createPeer( + PeerConnectionConfiguration.DEFAULT.withDisableAutoNegotiation(true))) { + peer.onLocalDescription.register((connection, sdp, type) -> descriptions.offer(sdp)); + peer.createDataChannel("libc-smoke"); + peer.setLocalDescription("offer"); + String description = descriptions.poll(10, TimeUnit.SECONDS); + if (description == null || !description.contains("a=fingerprint:sha-256")) { + throw new AssertionError("Native offer/certificate callback did not complete"); + } + if (!peer.closeAndAwait(Duration.ofSeconds(10))) { + throw new AssertionError("Native peer cleanup did not complete"); + } + } + System.out.println("Native loading, certificate, callback and cleanup passed: " + classifier); + } +} diff --git a/native-test/README.md b/native-test/README.md index 30ecd82..951e483 100644 --- a/native-test/README.md +++ b/native-test/README.md @@ -114,7 +114,7 @@ use native-generated certificates. From a clean committed tree, `scripts/package-development.sh [maven-directory]` runs native and JVM regressions and writes an immutable local artifact under `io.github.teamziax:libdatachannel-java:.0-dev.`. -The classifier is `linux-x86_64`. `provenance.json` records all three source +The classifier identifies the host libc and architecture, for example `linux-glibc-x86_64`. `provenance.json` records all three source SHAs and artifact hashes. This binary targets the current host's system OpenSSL/ABI; it is not a portable dockcross release. Nothing is uploaded. Rebuild headers and JNI together after changing the pinned native version. diff --git a/scripts/package-development.sh b/scripts/package-development.sh index 9b42676..c9302fe 100755 --- a/scripts/package-development.sh +++ b/scripts/package-development.sh @@ -15,10 +15,17 @@ version="${native_version}.0-dev.${revision}" -Plibdatachannel.development-version="$version" \ -Plibdatachannel.test-native-path="$PWD/build/native-probe/libdatachannel-java.so" python3 - "$output" "$version" "$revision" <<'PY' -import hashlib, json, pathlib, subprocess, sys, zipfile +import hashlib, json, pathlib, platform, subprocess, sys, zipfile output, version, revision = pathlib.Path(sys.argv[1]), sys.argv[2], sys.argv[3] root = output / 'dev/opencollab/libdatachannel-java' / version root.mkdir(parents=True, exist_ok=True) +architecture = platform.machine() +if architecture not in {"x86_64", "aarch64"}: + raise SystemExit(f"Unsupported development architecture: {architecture}") +mappings = pathlib.Path('/proc/self/maps').read_text() +libc = 'musl' if 'ld-musl-' in mappings or 'libc.musl-' in mappings else 'glibc' +classifier = f'linux-{libc}-{architecture}' + def add(jar, path, name): info = zipfile.ZipInfo(name, date_time=(1980, 1, 1, 0, 0, 0)) info.compress_type = zipfile.ZIP_DEFLATED @@ -29,7 +36,7 @@ with zipfile.ZipFile(root / f'libdatachannel-java-{version}.jar', 'w') as jar: if folder.exists(): for path in sorted(folder.rglob('*')): if path.is_file(): add(jar, path, path.relative_to(folder).as_posix()) -with zipfile.ZipFile(root / f'libdatachannel-java-{version}-linux-x86_64.jar', 'w') as jar: +with zipfile.ZipFile(root / f'libdatachannel-java-{version}-{classifier}.jar', 'w') as jar: add(jar, pathlib.Path('build/native-probe/libdatachannel-java.so'), 'native/libdatachannel-java.so') (root / f'libdatachannel-java-{version}.pom').write_text(f'''4.0.0 dev.opencollablibdatachannel-java{version} @@ -43,7 +50,7 @@ provenance = { 'bindingRevision': revision, 'libdatachannelRevision': head('jni/libdatachannel'), 'libjuiceRevision': head('jni/libdatachannel/deps/libjuice'), - 'platform': 'linux-x86_64', + 'platform': classifier, 'nativeBuild': 'system OpenSSL, Debug, current host ABI; not a portable release', 'checks': ['nativeTransportProbe', 'nativeCallbackCleanupProbe', 'nativeLoggingProbe', 'test'], 'sha256': {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in sorted(root.iterdir()) if path.is_file()}, diff --git a/src/main/java/tel/schich/libdatachannel/LinuxLibc.java b/src/main/java/tel/schich/libdatachannel/LinuxLibc.java new file mode 100644 index 0000000..c34714c --- /dev/null +++ b/src/main/java/tel/schich/libdatachannel/LinuxLibc.java @@ -0,0 +1,57 @@ +package tel.schich.libdatachannel; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; + +/** Detects the libc loaded by this JVM, even when both libcs are installed. */ +final class LinuxLibc { + static final String PROPERTY = "libdatachannel.libc"; + + private LinuxLibc() { + } + + static String detect() { + String override = System.getProperty(PROPERTY); + if (override != null) { + if (!override.equals("glibc") && !override.equals("musl")) { + throw new IllegalArgumentException(PROPERTY + " must be glibc or musl"); + } + return override; + } + try { + return fromMappings(Files.readAllLines(Path.of("/proc/self/maps"))); + } catch (IOException | SecurityException e) { + throw new IllegalStateException("Cannot identify the JVM's libc; set -D" + PROPERTY + + "=glibc or -D" + PROPERTY + "=musl", e); + } + } + + static String fromMappings(List mappings) { + boolean glibc = false; + boolean musl = false; + for (String mapping : mappings) { + int pathStart = mapping.indexOf('/'); + if (pathStart < 0) { + continue; + } + String path = mapping.substring(pathStart).replace(" (deleted)", ""); + String name = path.substring(path.lastIndexOf('/') + 1); + if (name.startsWith("ld-musl-") || name.startsWith("libc.musl-")) { + musl = true; + } else if (name.equals("libc.so.6") || name.matches("libc-[0-9]+\\.[0-9]+\\.so")) { + glibc = true; + } + } + // gcompat can map libc.so.6 into a musl JVM, so the musl loader takes precedence. + if (musl) { + return "musl"; + } + if (glibc) { + return "glibc"; + } + throw new IllegalStateException("Cannot identify the JVM's libc; set -D" + PROPERTY + + "=glibc or -D" + PROPERTY + "=musl"); + } +} diff --git a/src/main/java/tel/schich/libdatachannel/Platform.java b/src/main/java/tel/schich/libdatachannel/Platform.java index c4ea67c..21f179e 100644 --- a/src/main/java/tel/schich/libdatachannel/Platform.java +++ b/src/main/java/tel/schich/libdatachannel/Platform.java @@ -46,10 +46,10 @@ public static boolean isMacOS() { } public static OS getOS() { - if (isLinux()) { - return OS.LINUX; - } else if (isAndroid()) { + if (isAndroid()) { return OS.ANDROID; + } else if (isLinux()) { + return OS.LINUX; } else if (isMacOS()) { return OS.MACOS; } else if (isWindows()) { @@ -75,22 +75,36 @@ public static String classPathPropertyNameForLibrary(String name) { private static String archPrefixForOs() { switch (getOS()) { - case LINUX: - return "linux-"; - case WINDOWS: + case LINUX: { + return "linux-" + LinuxLibc.detect() + "-"; + } + case WINDOWS: { return "windows-"; - case ANDROID: + } + case ANDROID: { return "android-"; - case MACOS: + } + case MACOS: { return "macos-"; - default: + } + default: { return ""; + } } } private static String detectCpuArch() { String arch = System.getProperty("os.arch").toLowerCase(); - if (arch.contains("arm")) { + return detectCpuArch(arch, getOS()); + } + + static String detectCpuArch(String arch, OS os) { + if (arch.equals("aarch64") || arch.equals("arm64")) { + if (os == OS.MACOS) { + return "arm64"; + } + return "aarch64"; + } else if (arch.contains("arm")) { return "armv7"; } else if (arch.contains("86") || arch.contains("amd")) { if (arch.contains("64")) { @@ -102,11 +116,6 @@ private static String detectCpuArch() { return "riscv64"; } return "riscv32"; - } else if (arch.contains("aarch64") || arch.contains("arm64")) { - if (getOS() == OS.MACOS) { - return "arm64"; // macOS uses arm64 instead of aarch64 - } - return "aarch64"; } return arch; } @@ -133,7 +142,13 @@ private static void loadExplicitLibrary(String name, Class base) { return; } - final String sourceLibPath = classPathLocation(name, base, detectArch()); + String explicitClassPath = System.getProperty(classPathPropertyNameForLibrary(name)); + final String sourceLibPath; + if (explicitClassPath != null) { + sourceLibPath = explicitClassPath; + } else { + sourceLibPath = classPathLocation(name, base, detectArch()); + } LOGGER.trace("Loading native library {} from {}", name, sourceLibPath); try { final Path tempDirectory = Files.createTempDirectory(name + "-"); diff --git a/src/test/java/tel/schich/libdatachannel/LinuxLibcTest.java b/src/test/java/tel/schich/libdatachannel/LinuxLibcTest.java new file mode 100644 index 0000000..649e62b --- /dev/null +++ b/src/test/java/tel/schich/libdatachannel/LinuxLibcTest.java @@ -0,0 +1,50 @@ +package tel.schich.libdatachannel; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class LinuxLibcTest { + @AfterEach + void clearOverride() { + System.clearProperty(LinuxLibc.PROPERTY); + } + + @Test + void identifiesGlibcWithBothLibraryNamingSchemes() { + assertEquals("glibc", LinuxLibc.fromMappings(List.of("0000 r-xp /usr/lib64/libc.so.6"))); + assertEquals("glibc", LinuxLibc.fromMappings(List.of("0000 r-xp /lib/libc-2.31.so (deleted)"))); + } + + @Test + void identifiesMuslEvenWhenGcompatIsLoaded() { + assertEquals("musl", LinuxLibc.fromMappings(List.of( + "0000 r-xp /lib/libc.so.6", "0000 r-xp /lib/ld-musl-x86_64.so.1"))); + assertEquals("musl", LinuxLibc.fromMappings(List.of("0000 r-xp /lib/libc.musl-aarch64.so.1"))); + } + + @Test + void unknownMappingsRequireAnExplicitOverride() { + assertThrows(IllegalStateException.class, + () -> LinuxLibc.fromMappings(List.of("0000 r-xp /home/musl/libjvm.so"))); + System.setProperty(LinuxLibc.PROPERTY, "musl"); + assertEquals("musl", LinuxLibc.detect()); + System.setProperty(LinuxLibc.PROPERTY, "glibc"); + assertEquals("glibc", LinuxLibc.detect()); + System.setProperty(LinuxLibc.PROPERTY, "unknown"); + assertThrows(IllegalArgumentException.class, LinuxLibc::detect); + } + + @Test + void arm64AliasesDoNotSelectArmv7() { + assertEquals("aarch64", Platform.detectCpuArch("arm64", Platform.OS.LINUX)); + assertEquals("aarch64", Platform.detectCpuArch("aarch64", Platform.OS.LINUX)); + assertEquals("arm64", Platform.detectCpuArch("aarch64", Platform.OS.MACOS)); + assertEquals("armv7", Platform.detectCpuArch("armv7l", Platform.OS.LINUX)); + assertEquals("x86_64", Platform.detectCpuArch("amd64", Platform.OS.LINUX)); + } +}