flake.nix

{
  description = "nativelink";

  inputs = {
    nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
    flake-parts.url = "github:hercules-ci/flake-parts";
    git-hooks = {
      url = "github:cachix/git-hooks.nix";
      inputs.nixpkgs.follows = "nixpkgs";
    };
    rust-overlay = {
      url = "github:oxalica/rust-overlay";
      inputs.nixpkgs.follows = "nixpkgs";
    };
    crane = {
      url = "github:ipetkov/crane";
    };
    nix2container = {
      # TODO(SchahinRohani): Use a specific commit hash until nix2container is stable.
      url = "github:nlewo/nix2container/76be9608a7f4d6c985d28b0e7be903ae2547df3e";
      inputs.nixpkgs.follows = "nixpkgs";
    };
  };

  outputs = inputs @ {
    self,
    flake-parts,
    crane,
    rust-overlay,
    nix2container,
    ...
  }:
    flake-parts.lib.mkFlake {inherit inputs;} {
      systems = [
        "x86_64-linux"
        "x86_64-darwin"
        "aarch64-linux"
        "aarch64-darwin"
      ];
      imports = [
        inputs.git-hooks.flakeModule
        ./local-remote-execution/flake-module.nix
        ./tools/darwin/flake-module.nix
        ./tools/nixos/flake-module.nix
        ./flake-module.nix
      ];
      flake = {
        flakeModules = {
          default = ./flake-module.nix;
          darwin = ./tools/darwin/flake-module.nix;
          lre = ./local-remote-execution/flake-module.nix;
          nixos = ./tools/nixos/flake-module.nix;
        };
        overlays = {
          lre = import ./local-remote-execution/overlays/default.nix {inherit nix2container;};
          tools = import ./tools/public/default.nix {inherit nix2container;};
        };
        # TODO(jaroeichler): Keep template inputs on upstream.
        templates = {
          bazel = {
            path = ./templates/bazel;
            description = "Local remote execution with Bazel";
            welcomeText = ''
              # Getting started

              Enter the Nix environment with `nix develop`.
              Get your credentials for NativeLink and paste them into `user.bazelrc`.
              Run `bazel build hello-world` to build the example with local
              remote execution.

              See <https://www.nativelink.com/docs/explanations/lre> for further
              details on local remote execution.
            '';
          };
        };
      };
      perSystem = {
        config,
        pkgs,
        system,
        lib,
        ...
      }: let
        # On Linux we build fully static musl binaries, elsewhere the default stdenv.
        stdenvSelectorFor = q:
          if q.stdenv.targetPlatform.isLinux
          then q.pkgsMusl.stdenv
          else q.stdenv;
        craneLibFor = p:
          ((crane.mkLib p).overrideToolchain pkgs.lre.stableRustFor).overrideScope (_: _: {
            stdenvSelector = stdenvSelectorFor;
          });
        nightlyCraneLibFor = p:
          ((crane.mkLib p).overrideToolchain pkgs.lre.nightlyRustFor).overrideScope (_: _: {
            stdenvSelector = stdenvSelectorFor;
          });

        src = pkgs.lib.cleanSourceWith {
          src = (craneLibFor pkgs).path ./.;
          filter = path: type:
            (builtins.match "^.*(tests/.+\.json5|examples/.+\.json5|data/.+|nativelink-config/README\.md)" path != null)
            || ((craneLibFor pkgs).filterCargoSources path type);
        };

        # Warning: The different usages of `p` and `pkgs` are intentional as we
        # use crosscompilers and crosslinkers whose packagesets collapse with
        # the host's packageset. If you change this take care that you don't
        # accidentally explode the global closure size.
        commonArgsFor = p: let
          isLinuxBuild = p.stdenv.buildPlatform.isLinux;
          isLinuxTarget = p.stdenv.targetPlatform.isLinux;
          isCrossCompile = p.stdenv.targetPlatform.system != pkgs.stdenv.targetPlatform.system;
          # Map the nix system to the Rust target triple that we'd want to target
          # by default.
          targetArch =
            (
              nixSystem:
                {
                  "x86_64-linux" = "x86_64-unknown-linux-musl";
                  "aarch64-linux" = "aarch64-unknown-linux-musl";
                  "x86_64-darwin" = "x86_64-apple-darwin";
                  "aarch64-darwin" = "aarch64-apple-darwin";
                }.${
                  nixSystem
                } or (throw "Unsupported Nix host platform: ${nixSystem}")
            )
            p.stdenv.targetPlatform.system;

          # Full path to the linker for CARGO_TARGET_XXX_LINKER
          linkerPath =
            if isLinuxBuild && isLinuxTarget
            then "${pkgs.mold}/bin/ld.mold"
            else "${pkgs.llvmPackages_22.lld}/bin/ld.lld";

          linkerEnvVar = "CARGO_TARGET_${pkgs.lib.toUpper (pkgs.lib.replaceStrings ["-"] ["_"] targetArch)}_LINKER";
        in
          {
            inherit src;
            strictDeps = true;
            buildInputs =
              [p.cacert]
              ++ pkgs.lib.optionals p.stdenv.targetPlatform.isDarwin [
                p.apple-sdk_14
                p.libiconv
              ];
            nativeBuildInputs =
              [p.bashNonInteractive] # needed for some command tests
              ++ (
                if isLinuxBuild
                then [pkgs.mold]
                else [pkgs.llvmPackages_22.lld]
              )
              ++ pkgs.lib.optionals p.stdenv.targetPlatform.isDarwin [
                p.apple-sdk_14
                p.libiconv
              ];
            CARGO_BUILD_TARGET = targetArch;
          }
          // (pkgs.lib.optionalAttrs (isLinuxTarget && !isCrossCompile) {
            # customClang is only defined for the host compiler, so doesn't work for cross-compiling
            TARGET_CC = "${pkgs.lre.clang}/bin/customClang"; # So mimalloc gets the right compiler not defaulting to gcc
          })
          // (pkgs.lib.optionalAttrs isLinuxTarget {
            CARGO_BUILD_RUSTFLAGS = "-C target-feature=+crt-static";
            # FIXME(palfrey): Attempted workaround from https://github.com/llvm/llvm-project/issues/32849#issuecomment-2353071071 but doesn't work
            # CFLAGS = "-femit-all-decls";
            ${linkerEnvVar} = linkerPath;
          });

        # Additional target for external dependencies to simplify caching.
        cargoArtifactsFor = p: (craneLibFor p).buildDepsOnly (commonArgsFor p);
        nightlyCargoArtifactsFor = p: (nightlyCraneLibFor p).buildDepsOnly (commonArgsFor p);

        nativelinkFor = p:
          (craneLibFor p).buildPackage ((commonArgsFor p)
            // {
              cargoArtifacts = cargoArtifactsFor p;
              # If you're testing Nativelink locally, doing a dev profile will
              # massively speedup build times. Just don't commit/push anything build with dev!
              # CARGO_PROFILE = "dev";
            });

        nativeTargetPkgs =
          if pkgs.stdenv.hostPlatform.system == "x86_64-linux"
          then pkgs.pkgsCross.musl64
          else if pkgs.stdenv.hostPlatform.system == "aarch64-linux"
          then pkgs.pkgsCross.aarch64-multiplatform-musl
          else pkgs;

        nativelink = nativelinkFor nativeTargetPkgs;

        # These two can be built by all build platforms. This is not true for
        # darwin targets which are only buildable via native compilation.
        nativelink-aarch64-linux = nativelinkFor pkgs.pkgsCross.aarch64-multiplatform-musl;
        nativelink-x86_64-linux = nativelinkFor pkgs.pkgsCross.musl64;

        nativelink-is-executable-test = pkgs.callPackage ./tools/nativelink-is-executable-test.nix {inherit nativelink;};

        generate-toolchains = pkgs.callPackage ./tools/generate-toolchains.nix {};

        build-chromium-tests =
          pkgs.writeShellScriptBin
          "build-chromium-tests"
          ./deploy/chromium-example/build_chromium_tests.sh;

        docs = pkgs.callPackage ./tools/docs.nix {rust = pkgs.lre.stable-rust;};

        inherit (nix2container.packages.${system}.nix2container) pullImage;
        inherit (nix2container.packages.${system}.nix2container) buildImage;

        nativelinkImageFor = archPackages: arch:
          buildImage {
            inherit arch;
            name = "nativelink";
            copyToRoot = [
              (pkgs.buildEnv {
                name = "nativelink-buildEnv";
                paths = [archPackages];
                pathsToLink = ["/bin"];
              })
            ];
            config = {
              Entrypoint = [(pkgs.lib.getExe' archPackages "nativelink")];
              Labels = {
                "org.opencontainers.image.description" = "An RBE compatible, high-performance cache and remote executor.";
                "org.opencontainers.image.documentation" = "https://github.com/TraceMachina/nativelink";
                "org.opencontainers.image.licenses" = "FSL-1.1-Apache-2.0";
                "org.opencontainers.image.revision" = "${self.rev or self.dirtyRev or "dirty"}";
                "org.opencontainers.image.source" = "https://github.com/TraceMachina/nativelink";
                "org.opencontainers.image.title" = "NativeLink";
                "org.opencontainers.image.vendor" = "Trace Machina, Inc.";
              };
            };
          };

        nativelink-image-for-x64 = nativelinkImageFor nativelink-x86_64-linux "amd64";
        nativelink-image-for-aarch64 = nativelinkImageFor nativelink-aarch64-linux "arm64";

        nativelink-image =
          if pkgs.stdenv.isx86_64
          then nativelink-image-for-x64
          else nativelink-image-for-aarch64;

        nativelinkWorkerInitFor = archImage: archPackages: arch:
          pkgs.callPackage ./tools/nativelink-worker-init.nix {
            inherit buildImage self arch archPackages;
            nativelink-image = archImage;
          };

        nativelink-init-for-x64 = nativelinkWorkerInitFor nativelink-image-for-x64 pkgs.pkgsCross.musl64 "amd64";
        nativelink-init-for-aarch64 = nativelinkWorkerInitFor nativelink-image-for-aarch64 pkgs.pkgsCross.aarch64-multiplatform-musl "arm64";

        nativelink-worker-init =
          if pkgs.stdenv.isx86_64
          then nativelink-init-for-x64
          else nativelink-init-for-aarch64;

        createWorker = pkgs.nativelink-tools.lib.createWorker self;

        buck2-toolchain = let
          buck2-nightly-rust-version = "2026-03-24";
          buck2-nightly-rust = pkgs.rust-bin.nightly.${buck2-nightly-rust-version};
          buck2-rust = buck2-nightly-rust.default.override {extensions = ["rust-src"];};
        in
          pkgs.callPackage ./tools/create-worker-experimental.nix {
            inherit buildImage self;
            imageName = "buck2-toolchain";
            packagesForImage = [
              pkgs.coreutils
              pkgs.bash
              pkgs.go
              pkgs.diffutils
              pkgs.gnutar
              pkgs.gzip
              pkgs.python3
              pkgs.unzip
              pkgs.zstd
              pkgs.cargo-bloat
              pkgs.mold
              pkgs.reindeer
              pkgs.lld_22
              pkgs.clang_22
              buck2-rust
            ];
          };
        siso-chromium = buildImage {
          name = "siso-chromium";
          fromImage = pullImage {
            imageName = "gcr.io/chops-public-images-prod/rbe/siso-chromium/linux";
            imageDigest = "sha256:26de99218a1a8b527d4840490bcbf1690ee0b55c84316300b60776e6b3a03fe1";
            sha256 = "sha256-v2wctuZStb6eexcmJdkxKcGHjRk2LuZwyJvi/BerMyw=";
            tlsVerify = true;
            arch = "amd64";
            os = "linux";
          };
        };
        toolchain-drake = buildImage {
          name = "toolchain-drake";
          # imageDigest and sha256 are generated by toolchain-drake.sh for non-reproducible builds.
          fromImage = pullImage {
            imageName = "localhost:5001/toolchain-drake";
            imageDigest = ""; # DO NOT COMMIT DRAKE IMAGE_DIGEST VALUE
            sha256 = ""; # DO NOT COMMIT DRAKE SHA256 VALUE
            tlsVerify = false;
            arch = "amd64";
            os = "linux";
          };
        };
        toolchain-buck2 = buildImage {
          name = "toolchain-buck2";
          # imageDigest and sha256 are generated by toolchain-buck2.sh for non-reproducible builds.
          fromImage = pullImage {
            imageName = "localhost:5001/toolchain-buck2";
            imageDigest = ""; # DO NOT COMMIT BUCK2 IMAGE_DIGEST VALUE
            sha256 = ""; # DO NOT COMMIT BUCK2 SHA256 VALUE
            tlsVerify = false;
            arch = "amd64";
            os = "linux";
          };
        };

        nativelinkCoverageFor = p: let
          coverageArgs =
            (commonArgsFor p)
            // {
              # TODO(palfrey): For some reason we're triggering an edgecase where
              #                    mimalloc builds against glibc headers in coverage
              #                    builds. This leads to nonexistend __memcpy_chk and
              #                    __memset_chk symbols if fortification is enabled.
              #                    Our regular builds also have this issue, but we
              #                    should investigate further.
              hardeningDisable = ["fortify"];
            };
          cargoExtraArgs = builtins.concatStringsSep " " [
            "--workspace"
            "--locked"
            "--features nix"
            # "--branch" # FIXME(palfrey): because of https://github.com/llvm/llvm-project/issues/119558
            "--ignore-filename-regex '.*(genproto|vendor-cargo-deps|crates).*'"
          ];
        in
          (nightlyCraneLibFor p).mkCargoDerivation (coverageArgs
            // {
              # We build our own custom command so we can run with report for both html and text output
              # Mostly derived from the upstream cargoLlvmCov though
              # See https://github.com/ipetkov/crane/blob/59a82a1222dd3b2080b5cc52a1a2e8d5f1b77f37/lib/cargoLlvmCov.nix
              installPhaseCommand = "";
              buildPhaseCargoCommand = ''
                cargoWithProfile llvm-cov test ${cargoExtraArgs} --html --output-dir $out
                cargoWithProfile llvm-cov report --workspace --text --output-dir $out
              '';
              doInstallCargoArtifacts = false;
              pnameSuffix = "-llvm-cov";
              nativeBuildInputs = [(p.callPackage ./tools/cargo-llvm-cov/package.nix {})];

              cargoArtifacts = nightlyCargoArtifactsFor p;
              preConfigurePhases = ["tempHome"];
              tempHome = ''
                # Default home at this point is /homeless-shelter, which doesn't exist and so breaks things like
                # the Mongo downloader
                echo "HOME was ''${HOME}"
                export HOME=$(mktemp -d fake-homeXXXX --tmpdir)
                echo "HOME is now ''${HOME}"

                # FIXME: This is a giant pile of hacks, as what we should be doing is downloading mongodb and patching it to work with
                # Nix. Several hours of patchelf rummaging later, hitting issues like "_Unwind_GetRegionStart: symbol not found" we
                # instead have this hacky workaround. It adds a symlink to a Nix mongod where the extractor code expects to find it,
                # which will work. This is the wrong version (7.0.16 v.s. 7.0.11 currently, so not so bad) but TBH our codebase is mostly
                # fairly version agnostic so far and 7.0.11 is picked out of the air as the one we've used elsewhere.

                mkdir -p $HOME/.cache/mongo/extracted/7.0.11/mongodb-linux-x86_64-ubuntu2204-7.0.11/bin
                touch $HOME/.cache/mongo/extracted/7.0.11/extracted.marker # As needed by nativelink-store/tests/mongo_runner/mod.rs
                export MONGOD=$HOME/.cache/mongo/extracted/7.0.11/mongodb-linux-x86_64-ubuntu2204-7.0.11/bin/mongod
                ln -s ${p.mongodb}/bin/mongod ''${MONGOD}
                ''${MONGOD} --version
              '';
            });

        nativelinkCoverageForHost = nativelinkCoverageFor pkgs;
      in rec {
        _module.args.pkgs = import self.inputs.nixpkgs {
          inherit system;
          config.allowUnfreePredicate = pkg:
            builtins.elem (lib.getName pkg) [
              "mongodb"
            ];
          overlays = [
            self.overlays.lre
            self.overlays.tools
            (import rust-overlay)
            (import ./tools/rust-overlay-cut-libsecret.nix)
          ];
        };
        apps = {
          default = {
            type = "app";
            program = "${nativelink}/bin/nativelink";
          };
        };
        packages =
          rec {
            inherit
              nativelink
              nativelinkCoverageForHost
              nativelink-aarch64-linux
              nativelink-image
              nativelink-image-for-aarch64
              nativelink-image-for-x64
              nativelink-is-executable-test
              nativelink-worker-init
              nativelink-init-for-x64
              nativelink-init-for-aarch64
              nativelink-x86_64-linux
              ;

            # Used by the CI
            inherit (pkgs.nativelink-tools) local-image-test publish-ghcr create-multi-arch-image regctl-ghcr-login;

            default = nativelink;

            nativelink-worker-lre-cc = createWorker pkgs.lre.lre-cc.image;
            lre-java = pkgs.callPackage ./local-remote-execution/lre-java.nix {inherit buildImage;};
            rbe-autogen-lre-java = pkgs.rbe-autogen lre-java;
            nativelink-worker-lre-java = createWorker lre-java;
            nativelink-worker-lre-rs = createWorker pkgs.lre.lre-rs.image;
            nativelink-worker-siso-chromium = createWorker siso-chromium;
            nativelink-worker-toolchain-drake = createWorker toolchain-drake;
            nativelink-worker-toolchain-buck2 = createWorker toolchain-buck2;
            nativelink-worker-buck2-toolchain = buck2-toolchain;
            image = nativelink-image;

            inherit (pkgs) buildstream buck2 mongodb wait4x bazelisk;
            buildstream-with-nativelink-test = pkgs.callPackage integration_tests/buildstream/buildstream-with-nativelink-test.nix {
              inherit nativelink buildstream;
            };
            mongo-with-nativelink-test = pkgs.callPackage integration_tests/mongo/mongo-with-nativelink-test.nix {
              inherit nativelink mongodb wait4x bazelisk;
            };
            rbe-toolchain-with-nativelink-test = pkgs.callPackage toolchain-examples/rbe-toolchain-test.nix {
              inherit nativelink bazelisk;
            };
            buck2-with-nativelink-test = pkgs.callPackage integration_tests/buck2/buck2-with-nativelink-test.nix {
              inherit nativelink buck2;
            };
            update-module-hashes = pkgs.callPackage tools/updaters/rewrite-module.nix {
              python-with-requests = pkgs.python3.withPackages (ps:
                with ps; [
                  ps.requests
                ]);
            };
            generate-bazel-rc = pkgs.callPackage tools/generate-bazel-rc/build.nix {craneLib = craneLibFor pkgs;};
            generate-stores-config = pkgs.callPackage nativelink-config/generate-stores-config/build.nix {craneLib = craneLibFor pkgs;};
          }
          // (
            # It's not possible to crosscompile to darwin, not even between
            # x86_64-darwin and aarch64-darwin. We create these targets anyways
            # To keep them uniform with the linux targets if they're buildable.
            if pkgs.stdenv.system == "aarch64-darwin"
            then {
              nativelink-aarch64-darwin = nativelink;
            }
            else if pkgs.stdenv.system == "x86_64-darwin"
            then {
              nativelink-x86_64-darwin = nativelink;
            }
            else {}
          );
        pre-commit.settings = {
          hooks = import ./tools/pre-commit-hooks.nix {
            inherit pkgs;
            inherit (packages) generate-bazel-rc generate-stores-config;
            renovate-patched = pkgs.callPackage ./tools/renovate.nix {};
            nightly-rust = pkgs.rust-bin.nightly.${pkgs.lre.nightly-rust.meta.version};
          };
        };
        lre = {
          Env = with pkgs.lre;
            if pkgs.stdenv.isDarwin
            then lre-rs.meta.Env # C++ doesn't support Darwin yet.
            else (lre-cc.meta.Env ++ lre-rs.meta.Env);
          prefix =
            if pkgs.stdenv.isDarwin
            then "macos"
            else "linux";
        };
        nixos.path = with pkgs; [
          "/run/current-system/sw/bin"
          "${binutils.bintools}/bin"
          "${pkgs.lre.clang}/bin"
          "${git}/bin"

          # In the lre-rs image these are copied to `/bin` by the create-worker
          # function,
          #
          # Since we set `--incompatible_strict_action_env` in our .bazelrc we
          # default to `PATH=/bin:/usr/bin:/usr/local/bin` on non-NixOS systems.
          #
          # On NixOS we override that path with what we have in this list. We
          # could add `/bin` here, but using the explicit store paths adds
          # another layer of safety so that we don't mix local and remote tools
          # in cases where platform resolution doesn't behave as intended.
          #
          # Ideally, these shouldn't be in create-worker at all, and instead
          # should be their own lre-shell toolchain "below" lre-cc, rather than
          # a bolted-on-top layer in the final output.
          #
          # Note that these packages must be the same as the ones used in
          # `create-worker.nix`.
          "${bash}/bin"
          "${coreutils}/bin"
          "${gnused}/bin"
        ];
        devShells.default = pkgs.mkShell {
          packages = let
            bazel = pkgs.writeShellScriptBin "bazel" ''
              unset TMPDIR TMP
              exec ${pkgs.bazelisk}/bin/bazelisk "$@"
            '';
          in
            [
              # Development tooling
              pkgs.git
              pkgs.pre-commit
              pkgs.git-cliff
              pkgs.buck2
              packages.update-module-hashes
              pkgs.python3

              # Rust
              bazel
              pkgs.lre.stable-rust
              pkgs.lre.lre-rs.lre-rs-configs-gen
              pkgs.rust-analyzer

              ## Infrastructure
              pkgs.awscli2
              pkgs.google-cloud-sdk
              pkgs.skopeo
              pkgs.dive
              pkgs.cosign
              pkgs.kubectl
              pkgs.kubernetes-helm
              pkgs.cilium-cli
              pkgs.vale
              pkgs.trivy
              pkgs.docker-client
              pkgs.kind
              pkgs.tektoncd-cli
              pkgs.pulumi
              pkgs.pulumiPackages.pulumi-go
              pkgs.fluxcd
              pkgs.go
              pkgs.kustomize
              pkgs.kubectx

              # Web
              pkgs.bun
              pkgs.lychee
              pkgs.nodejs_22 # For pagefind search
              pkgs.playwright-driver
              pkgs.playwright-test

              # Additional tools from within our development environment.
              build-chromium-tests
              docs
              generate-toolchains
              pkgs.lre.clang
              pkgs.lre.lre-cc.lre-cc-configs-gen
              pkgs.nativelink-tools.local-image-test
              pkgs.nativelink-tools.create-local-image
              pkgs.nativelink-tools.create-multi-arch-image
              pkgs.attic-client
            ]
            ++ pkgs.lib.optionals pkgs.stdenv.isDarwin [
              pkgs.apple-sdk_14
              pkgs.libiconv
            ];

          shellHook =
            ''
              # Generate the .pre-commit-config.yaml symlink when entering the
              # development shell.
              ${config.pre-commit.installationScript}

              # Generate local-remote-execution.bazelrc which configures LRE toolchains when
              # running in the nix environment.
              ${config.lre.installationScript}

              # Generate nativelink.bazelrc which gives Bazel invocations access
              # to NativeLink's read-only cache.
              ${config.nativelink.installationScript}

              # If on NixOS, generate nixos.bazelrc, which adds the required
              # NixOS binary paths to the bazel environment.
              ${config.nixos.installationScript}

              # If on Darwin, generate darwin.bazelrc, which configures darwin
              # libs and frameworks.
              ${config.darwin.installationScript}

              # The Bazel and Cargo builds in nix require a Clang toolchain.
              # TODO(palfrey): The Bazel build currently uses the
              #                    irreproducible host C++ toolchain. Provide
              #                    this toolchain via nix for bitwise identical
              #                    binaries across machines.
              export CC=clang
              export PULUMI_K8S_AWAIT_ALL=true
              export PLAYWRIGHT_BROWSERS_PATH=${pkgs.playwright-driver.browsers}
              export PLAYWRIGHT_NODEJS_PATH=${pkgs.nodePackages_latest.nodejs}
            ''
            # TODO(palfrey): Generalize this.
            + pkgs.lib.optionalString (system == "x86_64-linux") ''
              export CC_x86_64_unknown_linux_gnu=customClang
            '';
        };
      };
    };
}