[pull] main from LibreChat-AI:main

.github/workflows/ci.yml

@@ -48,6 +48,24 @@ jobs:
           docker buildx build --check -f api/Dockerfile .
           docker buildx build --check -f docker/Dockerfile.worker-sandbox .
 
+  launcher-unit-tests:
+    name: Launcher Unit Tests
+    runs-on: ubuntu-latest
+    container: fedora:43
+    defaults:
+      run:
+        working-directory: launcher
+    steps:
+      - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
+
+      - name: Install Rust and libkrun
+        # Mirrors launcher/Dockerfile's builder stage; libkrun is only packaged
+        # for Fedora, and the guest-environment checks link against it.
+        run: dnf install -y --setopt=install_weak_deps=False rust cargo libkrun-devel gcc
+
+      - name: Cargo tests
+        run: cargo test
+
   api-unit-tests:
     name: API Unit Tests
     runs-on: ubuntu-latest

README.md

@@ -153,6 +153,15 @@ read-only root disk does not need modification at boot. Rebuild the runner
 image to pick up this layout change. A missing resolver handoff fails startup
 rather than leaving the guest with an unrelated public DNS server.
 
+libkrun delivers the guest environment on the kernel command line, which only
+carries single-line printable ASCII and is capped at 2048 bytes by the guest
+kernel. The launcher entrypoint therefore forwards only the `nameserver`,
+`search`, `domain`, `options` and `sortlist` directives, joined by `|`, and the
+guest wrapper expands them back into `/etc/resolv.conf` lines. The launcher
+rejects any forwarded variable that would not survive that trip (control
+characters, non-ASCII bytes, quoting the kernel would split, or an oversized
+environment) with a named error instead of a libkrun panic and restart loop.
+
 To validate a deployment, execute code that creates a file in `/mnt/data`,
 confirm the response includes its file reference, and download it. Recreate the
 egress gateway with a different container IP while leaving the runner alive,

api/src/guest-dns.sh

@@ -2,6 +2,10 @@
 # The guest root may be read-only. Bake the link, populate its target only
 # after /run is mounted, and leave direct NsJail/Lambda resolvers untouched.
 
+# launcher/entrypoint.sh joins resolver directives with this separator because
+# the handoff rides the guest kernel command line, which cannot carry newlines.
+RESOLV_FIELD_SEPARATOR='|'
+
 prepare_guest_dns() {
     local root="$1"
     mkdir -p "$root/run"
@@ -12,18 +16,20 @@ prepare_guest_dns() {
 configure_guest_dns() {
     local root="${1:-}"
     local target="$root/run/codeapi-resolver"
+    local resolv_conf="${SANDBOX_RESOLV_CONF:-}"
+    resolv_conf="${resolv_conf//"$RESOLV_FIELD_SEPARATOR"/$'\n'}"
     if [ ! -L "$root/etc/resolv.conf" ] || \
         [ "$(readlink "$root/etc/resolv.conf")" != '../run/codeapi-resolver/resolv.conf' ]; then
         return 0
     fi
-    if ! printf '%s\n' "${SANDBOX_RESOLV_CONF:-}" | grep -Eq '^[[:space:]]*nameserver[[:space:]]+[^[:space:]#]'; then
+    if ! printf '%s\n' "$resolv_conf" | grep -Eq '^[[:space:]]*nameserver[[:space:]]+[^[:space:]#]'; then
         echo 'ERROR: KVM guest requires resolver configuration from launcher-entrypoint.sh' >&2
         return 1
     fi
     # A fresh, root-owned directory prevents a sandbox UID from replacing DNS
     # configuration in the runtime mount. Never reuse a pre-existing entry.
     (umask 077; mkdir "$target") || return 1
-    printf '%s\n' "$SANDBOX_RESOLV_CONF" > "$target/resolv.conf" || return 1
+    printf '%s\n' "$resolv_conf" > "$target/resolv.conf" || return 1
     chmod 600 "$target/resolv.conf" || return 1
     unset SANDBOX_RESOLV_CONF
 }

api/src/hosted-app.test.ts

@@ -337,7 +337,9 @@ describe('HostedAppSupervisor', () => {
 
     expect(error).toBeInstanceOf(HostedAppError);
     expect(error.code).toBe('hosted_app_start_failed');
+    expect(error.message).toBe('hosted app exited');
     expect(supervisor.status()?.state).toBe('failed');
+    expect(supervisor.status()?.message).toBe('hosted app exited');
   });
 
   test('serializes quiesced workspace access and rejects it while an app is running', async () => {
@@ -424,9 +426,63 @@ describe('HostedAppSupervisor', () => {
     expect(error).toBeInstanceOf(HostedAppError);
     expect(error.code).toBe('hosted_app_cleanup_failed');
     expect(error.status).toBe(503);
+    expect(supervisor.status()).toMatchObject({
+      state: 'failed',
+      message: 'hosted app cleanup failed',
+    });
 
     permitCleanup = true;
-    await supervisor.shutdown();
+    const recovered = await supervisor.stop();
+    expect(recovered).toMatchObject({ state: 'stopped' });
+    expect(recovered).not.toHaveProperty('message');
+  });
+
+  test('allows checkpoint and restore to retry cleanup after stop fails', async () => {
+    const root = await workspace();
+    let permitCleanup = false;
+    const fixture = dependencies(root, {
+      killCgroup: async () => {
+        if (!permitCleanup) throw new Error('cgroup remains populated');
+      },
+    });
+    const supervisor = new HostedAppSupervisor(fixture.deps);
+    await supervisor.start(request());
+    await expect(supervisor.stop()).rejects.toMatchObject({
+      code: 'hosted_app_cleanup_failed',
+      status: 503,
+    });
+    const operations: string[] = [];
+
+    await expect(supervisor.withQuiescedWorkspace(async () => {
+      operations.push('unsafe checkpoint');
+    })).rejects.toMatchObject({
+      code: 'hosted_app_cleanup_failed',
+      status: 503,
+    });
+    expect(operations).toEqual([]);
+
+    permitCleanup = true;
+    await supervisor.withQuiescedWorkspace(async () => { operations.push('checkpoint'); });
+    await supervisor.withQuiescedWorkspace(async () => { operations.push('restore'); });
+
+    expect(operations).toEqual(['checkpoint', 'restore']);
+    expect(fixture.cgroupKills).toHaveLength(3);
+  });
+
+  test('surfaces replacement cleanup failure as retryable without spawning', async () => {
+    const root = await workspace();
+    const fixture = dependencies(root, {
+      killCgroup: async () => { throw new Error('cgroup remains populated'); },
+    });
+    const supervisor = new HostedAppSupervisor(fixture.deps);
+    await supervisor.start(request());
+
+    const error = await supervisor.start(request({ revision: 'rev-2' })).catch(value => value);
+
+    expect(error).toBeInstanceOf(HostedAppError);
+    expect(error).toMatchObject({ code: 'hosted_app_cleanup_failed', status: 503 });
+    expect(supervisor.status()?.state).toBe('failed');
+    expect(fixture.spawns).toHaveLength(1);
   });
 
   test('fails workspace mutation closed until a failed app cgroup is drained', async () => {

api/src/hosted-app.ts

@@ -458,18 +458,7 @@ export class HostedAppSupervisor {
   }
 
   async stop(): Promise<HostedAppStatus | undefined> {
-    return this.serialize(async () => {
-      try {
-        return await this.stopImpl();
-      } catch (error) {
-        logger.error({ err: error }, 'Hosted-app stop cleanup failed');
-        throw new HostedAppError(
-          'hosted_app_cleanup_failed',
-          'the hosted app could not be stopped safely',
-          503,
-        );
-      }
-    });
+    return this.serialize(() => this.stopImpl());
   }
 
   async shutdown(): Promise<void> {
@@ -729,42 +718,62 @@ export class HostedAppSupervisor {
   private async stopImpl(preserveActive = false): Promise<HostedAppStatus | undefined> {
     const active = this.active;
     if (!active) return undefined;
-    const child = active.process;
-    if (!child?.pid) {
+    try {
+      const child = active.process;
+      if (!child?.pid) {
+        await this.deps.killCgroup();
+        active.cgroupDrained = true;
+        active.status.state = 'stopped';
+        if (!preserveActive) delete active.status.message;
+        active.status.exited_at ??= this.deps.now().toISOString();
+        if (!preserveActive) this.active = undefined;
+        return publicStatus(active);
+      }
+
+      active.status.state = 'stopping';
+      this.deps.killProcessGroup(child.pid, 'SIGTERM');
+      const exited = new Promise<boolean>(resolve => child.once('exit', () => resolve(true)));
+      let timer: ReturnType<typeof setTimeout> | undefined;
+      const timedOut = new Promise<boolean>(resolve => {
+        timer = setTimeout(() => resolve(false), config.hosted_app_stop_timeout_ms);
+        timer.unref?.();
+      });
+      const stopped = await Promise.race([exited, timedOut]);
+      if (timer) clearTimeout(timer);
+      if (!stopped && active.process?.pid) {
+        await this.deps.killCgroup();
+        await Promise.race([
+          new Promise<void>(resolve => child.once('exit', () => resolve())),
+          new Promise<void>(resolve => setTimeout(resolve, config.hosted_app_stop_timeout_ms)),
+        ]);
+      }
+      /* Always sweep the cgroup: the tracked parent may have exited cleanly
+       * while a daemonized descendant stayed alive in a different process group. */
       await this.deps.killCgroup();
       active.cgroupDrained = true;
       active.status.state = 'stopped';
+      if (!preserveActive) delete active.status.message;
       active.status.exited_at ??= this.deps.now().toISOString();
+      const status = publicStatus(active);
       if (!preserveActive) this.active = undefined;
-      return publicStatus(active);
-    }
-
-    active.status.state = 'stopping';
-    this.deps.killProcessGroup(child.pid, 'SIGTERM');
-    const exited = new Promise<boolean>(resolve => child.once('exit', () => resolve(true)));
-    let timer: ReturnType<typeof setTimeout> | undefined;
-    const timedOut = new Promise<boolean>(resolve => {
-      timer = setTimeout(() => resolve(false), config.hosted_app_stop_timeout_ms);
-      timer.unref?.();
-    });
-    const stopped = await Promise.race([exited, timedOut]);
-    if (timer) clearTimeout(timer);
-    if (!stopped && active.process?.pid) {
-      await this.deps.killCgroup();
-      await Promise.race([
-        new Promise<void>(resolve => child.once('exit', () => resolve())),
-        new Promise<void>(resolve => setTimeout(resolve, config.hosted_app_stop_timeout_ms)),
-      ]);
+      return status;
+    } catch (error) {
+      /* Keep the failed record so checkpoint/restore can retry the cgroup
+       * sweep. A `stopping` record would permanently reject those operations
+       * before they reach the recoverable cleanup path. */
+      active.cgroupDrained = false;
+      active.status.state = 'failed';
+      active.status.message = 'hosted app cleanup failed';
+      logger.error(
+        { err: error, appId: active.request.app_id },
+        'Hosted-app stop cleanup failed',
+      );
+      throw new HostedAppError(
+        'hosted_app_cleanup_failed',
+        'the hosted app could not be stopped safely',
+        503,
+      );
     }
-    /* Always sweep the cgroup: the tracked parent may have exited cleanly
-     * while a daemonized descendant stayed alive in a different process group. */
-    await this.deps.killCgroup();
-    active.cgroupDrained = true;
-    active.status.state = 'stopped';
-    active.status.exited_at ??= this.deps.now().toISOString();
-    const status = publicStatus(active);
-    if (!preserveActive) this.active = undefined;
-    return status;
   }
 }
 

docs/lambda-microvm/README.md

@@ -232,7 +232,10 @@ Content-Type: application/json
 ```
 
 `GET /v1/hosted-apps/:app_id?runtime_session_hint=...` returns status and a
-fresh five-minute `preview_url`; `DELETE` on the same resource terminates the
+fresh five-minute `preview_url`; the authorization response loads a minimal
+same-origin handoff page before opening the app so the first request includes
+the host-only `SameSite=Strict` preview cookie even when LibreChat is on another
+site. `DELETE` on the same resource terminates the
 lease. A revision is immutable. Retrying the identical spec reasserts the
 resident process; changing code or launch settings requires a new revision and
 captures a new exact checkpoint. An ambiguous provider launch is replayed only

launcher/entrypoint.sh

@@ -5,8 +5,35 @@ set -e
 # names intact so new connections can resolve replacements after a restart.
 # Forward the resolver and search domains supplied by Docker or Kubernetes,
 # rather than pinning endpoint IPs or baking a deployment-specific nameserver.
-export SANDBOX_RESOLV_CONF="$(cat /etc/resolv.conf)"
-if ! printf '%s\n' "$SANDBOX_RESOLV_CONF" | grep -Eq '^[[:space:]]*nameserver[[:space:]]+[^[:space:]#]'; then
+#
+# libkrun places every guest environment entry on the kernel command line,
+# which accepts only single-line printable ASCII and is truncated by the guest
+# kernel past 2048 bytes. Keep the resolver directives alone, one per field,
+# joined by a separator that api/src/guest-dns.sh expands back into lines.
+RESOLV_FIELD_SEPARATOR='|'
+
+encode_resolv_conf() {
+    local LC_ALL=C
+    local line words encoded=''
+    while IFS= read -r line || [ -n "$line" ]; do
+        line="${line%$'\r'}"
+        if [[ ! "$line" =~ ^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]] ]]; then
+            continue
+        fi
+        read -ra words <<< "$line"
+        line="${words[*]}"
+        if [[ "$line" == *[!' '-'~']* || "$line" == *[\"$RESOLV_FIELD_SEPARATOR]* ]]; then
+            echo "ERROR: runner /etc/resolv.conf line cannot cross the kernel command line: $line" >&2
+            return 1
+        fi
+        encoded+="${encoded:+$RESOLV_FIELD_SEPARATOR}$line"
+    done
+    printf '%s' "$encoded"
+}
+
+SANDBOX_RESOLV_CONF="$(encode_resolv_conf < /etc/resolv.conf)"
+export SANDBOX_RESOLV_CONF
+if [[ "$RESOLV_FIELD_SEPARATOR$SANDBOX_RESOLV_CONF" != *"${RESOLV_FIELD_SEPARATOR}nameserver "[!\#]* ]]; then
     echo 'ERROR: runner /etc/resolv.conf has no nameserver' >&2
     exit 1
 fi