Feature/idp-43 - Analyse and explore the amber dandi documentation to allow the data storage from IDP in Ember

.gitignore

@@ -8,4 +8,6 @@ deployment/helm/
 skaffold.yaml
 .vscode
 .overrides
-.DS_Store
+.DS_Store
+CLAUDE.md
+CLAUDE.local.md

applications/idp-arc/frontend/.gitignore

@@ -0,0 +1,2 @@
+.env.*
+

applications/idp-arc/frontend/package.json

@@ -22,14 +22,16 @@
     "react": "^19.2.0",
     "react-dom": "^19.2.0",
     "react-i18next": "^17.0.2",
-    "react-router-dom": "^7.14.2"
+    "react-router-dom": "^7.14.2",
+    "spark-md5": "^3.0.2"
   },
   "devDependencies": {
     "@eslint/js": "^9.39.1",
     "@playwright/test": "^1.52.0",
     "@types/node": "^24.10.1",
     "@types/react": "^19.2.7",
     "@types/react-dom": "^19.2.3",
+    "@types/spark-md5": "^3.0.5",
     "@vitejs/plugin-react": "^5.1.1",
     "eslint": "^9.39.1",
     "eslint-plugin-react-hooks": "^7.0.1",
@@ -39,4 +41,4 @@
     "typescript-eslint": "^8.48.0",
     "vite": "^7.3.1"
   }
-}
+}

applications/idp-arc/frontend/src/app/container.ts

@@ -18,42 +18,46 @@
 
 import { KeycloakAuthClient } from '../infra/keycloakAuthClient'
 import { WorkspaceApiClient } from '../infra/workspaceApiClient'
-import { JupyterApiClient } from '../infra/jupyterApiClient'
+import { DandiApiClient } from '../infra/dandiApiClient'
 import { createLoadWorkspacesUseCase } from '../core/use-cases/loadWorkspaces'
-import { createCreateAndUploadUseCase } from '../core/use-cases/createAndUploadWorkspace'
+import { createCreateAndUploadToDandiUseCase } from '../core/use-cases/createAndUploadToDandi'
+import { createCreateWorkspaceUseCase } from '../core/use-cases/createWorkspace'
 
 // ─── Config ───────────────────────────────────────────────────────────────────
 // All environment-specific URLs live here (or read from import.meta.env in Vite).
 
-const BASE_DOMAIN    = 'v2dev.opensourcebrain.org'
-const WWW_BASE       = import.meta.env.DEV ? '/api-proxy' : `https://www.${BASE_DOMAIN}`
+const BASE_DOMAIN    = import.meta.env.VITE_OSB_BASE_DOMAIN ?? 'v2dev.opensourcebrain.org'
+const PROTOCOL       = import.meta.env.VITE_OSB_PROTOCOL ?? 'https'
+const WWW_BASE       = import.meta.env.DEV ? '/api-proxy' : `${PROTOCOL}://www.${BASE_DOMAIN}`
 const WORKSPACES_API = `${WWW_BASE}/proxy/workspaces/api`
 const WORKSPACES_LIST_URL =
   `${WWW_BASE}/proxy/workspaces/api/workspace?page=1&per_page=24&q=&tags=`
-const JUPYTER_BASE          = '/jupyter-proxy'
-// JupyterHub named-server suffix — the subdomain appname of the JupyterHub host.
-// For lab.v2dev.opensourcebrain.org workspace 764 spawns as server "764lab".
-const JUPYTER_SERVER_SUFFIX = 'lab'
-const FRONTEND_BASE  = `https://www.${BASE_DOMAIN}`
+const FRONTEND_BASE  = `${PROTOCOL}://www.${BASE_DOMAIN}`
 
 // ─── Infrastructure singletons ────────────────────────────────────────────────
 
 export const authClient = new KeycloakAuthClient({
-  url: 'https://accounts.v2dev.opensourcebrain.org',
-  realm: 'osb2dev',
+  url: import.meta.env.VITE_KEYCLOAK_URL ?? 'https://accounts.v2dev.opensourcebrain.org',
+  realm: import.meta.env.VITE_KEYCLOAK_REALM ?? 'osb2dev',
   clientId: 'idp-arc',
 })
 
 const workspaceApi = new WorkspaceApiClient(WORKSPACES_API, WORKSPACES_LIST_URL)
-const jupyterApi   = new JupyterApiClient(JUPYTER_BASE, BASE_DOMAIN)
+// DANDI upload endpoints live in OSB's `workspaces` app (the admin key has to sit wherever
+// they run, per Dario) — same API base as every other workspace call.
+const dandiApi     = new DandiApiClient(WORKSPACES_API)
 
 // ─── Use-cases (injected with their concrete dependencies) ────────────────────
 
 /** Refreshes the token then returns the workspace list. */
 export const loadWorkspaces = createLoadWorkspacesUseCase(authClient, workspaceApi)
 
-/** Runs the 4-step create-workspace + file-upload workflow. */
-export const createAndUpload = createCreateAndUploadUseCase(authClient, workspaceApi, jupyterApi, JUPYTER_SERVER_SUFFIX)
+/** Creates a new, empty workspace with the given name. */
+export const createWorkspace = createCreateWorkspaceUseCase(authClient, workspaceApi)
+
+/** DANDI-backed upload (Route A, see IDP-43 notes); `finalize` also runs the selected
+ * protocol's script server-side (jupyter_kernel_client.py in OSBv2's workspaces app). */
+export const createAndUploadToDandi = createCreateAndUploadToDandiUseCase(authClient, dandiApi)
 
 // ─── Helpers ──────────────────────────────────────────────────────────────────
 

applications/idp-arc/frontend/src/app/mockContainer.ts

@@ -10,21 +10,21 @@
 
 import { MockAuthClient } from '../infra/mocks/mockAuthClient'
 import { MockWorkspaceApiClient } from '../infra/mocks/mockWorkspaceApiClient'
-import { MockJupyterApiClient } from '../infra/mocks/mockJupyterApiClient'
+import { MockDandiApiClient } from '../infra/mocks/mockDandiApiClient'
 import { createLoadWorkspacesUseCase } from '../core/use-cases/loadWorkspaces'
-import { createCreateAndUploadUseCase } from '../core/use-cases/createAndUploadWorkspace'
+import { createCreateAndUploadToDandiUseCase } from '../core/use-cases/createAndUploadToDandi'
 
 // ─── Mock infrastructure singletons ──────────────────────────────────────────
 
 export const authClient = new MockAuthClient()
 
 const workspaceApi = new MockWorkspaceApiClient()
-const jupyterApi   = new MockJupyterApiClient()
+const dandiApi     = new MockDandiApiClient()
 
 // ─── Use-cases (same factory functions, different adapters) ───────────────────
 
 export const loadWorkspaces = createLoadWorkspacesUseCase(authClient, workspaceApi)
-export const createAndUpload = createCreateAndUploadUseCase(authClient, workspaceApi, jupyterApi)
+export const createAndUploadToDandi = createCreateAndUploadToDandiUseCase(authClient, dandiApi)
 
 // ─── Helpers ──────────────────────────────────────────────────────────────────
 

applications/idp-arc/frontend/src/components/DataUploadDialog.tsx

@@ -16,7 +16,7 @@ import CloseIcon from '@mui/icons-material/Close'
 import ArrowForwardIcon from '@mui/icons-material/ArrowForward'
 import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown'
 
-import { createAndUpload, getWorkspaceUrl, loadWorkspaces } from '../app/container'
+import { createAndUploadToDandi, getWorkspaceUrl, loadWorkspaces } from '../app/container'
 import { useAppContext } from '../AppContext'
 import type { Workspace } from '../core/types'
 import protocols from '../data/protocols.json'
@@ -92,54 +92,53 @@ export default function DataUploadDialog({ open, onClose, onAuthRequired }: Data
     window.focus()
   }
 
+  /** Slug for the asset path prefix — protocols.json has no stable id field, so derive one. */
+  const slugify = (s: string) => s.toLowerCase().trim().replace(/[^a-z0-9]+/g, '-').replace(/(^-|-$)/g, '')
+
+  /** The analysis script that ships into the workspace, chosen by the selected protocol.
+   * Every protocol currently points at the same placeholder script; the per-protocol pipelines
+   * replace the URLs in protocols.json without touching this code. */
+  const selectedScript = protocols.find((p) => p.name === (protocol || behavioralTask))
+
   const handleUpload = async () => {
     if (!file) return
     setForm((prev) => ({ ...prev, step: 'uploading', uploadMessage: '' }))
     abortRef.current = false
 
-    // `spawnedWorkspaceId` is set after the first upload attempt this session.
-    // On retry we reuse that workspace so the user can fix a stuck JupyterLab
-    // without causing a new workspace to be spawned on every attempt.
+    // `spawnedWorkspaceId` is set once finalize succeeds this session. Unlike the old
+    // JupyterLab flow, the workspace no longer exists until the DANDI upload has fully
+    // completed — it's the last thing `finalize` does, not the first step — so there is
+    // nothing to reuse on a retry before the first successful attempt.
     const isRetry = spawnedWorkspaceId !== undefined
     const selectedWorkspace = workspaces.find(w => String(w.id) === String(workspaceId))
     const newWorkspaceName = [behavioralTask, protocol].filter(Boolean).join(' — ') || 'New Workspace'
     const resolvedId = selectedWorkspace
       ? (typeof selectedWorkspace.id === 'string' ? parseInt(selectedWorkspace.id, 10) : selectedWorkspace.id)
       : undefined
-
-    // On retry reuse the previously spawned workspace; otherwise use the selected one.
     const uploadWorkspaceId = isRetry ? spawnedWorkspaceId : resolvedId
 
-    // Open the workspace tab only on the first attempt — on retry it is already open.
-    if (!isRetry && uploadWorkspaceId !== undefined) {
-      openWorkspaceTab(uploadWorkspaceId)
-    }
-
-    await createAndUpload(
+    // finalize is synchronous: it also spawns the workspace and runs the script, so this
+    // one call can block for minutes.
+    await createAndUploadToDandi(
       {
-        workspaceName: selectedWorkspace?.name ?? newWorkspaceName,
-        workspaceId: uploadWorkspaceId,
+        taskId: slugify(protocol || behavioralTask),
         file,
-        userId: tokenParsed?.sub as string,
-        // For new workspaces: track the id and open the tab the moment it is created.
-        onWorkspaceCreated: uploadWorkspaceId === undefined
-          ? (wsId: number) => {
-              setForm(prev => ({ ...prev, spawnedWorkspaceId: wsId }))
-              openWorkspaceTab(wsId)
-            }
-          : undefined,
+        workspaceId: uploadWorkspaceId,
+        workspaceName: selectedWorkspace?.name ?? newWorkspaceName,
+        scriptUrl: selectedScript?.scriptUrl,
+        scriptName: selectedScript?.scriptName,
       },
       (state) => {
-        // Capture the workspace id as soon as it is known so subsequent retries
-        // within this dialog session reuse the same workspace.
-        if (state.workspaceId !== undefined) {
-          setForm(prev => ({ ...prev, spawnedWorkspaceId: state.workspaceId }))
-        }
         if (state.phase === 'error' && state.error?.includes('sign in again')) {
           setForm((prev) => ({ ...prev, step: 'upload', uploadMessage: '' }))
           onAuthRequired?.()
           return
         }
+        // The workspace only exists once `finalize` succeeds — open its tab then, not earlier.
+        if (state.phase === 'done' && state.workspaceId !== undefined) {
+          setForm(prev => ({ ...prev, spawnedWorkspaceId: state.workspaceId }))
+          openWorkspaceTab(state.workspaceId)
+        }
         setForm((prev) => ({
           ...prev,
           uploadMessage: state.phase === 'error' ? (state.error ?? state.message) : state.message,
@@ -404,7 +403,7 @@ export default function DataUploadDialog({ open, onClose, onAuthRequired }: Data
               Your files has been successfully uploaded to Open Source Brain.
             </Typography>
             <Typography variant="body2" sx={{ opacity: 0.45 }}>
-              You can close this dialog.
+              {uploadMessage || 'You can close this dialog.'}
             </Typography>
           </Stack>
         )}

applications/idp-arc/frontend/src/core/dandiEtag.ts

@@ -0,0 +1,56 @@
+import SparkMD5 from 'spark-md5'
+
+/**
+ * Part size DANDI splits uploads into. Must match theirs exactly or the ETag won't agree.
+ *
+ * DANDI validates AssetBlob.etag against `^[0-9a-f]{32}-\d{1,5}$`, so the multipart form is
+ * required even for a single small part — it can't be skipped below the part size.
+ */
+export const DANDI_ETAG_PART_SIZE = 64 * 1024 * 1024 // 64 MiB
+
+export interface PartPlan {
+  partNumber: number
+  start: number
+  end: number
+  size: number
+}
+
+/** Splits a file size into DANDI's fixed-size parts. Always returns at least one part, even for a 0-byte file. */
+export function planParts(fileSize: number, partSize = DANDI_ETAG_PART_SIZE): PartPlan[] {
+  const parts: PartPlan[] = []
+  let start = 0
+  let partNumber = 1
+  while (start < fileSize) {
+    const end = Math.min(start + partSize, fileSize)
+    parts.push({ partNumber, start, end, size: end - start })
+    start = end
+    partNumber += 1
+  }
+  if (parts.length === 0) parts.push({ partNumber: 1, start: 0, end: 0, size: 0 })
+  return parts
+}
+
+/**
+ * Computes the dandi-etag: MD5 of each part, concatenate the raw (binary) digests, MD5 that
+ * concatenation, hex-encode, append `-<part count>`. Standard S3 multipart ETag algorithm.
+ */
+export async function computeDandiEtag(
+  file: File,
+  partSize = DANDI_ETAG_PART_SIZE,
+): Promise<{ etag: string; parts: PartPlan[] }> {
+  const parts = planParts(file.size, partSize)
+  let concatenatedRawDigests = ''
+
+  for (const part of parts) {
+    const buf = await file.slice(part.start, part.end).arrayBuffer()
+    const hasher = new SparkMD5.ArrayBuffer()
+    hasher.append(buf)
+    concatenatedRawDigests += hasher.end(true) // raw binary string — 16 bytes per part
+  }
+
+  const outer = new SparkMD5()
+  outer.appendBinary(concatenatedRawDigests)
+  const etag = `${outer.end()}-${parts.length}`
+
+  return { etag, parts }
+}

applications/idp-arc/frontend/src/core/ports/IDandiApi.ts

@@ -0,0 +1,56 @@
+/** Abstraction over the OSB endpoints that broker the DANDI upload. */
+
+export interface UploadPart {
+  partNumber: number
+  url: string
+}
+
+export interface UploadInitResult {
+  /** Absent when the content was already in DANDI — nothing was uploaded. */
+  uploadId?: string
+  path: string
+  /** Empty when deduplicated; the client then skips the S3 upload entirely. */
+  parts: UploadPart[]
+  /** Set only on the deduplicated path — the existing blob to attach a new asset to. */
+  blobId?: string
+}
+
+export interface UploadedPart {
+  partNumber: number
+  size: number
+  etag: string
+}
+
+export interface UploadFinalizeResult {
+  assetPath: string
+  dandisetUrl: string
+  workspaceId: number
+  /** Everything the protocol script printed, or why it didn't run. */
+  scriptOutput?: string
+}
+
+export interface FinalizeUploadInput {
+  /** Absent on the deduplicated path — nothing was uploaded, so there is nothing to complete. */
+  uploadId?: string
+  path: string
+  parts: UploadedPart[]
+  /** Existing workspace to attach to; omit to have the backend create one. */
+  workspaceId?: number
+  workspaceName?: string
+  blobId?: string
+  /** Analysis script to run. The backend fetches it, so it must be reachable from OSB's
+   * cluster — not a local address. */
+  scriptUrl?: string
+  /** Filename the script should land under in the workspace. */
+  scriptName?: string
+}
+
+export interface IDandiApi {
+  initUpload(token: string, taskId: string, filename: string, size: number, dandiEtag: string): Promise<UploadInitResult>
+
+  /** PUTs one part straight to S3 via its presigned URL. Returns S3's ETag for the part,
+   * read from the response header — requires the bucket to expose it via CORS. */
+  putPart(url: string, blob: Blob): Promise<string>
+
+  finalizeUpload(token: string, input: FinalizeUploadInput): Promise<UploadFinalizeResult>
+}

applications/idp-arc/frontend/src/core/ports/index.ts

@@ -1,3 +1,3 @@
 export type { IAuthClient } from './IAuthClient'
 export type { IWorkspaceApi } from './IWorkspaceApi'
-export type { IJupyterApi } from './IJupyterApi'
+export type { IDandiApi } from './IDandiApi'