From 0f8a5080e3643a3ad4ac2947fdaf4c19b2fefc33 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Thu, 20 Aug 2026 12:10:19 -0700 Subject: [PATCH 01/21] feat: add Custom Python Calculation notebook Runs an arbitrary user script against one or more materials. The script, its dependency list and any data files it reads are set in the params cells, uploaded to the account's object storage, and fetched onto the compute node by the Custom Python Script workflow from Standata. The script reads material.json, settings.json and its own files by relative path, and whatever it prints comes back as the result. Section 9 re-runs the saved workflow with different settings and no re-upload. The script is uploaded as a file rather than inlined into the workflow, so it is never passed through the template engine and reaches Python exactly as written. Adds a notebooks_utils file helper for the uploads, plus the fixed runner and an input setter that matches execution unit inputs by template name rather than position. Also fixes load_materials_from_folder, which built every material in one comprehension: a single file that is not a material raised out of the whole call, so a user dropping a data file their script reads into uploads/ broke material loading for every calculation notebook. One material lacking build metadata also silently downgraded all the others. Each file is now built on its own and anything no material class accepts is skipped with a warning. Co-Authored-By: Claude Opus 5 (1M context) --- other/materials_designer/uploads/radii.json | 18 + .../workflows/Introduction.ipynb | 3 +- .../workflows/custom_python_calculation.ipynb | 717 ++++++++++++++++++ .../core/entity/file/__init__.py | 0 .../notebooks_utils/core/entity/file/api.py | 84 ++ .../core/entity/material/io.py | 46 +- .../core/entity/workflow/api.py | 41 + tests/py/unit/core/entity/test_material_io.py | 55 ++ 8 files changed, 953 insertions(+), 11 deletions(-) create mode 100644 other/materials_designer/uploads/radii.json create mode 100644 other/materials_designer/workflows/custom_python_calculation.ipynb create mode 100644 src/py/mat3ra/notebooks_utils/core/entity/file/__init__.py create mode 100644 src/py/mat3ra/notebooks_utils/core/entity/file/api.py create mode 100644 tests/py/unit/core/entity/test_material_io.py diff --git a/other/materials_designer/uploads/radii.json b/other/materials_designer/uploads/radii.json new file mode 100644 index 000000000..69f11ff0e --- /dev/null +++ b/other/materials_designer/uploads/radii.json @@ -0,0 +1,18 @@ +{ + "H": 0.31, + "C": 0.76, + "N": 0.71, + "O": 0.66, + "F": 0.57, + "Si": 1.11, + "P": 1.07, + "S": 1.05, + "Cl": 1.02, + "Ge": 1.2, + "As": 1.19, + "Se": 1.2, + "Br": 1.2, + "Ni": 1.24, + "Cu": 1.32, + "Au": 1.36 +} diff --git a/other/materials_designer/workflows/Introduction.ipynb b/other/materials_designer/workflows/Introduction.ipynb index 51bef958b..531e08e6c 100644 --- a/other/materials_designer/workflows/Introduction.ipynb +++ b/other/materials_designer/workflows/Introduction.ipynb @@ -105,7 +105,8 @@ "## 9. Custom\n", "\n", "### 9.1. Python / Shell\n", - "#### 9.1.1. Custom Python and Shell workflows. *(to be added)*\n" + "#### [9.1.1. Custom Python calculation.](custom_python_calculation.ipynb)\n", + "#### 9.1.2. Custom Shell calculation. *(to be added)*\n" ] }, { diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb new file mode 100644 index 000000000..ca4d98df3 --- /dev/null +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -0,0 +1,717 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Custom Python Calculation\n", + "\n", + "Run your own Python script against one or more materials on the Mat3ra platform. The script and any\n", + "files it needs are uploaded to your object storage folder, a workflow fetches them onto the compute\n", + "node alongside the material, and whatever the script prints comes back as the result.\n", + "\n", + "

Usage

\n", + "\n", + "1. Put your script, its dependencies and any data files it reads in cell 1.2. below (or use the\n", + " default values).\n", + "1. Click \"Run\" > \"Run All\" to run all cells.\n", + "1. Wait for the jobs to complete.\n", + "1. Scroll down to view the results.\n", + "\n", + "## Summary\n", + "\n", + "1. Set up the environment and parameters: install packages (JupyterLite only) and configure the\n", + " script, its dependencies, its asset files, its settings, the materials, compute resources and job.\n", + "1. Authenticate and initialize API client: authenticate via browser, initialize the client, then\n", + " select account and project.\n", + "1. Create materials: materials are read from the `../uploads` folder — place files there manually or\n", + " run a material creation notebook first. If a material is not found by name, Standata is used as a\n", + " fallback. Each material is then saved to the platform.\n", + "1. Create workflow: upload the script and its asset files, then assemble a workflow that fetches\n", + " them, fetches the material, and runs the script. Optionally save the workflow to the collection.\n", + "1. Configure compute: get list of clusters and create compute configuration with selected cluster,\n", + " queue, and number of processors.\n", + "1. Create one job per material from the material, workflow, project and compute configuration.\n", + "1. Submit the jobs and monitor the status: submit and wait for completion.\n", + "1. Retrieve results: read each job's standard output and display the values it printed.\n", + "1. Reuse the saved workflow with a different material and different settings, without uploading\n", + " anything again.\n", + "\n", + "## How the script receives its inputs\n", + "\n", + "Everything lands in the job's working directory, so the script reads it all by **relative path**:\n", + "\n", + "| File | Written by | Contents |\n", + "| --- | --- | --- |\n", + "| `material.json` | the workflow | the job's material, as stored on the platform |\n", + "| `settings.json` | the workflow | the `SETTINGS` dictionary below |\n", + "| your asset files | the workflow | uploaded verbatim from `USER_ASSET_FILES` |\n", + "\n", + "The script's standard output is the result. Print JSON and this notebook renders it as a table." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## 1. Set up the environment and parameters\n", + "### 1.1. Install packages (JupyterLite)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.packages import install_packages\n", + "\n", + "await install_packages(\"made|api_examples\")" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "### 1.2. Set parameters and configurations for the workflow and job\n", + "\n", + "`USER_ASSET_FILES` names files your script opens. Put them in the `../uploads` folder first — drag\n", + "them into the JupyterLite file browser — and this notebook uploads them alongside your script." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from datetime import datetime\n", + "\n", + "from mat3ra.ide.compute import QueueName\n", + "\n", + "# 2. Auth and organization parameters\n", + "# Set organization name to use it as the owner, otherwise your personal account is used\n", + "ORGANIZATION_NAME = None\n", + "\n", + "# 3. Material parameters\n", + "FOLDER = \"../uploads\"\n", + "MATERIAL_NAMES = [\"Silicon\"] # One job is created per material\n", + "\n", + "# 4. Script parameters\n", + "USER_REQUIREMENTS = [\"numpy<2\"] # Installed into a virtual environment on the compute node\n", + "USER_ASSET_FILES = [\"radii.json\"] # Files the script opens, taken from FOLDER\n", + "SETTINGS = {\"cutoff_scale\": 1.2} # Passed to the script as settings.json\n", + "\n", + "# 5. Workflow parameters\n", + "WORKFLOW_SEARCH_TERM = \"custom_script.json\" # Search term for Workflows Standata\n", + "APPLICATION_NAME = \"python\"\n", + "MY_WORKFLOW_NAME = \"Custom Python Calculation\"\n", + "save_to_collection = True\n", + "\n", + "# 6. Compute parameters\n", + "CLUSTER_NAME = None # specify full or partial name i.e. \"cluster-001\" to select\n", + "QUEUE_NAME = QueueName.D\n", + "PPN = 1\n", + "\n", + "# 7. Job parameters\n", + "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", + "POLL_INTERVAL = 30 # seconds\n", + "\n", + "# 8. Reuse parameters (section 9) - re-run the saved workflow without uploading anything again\n", + "REUSE_MATERIAL_NAME = \"Graphene\"\n", + "REUSE_SETTINGS = {\"cutoff_scale\": 1.3}" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "### 1.3. Set the script to run\n", + "\n", + "This is the calculation. It runs on the compute node with `material.json`, `settings.json` and your\n", + "asset files beside it, and whatever it prints becomes the result. Replace it with your own.\n", + "\n", + "The script is uploaded as a file and fetched onto the node, never inlined into the workflow, so its\n", + "contents reach Python exactly as written - text that looks like a template placeholder is left\n", + "alone." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "USER_SCRIPT = r\"\"\"\n", + "import itertools\n", + "import json\n", + "\n", + "import numpy as np\n", + "\n", + "material = json.load(open(\"material.json\"))\n", + "settings = json.load(open(\"settings.json\"))\n", + "radii = json.load(open(\"radii.json\"))\n", + "\n", + "# The platform stores the cell as lengths and angles, so build the vectors from them.\n", + "lattice = material[\"lattice\"]\n", + "a, b, c = lattice[\"a\"], lattice[\"b\"], lattice[\"c\"]\n", + "alpha, beta, gamma = (np.radians(lattice[key]) for key in (\"alpha\", \"beta\", \"gamma\"))\n", + "c_x = c * np.cos(beta)\n", + "c_y = c * (np.cos(alpha) - np.cos(beta) * np.cos(gamma)) / np.sin(gamma)\n", + "vectors = np.array(\n", + " [\n", + " [a, 0.0, 0.0],\n", + " [b * np.cos(gamma), b * np.sin(gamma), 0.0],\n", + " [c_x, c_y, np.sqrt(max(c**2 - c_x**2 - c_y**2, 0.0))],\n", + " ]\n", + ")\n", + "\n", + "elements = [element[\"value\"] for element in material[\"basis\"][\"elements\"]]\n", + "crystal = np.array([point[\"value\"] for point in material[\"basis\"][\"coordinates\"]], dtype=float)\n", + "cartesian = crystal @ vectors\n", + "\n", + "# Count neighbours within scale * (r_i + r_j), including atoms in the neighbouring cells.\n", + "scale = settings[\"cutoff_scale\"]\n", + "images = [np.array(shift) @ vectors for shift in itertools.product((-1, 0, 1), repeat=3)]\n", + "\n", + "coordination = {}\n", + "for element_i, position_i in zip(elements, cartesian):\n", + " neighbors = 0\n", + " for element_j, position_j in zip(elements, cartesian):\n", + " cutoff = scale * (radii[element_i] + radii[element_j])\n", + " for image in images:\n", + " distance = np.linalg.norm(position_i - position_j - image)\n", + " if 0.01 < distance < cutoff:\n", + " neighbors += 1\n", + " coordination[element_i] = neighbors\n", + "\n", + "print(json.dumps({\n", + " \"formula\": material.get(\"formula\"),\n", + " \"n_atoms\": len(elements),\n", + " \"coordination\": coordination,\n", + "}))\n", + "\"\"\"" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "## 2. Authenticate and initialize API client\n", + "### 2.1. Authenticate\n", + "Authenticate in the browser and have credentials stored in environment variable \"OIDC_ACCESS_TOKEN\"." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.auth import authenticate\n", + "\n", + "\n", + "await authenticate()" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "### 2.2. Initialize API Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.api_client import APIClient\n", + "\n", + "client = APIClient.authenticate()\n", + "client" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "### 2.3. Select account to work under" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "client.list_accounts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "selected_account = client.my_account\n", + "\n", + "if ORGANIZATION_NAME:\n", + " selected_account = client.get_account(name=ORGANIZATION_NAME)\n", + "\n", + "ACCOUNT_ID = selected_account.id\n", + "print(f\"Using account: {selected_account.name} ({ACCOUNT_ID})\")" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "### 2.4. Select project" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "## 3. Create materials\n", + "### 3.1. Load materials from local files (or Standata)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.made.material import Material\n", + "from mat3ra.standata.materials import Materials\n", + "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", + "from mat3ra.notebooks_utils.material import load_material_from_folder\n", + "\n", + "materials = [\n", + " load_material_from_folder(FOLDER, name) or Material.create(Materials.get_by_name_first_match(name))\n", + " for name in MATERIAL_NAMES\n", + "]\n", + "visualize([{\"material\": material, \"title\": material.name} for material in materials])" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "### 3.2. Save materials to the platform" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.core.entity.material.api import get_or_create_material\n", + "\n", + "saved_materials = [\n", + " Material.create(get_or_create_material(client, material, ACCOUNT_ID)) for material in materials\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "20", + "metadata": {}, + "source": [ + "## 4. Create workflow and set its parameters\n", + "### 4.1. Upload the script and its asset files\n", + "\n", + "The files go to your account's object storage folder (\"Dropbox\"), which the compute node reads them\n", + "from. An upload travels inside the request body, which the API caps at 50 MB, and JupyterLite holds\n", + "the content in memory before sending it. For anything large, upload it through the Dropbox page in\n", + "the web interface instead and name it in `USER_ASSET_FILES` all the same." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from mat3ra.notebooks_utils.core.entity.file.api import upload_files\n", + "\n", + "files_to_upload = {\"user_script.py\": USER_SCRIPT}\n", + "for name in USER_ASSET_FILES:\n", + " with open(os.path.join(FOLDER, name)) as file:\n", + " files_to_upload[name] = file.read()\n", + "\n", + "uploaded_files = upload_files(client, files_to_upload, ACCOUNT_ID)" + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, + "source": [ + "### 4.2. Create workflow from standard workflows and preview it\n", + "\n", + "The `Custom Python Script` workflow already carries the unit chain this needs: fetch the uploaded\n", + "files, fetch the material, put both into the workflow scope, then run the script. Four things are\n", + "filled in per job — the objects to fetch, the settings, the runner that hands your script its\n", + "inputs, and the dependency list." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.standata.applications import ApplicationStandata\n", + "from mat3ra.ade.application import Application\n", + "from mat3ra.standata.workflows import WorkflowStandata\n", + "from mat3ra.wode.workflows import Workflow\n", + "from mat3ra.notebooks_utils.core.entity.file.api import to_object_storage_input\n", + "from mat3ra.notebooks_utils.core.entity.workflow.api import CUSTOM_SCRIPT_RUNNER, set_execution_unit_input\n", + "from mat3ra.notebooks_utils.ipython.entity.workflow.visualize import visualize_workflow\n", + "\n", + "app_config = ApplicationStandata.get_by_name_first_match(APPLICATION_NAME)\n", + "app = Application(**app_config)\n", + "\n", + "workflow_config = WorkflowStandata.filter_by_application(app.name).get_by_name_first_match(WORKFLOW_SEARCH_TERM)\n", + "workflow_config[\"name\"] = MY_WORKFLOW_NAME\n", + "subworkflow = workflow_config[\"subworkflows\"][0]\n", + "subworkflow[\"name\"] = MY_WORKFLOW_NAME\n", + "units = {unit[\"name\"]: unit for unit in subworkflow[\"units\"]}\n", + "\n", + "units[\"io-user-files\"][\"input\"] = [to_object_storage_input(file) for file in uploaded_files]\n", + "units[\"assign-settings\"][\"value\"] = json.dumps(SETTINGS)\n", + "\n", + "set_execution_unit_input(units[\"custom_script\"], \"script.py\", CUSTOM_SCRIPT_RUNNER)\n", + "set_execution_unit_input(units[\"custom_script\"], \"requirements.txt\", \"\\n\".join(USER_REQUIREMENTS) + \"\\n\")\n", + "\n", + "workflow = Workflow.create(workflow_config)\n", + "visualize_workflow(workflow)" + ] + }, + { + "cell_type": "markdown", + "id": "24", + "metadata": {}, + "source": [ + "### 4.3. Save workflow to collection\n", + "\n", + "Saving it makes the workflow reusable: section 9 loads it back and runs it against another material\n", + "with different settings, without uploading anything again." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25", + "metadata": {}, + "outputs": [], + "source": [ + "saved_workflow = None\n", + "if save_to_collection:\n", + " saved_workflow = Workflow.create(\n", + " client.workflows.create(workflow.to_dict_without_special_keys(), owner_id=ACCOUNT_ID)\n", + " )\n", + " print(f\"✅ Workflow saved to collection: {saved_workflow.id}\")" + ] + }, + { + "cell_type": "markdown", + "id": "26", + "metadata": {}, + "source": [ + "## 5. Create the compute configuration\n", + "### 5.1. Get list of clusters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [ + "clusters = client.clusters.list()\n", + "print(f\"Available clusters: {[c['hostname'] for c in clusters]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "28", + "metadata": {}, + "source": [ + "### 5.2. Create compute configuration for the jobs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.ide.compute import Compute\n", + "\n", + "# Select cluster: use specified name if provided, otherwise use first available\n", + "if CLUSTER_NAME:\n", + " cluster = next((c for c in clusters if CLUSTER_NAME in c[\"hostname\"]), None)\n", + "else:\n", + " cluster = clusters[0]\n", + "\n", + "compute = Compute(\n", + " cluster=cluster,\n", + " queue=QUEUE_NAME,\n", + " ppn=PPN,\n", + ")\n", + "print(f\"Using cluster: {compute.cluster.hostname}, queue: {QUEUE_NAME}, ppn: {PPN}\")" + ] + }, + { + "cell_type": "markdown", + "id": "30", + "metadata": {}, + "source": [ + "## 6. Create the jobs with material and workflow configuration\n", + "### 6.1. Create one job per material" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.job import create_job\n", + "from mat3ra.notebooks_utils.ui import display_JSON\n", + "\n", + "jobs = []\n", + "for saved_material in saved_materials:\n", + " job_response = create_job(\n", + " api_client=client,\n", + " materials=[saved_material],\n", + " workflow=saved_workflow or workflow,\n", + " project_id=project_id,\n", + " owner_id=ACCOUNT_ID,\n", + " prefix=f\"{MY_WORKFLOW_NAME} {saved_material.formula} {timestamp}\",\n", + " compute=compute.to_dict(),\n", + " )\n", + " jobs.append(job_response if not isinstance(job_response, list) else job_response[0])\n", + "\n", + "job_ids = [job[\"_id\"] for job in jobs]\n", + "print(f\"✅ Created {len(job_ids)} jobs: {job_ids}\")\n", + "display_JSON(jobs[0])" + ] + }, + { + "cell_type": "markdown", + "id": "32", + "metadata": {}, + "source": [ + "## 7. Submit the jobs and monitor the status" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.api.job import submit_jobs\n", + "\n", + "submit_jobs(client.jobs, job_ids)\n", + "print(f\"✅ Submitted {len(job_ids)} jobs successfully!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.api.job import wait_for_jobs_to_finish_async\n", + "\n", + "await wait_for_jobs_to_finish_async(client.jobs, job_ids, poll_interval=POLL_INTERVAL)" + ] + }, + { + "cell_type": "markdown", + "id": "35", + "metadata": {}, + "source": [ + "## 8. Retrieve results\n", + "\n", + "Each job's standard output is the script's own output. Lines that parse as JSON become table\n", + "columns; everything else is shown as printed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "36", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "from mat3ra.notebooks_utils.io import read_from_url\n", + "\n", + "\n", + "async def read_job_stdout(job_id):\n", + " \"\"\"Contents of the execution unit's .out file, or why the job produced none.\"\"\"\n", + " files = client.jobs.list_files(job_id)\n", + " stdout_file = next((file for file in files if file[\"key\"].endswith(\".out\")), None)\n", + " if stdout_file is None:\n", + " job = client.jobs.get(job_id)\n", + " errors = job.get(\"compute\", {}).get(\"errors\", [])\n", + " return f\"No output. Job status: {job['status']}. {json.dumps(errors, indent=2)}\"\n", + " return await read_from_url(stdout_file[\"signedUrl\"])\n", + "\n", + "\n", + "results = []\n", + "for saved_material, job in zip(saved_materials, jobs):\n", + " stdout = await read_job_stdout(job[\"_id\"])\n", + " print(f\"--- {job['name']} ---\\n{stdout}\")\n", + " for line in stdout.splitlines():\n", + " try:\n", + " results.append({\"material\": saved_material.name, **json.loads(line)})\n", + " except json.JSONDecodeError:\n", + " continue\n", + "\n", + "pd.DataFrame(results)" + ] + }, + { + "cell_type": "markdown", + "id": "37", + "metadata": {}, + "source": [ + "## 9. Reuse the saved workflow\n", + "\n", + "The saved workflow keeps pointing at the files already in object storage, so running it again needs\n", + "no upload — only the settings and the material change. `REUSE_MATERIAL_NAME` and `REUSE_SETTINGS`\n", + "are set in section 1.2.\n", + "\n", + "### 9.1. Load the saved workflow and change its settings" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38", + "metadata": {}, + "outputs": [], + "source": [ + "reuse_workflow_config = client.workflows.get(saved_workflow.id) if save_to_collection else None\n", + "\n", + "if reuse_workflow_config:\n", + " reuse_units = {unit[\"name\"]: unit for unit in reuse_workflow_config[\"subworkflows\"][0][\"units\"]}\n", + " reuse_units[\"assign-settings\"][\"value\"] = json.dumps(REUSE_SETTINGS)\n", + " reuse_workflow = Workflow.create(reuse_workflow_config)\n", + " print(f\"Reusing workflow {saved_workflow.id} with SETTINGS = {REUSE_SETTINGS}\")" + ] + }, + { + "cell_type": "markdown", + "id": "39", + "metadata": {}, + "source": [ + "### 9.2. Run it against another material" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [ + "if reuse_workflow_config:\n", + " reuse_material = Material.create(\n", + " get_or_create_material(\n", + " client,\n", + " load_material_from_folder(FOLDER, REUSE_MATERIAL_NAME)\n", + " or Material.create(Materials.get_by_name_first_match(REUSE_MATERIAL_NAME)),\n", + " ACCOUNT_ID,\n", + " )\n", + " )\n", + " reuse_job = create_job(\n", + " api_client=client,\n", + " materials=[reuse_material],\n", + " workflow=reuse_workflow,\n", + " project_id=project_id,\n", + " owner_id=ACCOUNT_ID,\n", + " prefix=f\"{MY_WORKFLOW_NAME} {reuse_material.formula} {timestamp} (reuse)\",\n", + " compute=compute.to_dict(),\n", + " )\n", + " reuse_job_id = (reuse_job if not isinstance(reuse_job, list) else reuse_job[0])[\"_id\"]\n", + " client.jobs.submit(reuse_job_id)\n", + " await wait_for_jobs_to_finish_async(client.jobs, [reuse_job_id], poll_interval=POLL_INTERVAL)\n", + " print(await read_job_stdout(reuse_job_id))" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/py/mat3ra/notebooks_utils/core/entity/file/__init__.py b/src/py/mat3ra/notebooks_utils/core/entity/file/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py new file mode 100644 index 000000000..1943b50f3 --- /dev/null +++ b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py @@ -0,0 +1,84 @@ +import json +from typing import Dict, List + +from mat3ra.api_client import APIClient +from mat3ra.api_client.endpoints import BaseEndpoint + +# The execution unit writes its own `script.py` and `requirements.txt` after the IO unit has +# fetched the uploaded files, so anything uploaded under those names is overwritten before it runs. +RESERVED_FILENAMES = ("script.py", "requirements.txt") + + +def _files_endpoint(api_client: APIClient) -> BaseEndpoint: + """ + A raw endpoint for `/files`, which mat3ra-api-client does not model yet. + + Args: + api_client (APIClient): API client instance carrying the authorization context. + + Returns: + BaseEndpoint: Endpoint bound to the same host, version and credentials as the client. + """ + return BaseEndpoint( + api_client.host, + api_client.port, + version=api_client.version, + secure=api_client.secure, + auth=api_client.auth, + ) + + +def upload_files(api_client: APIClient, files: Dict[str, str], account_id: str) -> List[dict]: + """ + Uploads text files to the account's object storage ("Dropbox") folder. + + Args: + api_client (APIClient): API client instance carrying the authorization context. + files (dict): File name (relative to the account folder) mapped to its text content. + account_id (str): Account to upload under. + + Returns: + list[dict]: One cloud file record per upload, with `key`, `size`, `bucket`, `region` + and `provider`. + + Raises: + ValueError: If a file name collides with one the execution unit writes itself. + """ + reserved = [name for name in files if name.split("/")[-1] in RESERVED_FILENAMES] + if reserved: + raise ValueError(f"Rename {reserved}: {RESERVED_FILENAMES} are written by the workflow itself.") + + endpoint = _files_endpoint(api_client) + headers = endpoint.get_headers(api_client.auth.account_id or "", api_client.auth.auth_token or "") + + uploaded = [] + for name, content in files.items(): + payload = {"name": name, "body": content, "accountId": account_id} + record = endpoint.request("POST", "files", data=json.dumps(payload), headers=headers) + print(f"⬆️ Uploaded {record['key']} ({record['size']} bytes)") + uploaded.append(record) + return uploaded + + +def to_object_storage_input(cloud_file: dict) -> dict: + """ + Converts an upload record into an `object_storage` input item for a workflow IO unit. + + Args: + cloud_file (dict): A record returned by `upload_files`. + + Returns: + dict: IO unit input item. The runner fetches it into the job's working directory under + `basename`, which is what makes the user script's relative paths resolve. + """ + return { + "type": "object_storage", + "basename": cloud_file["key"].split("/")[-1], + "pathname": "", + "objectData": { + "NAME": cloud_file["key"], + "PROVIDER": cloud_file["provider"], + "CONTAINER": cloud_file["bucket"], + "REGION": cloud_file["region"], + }, + } diff --git a/src/py/mat3ra/notebooks_utils/core/entity/material/io.py b/src/py/mat3ra/notebooks_utils/core/entity/material/io.py index 5c60dd5fd..11866eee9 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/material/io.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/material/io.py @@ -1,7 +1,7 @@ import inspect import json import os -from typing import Any, List, Optional +from typing import Any, Dict, List, Optional from mat3ra.made.material import Material from mat3ra.made.tools.build_components import MaterialWithBuildMetadata @@ -64,6 +64,28 @@ def set_materials(materials: List[Any], folder_path: str = UPLOADS_FOLDER): ) +def _create_material_or_none(config: Dict[str, Any]) -> Optional[Any]: + """ + Builds a material from a config, or returns None when the config is not one. + + The folder holds whatever the user put there — a data file a script reads is as likely as a + material — so a config no material class accepts is skipped instead of failing the whole load. + Build metadata is preferred per file, so one plain material does not strip it from the rest. + + Args: + config (dict): Parsed contents of a JSON file from the folder. + + Returns: + Material | MaterialWithBuildMetadata | None: The material, or None if the config is not one. + """ + for material_cls in (MaterialWithBuildMetadata, Material): + try: + return material_cls.create(config) + except Exception: + continue + return None + + def load_materials_from_folder(folder_path: Optional[str] = None, verbose: bool = True) -> List[Any]: """ Load materials from the specified folder or from the UPLOADS_FOLDER by default. @@ -84,7 +106,6 @@ def load_materials_from_folder(folder_path: Optional[str] = None, verbose: bool data_from_host = [] try: - index = 0 for filename in sorted(os.listdir(folder_path)): if filename.endswith(".json"): file_path = os.path.join(folder_path, filename) @@ -98,18 +119,23 @@ def load_materials_from_folder(folder_path: Optional[str] = None, verbose: bool force_verbose=verbose, ) continue - name = os.path.splitext(filename)[0] - log(f"{index}: {name}", SeverityLevelEnum.INFO, force_verbose=verbose) - index += 1 - data_from_host.append(data) + data_from_host.append((os.path.splitext(filename)[0], data)) except FileNotFoundError: log(f"No data found in the '{folder_path}' folder.", SeverityLevelEnum.ERROR, force_verbose=verbose) return [] - try: - materials = [MaterialWithBuildMetadata.create(item) for item in data_from_host] - except Exception: - materials = [Material.create(item) for item in data_from_host] + materials: List[Any] = [] + for name, item in data_from_host: + material = _create_material_or_none(item) + if material is None: + log( + f"Skipping '{name}.json': not a material.", + SeverityLevelEnum.WARNING, + force_verbose=verbose, + ) + continue + log(f"{len(materials)}: {name}", SeverityLevelEnum.INFO, force_verbose=verbose) + materials.append(material) if materials: log( diff --git a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py index 8694cfced..678d1deef 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py @@ -37,3 +37,44 @@ def copy_bank_workflow_by_system_name(endpoint: BankWorkflowEndpoints, system_na """ bank_workflow_id = endpoint.list({"systemName": system_name})[0]["_id"] return endpoint.copy(bank_workflow_id, account_id)["_id"] + + +# Written by the runner into the job's working directory, next to the files the IO unit fetched. +CUSTOM_SCRIPT_RUNNER = '''import json + +with open("material.json", "w") as file: + json.dump(json.loads(r"""{{ MATERIAL | default({}) | tojson }}"""), file) + +with open("settings.json", "w") as file: + json.dump(json.loads(r"""{{ SETTINGS | default({}) | tojson }}"""), file) + +with open("user_script.py") as file: + exec(compile(file.read(), "user_script.py", "exec")) +''' + + +def set_execution_unit_input(unit: dict, template_name: str, content: str) -> None: + """ + Replaces one input file of an execution unit with fixed content. + + `isManuallyChanged` stops the platform re-rendering the file at job creation, which is what + lets the runner keep its `{{ ... }}` placeholders for the compute node to resolve, and keeps a + user's requirements verbatim. Inputs are matched by template name rather than position, so + adding a file to the flavor cannot silently write content into the wrong one. + + Args: + unit (dict): Execution unit config from a workflow. + template_name (str): Name of the input file to replace, e.g. "script.py". + content (str): Content to write. + + Raises: + KeyError: If the unit has no input with that template name. + """ + for unit_input in unit["input"]: + if unit_input["template"]["name"] == template_name: + unit_input["template"]["content"] = content + unit_input["rendered"] = content + unit_input["isManuallyChanged"] = True + return + available = [unit_input["template"]["name"] for unit_input in unit["input"]] + raise KeyError(f"No input named '{template_name}' in unit '{unit.get('name')}'. Available: {available}") diff --git a/tests/py/unit/core/entity/test_material_io.py b/tests/py/unit/core/entity/test_material_io.py new file mode 100644 index 000000000..fec281162 --- /dev/null +++ b/tests/py/unit/core/entity/test_material_io.py @@ -0,0 +1,55 @@ +import json +from typing import Any, Dict + +from mat3ra.notebooks_utils.core.entity.material.io import load_material_from_folder, load_materials_from_folder + +SILICON: Dict[str, Any] = { + "name": "Silicon", + "lattice": { + "a": 3.867, + "b": 3.867, + "c": 3.867, + "alpha": 60.0, + "beta": 60.0, + "gamma": 60.0, + "units": {"length": "angstrom", "angle": "degree"}, + "type": "FCC", + }, + "basis": { + "elements": [{"id": 0, "value": "Si"}, {"id": 1, "value": "Si"}], + "coordinates": [{"id": 0, "value": [0.0, 0.0, 0.0]}, {"id": 1, "value": [0.25, 0.25, 0.25]}], + "units": "crystal", + }, +} + +# The kind of file a user script reads - valid JSON, not a material. +RADII: Dict[str, Any] = {"Si": 1.11, "C": 0.76} + + +def _write(folder, name, payload): + path = folder / name + path.write_text(json.dumps(payload)) + return path + + +def test_non_material_json_is_skipped_not_fatal(tmp_path): + """A data file beside the materials must not fail the whole load.""" + _write(tmp_path, "silicon.json", SILICON) + _write(tmp_path, "radii.json", RADII) + + materials = load_materials_from_folder(str(tmp_path), verbose=False) + + assert [material.name for material in materials] == ["Silicon"] + + +def test_lookup_by_name_still_works_alongside_a_data_file(tmp_path): + _write(tmp_path, "silicon.json", SILICON) + _write(tmp_path, "radii.json", RADII) + + assert load_material_from_folder(str(tmp_path), "Silicon", verbose=False).name == "Silicon" + + +def test_folder_of_only_data_files_yields_nothing(tmp_path): + _write(tmp_path, "radii.json", RADII) + + assert load_materials_from_folder(str(tmp_path), verbose=False) == [] From 32478b288b122d0c1f378bb7e269e62977249458 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Thu, 20 Aug 2026 12:26:31 -0700 Subject: [PATCH 02/21] fix: install ipython for python unit tests notebooks_utils.io imports IPython eagerly through pyodide.io, so any test importing a module that reaches it for get_data or set_data - core.entity.material.io among them - fails to collect with ModuleNotFoundError under CI, which installs only ".[tests]". IPython was declared in the jupyterlite extra alone. Co-Authored-By: Claude Opus 5 (1M context) --- pyproject.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index bd6adf9b2..345457053 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,6 +71,9 @@ tests = [ "pytest", "pytest-asyncio", "pytest-cov", + # notebooks_utils.io imports IPython eagerly (via pyodide.io), so anything reaching it for + # get_data/set_data - core.entity.material.io, for one - needs IPython importable under test. + "ipython>=8.0", "mat3ra-notebooks-utils[workflows]", ] docs = [ From 655dc29338bf6b605d58b1e1224795fec2ede745 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Thu, 20 Aug 2026 13:01:32 -0700 Subject: [PATCH 03/21] fix: guard uploads against name collisions and missing assets Review feedback on #360. An entry in USER_ASSET_FILES named user_script.py silently replaced the script, and a missing asset raised a bare FileNotFoundError that named neither the file nor the parameter to fix - a trap hit during development. RESERVED_FILENAMES also covered only the two files the execution unit renders, not material.json and settings.json, which the runner writes after the IO unit has fetched the uploads. An asset under either name was replaced at run time with no warning. Co-Authored-By: Claude Opus 5 (1M context) --- .../workflows/custom_python_calculation.ipynb | 7 ++++++- src/py/mat3ra/notebooks_utils/core/entity/file/api.py | 8 +++++--- src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py | 2 ++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index ca4d98df3..082b2b6b1 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -376,7 +376,12 @@ "\n", "files_to_upload = {\"user_script.py\": USER_SCRIPT}\n", "for name in USER_ASSET_FILES:\n", - " with open(os.path.join(FOLDER, name)) as file:\n", + " if name in files_to_upload:\n", + " raise ValueError(f\"Rename '{name}' in USER_ASSET_FILES: the script is uploaded under that name.\")\n", + " path = os.path.join(FOLDER, name)\n", + " if not os.path.exists(path):\n", + " raise FileNotFoundError(f\"'{name}' is listed in USER_ASSET_FILES but is not in {FOLDER}.\")\n", + " with open(path) as file:\n", " files_to_upload[name] = file.read()\n", "\n", "uploaded_files = upload_files(client, files_to_upload, ACCOUNT_ID)" diff --git a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py index 1943b50f3..9e5c1a79b 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py @@ -4,9 +4,11 @@ from mat3ra.api_client import APIClient from mat3ra.api_client.endpoints import BaseEndpoint -# The execution unit writes its own `script.py` and `requirements.txt` after the IO unit has -# fetched the uploaded files, so anything uploaded under those names is overwritten before it runs. -RESERVED_FILENAMES = ("script.py", "requirements.txt") +# All of these land in the job's working directory after the IO unit has fetched the uploaded +# files, so an upload under any of them is overwritten before the user's script runs: the execution +# unit renders `script.py` and `requirements.txt`, and the runner it renders writes `material.json` +# and `settings.json`. Keep in step with CUSTOM_SCRIPT_RUNNER in ../workflow/api.py. +RESERVED_FILENAMES = ("script.py", "requirements.txt", "material.json", "settings.json") def _files_endpoint(api_client: APIClient) -> BaseEndpoint: diff --git a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py index 678d1deef..61eaecbcd 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py @@ -40,6 +40,8 @@ def copy_bank_workflow_by_system_name(endpoint: BankWorkflowEndpoints, system_na # Written by the runner into the job's working directory, next to the files the IO unit fetched. +# The filenames it writes are also listed in RESERVED_FILENAMES in ../file/api.py, which refuses an +# upload that would be overwritten by them; change both together. CUSTOM_SCRIPT_RUNNER = '''import json with open("material.json", "w") as file: From 3048771dcc4f673185d539997190099d5cafc654 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Thu, 20 Aug 2026 20:14:16 -0700 Subject: [PATCH 04/21] refactor: carry settings as an uploaded file, not a workflow unit The assign-settings unit is gone from the standata workflow: SETTINGS now travels as a settings.json uploaded next to the script and fetched by the same io unit. The chain drops to four units, the runner writes only material.json, and reusing a saved workflow no longer edits the workflow at all - upload a new settings.json over the old key and submit. settings.json leaves RESERVED_FILENAMES accordingly: the workflow no longer writes it, the notebook uploads it. Co-Authored-By: Claude Opus 5 (1M context) --- .../workflows/custom_python_calculation.ipynb | 33 ++++++++----------- .../notebooks_utils/core/entity/file/api.py | 6 ++-- .../core/entity/workflow/api.py | 7 ++-- 3 files changed, 19 insertions(+), 27 deletions(-) diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index 082b2b6b1..4e310c85e 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -45,8 +45,8 @@ "| File | Written by | Contents |\n", "| --- | --- | --- |\n", "| `material.json` | the workflow | the job's material, as stored on the platform |\n", - "| `settings.json` | the workflow | the `SETTINGS` dictionary below |\n", - "| your asset files | the workflow | uploaded verbatim from `USER_ASSET_FILES` |\n", + "| `settings.json` | this notebook | the `SETTINGS` dictionary below, uploaded next to the script |\n", + "| your asset files | this notebook | uploaded verbatim from `USER_ASSET_FILES` |\n", "\n", "The script's standard output is the result. Print JSON and this notebook renders it as a table." ] @@ -106,7 +106,7 @@ "# 4. Script parameters\n", "USER_REQUIREMENTS = [\"numpy<2\"] # Installed into a virtual environment on the compute node\n", "USER_ASSET_FILES = [\"radii.json\"] # Files the script opens, taken from FOLDER\n", - "SETTINGS = {\"cutoff_scale\": 1.2} # Passed to the script as settings.json\n", + "SETTINGS = {\"cutoff_scale\": 1.2} # Uploaded as settings.json next to the script\n", "\n", "# 5. Workflow parameters\n", "WORKFLOW_SEARCH_TERM = \"custom_script.json\" # Search term for Workflows Standata\n", @@ -374,7 +374,7 @@ "\n", "from mat3ra.notebooks_utils.core.entity.file.api import upload_files\n", "\n", - "files_to_upload = {\"user_script.py\": USER_SCRIPT}\n", + "files_to_upload = {\"user_script.py\": USER_SCRIPT, \"settings.json\": json.dumps(SETTINGS)}\n", "for name in USER_ASSET_FILES:\n", " if name in files_to_upload:\n", " raise ValueError(f\"Rename '{name}' in USER_ASSET_FILES: the script is uploaded under that name.\")\n", @@ -395,9 +395,8 @@ "### 4.2. Create workflow from standard workflows and preview it\n", "\n", "The `Custom Python Script` workflow already carries the unit chain this needs: fetch the uploaded\n", - "files, fetch the material, put both into the workflow scope, then run the script. Four things are\n", - "filled in per job — the objects to fetch, the settings, the runner that hands your script its\n", - "inputs, and the dependency list." + "files, fetch the material, run the script. Three things are filled in per job — the objects to\n", + "fetch, the runner that hands your script its inputs, and the dependency list." ] }, { @@ -425,7 +424,6 @@ "units = {unit[\"name\"]: unit for unit in subworkflow[\"units\"]}\n", "\n", "units[\"io-user-files\"][\"input\"] = [to_object_storage_input(file) for file in uploaded_files]\n", - "units[\"assign-settings\"][\"value\"] = json.dumps(SETTINGS)\n", "\n", "set_execution_unit_input(units[\"custom_script\"], \"script.py\", CUSTOM_SCRIPT_RUNNER)\n", "set_execution_unit_input(units[\"custom_script\"], \"requirements.txt\", \"\\n\".join(USER_REQUIREMENTS) + \"\\n\")\n", @@ -635,11 +633,11 @@ "source": [ "## 9. Reuse the saved workflow\n", "\n", - "The saved workflow keeps pointing at the files already in object storage, so running it again needs\n", - "no upload — only the settings and the material change. `REUSE_MATERIAL_NAME` and `REUSE_SETTINGS`\n", - "are set in section 1.2.\n", + "The saved workflow keeps pointing at the files already in object storage, so re-running it needs no\n", + "change to the workflow itself — upload a new `settings.json` over the old one and pick another\n", + "material. `REUSE_MATERIAL_NAME` and `REUSE_SETTINGS` are set in section 1.2.\n", "\n", - "### 9.1. Load the saved workflow and change its settings" + "### 9.1. Upload the new settings" ] }, { @@ -649,12 +647,9 @@ "metadata": {}, "outputs": [], "source": [ - "reuse_workflow_config = client.workflows.get(saved_workflow.id) if save_to_collection else None\n", - "\n", - "if reuse_workflow_config:\n", - " reuse_units = {unit[\"name\"]: unit for unit in reuse_workflow_config[\"subworkflows\"][0][\"units\"]}\n", - " reuse_units[\"assign-settings\"][\"value\"] = json.dumps(REUSE_SETTINGS)\n", - " reuse_workflow = Workflow.create(reuse_workflow_config)\n", + "if save_to_collection:\n", + " upload_files(client, {\"settings.json\": json.dumps(REUSE_SETTINGS)}, ACCOUNT_ID)\n", + " reuse_workflow = Workflow.create(client.workflows.get(saved_workflow.id))\n", " print(f\"Reusing workflow {saved_workflow.id} with SETTINGS = {REUSE_SETTINGS}\")" ] }, @@ -673,7 +668,7 @@ "metadata": {}, "outputs": [], "source": [ - "if reuse_workflow_config:\n", + "if save_to_collection:\n", " reuse_material = Material.create(\n", " get_or_create_material(\n", " client,\n", diff --git a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py index 9e5c1a79b..388d609b6 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py @@ -6,9 +6,9 @@ # All of these land in the job's working directory after the IO unit has fetched the uploaded # files, so an upload under any of them is overwritten before the user's script runs: the execution -# unit renders `script.py` and `requirements.txt`, and the runner it renders writes `material.json` -# and `settings.json`. Keep in step with CUSTOM_SCRIPT_RUNNER in ../workflow/api.py. -RESERVED_FILENAMES = ("script.py", "requirements.txt", "material.json", "settings.json") +# unit renders `script.py` and `requirements.txt`, and the runner it renders writes `material.json`. +# Keep in step with CUSTOM_SCRIPT_RUNNER in ../workflow/api.py. +RESERVED_FILENAMES = ("script.py", "requirements.txt", "material.json") def _files_endpoint(api_client: APIClient) -> BaseEndpoint: diff --git a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py index 61eaecbcd..3ac427d25 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py @@ -40,16 +40,13 @@ def copy_bank_workflow_by_system_name(endpoint: BankWorkflowEndpoints, system_na # Written by the runner into the job's working directory, next to the files the IO unit fetched. -# The filenames it writes are also listed in RESERVED_FILENAMES in ../file/api.py, which refuses an -# upload that would be overwritten by them; change both together. +# The filename it writes is also listed in RESERVED_FILENAMES in ../file/api.py, which refuses an +# upload that would be overwritten by it; change both together. CUSTOM_SCRIPT_RUNNER = '''import json with open("material.json", "w") as file: json.dump(json.loads(r"""{{ MATERIAL | default({}) | tojson }}"""), file) -with open("settings.json", "w") as file: - json.dump(json.loads(r"""{{ SETTINGS | default({}) | tojson }}"""), file) - with open("user_script.py") as file: exec(compile(file.read(), "user_script.py", "exec")) ''' From 8b803815ade4b32136846aacc2cc6e56af7f5790 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Thu, 20 Aug 2026 20:30:48 -0700 Subject: [PATCH 05/21] refactor: drop the reuse section - the saved workflow is the reuse story The workflow stays in the collection pointing at the uploaded files, runnable from the UI or another notebook against any material; changing parameters is uploading a new settings.json. A dedicated demo section with its own REUSE_* params was scaffolding, not something a user needs. Co-Authored-By: Claude Opus 5 (1M context) --- .../workflows/custom_python_calculation.ipynb | 79 +------------------ 1 file changed, 4 insertions(+), 75 deletions(-) diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index 4e310c85e..fbe75455b 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -35,8 +35,6 @@ "1. Create one job per material from the material, workflow, project and compute configuration.\n", "1. Submit the jobs and monitor the status: submit and wait for completion.\n", "1. Retrieve results: read each job's standard output and display the values it printed.\n", - "1. Reuse the saved workflow with a different material and different settings, without uploading\n", - " anything again.\n", "\n", "## How the script receives its inputs\n", "\n", @@ -121,11 +119,7 @@ "\n", "# 7. Job parameters\n", "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", - "POLL_INTERVAL = 30 # seconds\n", - "\n", - "# 8. Reuse parameters (section 9) - re-run the saved workflow without uploading anything again\n", - "REUSE_MATERIAL_NAME = \"Graphene\"\n", - "REUSE_SETTINGS = {\"cutoff_scale\": 1.3}" + "POLL_INTERVAL = 30 # seconds" ] }, { @@ -439,8 +433,9 @@ "source": [ "### 4.3. Save workflow to collection\n", "\n", - "Saving it makes the workflow reusable: section 9 loads it back and runs it against another material\n", - "with different settings, without uploading anything again." + "Saving it makes the workflow reusable: it stays in your collection pointing at the uploaded files,\n", + "so it can be run again — from the UI or another notebook — against any material. To change the\n", + "script's parameters, upload a new `settings.json` over the old one." ] }, { @@ -625,72 +620,6 @@ "\n", "pd.DataFrame(results)" ] - }, - { - "cell_type": "markdown", - "id": "37", - "metadata": {}, - "source": [ - "## 9. Reuse the saved workflow\n", - "\n", - "The saved workflow keeps pointing at the files already in object storage, so re-running it needs no\n", - "change to the workflow itself — upload a new `settings.json` over the old one and pick another\n", - "material. `REUSE_MATERIAL_NAME` and `REUSE_SETTINGS` are set in section 1.2.\n", - "\n", - "### 9.1. Upload the new settings" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "38", - "metadata": {}, - "outputs": [], - "source": [ - "if save_to_collection:\n", - " upload_files(client, {\"settings.json\": json.dumps(REUSE_SETTINGS)}, ACCOUNT_ID)\n", - " reuse_workflow = Workflow.create(client.workflows.get(saved_workflow.id))\n", - " print(f\"Reusing workflow {saved_workflow.id} with SETTINGS = {REUSE_SETTINGS}\")" - ] - }, - { - "cell_type": "markdown", - "id": "39", - "metadata": {}, - "source": [ - "### 9.2. Run it against another material" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "40", - "metadata": {}, - "outputs": [], - "source": [ - "if save_to_collection:\n", - " reuse_material = Material.create(\n", - " get_or_create_material(\n", - " client,\n", - " load_material_from_folder(FOLDER, REUSE_MATERIAL_NAME)\n", - " or Material.create(Materials.get_by_name_first_match(REUSE_MATERIAL_NAME)),\n", - " ACCOUNT_ID,\n", - " )\n", - " )\n", - " reuse_job = create_job(\n", - " api_client=client,\n", - " materials=[reuse_material],\n", - " workflow=reuse_workflow,\n", - " project_id=project_id,\n", - " owner_id=ACCOUNT_ID,\n", - " prefix=f\"{MY_WORKFLOW_NAME} {reuse_material.formula} {timestamp} (reuse)\",\n", - " compute=compute.to_dict(),\n", - " )\n", - " reuse_job_id = (reuse_job if not isinstance(reuse_job, list) else reuse_job[0])[\"_id\"]\n", - " client.jobs.submit(reuse_job_id)\n", - " await wait_for_jobs_to_finish_async(client.jobs, [reuse_job_id], poll_interval=POLL_INTERVAL)\n", - " print(await read_job_stdout(reuse_job_id))" - ] } ], "metadata": { From 24242190ffe9b85b56cc53ec94026511cf1f1dca Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Thu, 20 Aug 2026 20:33:39 -0700 Subject: [PATCH 06/21] fix: name the actual conflict in the upload collision message For settings.json the error claimed the script used that name; the conflict is the notebook's own settings upload. Review feedback on #360. Co-Authored-By: Claude Opus 5 (1M context) --- .../workflows/custom_python_calculation.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index fbe75455b..da18d6ed7 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -371,7 +371,7 @@ "files_to_upload = {\"user_script.py\": USER_SCRIPT, \"settings.json\": json.dumps(SETTINGS)}\n", "for name in USER_ASSET_FILES:\n", " if name in files_to_upload:\n", - " raise ValueError(f\"Rename '{name}' in USER_ASSET_FILES: the script is uploaded under that name.\")\n", + " raise ValueError(f\"Rename '{name}' in USER_ASSET_FILES: this notebook already uploads a file by that name.\")\n", " path = os.path.join(FOLDER, name)\n", " if not os.path.exists(path):\n", " raise FileNotFoundError(f\"'{name}' is listed in USER_ASSET_FILES but is not in {FOLDER}.\")\n", From 2cdcc4ed7857f0bffb4e11cc764da10c579ed6aa Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Thu, 20 Aug 2026 22:28:23 -0700 Subject: [PATCH 07/21] test: cover the file upload helper Closes the review blocker on #360. The upload payload shape, the reserved-name refusal, and to_object_storage_input were verified only by a three-minute cluster run; to_object_storage_input in particular is a pure function whose five fields rupy requires with no defaults. Co-Authored-By: Claude Opus 5 (1M context) --- tests/py/unit/core/entity/test_file_api.py | 81 ++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 tests/py/unit/core/entity/test_file_api.py diff --git a/tests/py/unit/core/entity/test_file_api.py b/tests/py/unit/core/entity/test_file_api.py new file mode 100644 index 000000000..c5ef8346b --- /dev/null +++ b/tests/py/unit/core/entity/test_file_api.py @@ -0,0 +1,81 @@ +import json +from typing import Any, Dict +from unittest.mock import MagicMock + +import pytest +from mat3ra.notebooks_utils.core.entity.file.api import RESERVED_FILENAMES, to_object_storage_input, upload_files + +ACCOUNT_ID = "account-1" + +CLOUD_FILE: Dict[str, Any] = { + "key": "my-account/user_script.py", + "size": 14, + "bucket": "test-bucket", + "region": "us-west-2", + "provider": "aws", + "lastModified": 1609891535, + "name": "user_script.py", +} + + +def _client(*records): + """An APIClient whose files endpoint returns the given records, one per request.""" + client = MagicMock() + client.host, client.port, client.version, client.secure = "localhost", 3000, "2018-10-01", False + client.auth.account_id, client.auth.auth_token = ACCOUNT_ID, "token" + client.auth.access_token = None + return client + + +def test_upload_posts_name_and_body_per_file(monkeypatch): + requests = [] + + def request(method, path, data=None, headers=None): + requests.append((method, path, json.loads(data))) + return CLOUD_FILE + + endpoint = MagicMock() + endpoint.request.side_effect = request + endpoint.get_headers.return_value = {} + monkeypatch.setattr("mat3ra.notebooks_utils.core.entity.file.api._files_endpoint", lambda _: endpoint) + + uploaded = upload_files(_client(), {"user_script.py": "print(1)", "radii.json": "{}"}, ACCOUNT_ID) + + assert [r[:2] for r in requests] == [("POST", "files"), ("POST", "files")] + assert [r[2] for r in requests] == [ + {"name": "user_script.py", "body": "print(1)", "accountId": ACCOUNT_ID}, + {"name": "radii.json", "body": "{}", "accountId": ACCOUNT_ID}, + ] + assert uploaded == [CLOUD_FILE, CLOUD_FILE] + + +@pytest.mark.parametrize("name", RESERVED_FILENAMES) +def test_upload_refuses_a_name_the_job_would_overwrite(name, monkeypatch): + endpoint = MagicMock() + monkeypatch.setattr("mat3ra.notebooks_utils.core.entity.file.api._files_endpoint", lambda _: endpoint) + + with pytest.raises(ValueError, match=name): + upload_files(_client(), {name: "x"}, ACCOUNT_ID) + + endpoint.request.assert_not_called() + + +def test_object_storage_input_carries_every_field_the_runner_needs(): + """rupy requires all of NAME/PROVIDER/CONTAINER/REGION plus a basename, with no defaults.""" + assert to_object_storage_input(CLOUD_FILE) == { + "type": "object_storage", + "basename": "user_script.py", + "pathname": "", + "objectData": { + "NAME": "my-account/user_script.py", + "PROVIDER": "aws", + "CONTAINER": "test-bucket", + "REGION": "us-west-2", + }, + } + + +def test_object_storage_basename_is_the_name_the_script_opens(): + nested = {**CLOUD_FILE, "key": "my-account/assets/radii.json"} + + assert to_object_storage_input(nested)["basename"] == "radii.json" From 8a326f4d2f2bdcb1a3d962850db3de4cfef78339 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Fri, 21 Aug 2026 18:15:55 -0700 Subject: [PATCH 08/21] refactor(notebook): strip the example-specific parameters from the defaults The notebook shipped with one worked example baked into its defaults - a radii table as a data file, a cutoff_scale setting uploaded as settings.json, and a 40-line coordination-number script - which read as required machinery rather than as an example. The defaults are now the minimum a custom calculation needs: one material, no data files, no dependencies, and a stdlib-only script that reports the elements, the atom count and the cell volume. settings.json is gone; a script's parameters belong in the script, which sits in its own cell. USER_ASSET_FILES and USER_REQUIREMENTS keep the mechanisms, with an example in a comment. The Cypress feature covers the richer path by overriding both cells. --- .../workflows/custom_python_calculation.ipynb | 82 +++++++------------ 1 file changed, 29 insertions(+), 53 deletions(-) diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index da18d6ed7..556290074 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -13,8 +13,9 @@ "\n", "

Usage

\n", "\n", - "1. Put your script, its dependencies and any data files it reads in cell 1.2. below (or use the\n", - " default values).\n", + "1. Set the materials and, if the script needs them, its dependencies and data files in cell 1.2.\n", + " below.\n", + "1. Replace the script in cell 1.3. with your own (or keep the default one).\n", "1. Click \"Run\" > \"Run All\" to run all cells.\n", "1. Wait for the jobs to complete.\n", "1. Scroll down to view the results.\n", @@ -22,13 +23,13 @@ "## Summary\n", "\n", "1. Set up the environment and parameters: install packages (JupyterLite only) and configure the\n", - " script, its dependencies, its asset files, its settings, the materials, compute resources and job.\n", + " materials, the script, its dependencies, its data files, compute resources and job.\n", "1. Authenticate and initialize API client: authenticate via browser, initialize the client, then\n", " select account and project.\n", "1. Create materials: materials are read from the `../uploads` folder — place files there manually or\n", " run a material creation notebook first. If a material is not found by name, Standata is used as a\n", " fallback. Each material is then saved to the platform.\n", - "1. Create workflow: upload the script and its asset files, then assemble a workflow that fetches\n", + "1. Create workflow: upload the script and its data files, then assemble a workflow that fetches\n", " them, fetches the material, and runs the script. Optionally save the workflow to the collection.\n", "1. Configure compute: get list of clusters and create compute configuration with selected cluster,\n", " queue, and number of processors.\n", @@ -43,8 +44,7 @@ "| File | Written by | Contents |\n", "| --- | --- | --- |\n", "| `material.json` | the workflow | the job's material, as stored on the platform |\n", - "| `settings.json` | this notebook | the `SETTINGS` dictionary below, uploaded next to the script |\n", - "| your asset files | this notebook | uploaded verbatim from `USER_ASSET_FILES` |\n", + "| your data files | this notebook | uploaded verbatim from `USER_ASSET_FILES` |\n", "\n", "The script's standard output is the result. Print JSON and this notebook renders it as a table." ] @@ -77,8 +77,8 @@ "source": [ "### 1.2. Set parameters and configurations for the workflow and job\n", "\n", - "`USER_ASSET_FILES` names files your script opens. Put them in the `../uploads` folder first — drag\n", - "them into the JupyterLite file browser — and this notebook uploads them alongside your script." + "`USER_ASSET_FILES` names data files the script opens. Put them in the `../uploads` folder first —\n", + "drag them into the JupyterLite file browser — and this notebook uploads them alongside the script." ] }, { @@ -102,9 +102,8 @@ "MATERIAL_NAMES = [\"Silicon\"] # One job is created per material\n", "\n", "# 4. Script parameters\n", - "USER_REQUIREMENTS = [\"numpy<2\"] # Installed into a virtual environment on the compute node\n", - "USER_ASSET_FILES = [\"radii.json\"] # Files the script opens, taken from FOLDER\n", - "SETTINGS = {\"cutoff_scale\": 1.2} # Uploaded as settings.json next to the script\n", + "USER_REQUIREMENTS = [] # e.g. [\"numpy<2\"], installed into a virtual environment on the compute node\n", + "USER_ASSET_FILES = [] # e.g. [\"radii.json\"], data files the script opens, taken from FOLDER\n", "\n", "# 5. Workflow parameters\n", "WORKFLOW_SEARCH_TERM = \"custom_script.json\" # Search term for Workflows Standata\n", @@ -129,8 +128,8 @@ "source": [ "### 1.3. Set the script to run\n", "\n", - "This is the calculation. It runs on the compute node with `material.json`, `settings.json` and your\n", - "asset files beside it, and whatever it prints becomes the result. Replace it with your own.\n", + "This is the calculation. It runs on the compute node with `material.json` and any data files beside\n", + "it, and whatever it prints becomes the result. Replace it with your own.\n", "\n", "The script is uploaded as a file and fetched onto the node, never inlined into the workflow, so its\n", "contents reach Python exactly as written - text that looks like a template placeholder is left\n", @@ -145,52 +144,29 @@ "outputs": [], "source": [ "USER_SCRIPT = r\"\"\"\n", - "import itertools\n", "import json\n", - "\n", - "import numpy as np\n", + "import math\n", "\n", "material = json.load(open(\"material.json\"))\n", - "settings = json.load(open(\"settings.json\"))\n", - "radii = json.load(open(\"radii.json\"))\n", "\n", - "# The platform stores the cell as lengths and angles, so build the vectors from them.\n", + "# The platform stores the unit cell as lengths and angles, so the volume comes from those.\n", "lattice = material[\"lattice\"]\n", "a, b, c = lattice[\"a\"], lattice[\"b\"], lattice[\"c\"]\n", - "alpha, beta, gamma = (np.radians(lattice[key]) for key in (\"alpha\", \"beta\", \"gamma\"))\n", - "c_x = c * np.cos(beta)\n", - "c_y = c * (np.cos(alpha) - np.cos(beta) * np.cos(gamma)) / np.sin(gamma)\n", - "vectors = np.array(\n", - " [\n", - " [a, 0.0, 0.0],\n", - " [b * np.cos(gamma), b * np.sin(gamma), 0.0],\n", - " [c_x, c_y, np.sqrt(max(c**2 - c_x**2 - c_y**2, 0.0))],\n", - " ]\n", + "cos_alpha, cos_beta, cos_gamma = (math.cos(math.radians(lattice[key])) for key in (\"alpha\", \"beta\", \"gamma\"))\n", + "volume = (\n", + " a\n", + " * b\n", + " * c\n", + " * math.sqrt(1 - cos_alpha**2 - cos_beta**2 - cos_gamma**2 + 2 * cos_alpha * cos_beta * cos_gamma)\n", ")\n", "\n", "elements = [element[\"value\"] for element in material[\"basis\"][\"elements\"]]\n", - "crystal = np.array([point[\"value\"] for point in material[\"basis\"][\"coordinates\"]], dtype=float)\n", - "cartesian = crystal @ vectors\n", - "\n", - "# Count neighbours within scale * (r_i + r_j), including atoms in the neighbouring cells.\n", - "scale = settings[\"cutoff_scale\"]\n", - "images = [np.array(shift) @ vectors for shift in itertools.product((-1, 0, 1), repeat=3)]\n", - "\n", - "coordination = {}\n", - "for element_i, position_i in zip(elements, cartesian):\n", - " neighbors = 0\n", - " for element_j, position_j in zip(elements, cartesian):\n", - " cutoff = scale * (radii[element_i] + radii[element_j])\n", - " for image in images:\n", - " distance = np.linalg.norm(position_i - position_j - image)\n", - " if 0.01 < distance < cutoff:\n", - " neighbors += 1\n", - " coordination[element_i] = neighbors\n", "\n", "print(json.dumps({\n", - " \"formula\": material.get(\"formula\"),\n", + " \"elements\": sorted(set(elements)),\n", " \"n_atoms\": len(elements),\n", - " \"coordination\": coordination,\n", + " \"volume\": round(volume, 4),\n", + " \"atoms_per_volume\": round(len(elements) / volume, 4),\n", "}))\n", "\"\"\"" ] @@ -348,8 +324,7 @@ "id": "20", "metadata": {}, "source": [ - "## 4. Create workflow and set its parameters\n", - "### 4.1. Upload the script and its asset files\n", + "### 4.1. Upload the script and its data files\n", "\n", "The files go to your account's object storage folder (\"Dropbox\"), which the compute node reads them\n", "from. An upload travels inside the request body, which the API caps at 50 MB, and JupyterLite holds\n", @@ -368,7 +343,7 @@ "\n", "from mat3ra.notebooks_utils.core.entity.file.api import upload_files\n", "\n", - "files_to_upload = {\"user_script.py\": USER_SCRIPT, \"settings.json\": json.dumps(SETTINGS)}\n", + "files_to_upload = {\"user_script.py\": USER_SCRIPT}\n", "for name in USER_ASSET_FILES:\n", " if name in files_to_upload:\n", " raise ValueError(f\"Rename '{name}' in USER_ASSET_FILES: this notebook already uploads a file by that name.\")\n", @@ -390,7 +365,7 @@ "\n", "The `Custom Python Script` workflow already carries the unit chain this needs: fetch the uploaded\n", "files, fetch the material, run the script. Three things are filled in per job — the objects to\n", - "fetch, the runner that hands your script its inputs, and the dependency list." + "fetch, the runner that hands the script its inputs, and the dependency list." ] }, { @@ -434,8 +409,9 @@ "### 4.3. Save workflow to collection\n", "\n", "Saving it makes the workflow reusable: it stays in your collection pointing at the uploaded files,\n", - "so it can be run again — from the UI or another notebook — against any material. To change the\n", - "script's parameters, upload a new `settings.json` over the old one." + "so it can be run again — from the UI or another notebook — against any material. Re-running this\n", + "notebook overwrites the uploaded files in place, so a saved workflow picks up an edited script\n", + "without being rebuilt." ] }, { From 27dd1a07532811b282335cef9409746c1fca8b69 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Fri, 21 Aug 2026 18:37:39 -0700 Subject: [PATCH 09/21] =?UTF-8?q?fix(notebook):=20make=20the=20data-file?= =?UTF-8?q?=20contract=20explicit=20=E2=80=94=20UTF-8=20text,=20read=20as?= =?UTF-8?q?=20bytes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Asset files were opened in text mode, so a binary file died on a decode error inside the read and text-mode newline translation meant the upload was not byte-for-byte the file. Reading bytes and decoding UTF-8 strictly fixes both: the upload is verbatim, and a binary file now fails with a message that says where such a file goes instead. An upload travels as a string in a JSON body, so the text-only contract is a property of the endpoint, not a notebook choice. It is stated where the parameter is set, and §4.1 carries the two-line recipe for a file already sitting in the object storage folder - too large or not text - which USER_ASSET_FILES cannot cover. Raised by CodeRabbit on PR #360. --- .../workflows/custom_python_calculation.ipynb | 30 ++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index 556290074..6ed3036da 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -77,8 +77,9 @@ "source": [ "### 1.2. Set parameters and configurations for the workflow and job\n", "\n", - "`USER_ASSET_FILES` names data files the script opens. Put them in the `../uploads` folder first —\n", - "drag them into the JupyterLite file browser — and this notebook uploads them alongside the script." + "`USER_ASSET_FILES` names UTF-8 text files the script opens. Put them in the `../uploads` folder\n", + "first — drag them into the JupyterLite file browser — and this notebook uploads them alongside the\n", + "script." ] }, { @@ -327,9 +328,17 @@ "### 4.1. Upload the script and its data files\n", "\n", "The files go to your account's object storage folder (\"Dropbox\"), which the compute node reads them\n", - "from. An upload travels inside the request body, which the API caps at 50 MB, and JupyterLite holds\n", - "the content in memory before sending it. For anything large, upload it through the Dropbox page in\n", - "the web interface instead and name it in `USER_ASSET_FILES` all the same." + "from. An upload travels as a string inside the request body, which the API caps at 50 MB, and\n", + "JupyterLite holds the content in memory before sending it.\n", + "\n", + "A file that is too large for that, or that is not UTF-8 text, is uploaded through the Dropbox page\n", + "in the web interface instead. It is then already in the folder, so rather than listing it in\n", + "`USER_ASSET_FILES`, add its storage record to `uploaded_files` after the cell below — reusing the\n", + "record of the script, which sits in the same folder:\n", + "\n", + "```python\n", + "uploaded_files.append({**uploaded_files[0], \"key\": f\"{os.path.dirname(uploaded_files[0]['key'])}/big_file.dat\"})\n", + "```" ] }, { @@ -350,8 +359,15 @@ " path = os.path.join(FOLDER, name)\n", " if not os.path.exists(path):\n", " raise FileNotFoundError(f\"'{name}' is listed in USER_ASSET_FILES but is not in {FOLDER}.\")\n", - " with open(path) as file:\n", - " files_to_upload[name] = file.read()\n", + " with open(path, \"rb\") as file:\n", + " content = file.read()\n", + " try:\n", + " files_to_upload[name] = content.decode(\"utf-8\")\n", + " except UnicodeDecodeError:\n", + " raise ValueError(\n", + " f\"'{name}' is not UTF-8 text. An upload travels as a JSON string, so binary files go \"\n", + " f\"through the Dropbox page of the web interface instead, into the same folder.\"\n", + " )\n", "\n", "uploaded_files = upload_files(client, files_to_upload, ACCOUNT_ID)" ] From 9b7901ec497465e5658b8686369aa89b58fce2d8 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Sun, 23 Aug 2026 19:35:16 -0700 Subject: [PATCH 10/21] feat(notebook): band structure with a user-uploaded pseudopotential The example Vsevolod asked for on 2026-08-23: upload a custom pseudopotential (Si.upf, ONCVPSP NC PBE, ships in uploads/) through the files endpoint and run a Quantum ESPRESSO band structure for Si that consumes it - driven entirely from Python. Three verified pieces line it up, none of them new platform machinery: - pw templates render pseudo_dir = JOB_WORK_DIR/pseudo; - an object_storage io unit with pathname "pseudo" fetches the upload exactly there (rupy joins work_dir/pathname/basename), and survives job creation intact; - the ATOMIC_SPECIES cards are pointed at the file by the documented expert-mode input edit with isManuallyChanged, which survives job update while a method.data.pseudo edit does not (formMethodData re-resolves it from metaProperties on every create and update - probed, not guessed). to_object_storage_input grows an optional pathname (default keeps current behaviour); unit-tested. The io-unit attachment is idempotent across re-runs. The Cypress feature asserting the consumed inputs name Si.upf is in web-app. The local runtime gate is queued behind the compute broker, which is currently unreachable (app-side SYN_SENT, heartbeats stopped 08-21); results will be reported when it returns. --- other/materials_designer/uploads/Si.upf | 3 + .../workflows/Introduction.ipynb | 3 +- ...and_structure_custom_pseudopotential.ipynb | 679 ++++++++++++++++++ .../notebooks_utils/core/entity/file/api.py | 9 +- tests/py/unit/core/entity/test_file_api.py | 8 + 5 files changed, 698 insertions(+), 4 deletions(-) create mode 100644 other/materials_designer/uploads/Si.upf create mode 100644 other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb diff --git a/other/materials_designer/uploads/Si.upf b/other/materials_designer/uploads/Si.upf new file mode 100644 index 000000000..4fb6731d6 --- /dev/null +++ b/other/materials_designer/uploads/Si.upf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:39822757f53f36e3bf3bfb779356152a8d3f21199c7db9dd5a931e5d18c45282 +size 225602 diff --git a/other/materials_designer/workflows/Introduction.ipynb b/other/materials_designer/workflows/Introduction.ipynb index 531e08e6c..8531e9828 100644 --- a/other/materials_designer/workflows/Introduction.ipynb +++ b/other/materials_designer/workflows/Introduction.ipynb @@ -106,7 +106,8 @@ "\n", "### 9.1. Python / Shell\n", "#### [9.1.1. Custom Python calculation.](custom_python_calculation.ipynb)\n", - "#### 9.1.2. Custom Shell calculation. *(to be added)*\n" + "#### [9.1.2. Band structure with a custom pseudopotential.](band_structure_custom_pseudopotential.ipynb)\n", + "#### 9.1.3. Custom Shell calculation. *(to be added)*\n" ] }, { diff --git a/other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb b/other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb new file mode 100644 index 000000000..c92726f81 --- /dev/null +++ b/other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb @@ -0,0 +1,679 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Band Structure with a Custom Pseudopotential\n", + "\n", + "Calculate the electronic band structure of a material with a pseudopotential you provide, driven\n", + "entirely from Python. The pseudopotential is uploaded to your object storage folder (\"Dropbox\"),\n", + "a workflow unit fetches it onto the compute node next to the calculation, and the Quantum\n", + "ESPRESSO input is pointed at it.\n", + "\n", + "

Usage

\n", + "\n", + "1. Put your pseudopotential file in the `../uploads` folder (a Si example, `Si.upf`, ships with\n", + " this notebook) and set the parameters in cell 1.2. below.\n", + "1. Click \"Run\" > \"Run All\" to run all cells.\n", + "1. Wait for the job to complete.\n", + "1. Scroll down to view the result.\n", + "\n", + "## Summary\n", + "\n", + "1. Set up the environment and parameters: install packages (JupyterLite only) and configure the\n", + " material, the pseudopotential file, workflow, compute resources, and job.\n", + "1. Authenticate and initialize API client: authenticate via browser, initialize the client, then\n", + " select account and project.\n", + "1. Create material: read from the `../uploads` folder, with Standata as a fallback, then saved to\n", + " the platform.\n", + "1. Configure workflow: upload the pseudopotential, load the band structure workflow from Standata,\n", + " set the model, and attach a unit that fetches the uploaded file onto the compute node.\n", + "1. Configure compute: get list of clusters and create compute configuration.\n", + "1. Create the job, then point its Quantum ESPRESSO inputs at the uploaded pseudopotential.\n", + "1. Submit the job and monitor the status.\n", + "1. Retrieve results: confirm which pseudopotential was used and display the band structure.\n", + "\n", + "## How the custom pseudopotential reaches the calculation\n", + "\n", + "Quantum ESPRESSO reads pseudopotentials from the job's `pseudo` directory (`pseudo_dir` in the\n", + "input). Three pieces line up to put your file there and make the calculation use it:\n", + "\n", + "| Piece | What it does |\n", + "| --- | --- |\n", + "| upload (§4.1) | puts the file in your account's object storage folder |\n", + "| io unit (§4.5) | fetches it into the job's `pseudo` directory on the compute node |\n", + "| input edit (§6.2) | names the file in the `ATOMIC_SPECIES` card of each pw.x input |\n", + "\n", + "The default pseudopotential the platform would have picked is simply never referenced." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## 1. Set up the environment and parameters\n", + "### 1.1. Install packages (JupyterLite)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.packages import install_packages\n", + "\n", + "await install_packages(\"made|api_examples\")" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "### 1.2. Set parameters\n", + "\n", + "`PSEUDO_FILE` names a UPF file in the `../uploads` folder — drag your own into the JupyterLite\n", + "file browser to replace the shipped example. `PSEUDO_ELEMENT` is the element it is for, and the\n", + "model parameters below describe it (the shipped `Si.upf` is norm-conserving PBE)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "from datetime import datetime\n", + "from mat3ra.ide.compute import QueueName\n", + "\n", + "# 2. Auth and organization parameters\n", + "ORGANIZATION_NAME = None\n", + "\n", + "# 3. Material parameters\n", + "FOLDER = \"../uploads\"\n", + "MATERIAL_NAME = \"Silicon\"\n", + "\n", + "# 4. Pseudopotential parameters\n", + "PSEUDO_FILE = \"Si.upf\"\n", + "PSEUDO_ELEMENT = \"Si\"\n", + "\n", + "# 4. Workflow parameters\n", + "APPLICATION_NAME = \"espresso\"\n", + "WORKFLOW_SEARCH_TERM = \"band_structure.json\"\n", + "MY_WORKFLOW_NAME = \"Band Structure - Custom Pseudopotential\"\n", + "\n", + "# Model parameters, describing the uploaded pseudopotential\n", + "MODEL_SUBTYPE = \"gga\" # \"gga\" or \"lda\"\n", + "FUNCTIONAL = \"pbe\" # for gga: \"pbe\", \"pbesol\"; for lda: \"pz\"\n", + "PSEUDOPOTENTIAL_TYPE = \"nc\" # \"us\" (ultrasoft), \"nc\" (norm-conserving), \"paw\"\n", + "\n", + "# Energy cutoffs\n", + "ECUTWFC = 40\n", + "ECUTRHO = 200\n", + "\n", + "# 5. Compute parameters\n", + "CLUSTER_NAME = None # specify full or partial name i.e. \"cluster-001\" to select\n", + "QUEUE_NAME = QueueName.D\n", + "PPN = 1\n", + "\n", + "# 6. Job parameters\n", + "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", + "POLL_INTERVAL = 30 # seconds" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "## 2. Authenticate and initialize API client\n", + "### 2.1. Authenticate\n", + "Authenticate in the browser and have credentials stored in environment variable \"OIDC_ACCESS_TOKEN\"." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.auth import authenticate\n", + "\n", + "await authenticate()" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "### 2.2. Initialize API client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.api_client import APIClient\n", + "\n", + "client = APIClient.authenticate()\n", + "client" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "### 2.3. Select account" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "client.list_accounts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "11", + "metadata": {}, + "outputs": [], + "source": [ + "selected_account = client.my_account\n", + "\n", + "if ORGANIZATION_NAME:\n", + " selected_account = client.get_account(name=ORGANIZATION_NAME)\n", + "\n", + "ACCOUNT_ID = selected_account.id\n", + "print(f\"\\u2705 Selected account ID: {ACCOUNT_ID}, name: {selected_account.name}\")" + ] + }, + { + "cell_type": "markdown", + "id": "12", + "metadata": {}, + "source": [ + "### 2.4. Select project" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"\\u2705 Using project: {projects[0]['name']} ({project_id})\")" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "## 3. Create material\n", + "### 3.1. Load material from local file (or Standata)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.made.material import Material\n", + "from mat3ra.standata.materials import Materials\n", + "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", + "from mat3ra.notebooks_utils.material import load_material_from_folder\n", + "\n", + "material = load_material_from_folder(FOLDER, MATERIAL_NAME) or Material.create(\n", + " Materials.get_by_name_first_match(MATERIAL_NAME))\n", + "\n", + "visualize(material)" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "### 3.2. Save material to the platform" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.core.entity.material.api import get_or_create_material\n", + "\n", + "material.basis.set_labels_from_list([])\n", + "saved_material_response = get_or_create_material(client, material, ACCOUNT_ID)\n", + "saved_material = Material.create(saved_material_response)" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "## 4. Configure workflow\n", + "### 4.1. Upload the pseudopotential\n", + "\n", + "The file goes to your account's object storage folder, which the compute node fetches it from.\n", + "An upload travels as a string inside the request body, which the API caps at 50 MB, and must be\n", + "UTF-8 text — UPF files are XML-like text, so they qualify. Anything larger goes to the same\n", + "folder through the Dropbox page in the web interface instead." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from mat3ra.notebooks_utils.core.entity.file.api import upload_files\n", + "\n", + "path = os.path.join(FOLDER, PSEUDO_FILE)\n", + "if not os.path.exists(path):\n", + " raise FileNotFoundError(f\"'{PSEUDO_FILE}' is not in {FOLDER}. Drag it into the file browser first.\")\n", + "with open(path, \"rb\") as file:\n", + " content = file.read().decode(\"utf-8\")\n", + "\n", + "uploaded_pseudo = upload_files(client, {PSEUDO_FILE: content}, ACCOUNT_ID)[0]" + ] + }, + { + "cell_type": "markdown", + "id": "20", + "metadata": {}, + "source": [ + "### 4.2. Load workflow from Standata and preview it" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.standata.applications import ApplicationStandata\n", + "from mat3ra.ade.application import Application\n", + "from mat3ra.standata.workflows import WorkflowStandata\n", + "from mat3ra.wode.workflows import Workflow\n", + "from mat3ra.notebooks_utils.ipython.entity.workflow.visualize import visualize_workflow\n", + "\n", + "app_config = ApplicationStandata.get_by_name_first_match(APPLICATION_NAME)\n", + "app = Application(**app_config)\n", + "\n", + "workflow_config = WorkflowStandata.filter_by_application(app.name).get_by_name_first_match(WORKFLOW_SEARCH_TERM)\n", + "workflow = Workflow.create(workflow_config)\n", + "workflow.name = MY_WORKFLOW_NAME\n", + "\n", + "visualize_workflow(workflow)" + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, + "source": [ + "### 4.3. Set the model to match the uploaded pseudopotential\n", + "\n", + "The method subtype tells the platform what kind of pseudopotential the calculation uses, and the\n", + "cutoffs should suit it — norm-conserving potentials typically want a higher wavefunction cutoff\n", + "than ultrasoft ones." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.standata.model_tree import ModelTreeStandata\n", + "from mat3ra.mode import ModelFactory\n", + "from mat3ra.wode.context.providers import PlanewaveCutoffsContextProvider\n", + "\n", + "model_config = ModelTreeStandata.get_model_by_parameters(\n", + " type=\"dft\", subtype=MODEL_SUBTYPE, functional=FUNCTIONAL\n", + ")\n", + "model_config[\"method\"] = {\"type\": \"pseudopotential\", \"subtype\": PSEUDOPOTENTIAL_TYPE}\n", + "model = ModelFactory.create(model_config)\n", + "\n", + "for subworkflow in workflow.subworkflows:\n", + " subworkflow.model = model\n", + "\n", + "cutoffs_context = PlanewaveCutoffsContextProvider(\n", + " wavefunction=ECUTWFC, density=ECUTRHO, isEdited=True\n", + ").get_context_item_data()\n", + "for unit_name in [\"pw_scf\", \"pw_bands\"]:\n", + " unit = workflow.subworkflows[0].get_unit_by_name(name=unit_name)\n", + " if unit:\n", + " unit.add_context(cutoffs_context)\n", + " workflow.subworkflows[0].set_unit(unit)" + ] + }, + { + "cell_type": "markdown", + "id": "24", + "metadata": {}, + "source": [ + "### 4.4. Save workflow to collection" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.core.entity.workflow.api import get_or_create_workflow\n", + "\n", + "saved_workflow_response = get_or_create_workflow(client, workflow, ACCOUNT_ID)\n", + "workflow_id = saved_workflow_response[\"_id\"]\n", + "print(f\"Workflow ID: {workflow_id}\")" + ] + }, + { + "cell_type": "markdown", + "id": "26", + "metadata": {}, + "source": [ + "### 4.5. Attach a unit that fetches the pseudopotential onto the compute node\n", + "\n", + "An `io` unit is prepended to the saved workflow. It downloads the uploaded file into the job's\n", + "`pseudo` subdirectory — the exact directory `pseudo_dir` in the Quantum ESPRESSO inputs points\n", + "at — before the first pw.x unit runs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.core.entity.file.api import to_object_storage_input\n", + "\n", + "saved_workflow = client.workflows.get(workflow_id)\n", + "subworkflow = saved_workflow[\"subworkflows\"][0]\n", + "first_unit = subworkflow[\"units\"][0]\n", + "pseudo_input = [to_object_storage_input(uploaded_pseudo, pathname=\"pseudo\")]\n", + "\n", + "if first_unit[\"name\"] == \"io-pseudo\":\n", + " # Already attached by a previous run of this notebook - refresh it to the latest upload.\n", + " first_unit[\"input\"] = pseudo_input\n", + "else:\n", + " io_unit = {\n", + " \"type\": \"io\",\n", + " \"name\": \"io-pseudo\",\n", + " \"subtype\": \"input\",\n", + " \"source\": \"object_storage\",\n", + " \"input\": pseudo_input,\n", + " # Nothing references this id, but a unit needs one to sit in the flowchart; namespaced by\n", + " # workflow so it cannot clash.\n", + " \"flowchartId\": \"band-structure-custom-pseudo-io\",\n", + " \"head\": True,\n", + " \"next\": first_unit[\"flowchartId\"],\n", + " \"status\": \"idle\",\n", + " \"statusTrack\": [],\n", + " \"results\": [],\n", + " \"monitors\": [],\n", + " \"preProcessors\": [],\n", + " \"postProcessors\": [],\n", + " }\n", + " first_unit[\"head\"] = False\n", + " subworkflow[\"units\"].insert(0, io_unit)\n", + "\n", + "client.workflows.update(workflow_id, saved_workflow)\n", + "workflow_with_pseudo = client.workflows.get(workflow_id)\n", + "print(\"Units:\", [unit[\"name\"] for unit in workflow_with_pseudo[\"subworkflows\"][0][\"units\"]])" + ] + }, + { + "cell_type": "markdown", + "id": "28", + "metadata": {}, + "source": [ + "## 5. Create the compute configuration\n", + "### 5.1. Get list of clusters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "clusters = client.clusters.list()\n", + "print(f\"Available clusters: {[c['hostname'] for c in clusters]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "30", + "metadata": {}, + "source": [ + "### 5.2. Create compute configuration" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.ide.compute import Compute\n", + "\n", + "if CLUSTER_NAME:\n", + " cluster = next((c for c in clusters if CLUSTER_NAME in c[\"hostname\"]), None)\n", + "else:\n", + " cluster = clusters[0]\n", + "\n", + "compute = Compute(\n", + " cluster=cluster,\n", + " queue=QUEUE_NAME,\n", + " ppn=PPN\n", + ")\n", + "print(f\"Using cluster: {compute.cluster.hostname}, queue: {QUEUE_NAME}, ppn: {PPN}\")" + ] + }, + { + "cell_type": "markdown", + "id": "32", + "metadata": {}, + "source": [ + "## 6. Create the job\n", + "### 6.1. Create job" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.job import create_job\n", + "from mat3ra.notebooks_utils.ui import display_JSON\n", + "\n", + "job_name = MY_WORKFLOW_NAME + \" \" + saved_material.formula + \" \" + timestamp\n", + "job_response = create_job(\n", + " api_client=client,\n", + " materials=[saved_material],\n", + " workflow=workflow_with_pseudo,\n", + " project_id=project_id,\n", + " owner_id=ACCOUNT_ID,\n", + " prefix=job_name,\n", + " compute=compute.to_dict(),\n", + ")\n", + "\n", + "job_id = job_response[\"_id\"]\n", + "print(\"✅ Job created successfully!\")\n", + "print(f\"Job ID: {job_id}\")\n", + "display_JSON(job_response)" + ] + }, + { + "cell_type": "markdown", + "id": "34", + "metadata": {}, + "source": [ + "### 6.2. Point the inputs at the uploaded pseudopotential\n", + "\n", + "At job creation the platform fills the `ATOMIC_SPECIES` card of each pw.x input with its default\n", + "pseudopotential for the element. The lines are edited here to name the uploaded file instead —\n", + "the expert-mode input edit the platform documentation describes — and `isManuallyChanged` keeps\n", + "the edit from being re-rendered." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "35", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "\n", + "job = client.jobs.get(job_id)\n", + "pattern = re.compile(rf\"^(\\s*{PSEUDO_ELEMENT}\\s+[\\d.]+\\s+)\\S+\", re.MULTILINE)\n", + "\n", + "for unit in job[\"workflow\"][\"subworkflows\"][0][\"units\"]:\n", + " for unit_input in unit.get(\"input\", []):\n", + " rendered = unit_input.get(\"rendered\") or \"\"\n", + " if \"ATOMIC_SPECIES\" not in rendered:\n", + " continue\n", + " unit_input[\"rendered\"] = pattern.sub(rf\"\\g<1>{PSEUDO_FILE}\", rendered)\n", + " unit_input[\"isManuallyChanged\"] = True\n", + " print(f\"✅ {unit['name']} now uses {PSEUDO_FILE}\")\n", + "\n", + "client.jobs.update(job_id, job)" + ] + }, + { + "cell_type": "markdown", + "id": "36", + "metadata": {}, + "source": [ + "## 7. Submit the job and monitor the status" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "37", + "metadata": {}, + "outputs": [], + "source": [ + "client.jobs.submit(job_id)\n", + "print(f\"✅ Job {job_id} submitted successfully!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.api.job import wait_for_jobs_to_finish_async\n", + "\n", + "await wait_for_jobs_to_finish_async(client.jobs, [job_id], poll_interval=POLL_INTERVAL)" + ] + }, + { + "cell_type": "markdown", + "id": "39", + "metadata": {}, + "source": [ + "## 8. Retrieve and visualize results\n", + "### 8.1. Confirm the pseudopotential that was used\n", + "\n", + "The `ATOMIC_SPECIES` card of each pw.x input, exactly as the calculation consumed it." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [ + "finished_job = client.jobs.get(job_id)\n", + "for unit in finished_job[\"workflow\"][\"subworkflows\"][0][\"units\"]:\n", + " for unit_input in unit.get(\"input\", []):\n", + " rendered = unit_input.get(\"rendered\") or \"\"\n", + " if \"ATOMIC_SPECIES\" in rendered:\n", + " card = rendered[rendered.index(\"ATOMIC_SPECIES\"):].split(\"\\n\\n\")[0]\n", + " print(f\"--- {unit['name']} ---\\n{card}\")" + ] + }, + { + "cell_type": "markdown", + "id": "41", + "metadata": {}, + "source": [ + "### 8.2. Band Structure" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "42", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.prode import PropertyName\n", + "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", + "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", + "\n", + "band_structure_data = get_properties_for_job(client, job_id, property_name=PropertyName.non_scalar.band_structure.value)\n", + "visualize_properties(band_structure_data, title=\"Band Structure\", extra_config={\"material\": material.to_dict()})" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py index 388d609b6..94eabe9e0 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py @@ -62,21 +62,24 @@ def upload_files(api_client: APIClient, files: Dict[str, str], account_id: str) return uploaded -def to_object_storage_input(cloud_file: dict) -> dict: +def to_object_storage_input(cloud_file: dict, pathname: str = "") -> dict: """ Converts an upload record into an `object_storage` input item for a workflow IO unit. Args: cloud_file (dict): A record returned by `upload_files`. + pathname (str): Subdirectory of the job's working directory to fetch into. The runner + creates it if needed. E.g. "pseudo" places the file where Quantum ESPRESSO's + `pseudo_dir` points. Returns: dict: IO unit input item. The runner fetches it into the job's working directory under - `basename`, which is what makes the user script's relative paths resolve. + `pathname`/`basename`, which is what makes relative paths resolve. """ return { "type": "object_storage", "basename": cloud_file["key"].split("/")[-1], - "pathname": "", + "pathname": pathname, "objectData": { "NAME": cloud_file["key"], "PROVIDER": cloud_file["provider"], diff --git a/tests/py/unit/core/entity/test_file_api.py b/tests/py/unit/core/entity/test_file_api.py index c5ef8346b..28c96a5e1 100644 --- a/tests/py/unit/core/entity/test_file_api.py +++ b/tests/py/unit/core/entity/test_file_api.py @@ -79,3 +79,11 @@ def test_object_storage_basename_is_the_name_the_script_opens(): nested = {**CLOUD_FILE, "key": "my-account/assets/radii.json"} assert to_object_storage_input(nested)["basename"] == "radii.json" + + +def test_object_storage_pathname_targets_a_subdirectory(): + """A pathname of "pseudo" fetches the file into /pseudo — where QE's pseudo_dir points.""" + item = to_object_storage_input(CLOUD_FILE, pathname="pseudo") + + assert item["pathname"] == "pseudo" + assert item["basename"] == "user_script.py" From 53b55067a32e85c0d83a57eb9de8fc9d7efcb6dc Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Sun, 23 Aug 2026 21:15:23 -0700 Subject: [PATCH 11/21] fix(notebook): scope the pseudopotential substitution to the ATOMIC_SPECIES card The bare multiline "element number token" pattern also matches ATOMIC_POSITIONS lines, where it overwrote the second coordinate with the filename - reproduced on a real rendered input: "Si 0.000000000 Si.upf 0.000000000". The substitution now runs only on the ATOMIC_SPECIES card slice, and the check against a real job document full-diffs the input: exactly one token changes per pw unit. Raised by CodeRabbit on PR #360 (a genuine catch - the earlier verification asserted the SPECIES card and never diffed the rest). --- .../band_structure_custom_pseudopotential.ipynb | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb b/other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb index c92726f81..8ecfeca37 100644 --- a/other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb +++ b/other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb @@ -567,6 +567,9 @@ "import re\n", "\n", "job = client.jobs.get(job_id)\n", + "# Element + mass + filename, matched only inside the ATOMIC_SPECIES card - the same\n", + "# \"element number number\" shape also occurs in ATOMIC_POSITIONS, where a bare substitution\n", + "# would overwrite a coordinate.\n", "pattern = re.compile(rf\"^(\\s*{PSEUDO_ELEMENT}\\s+[\\d.]+\\s+)\\S+\", re.MULTILINE)\n", "\n", "for unit in job[\"workflow\"][\"subworkflows\"][0][\"units\"]:\n", @@ -574,7 +577,11 @@ " rendered = unit_input.get(\"rendered\") or \"\"\n", " if \"ATOMIC_SPECIES\" not in rendered:\n", " continue\n", - " unit_input[\"rendered\"] = pattern.sub(rf\"\\g<1>{PSEUDO_FILE}\", rendered)\n", + " start = rendered.index(\"ATOMIC_SPECIES\")\n", + " end = rendered.find(\"\\n\\n\", start)\n", + " end = len(rendered) if end == -1 else end\n", + " card = pattern.sub(rf\"\\g<1>{PSEUDO_FILE}\", rendered[start:end])\n", + " unit_input[\"rendered\"] = rendered[:start] + card + rendered[end:]\n", " unit_input[\"isManuallyChanged\"] = True\n", " print(f\"✅ {unit['name']} now uses {PSEUDO_FILE}\")\n", "\n", From 28b681c62877ebbfb682ed71b8bc2cda537aab87 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Sun, 23 Aug 2026 22:21:01 -0700 Subject: [PATCH 12/21] fix(notebook): attach the io unit before saving, not by updating the saved workflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first live run failed at §4.5: PATCH /workflows/:id 500s for an API-created workflow because the update handler scopes its lookup to workflows inSet of the caller's sets - a bank-copied workflow is in the account's set, a created one is not (ENTITY_NOT_FOUND surfaced as the known null-return 500). Reproduced against the live instance; the earlier probe had validated the update on a bank copy only. The io unit is now inserted into the workflow config before Workflow.create, so the single create call stores the complete four-unit workflow - verified live: created workflow carries io-pseudo at head, and a job from it renders with the chain intact. Also switched the save from get_or_create_workflow to a plain create: the list query the helper filters by is ignored by the platform (recorded finding), so its "reuse" can return an arbitrary workflow - observed returning a Total Energy workflow - which would silently drop the io unit. --- ...and_structure_custom_pseudopotential.ipynb | 126 +++++++----------- 1 file changed, 48 insertions(+), 78 deletions(-) diff --git a/other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb b/other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb index 8ecfeca37..36c3018a6 100644 --- a/other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb +++ b/other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb @@ -43,7 +43,7 @@ "| Piece | What it does |\n", "| --- | --- |\n", "| upload (§4.1) | puts the file in your account's object storage folder |\n", - "| io unit (§4.5) | fetches it into the job's `pseudo` directory on the compute node |\n", + "| io unit (§4.2) | fetches it into the job's `pseudo` directory on the compute node |\n", "| input edit (§6.2) | names the file in the `ATOMIC_SPECIES` card of each pw.x input |\n", "\n", "The default pseudopotential the platform would have picked is simply never referenced." @@ -312,7 +312,11 @@ "id": "20", "metadata": {}, "source": [ - "### 4.2. Load workflow from Standata and preview it" + "### 4.2. Load workflow from Standata, attach the fetch of the pseudopotential, and preview\n", + "\n", + "An `io` unit is placed at the head of the workflow before it is built. It downloads the uploaded\n", + "file into the job's `pseudo` subdirectory — the exact directory `pseudo_dir` in the Quantum\n", + "ESPRESSO inputs points at — before the first pw.x unit runs." ] }, { @@ -326,12 +330,36 @@ "from mat3ra.ade.application import Application\n", "from mat3ra.standata.workflows import WorkflowStandata\n", "from mat3ra.wode.workflows import Workflow\n", + "from mat3ra.notebooks_utils.core.entity.file.api import to_object_storage_input\n", "from mat3ra.notebooks_utils.ipython.entity.workflow.visualize import visualize_workflow\n", "\n", "app_config = ApplicationStandata.get_by_name_first_match(APPLICATION_NAME)\n", "app = Application(**app_config)\n", "\n", "workflow_config = WorkflowStandata.filter_by_application(app.name).get_by_name_first_match(WORKFLOW_SEARCH_TERM)\n", + "subworkflow = workflow_config[\"subworkflows\"][0]\n", + "first_unit = subworkflow[\"units\"][0]\n", + "io_unit = {\n", + " \"type\": \"io\",\n", + " \"name\": \"io-pseudo\",\n", + " \"subtype\": \"input\",\n", + " \"source\": \"object_storage\",\n", + " \"input\": [to_object_storage_input(uploaded_pseudo, pathname=\"pseudo\")],\n", + " # Nothing references this id, but a unit needs one to sit in the flowchart; namespaced by\n", + " # workflow so it cannot clash.\n", + " \"flowchartId\": \"band-structure-custom-pseudo-io\",\n", + " \"head\": True,\n", + " \"next\": first_unit[\"flowchartId\"],\n", + " \"status\": \"idle\",\n", + " \"statusTrack\": [],\n", + " \"results\": [],\n", + " \"monitors\": [],\n", + " \"preProcessors\": [],\n", + " \"postProcessors\": [],\n", + "}\n", + "first_unit[\"head\"] = False\n", + "subworkflow[\"units\"].insert(0, io_unit)\n", + "\n", "workflow = Workflow.create(workflow_config)\n", "workflow.name = MY_WORKFLOW_NAME\n", "\n", @@ -395,73 +423,15 @@ "metadata": {}, "outputs": [], "source": [ - "from mat3ra.notebooks_utils.core.entity.workflow.api import get_or_create_workflow\n", - "\n", - "saved_workflow_response = get_or_create_workflow(client, workflow, ACCOUNT_ID)\n", - "workflow_id = saved_workflow_response[\"_id\"]\n", - "print(f\"Workflow ID: {workflow_id}\")" + "saved_workflow = client.workflows.create(workflow.to_dict_without_special_keys(), owner_id=ACCOUNT_ID)\n", + "workflow_id = saved_workflow[\"_id\"]\n", + "print(f\"✅ Workflow saved to collection: {workflow_id}\")" ] }, { "cell_type": "markdown", "id": "26", "metadata": {}, - "source": [ - "### 4.5. Attach a unit that fetches the pseudopotential onto the compute node\n", - "\n", - "An `io` unit is prepended to the saved workflow. It downloads the uploaded file into the job's\n", - "`pseudo` subdirectory — the exact directory `pseudo_dir` in the Quantum ESPRESSO inputs points\n", - "at — before the first pw.x unit runs." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "27", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.notebooks_utils.core.entity.file.api import to_object_storage_input\n", - "\n", - "saved_workflow = client.workflows.get(workflow_id)\n", - "subworkflow = saved_workflow[\"subworkflows\"][0]\n", - "first_unit = subworkflow[\"units\"][0]\n", - "pseudo_input = [to_object_storage_input(uploaded_pseudo, pathname=\"pseudo\")]\n", - "\n", - "if first_unit[\"name\"] == \"io-pseudo\":\n", - " # Already attached by a previous run of this notebook - refresh it to the latest upload.\n", - " first_unit[\"input\"] = pseudo_input\n", - "else:\n", - " io_unit = {\n", - " \"type\": \"io\",\n", - " \"name\": \"io-pseudo\",\n", - " \"subtype\": \"input\",\n", - " \"source\": \"object_storage\",\n", - " \"input\": pseudo_input,\n", - " # Nothing references this id, but a unit needs one to sit in the flowchart; namespaced by\n", - " # workflow so it cannot clash.\n", - " \"flowchartId\": \"band-structure-custom-pseudo-io\",\n", - " \"head\": True,\n", - " \"next\": first_unit[\"flowchartId\"],\n", - " \"status\": \"idle\",\n", - " \"statusTrack\": [],\n", - " \"results\": [],\n", - " \"monitors\": [],\n", - " \"preProcessors\": [],\n", - " \"postProcessors\": [],\n", - " }\n", - " first_unit[\"head\"] = False\n", - " subworkflow[\"units\"].insert(0, io_unit)\n", - "\n", - "client.workflows.update(workflow_id, saved_workflow)\n", - "workflow_with_pseudo = client.workflows.get(workflow_id)\n", - "print(\"Units:\", [unit[\"name\"] for unit in workflow_with_pseudo[\"subworkflows\"][0][\"units\"]])" - ] - }, - { - "cell_type": "markdown", - "id": "28", - "metadata": {}, "source": [ "## 5. Create the compute configuration\n", "### 5.1. Get list of clusters" @@ -470,7 +440,7 @@ { "cell_type": "code", "execution_count": null, - "id": "29", + "id": "27", "metadata": {}, "outputs": [], "source": [ @@ -480,7 +450,7 @@ }, { "cell_type": "markdown", - "id": "30", + "id": "28", "metadata": {}, "source": [ "### 5.2. Create compute configuration" @@ -489,7 +459,7 @@ { "cell_type": "code", "execution_count": null, - "id": "31", + "id": "29", "metadata": {}, "outputs": [], "source": [ @@ -510,7 +480,7 @@ }, { "cell_type": "markdown", - "id": "32", + "id": "30", "metadata": {}, "source": [ "## 6. Create the job\n", @@ -520,7 +490,7 @@ { "cell_type": "code", "execution_count": null, - "id": "33", + "id": "31", "metadata": {}, "outputs": [], "source": [ @@ -531,7 +501,7 @@ "job_response = create_job(\n", " api_client=client,\n", " materials=[saved_material],\n", - " workflow=workflow_with_pseudo,\n", + " workflow=saved_workflow,\n", " project_id=project_id,\n", " owner_id=ACCOUNT_ID,\n", " prefix=job_name,\n", @@ -546,7 +516,7 @@ }, { "cell_type": "markdown", - "id": "34", + "id": "32", "metadata": {}, "source": [ "### 6.2. Point the inputs at the uploaded pseudopotential\n", @@ -560,7 +530,7 @@ { "cell_type": "code", "execution_count": null, - "id": "35", + "id": "33", "metadata": {}, "outputs": [], "source": [ @@ -590,7 +560,7 @@ }, { "cell_type": "markdown", - "id": "36", + "id": "34", "metadata": {}, "source": [ "## 7. Submit the job and monitor the status" @@ -599,7 +569,7 @@ { "cell_type": "code", "execution_count": null, - "id": "37", + "id": "35", "metadata": {}, "outputs": [], "source": [ @@ -610,7 +580,7 @@ { "cell_type": "code", "execution_count": null, - "id": "38", + "id": "36", "metadata": {}, "outputs": [], "source": [ @@ -621,7 +591,7 @@ }, { "cell_type": "markdown", - "id": "39", + "id": "37", "metadata": {}, "source": [ "## 8. Retrieve and visualize results\n", @@ -633,7 +603,7 @@ { "cell_type": "code", "execution_count": null, - "id": "40", + "id": "38", "metadata": {}, "outputs": [], "source": [ @@ -648,7 +618,7 @@ }, { "cell_type": "markdown", - "id": "41", + "id": "39", "metadata": {}, "source": [ "### 8.2. Band Structure" @@ -657,7 +627,7 @@ { "cell_type": "code", "execution_count": null, - "id": "42", + "id": "40", "metadata": {}, "outputs": [], "source": [ From 77aee20a8d7089b0808eead58d3583f5e069647e Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 24 Aug 2026 09:38:51 -0700 Subject: [PATCH 13/21] refactor(notebook): fold the pseudopotential example into custom_python_calculation Per Vsevolod: "It needs to be custom python! Not separate arbitrary thing. We upload pseudo thru it, we run the QE job with it. That's the idea." band_structure_custom_pseudopotential.ipynb is deleted; its verified mechanics move into custom_python_calculation.ipynb as section 9: upload Si.upf through the same upload call as section 4.1, prepend the io unit that fetches it into the job's pseudo directory, point the ATOMIC_SPECIES cards at it, run the Quantum ESPRESSO band structure, print the consumed cards and plot the bands. One notebook, one upload path, and the example demonstrates that an upload is an ordinary platform file any application can consume. TOC entry for the separate notebook removed. --- .../workflows/Introduction.ipynb | 3 +- ...and_structure_custom_pseudopotential.ipynb | 656 ------------------ .../workflows/custom_python_calculation.ipynb | 177 +++++ 3 files changed, 178 insertions(+), 658 deletions(-) delete mode 100644 other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb diff --git a/other/materials_designer/workflows/Introduction.ipynb b/other/materials_designer/workflows/Introduction.ipynb index 8531e9828..531e08e6c 100644 --- a/other/materials_designer/workflows/Introduction.ipynb +++ b/other/materials_designer/workflows/Introduction.ipynb @@ -106,8 +106,7 @@ "\n", "### 9.1. Python / Shell\n", "#### [9.1.1. Custom Python calculation.](custom_python_calculation.ipynb)\n", - "#### [9.1.2. Band structure with a custom pseudopotential.](band_structure_custom_pseudopotential.ipynb)\n", - "#### 9.1.3. Custom Shell calculation. *(to be added)*\n" + "#### 9.1.2. Custom Shell calculation. *(to be added)*\n" ] }, { diff --git a/other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb b/other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb deleted file mode 100644 index 36c3018a6..000000000 --- a/other/materials_designer/workflows/band_structure_custom_pseudopotential.ipynb +++ /dev/null @@ -1,656 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "0", - "metadata": {}, - "source": [ - "# Band Structure with a Custom Pseudopotential\n", - "\n", - "Calculate the electronic band structure of a material with a pseudopotential you provide, driven\n", - "entirely from Python. The pseudopotential is uploaded to your object storage folder (\"Dropbox\"),\n", - "a workflow unit fetches it onto the compute node next to the calculation, and the Quantum\n", - "ESPRESSO input is pointed at it.\n", - "\n", - "

Usage

\n", - "\n", - "1. Put your pseudopotential file in the `../uploads` folder (a Si example, `Si.upf`, ships with\n", - " this notebook) and set the parameters in cell 1.2. below.\n", - "1. Click \"Run\" > \"Run All\" to run all cells.\n", - "1. Wait for the job to complete.\n", - "1. Scroll down to view the result.\n", - "\n", - "## Summary\n", - "\n", - "1. Set up the environment and parameters: install packages (JupyterLite only) and configure the\n", - " material, the pseudopotential file, workflow, compute resources, and job.\n", - "1. Authenticate and initialize API client: authenticate via browser, initialize the client, then\n", - " select account and project.\n", - "1. Create material: read from the `../uploads` folder, with Standata as a fallback, then saved to\n", - " the platform.\n", - "1. Configure workflow: upload the pseudopotential, load the band structure workflow from Standata,\n", - " set the model, and attach a unit that fetches the uploaded file onto the compute node.\n", - "1. Configure compute: get list of clusters and create compute configuration.\n", - "1. Create the job, then point its Quantum ESPRESSO inputs at the uploaded pseudopotential.\n", - "1. Submit the job and monitor the status.\n", - "1. Retrieve results: confirm which pseudopotential was used and display the band structure.\n", - "\n", - "## How the custom pseudopotential reaches the calculation\n", - "\n", - "Quantum ESPRESSO reads pseudopotentials from the job's `pseudo` directory (`pseudo_dir` in the\n", - "input). Three pieces line up to put your file there and make the calculation use it:\n", - "\n", - "| Piece | What it does |\n", - "| --- | --- |\n", - "| upload (§4.1) | puts the file in your account's object storage folder |\n", - "| io unit (§4.2) | fetches it into the job's `pseudo` directory on the compute node |\n", - "| input edit (§6.2) | names the file in the `ATOMIC_SPECIES` card of each pw.x input |\n", - "\n", - "The default pseudopotential the platform would have picked is simply never referenced." - ] - }, - { - "cell_type": "markdown", - "id": "1", - "metadata": {}, - "source": [ - "## 1. Set up the environment and parameters\n", - "### 1.1. Install packages (JupyterLite)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "2", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.notebooks_utils.packages import install_packages\n", - "\n", - "await install_packages(\"made|api_examples\")" - ] - }, - { - "cell_type": "markdown", - "id": "3", - "metadata": {}, - "source": [ - "### 1.2. Set parameters\n", - "\n", - "`PSEUDO_FILE` names a UPF file in the `../uploads` folder — drag your own into the JupyterLite\n", - "file browser to replace the shipped example. `PSEUDO_ELEMENT` is the element it is for, and the\n", - "model parameters below describe it (the shipped `Si.upf` is norm-conserving PBE)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4", - "metadata": {}, - "outputs": [], - "source": [ - "from datetime import datetime\n", - "from mat3ra.ide.compute import QueueName\n", - "\n", - "# 2. Auth and organization parameters\n", - "ORGANIZATION_NAME = None\n", - "\n", - "# 3. Material parameters\n", - "FOLDER = \"../uploads\"\n", - "MATERIAL_NAME = \"Silicon\"\n", - "\n", - "# 4. Pseudopotential parameters\n", - "PSEUDO_FILE = \"Si.upf\"\n", - "PSEUDO_ELEMENT = \"Si\"\n", - "\n", - "# 4. Workflow parameters\n", - "APPLICATION_NAME = \"espresso\"\n", - "WORKFLOW_SEARCH_TERM = \"band_structure.json\"\n", - "MY_WORKFLOW_NAME = \"Band Structure - Custom Pseudopotential\"\n", - "\n", - "# Model parameters, describing the uploaded pseudopotential\n", - "MODEL_SUBTYPE = \"gga\" # \"gga\" or \"lda\"\n", - "FUNCTIONAL = \"pbe\" # for gga: \"pbe\", \"pbesol\"; for lda: \"pz\"\n", - "PSEUDOPOTENTIAL_TYPE = \"nc\" # \"us\" (ultrasoft), \"nc\" (norm-conserving), \"paw\"\n", - "\n", - "# Energy cutoffs\n", - "ECUTWFC = 40\n", - "ECUTRHO = 200\n", - "\n", - "# 5. Compute parameters\n", - "CLUSTER_NAME = None # specify full or partial name i.e. \"cluster-001\" to select\n", - "QUEUE_NAME = QueueName.D\n", - "PPN = 1\n", - "\n", - "# 6. Job parameters\n", - "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", - "POLL_INTERVAL = 30 # seconds" - ] - }, - { - "cell_type": "markdown", - "id": "5", - "metadata": {}, - "source": [ - "## 2. Authenticate and initialize API client\n", - "### 2.1. Authenticate\n", - "Authenticate in the browser and have credentials stored in environment variable \"OIDC_ACCESS_TOKEN\"." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.notebooks_utils.auth import authenticate\n", - "\n", - "await authenticate()" - ] - }, - { - "cell_type": "markdown", - "id": "7", - "metadata": {}, - "source": [ - "### 2.2. Initialize API client" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "8", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.api_client import APIClient\n", - "\n", - "client = APIClient.authenticate()\n", - "client" - ] - }, - { - "cell_type": "markdown", - "id": "9", - "metadata": {}, - "source": [ - "### 2.3. Select account" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "10", - "metadata": {}, - "outputs": [], - "source": [ - "client.list_accounts()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "11", - "metadata": {}, - "outputs": [], - "source": [ - "selected_account = client.my_account\n", - "\n", - "if ORGANIZATION_NAME:\n", - " selected_account = client.get_account(name=ORGANIZATION_NAME)\n", - "\n", - "ACCOUNT_ID = selected_account.id\n", - "print(f\"\\u2705 Selected account ID: {ACCOUNT_ID}, name: {selected_account.name}\")" - ] - }, - { - "cell_type": "markdown", - "id": "12", - "metadata": {}, - "source": [ - "### 2.4. Select project" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "13", - "metadata": {}, - "outputs": [], - "source": [ - "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", - "project_id = projects[0][\"_id\"]\n", - "print(f\"\\u2705 Using project: {projects[0]['name']} ({project_id})\")" - ] - }, - { - "cell_type": "markdown", - "id": "14", - "metadata": {}, - "source": [ - "## 3. Create material\n", - "### 3.1. Load material from local file (or Standata)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "15", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.made.material import Material\n", - "from mat3ra.standata.materials import Materials\n", - "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", - "from mat3ra.notebooks_utils.material import load_material_from_folder\n", - "\n", - "material = load_material_from_folder(FOLDER, MATERIAL_NAME) or Material.create(\n", - " Materials.get_by_name_first_match(MATERIAL_NAME))\n", - "\n", - "visualize(material)" - ] - }, - { - "cell_type": "markdown", - "id": "16", - "metadata": {}, - "source": [ - "### 3.2. Save material to the platform" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "17", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.notebooks_utils.core.entity.material.api import get_or_create_material\n", - "\n", - "material.basis.set_labels_from_list([])\n", - "saved_material_response = get_or_create_material(client, material, ACCOUNT_ID)\n", - "saved_material = Material.create(saved_material_response)" - ] - }, - { - "cell_type": "markdown", - "id": "18", - "metadata": {}, - "source": [ - "## 4. Configure workflow\n", - "### 4.1. Upload the pseudopotential\n", - "\n", - "The file goes to your account's object storage folder, which the compute node fetches it from.\n", - "An upload travels as a string inside the request body, which the API caps at 50 MB, and must be\n", - "UTF-8 text — UPF files are XML-like text, so they qualify. Anything larger goes to the same\n", - "folder through the Dropbox page in the web interface instead." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "19", - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "\n", - "from mat3ra.notebooks_utils.core.entity.file.api import upload_files\n", - "\n", - "path = os.path.join(FOLDER, PSEUDO_FILE)\n", - "if not os.path.exists(path):\n", - " raise FileNotFoundError(f\"'{PSEUDO_FILE}' is not in {FOLDER}. Drag it into the file browser first.\")\n", - "with open(path, \"rb\") as file:\n", - " content = file.read().decode(\"utf-8\")\n", - "\n", - "uploaded_pseudo = upload_files(client, {PSEUDO_FILE: content}, ACCOUNT_ID)[0]" - ] - }, - { - "cell_type": "markdown", - "id": "20", - "metadata": {}, - "source": [ - "### 4.2. Load workflow from Standata, attach the fetch of the pseudopotential, and preview\n", - "\n", - "An `io` unit is placed at the head of the workflow before it is built. It downloads the uploaded\n", - "file into the job's `pseudo` subdirectory — the exact directory `pseudo_dir` in the Quantum\n", - "ESPRESSO inputs points at — before the first pw.x unit runs." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "21", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.standata.applications import ApplicationStandata\n", - "from mat3ra.ade.application import Application\n", - "from mat3ra.standata.workflows import WorkflowStandata\n", - "from mat3ra.wode.workflows import Workflow\n", - "from mat3ra.notebooks_utils.core.entity.file.api import to_object_storage_input\n", - "from mat3ra.notebooks_utils.ipython.entity.workflow.visualize import visualize_workflow\n", - "\n", - "app_config = ApplicationStandata.get_by_name_first_match(APPLICATION_NAME)\n", - "app = Application(**app_config)\n", - "\n", - "workflow_config = WorkflowStandata.filter_by_application(app.name).get_by_name_first_match(WORKFLOW_SEARCH_TERM)\n", - "subworkflow = workflow_config[\"subworkflows\"][0]\n", - "first_unit = subworkflow[\"units\"][0]\n", - "io_unit = {\n", - " \"type\": \"io\",\n", - " \"name\": \"io-pseudo\",\n", - " \"subtype\": \"input\",\n", - " \"source\": \"object_storage\",\n", - " \"input\": [to_object_storage_input(uploaded_pseudo, pathname=\"pseudo\")],\n", - " # Nothing references this id, but a unit needs one to sit in the flowchart; namespaced by\n", - " # workflow so it cannot clash.\n", - " \"flowchartId\": \"band-structure-custom-pseudo-io\",\n", - " \"head\": True,\n", - " \"next\": first_unit[\"flowchartId\"],\n", - " \"status\": \"idle\",\n", - " \"statusTrack\": [],\n", - " \"results\": [],\n", - " \"monitors\": [],\n", - " \"preProcessors\": [],\n", - " \"postProcessors\": [],\n", - "}\n", - "first_unit[\"head\"] = False\n", - "subworkflow[\"units\"].insert(0, io_unit)\n", - "\n", - "workflow = Workflow.create(workflow_config)\n", - "workflow.name = MY_WORKFLOW_NAME\n", - "\n", - "visualize_workflow(workflow)" - ] - }, - { - "cell_type": "markdown", - "id": "22", - "metadata": {}, - "source": [ - "### 4.3. Set the model to match the uploaded pseudopotential\n", - "\n", - "The method subtype tells the platform what kind of pseudopotential the calculation uses, and the\n", - "cutoffs should suit it — norm-conserving potentials typically want a higher wavefunction cutoff\n", - "than ultrasoft ones." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "23", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.standata.model_tree import ModelTreeStandata\n", - "from mat3ra.mode import ModelFactory\n", - "from mat3ra.wode.context.providers import PlanewaveCutoffsContextProvider\n", - "\n", - "model_config = ModelTreeStandata.get_model_by_parameters(\n", - " type=\"dft\", subtype=MODEL_SUBTYPE, functional=FUNCTIONAL\n", - ")\n", - "model_config[\"method\"] = {\"type\": \"pseudopotential\", \"subtype\": PSEUDOPOTENTIAL_TYPE}\n", - "model = ModelFactory.create(model_config)\n", - "\n", - "for subworkflow in workflow.subworkflows:\n", - " subworkflow.model = model\n", - "\n", - "cutoffs_context = PlanewaveCutoffsContextProvider(\n", - " wavefunction=ECUTWFC, density=ECUTRHO, isEdited=True\n", - ").get_context_item_data()\n", - "for unit_name in [\"pw_scf\", \"pw_bands\"]:\n", - " unit = workflow.subworkflows[0].get_unit_by_name(name=unit_name)\n", - " if unit:\n", - " unit.add_context(cutoffs_context)\n", - " workflow.subworkflows[0].set_unit(unit)" - ] - }, - { - "cell_type": "markdown", - "id": "24", - "metadata": {}, - "source": [ - "### 4.4. Save workflow to collection" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "25", - "metadata": {}, - "outputs": [], - "source": [ - "saved_workflow = client.workflows.create(workflow.to_dict_without_special_keys(), owner_id=ACCOUNT_ID)\n", - "workflow_id = saved_workflow[\"_id\"]\n", - "print(f\"✅ Workflow saved to collection: {workflow_id}\")" - ] - }, - { - "cell_type": "markdown", - "id": "26", - "metadata": {}, - "source": [ - "## 5. Create the compute configuration\n", - "### 5.1. Get list of clusters" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "27", - "metadata": {}, - "outputs": [], - "source": [ - "clusters = client.clusters.list()\n", - "print(f\"Available clusters: {[c['hostname'] for c in clusters]}\")" - ] - }, - { - "cell_type": "markdown", - "id": "28", - "metadata": {}, - "source": [ - "### 5.2. Create compute configuration" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "29", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.ide.compute import Compute\n", - "\n", - "if CLUSTER_NAME:\n", - " cluster = next((c for c in clusters if CLUSTER_NAME in c[\"hostname\"]), None)\n", - "else:\n", - " cluster = clusters[0]\n", - "\n", - "compute = Compute(\n", - " cluster=cluster,\n", - " queue=QUEUE_NAME,\n", - " ppn=PPN\n", - ")\n", - "print(f\"Using cluster: {compute.cluster.hostname}, queue: {QUEUE_NAME}, ppn: {PPN}\")" - ] - }, - { - "cell_type": "markdown", - "id": "30", - "metadata": {}, - "source": [ - "## 6. Create the job\n", - "### 6.1. Create job" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "31", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.notebooks_utils.job import create_job\n", - "from mat3ra.notebooks_utils.ui import display_JSON\n", - "\n", - "job_name = MY_WORKFLOW_NAME + \" \" + saved_material.formula + \" \" + timestamp\n", - "job_response = create_job(\n", - " api_client=client,\n", - " materials=[saved_material],\n", - " workflow=saved_workflow,\n", - " project_id=project_id,\n", - " owner_id=ACCOUNT_ID,\n", - " prefix=job_name,\n", - " compute=compute.to_dict(),\n", - ")\n", - "\n", - "job_id = job_response[\"_id\"]\n", - "print(\"✅ Job created successfully!\")\n", - "print(f\"Job ID: {job_id}\")\n", - "display_JSON(job_response)" - ] - }, - { - "cell_type": "markdown", - "id": "32", - "metadata": {}, - "source": [ - "### 6.2. Point the inputs at the uploaded pseudopotential\n", - "\n", - "At job creation the platform fills the `ATOMIC_SPECIES` card of each pw.x input with its default\n", - "pseudopotential for the element. The lines are edited here to name the uploaded file instead —\n", - "the expert-mode input edit the platform documentation describes — and `isManuallyChanged` keeps\n", - "the edit from being re-rendered." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "33", - "metadata": {}, - "outputs": [], - "source": [ - "import re\n", - "\n", - "job = client.jobs.get(job_id)\n", - "# Element + mass + filename, matched only inside the ATOMIC_SPECIES card - the same\n", - "# \"element number number\" shape also occurs in ATOMIC_POSITIONS, where a bare substitution\n", - "# would overwrite a coordinate.\n", - "pattern = re.compile(rf\"^(\\s*{PSEUDO_ELEMENT}\\s+[\\d.]+\\s+)\\S+\", re.MULTILINE)\n", - "\n", - "for unit in job[\"workflow\"][\"subworkflows\"][0][\"units\"]:\n", - " for unit_input in unit.get(\"input\", []):\n", - " rendered = unit_input.get(\"rendered\") or \"\"\n", - " if \"ATOMIC_SPECIES\" not in rendered:\n", - " continue\n", - " start = rendered.index(\"ATOMIC_SPECIES\")\n", - " end = rendered.find(\"\\n\\n\", start)\n", - " end = len(rendered) if end == -1 else end\n", - " card = pattern.sub(rf\"\\g<1>{PSEUDO_FILE}\", rendered[start:end])\n", - " unit_input[\"rendered\"] = rendered[:start] + card + rendered[end:]\n", - " unit_input[\"isManuallyChanged\"] = True\n", - " print(f\"✅ {unit['name']} now uses {PSEUDO_FILE}\")\n", - "\n", - "client.jobs.update(job_id, job)" - ] - }, - { - "cell_type": "markdown", - "id": "34", - "metadata": {}, - "source": [ - "## 7. Submit the job and monitor the status" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "35", - "metadata": {}, - "outputs": [], - "source": [ - "client.jobs.submit(job_id)\n", - "print(f\"✅ Job {job_id} submitted successfully!\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "36", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.notebooks_utils.api.job import wait_for_jobs_to_finish_async\n", - "\n", - "await wait_for_jobs_to_finish_async(client.jobs, [job_id], poll_interval=POLL_INTERVAL)" - ] - }, - { - "cell_type": "markdown", - "id": "37", - "metadata": {}, - "source": [ - "## 8. Retrieve and visualize results\n", - "### 8.1. Confirm the pseudopotential that was used\n", - "\n", - "The `ATOMIC_SPECIES` card of each pw.x input, exactly as the calculation consumed it." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "38", - "metadata": {}, - "outputs": [], - "source": [ - "finished_job = client.jobs.get(job_id)\n", - "for unit in finished_job[\"workflow\"][\"subworkflows\"][0][\"units\"]:\n", - " for unit_input in unit.get(\"input\", []):\n", - " rendered = unit_input.get(\"rendered\") or \"\"\n", - " if \"ATOMIC_SPECIES\" in rendered:\n", - " card = rendered[rendered.index(\"ATOMIC_SPECIES\"):].split(\"\\n\\n\")[0]\n", - " print(f\"--- {unit['name']} ---\\n{card}\")" - ] - }, - { - "cell_type": "markdown", - "id": "39", - "metadata": {}, - "source": [ - "### 8.2. Band Structure" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "40", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.prode import PropertyName\n", - "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", - "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", - "\n", - "band_structure_data = get_properties_for_job(client, job_id, property_name=PropertyName.non_scalar.band_structure.value)\n", - "visualize_properties(band_structure_data, title=\"Band Structure\", extra_config={\"material\": material.to_dict()})" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "version": "3.11.2" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -} diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index 6ed3036da..0d4504752 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -36,6 +36,8 @@ "1. Create one job per material from the material, workflow, project and compute configuration.\n", "1. Submit the jobs and monitor the status: submit and wait for completion.\n", "1. Retrieve results: read each job's standard output and display the values it printed.\n", + "1. Example: upload a pseudopotential through the same call and run a Quantum ESPRESSO band\n", + " structure that consumes it.\n", "\n", "## How the script receives its inputs\n", "\n", @@ -612,6 +614,181 @@ "\n", "pd.DataFrame(results)" ] + }, + { + "cell_type": "markdown", + "id": "37", + "metadata": {}, + "source": [ + "## 9. Example: run a Quantum ESPRESSO job with an uploaded pseudopotential\n", + "\n", + "An upload is an ordinary platform file, so it is not limited to feeding the custom script above —\n", + "any workflow can fetch it. This example uploads a pseudopotential (`Si.upf` ships in the\n", + "`../uploads` folder; replace it with your own) and runs a Quantum ESPRESSO band structure for\n", + "silicon that consumes it:\n", + "\n", + "1. the file goes up through the same upload call as section 4.1.;\n", + "1. an `io` unit at the head of the band structure workflow downloads it into the job's `pseudo`\n", + " directory — exactly where `pseudo_dir` in the Quantum ESPRESSO inputs points;\n", + "1. the `ATOMIC_SPECIES` card of each pw.x input is pointed at the file — the expert-mode input\n", + " edit the platform documentation describes — and `isManuallyChanged` keeps the edit from being\n", + " re-rendered." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "38", + "metadata": {}, + "outputs": [], + "source": [ + "PSEUDO_FILE = \"Si.upf\" # norm-conserving, PBE; the model below is set to match\n", + "PSEUDO_ELEMENT = \"Si\"\n", + "\n", + "with open(os.path.join(FOLDER, PSEUDO_FILE), \"rb\") as file:\n", + " uploaded_pseudo = upload_files(client, {PSEUDO_FILE: file.read().decode(\"utf-8\")}, ACCOUNT_ID)[0]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "39", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.standata.model_tree import ModelTreeStandata\n", + "from mat3ra.mode import ModelFactory\n", + "from mat3ra.wode.context.providers import PlanewaveCutoffsContextProvider\n", + "\n", + "bs_config = WorkflowStandata.filter_by_application(\"espresso\").get_by_name_first_match(\"band_structure.json\")\n", + "bs_subworkflow = bs_config[\"subworkflows\"][0]\n", + "first_unit = bs_subworkflow[\"units\"][0]\n", + "io_unit = {\n", + " \"type\": \"io\",\n", + " \"name\": \"io-pseudo\",\n", + " \"subtype\": \"input\",\n", + " \"source\": \"object_storage\",\n", + " \"input\": [to_object_storage_input(uploaded_pseudo, pathname=\"pseudo\")],\n", + " # Nothing references this id, but a unit needs one to sit in the flowchart; namespaced by\n", + " # workflow so it cannot clash.\n", + " \"flowchartId\": \"band-structure-custom-pseudo-io\",\n", + " \"head\": True,\n", + " \"next\": first_unit[\"flowchartId\"],\n", + " \"status\": \"idle\",\n", + " \"statusTrack\": [],\n", + " \"results\": [],\n", + " \"monitors\": [],\n", + " \"preProcessors\": [],\n", + " \"postProcessors\": [],\n", + "}\n", + "first_unit[\"head\"] = False\n", + "bs_subworkflow[\"units\"].insert(0, io_unit)\n", + "\n", + "bs_workflow = Workflow.create(bs_config)\n", + "bs_workflow.name = \"Band Structure - Custom Pseudopotential\"\n", + "\n", + "model_config = ModelTreeStandata.get_model_by_parameters(type=\"dft\", subtype=\"gga\", functional=\"pbe\")\n", + "model_config[\"method\"] = {\"type\": \"pseudopotential\", \"subtype\": \"nc\"}\n", + "model = ModelFactory.create(model_config)\n", + "for bs_sw in bs_workflow.subworkflows:\n", + " bs_sw.model = model\n", + "cutoffs_context = PlanewaveCutoffsContextProvider(wavefunction=40, density=200, isEdited=True).get_context_item_data()\n", + "for unit_name in [\"pw_scf\", \"pw_bands\"]:\n", + " unit = bs_workflow.subworkflows[0].get_unit_by_name(name=unit_name)\n", + " if unit:\n", + " unit.add_context(cutoffs_context)\n", + " bs_workflow.subworkflows[0].set_unit(unit)\n", + "\n", + "saved_bs_workflow = client.workflows.create(bs_workflow.to_dict_without_special_keys(), owner_id=ACCOUNT_ID)\n", + "print(f\"✅ Workflow saved to collection: {saved_bs_workflow['_id']}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "40", + "metadata": {}, + "outputs": [], + "source": [ + "bs_material = Material.create(\n", + " get_or_create_material(client, Material.create(Materials.get_by_name_first_match(\"Silicon\")), ACCOUNT_ID)\n", + ")\n", + "bs_job = create_job(\n", + " api_client=client,\n", + " materials=[bs_material],\n", + " workflow=saved_bs_workflow,\n", + " project_id=project_id,\n", + " owner_id=ACCOUNT_ID,\n", + " prefix=f\"Band Structure Custom Pseudopotential {timestamp}\",\n", + " compute=compute.to_dict(),\n", + ")\n", + "bs_job_id = bs_job[\"_id\"]\n", + "print(f\"✅ Job created: {bs_job_id}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "41", + "metadata": {}, + "outputs": [], + "source": [ + "import re\n", + "\n", + "bs_job = client.jobs.get(bs_job_id)\n", + "# Element + mass + filename, matched only inside the ATOMIC_SPECIES card - the same\n", + "# \"element number number\" shape also occurs in ATOMIC_POSITIONS, where a bare substitution\n", + "# would overwrite a coordinate.\n", + "pattern = re.compile(rf\"^(\\s*{PSEUDO_ELEMENT}\\s+[\\d.]+\\s+)\\S+\", re.MULTILINE)\n", + "\n", + "for unit in bs_job[\"workflow\"][\"subworkflows\"][0][\"units\"]:\n", + " for unit_input in unit.get(\"input\", []):\n", + " rendered = unit_input.get(\"rendered\") or \"\"\n", + " if \"ATOMIC_SPECIES\" not in rendered:\n", + " continue\n", + " start = rendered.index(\"ATOMIC_SPECIES\")\n", + " end = rendered.find(\"\\n\\n\", start)\n", + " end = len(rendered) if end == -1 else end\n", + " card = pattern.sub(rf\"\\g<1>{PSEUDO_FILE}\", rendered[start:end])\n", + " unit_input[\"rendered\"] = rendered[:start] + card + rendered[end:]\n", + " unit_input[\"isManuallyChanged\"] = True\n", + " print(f\"✅ {unit['name']} now uses {PSEUDO_FILE}\")\n", + "\n", + "client.jobs.update(bs_job_id, bs_job)\n", + "client.jobs.submit(bs_job_id)\n", + "await wait_for_jobs_to_finish_async(client.jobs, [bs_job_id], poll_interval=POLL_INTERVAL)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "42", + "metadata": {}, + "outputs": [], + "source": [ + "finished_bs_job = client.jobs.get(bs_job_id)\n", + "for unit in finished_bs_job[\"workflow\"][\"subworkflows\"][0][\"units\"]:\n", + " for unit_input in unit.get(\"input\", []):\n", + " rendered = unit_input.get(\"rendered\") or \"\"\n", + " if \"ATOMIC_SPECIES\" in rendered:\n", + " card = rendered[rendered.index(\"ATOMIC_SPECIES\"):].split(\"\\n\\n\")[0]\n", + " print(f\"--- {unit['name']} ---\\n{card}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "43", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.prode import PropertyName\n", + "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", + "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", + "\n", + "band_structure_data = get_properties_for_job(client, bs_job_id, property_name=PropertyName.non_scalar.band_structure.value)\n", + "visualize_properties(band_structure_data, title=\"Band Structure\", extra_config={\"material\": bs_material.to_dict()})" + ] } ], "metadata": { From 71fc1a25f31b5b17a42ed630b9db4b284553ae9c Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 24 Aug 2026 09:43:17 -0700 Subject: [PATCH 14/21] fix(notebook): anchor the pseudopotential substitution on the .upf extension CodeRabbit's follow-up on the card-slice approach was right that a blank line is a fragile boundary: cards touching each other would leak the substitution into ATOMIC_POSITIONS again. Requiring the replaced token to be a UPF filename kills the whole class without slicing - position lines end in numbers, never ".upf" - and drops four lines. Verified by full token diff on real rendered inputs and on a synthetic no-blank-line variant. --- .../workflows/custom_python_calculation.ipynb | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index 0d4504752..3a97e7880 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -736,21 +736,17 @@ "import re\n", "\n", "bs_job = client.jobs.get(bs_job_id)\n", - "# Element + mass + filename, matched only inside the ATOMIC_SPECIES card - the same\n", - "# \"element number number\" shape also occurs in ATOMIC_POSITIONS, where a bare substitution\n", - "# would overwrite a coordinate.\n", - "pattern = re.compile(rf\"^(\\s*{PSEUDO_ELEMENT}\\s+[\\d.]+\\s+)\\S+\", re.MULTILINE)\n", + "# Element + mass + a UPF filename. Anchoring on the \".upf\" extension keeps the substitution off\n", + "# ATOMIC_POSITIONS lines, whose \"element number number\" shape would otherwise also match, and\n", + "# needs no assumptions about how the cards are separated.\n", + "pattern = re.compile(rf\"^(\\s*{PSEUDO_ELEMENT}\\s+[\\d.]+\\s+)\\S+\\.upf\\s*$\", re.MULTILINE | re.IGNORECASE)\n", "\n", "for unit in bs_job[\"workflow\"][\"subworkflows\"][0][\"units\"]:\n", " for unit_input in unit.get(\"input\", []):\n", " rendered = unit_input.get(\"rendered\") or \"\"\n", " if \"ATOMIC_SPECIES\" not in rendered:\n", " continue\n", - " start = rendered.index(\"ATOMIC_SPECIES\")\n", - " end = rendered.find(\"\\n\\n\", start)\n", - " end = len(rendered) if end == -1 else end\n", - " card = pattern.sub(rf\"\\g<1>{PSEUDO_FILE}\", rendered[start:end])\n", - " unit_input[\"rendered\"] = rendered[:start] + card + rendered[end:]\n", + " unit_input[\"rendered\"] = pattern.sub(rf\"\\g<1>{PSEUDO_FILE}\", rendered)\n", " unit_input[\"isManuallyChanged\"] = True\n", " print(f\"✅ {unit['name']} now uses {PSEUDO_FILE}\")\n", "\n", From 0a724f37113687d4dd16dea1345a7195ed0ff87e Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 24 Aug 2026 14:28:57 -0700 Subject: [PATCH 15/21] feat(notebook): Custom Shell Calculation - the shell twin, with the pseudopotential demo Per Vsevolod: the QE-with-uploaded-pseudo example runs through a custom SHELL notebook that does what the python one does, "with a difference in app used and the body". custom_python_calculation.ipynb reverts to its clean 37-cell state (section 9 and the rendered-input patching die here); the new custom_shell_calculation.ipynb mirrors it cell for cell on the new standata Custom Shell Script workflow. The default body is the correction's example end to end: upload Si.upf through the notebook's own upload cell, build pw.x inputs from material.json with pseudo_dir './', module add espresso, run SCF + bands (the invocation is the platform's own job_espresso_pw_scf.sh precedent), and print QE's own proof line - "read from file: ./Si.upf" - plus the band edges at Gamma as JSON. The shell runner (CUSTOM_SCRIPT_RUNNER_SH) writes material.json and *sources* user_script.sh so `module` stays available. hello_world.sh, its on-disk name, joins RESERVED_FILENAMES (tested). Input building verified locally against the real Silicon material: valid pw inputs, ATOMIC_SPECIES names Si.upf. --- .../workflows/Introduction.ipynb | 2 +- .../workflows/custom_python_calculation.ipynb | 173 ----- .../workflows/custom_shell_calculation.ipynb | 697 ++++++++++++++++++ .../notebooks_utils/core/entity/file/api.py | 5 +- .../core/entity/workflow/api.py | 13 + tests/py/unit/core/entity/test_file_api.py | 6 + 6 files changed, 720 insertions(+), 176 deletions(-) create mode 100644 other/materials_designer/workflows/custom_shell_calculation.ipynb diff --git a/other/materials_designer/workflows/Introduction.ipynb b/other/materials_designer/workflows/Introduction.ipynb index 531e08e6c..5a9cd46d3 100644 --- a/other/materials_designer/workflows/Introduction.ipynb +++ b/other/materials_designer/workflows/Introduction.ipynb @@ -106,7 +106,7 @@ "\n", "### 9.1. Python / Shell\n", "#### [9.1.1. Custom Python calculation.](custom_python_calculation.ipynb)\n", - "#### 9.1.2. Custom Shell calculation. *(to be added)*\n" + "#### [9.1.2. Custom Shell calculation.](custom_shell_calculation.ipynb)\n" ] }, { diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index 3a97e7880..6ed3036da 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -36,8 +36,6 @@ "1. Create one job per material from the material, workflow, project and compute configuration.\n", "1. Submit the jobs and monitor the status: submit and wait for completion.\n", "1. Retrieve results: read each job's standard output and display the values it printed.\n", - "1. Example: upload a pseudopotential through the same call and run a Quantum ESPRESSO band\n", - " structure that consumes it.\n", "\n", "## How the script receives its inputs\n", "\n", @@ -614,177 +612,6 @@ "\n", "pd.DataFrame(results)" ] - }, - { - "cell_type": "markdown", - "id": "37", - "metadata": {}, - "source": [ - "## 9. Example: run a Quantum ESPRESSO job with an uploaded pseudopotential\n", - "\n", - "An upload is an ordinary platform file, so it is not limited to feeding the custom script above —\n", - "any workflow can fetch it. This example uploads a pseudopotential (`Si.upf` ships in the\n", - "`../uploads` folder; replace it with your own) and runs a Quantum ESPRESSO band structure for\n", - "silicon that consumes it:\n", - "\n", - "1. the file goes up through the same upload call as section 4.1.;\n", - "1. an `io` unit at the head of the band structure workflow downloads it into the job's `pseudo`\n", - " directory — exactly where `pseudo_dir` in the Quantum ESPRESSO inputs points;\n", - "1. the `ATOMIC_SPECIES` card of each pw.x input is pointed at the file — the expert-mode input\n", - " edit the platform documentation describes — and `isManuallyChanged` keeps the edit from being\n", - " re-rendered." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "38", - "metadata": {}, - "outputs": [], - "source": [ - "PSEUDO_FILE = \"Si.upf\" # norm-conserving, PBE; the model below is set to match\n", - "PSEUDO_ELEMENT = \"Si\"\n", - "\n", - "with open(os.path.join(FOLDER, PSEUDO_FILE), \"rb\") as file:\n", - " uploaded_pseudo = upload_files(client, {PSEUDO_FILE: file.read().decode(\"utf-8\")}, ACCOUNT_ID)[0]" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "39", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.standata.model_tree import ModelTreeStandata\n", - "from mat3ra.mode import ModelFactory\n", - "from mat3ra.wode.context.providers import PlanewaveCutoffsContextProvider\n", - "\n", - "bs_config = WorkflowStandata.filter_by_application(\"espresso\").get_by_name_first_match(\"band_structure.json\")\n", - "bs_subworkflow = bs_config[\"subworkflows\"][0]\n", - "first_unit = bs_subworkflow[\"units\"][0]\n", - "io_unit = {\n", - " \"type\": \"io\",\n", - " \"name\": \"io-pseudo\",\n", - " \"subtype\": \"input\",\n", - " \"source\": \"object_storage\",\n", - " \"input\": [to_object_storage_input(uploaded_pseudo, pathname=\"pseudo\")],\n", - " # Nothing references this id, but a unit needs one to sit in the flowchart; namespaced by\n", - " # workflow so it cannot clash.\n", - " \"flowchartId\": \"band-structure-custom-pseudo-io\",\n", - " \"head\": True,\n", - " \"next\": first_unit[\"flowchartId\"],\n", - " \"status\": \"idle\",\n", - " \"statusTrack\": [],\n", - " \"results\": [],\n", - " \"monitors\": [],\n", - " \"preProcessors\": [],\n", - " \"postProcessors\": [],\n", - "}\n", - "first_unit[\"head\"] = False\n", - "bs_subworkflow[\"units\"].insert(0, io_unit)\n", - "\n", - "bs_workflow = Workflow.create(bs_config)\n", - "bs_workflow.name = \"Band Structure - Custom Pseudopotential\"\n", - "\n", - "model_config = ModelTreeStandata.get_model_by_parameters(type=\"dft\", subtype=\"gga\", functional=\"pbe\")\n", - "model_config[\"method\"] = {\"type\": \"pseudopotential\", \"subtype\": \"nc\"}\n", - "model = ModelFactory.create(model_config)\n", - "for bs_sw in bs_workflow.subworkflows:\n", - " bs_sw.model = model\n", - "cutoffs_context = PlanewaveCutoffsContextProvider(wavefunction=40, density=200, isEdited=True).get_context_item_data()\n", - "for unit_name in [\"pw_scf\", \"pw_bands\"]:\n", - " unit = bs_workflow.subworkflows[0].get_unit_by_name(name=unit_name)\n", - " if unit:\n", - " unit.add_context(cutoffs_context)\n", - " bs_workflow.subworkflows[0].set_unit(unit)\n", - "\n", - "saved_bs_workflow = client.workflows.create(bs_workflow.to_dict_without_special_keys(), owner_id=ACCOUNT_ID)\n", - "print(f\"✅ Workflow saved to collection: {saved_bs_workflow['_id']}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "40", - "metadata": {}, - "outputs": [], - "source": [ - "bs_material = Material.create(\n", - " get_or_create_material(client, Material.create(Materials.get_by_name_first_match(\"Silicon\")), ACCOUNT_ID)\n", - ")\n", - "bs_job = create_job(\n", - " api_client=client,\n", - " materials=[bs_material],\n", - " workflow=saved_bs_workflow,\n", - " project_id=project_id,\n", - " owner_id=ACCOUNT_ID,\n", - " prefix=f\"Band Structure Custom Pseudopotential {timestamp}\",\n", - " compute=compute.to_dict(),\n", - ")\n", - "bs_job_id = bs_job[\"_id\"]\n", - "print(f\"✅ Job created: {bs_job_id}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "41", - "metadata": {}, - "outputs": [], - "source": [ - "import re\n", - "\n", - "bs_job = client.jobs.get(bs_job_id)\n", - "# Element + mass + a UPF filename. Anchoring on the \".upf\" extension keeps the substitution off\n", - "# ATOMIC_POSITIONS lines, whose \"element number number\" shape would otherwise also match, and\n", - "# needs no assumptions about how the cards are separated.\n", - "pattern = re.compile(rf\"^(\\s*{PSEUDO_ELEMENT}\\s+[\\d.]+\\s+)\\S+\\.upf\\s*$\", re.MULTILINE | re.IGNORECASE)\n", - "\n", - "for unit in bs_job[\"workflow\"][\"subworkflows\"][0][\"units\"]:\n", - " for unit_input in unit.get(\"input\", []):\n", - " rendered = unit_input.get(\"rendered\") or \"\"\n", - " if \"ATOMIC_SPECIES\" not in rendered:\n", - " continue\n", - " unit_input[\"rendered\"] = pattern.sub(rf\"\\g<1>{PSEUDO_FILE}\", rendered)\n", - " unit_input[\"isManuallyChanged\"] = True\n", - " print(f\"✅ {unit['name']} now uses {PSEUDO_FILE}\")\n", - "\n", - "client.jobs.update(bs_job_id, bs_job)\n", - "client.jobs.submit(bs_job_id)\n", - "await wait_for_jobs_to_finish_async(client.jobs, [bs_job_id], poll_interval=POLL_INTERVAL)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "42", - "metadata": {}, - "outputs": [], - "source": [ - "finished_bs_job = client.jobs.get(bs_job_id)\n", - "for unit in finished_bs_job[\"workflow\"][\"subworkflows\"][0][\"units\"]:\n", - " for unit_input in unit.get(\"input\", []):\n", - " rendered = unit_input.get(\"rendered\") or \"\"\n", - " if \"ATOMIC_SPECIES\" in rendered:\n", - " card = rendered[rendered.index(\"ATOMIC_SPECIES\"):].split(\"\\n\\n\")[0]\n", - " print(f\"--- {unit['name']} ---\\n{card}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "43", - "metadata": {}, - "outputs": [], - "source": [ - "from mat3ra.prode import PropertyName\n", - "from mat3ra.notebooks_utils.core.entity.property.api import get_properties_for_job\n", - "from mat3ra.notebooks_utils.ipython.entity.property.visualize import visualize_properties\n", - "\n", - "band_structure_data = get_properties_for_job(client, bs_job_id, property_name=PropertyName.non_scalar.band_structure.value)\n", - "visualize_properties(band_structure_data, title=\"Band Structure\", extra_config={\"material\": bs_material.to_dict()})" - ] } ], "metadata": { diff --git a/other/materials_designer/workflows/custom_shell_calculation.ipynb b/other/materials_designer/workflows/custom_shell_calculation.ipynb new file mode 100644 index 000000000..05fb9753a --- /dev/null +++ b/other/materials_designer/workflows/custom_shell_calculation.ipynb @@ -0,0 +1,697 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "0", + "metadata": {}, + "source": [ + "# Custom Shell Calculation\n", + "\n", + "Run your own shell script against one or more materials on the Mat3ra platform — the shell twin of\n", + "the Custom Python Calculation notebook. The script and any files it needs are uploaded to your\n", + "object storage folder, a workflow fetches them onto the compute node alongside the material, and\n", + "whatever the script prints comes back as the result. A shell script can invoke any application\n", + "installed on the node; the default example runs a Quantum ESPRESSO band structure for silicon with\n", + "an uploaded pseudopotential.\n", + "\n", + "

Usage

\n", + "\n", + "1. Set the materials and any data files the script reads in cell 1.2. below.\n", + "1. Replace the script in cell 1.3. with your own (or keep the default one).\n", + "1. Click \"Run\" > \"Run All\" to run all cells.\n", + "1. Wait for the jobs to complete.\n", + "1. Scroll down to view the results.\n", + "\n", + "## Summary\n", + "\n", + "1. Set up the environment and parameters: install packages (JupyterLite only) and configure the\n", + " materials, the script, its data files, compute resources and job.\n", + "1. Authenticate and initialize API client: authenticate via browser, initialize the client, then\n", + " select account and project.\n", + "1. Create materials: materials are read from the `../uploads` folder — place files there manually or\n", + " run a material creation notebook first. If a material is not found by name, Standata is used as a\n", + " fallback. Each material is then saved to the platform.\n", + "1. Create workflow: upload the script and its data files, then assemble a workflow that fetches\n", + " them, fetches the material, and runs the script. Optionally save the workflow to the collection.\n", + "1. Configure compute: get list of clusters and create compute configuration with selected cluster,\n", + " queue, and number of processors.\n", + "1. Create one job per material from the material, workflow, project and compute configuration.\n", + "1. Submit the jobs and monitor the status: submit and wait for completion.\n", + "1. Retrieve results: read each job's standard output and display the values it printed.\n", + "\n", + "## How the script receives its inputs\n", + "\n", + "Everything lands in the job's working directory, so the script reads it all by **relative path**:\n", + "\n", + "| File | Written by | Contents |\n", + "| --- | --- | --- |\n", + "| `material.json` | the workflow | the job's material, as stored on the platform |\n", + "| your data files | this notebook | uploaded verbatim from `USER_ASSET_FILES` |\n", + "\n", + "The script runs in a shell where `module` is available, so node-side applications load the same way\n", + "they do in a command-line job (e.g. `module add espresso`). The script's standard output is the\n", + "result. Print JSON and this notebook renders it as a table." + ] + }, + { + "cell_type": "markdown", + "id": "1", + "metadata": {}, + "source": [ + "## 1. Set up the environment and parameters\n", + "### 1.1. Install packages (JupyterLite)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.packages import install_packages\n", + "\n", + "await install_packages(\"made|api_examples\")" + ] + }, + { + "cell_type": "markdown", + "id": "3", + "metadata": {}, + "source": [ + "### 1.2. Set parameters and configurations for the workflow and job\n", + "\n", + "`USER_ASSET_FILES` names UTF-8 text files the script opens. Put them in the `../uploads` folder\n", + "first — drag them into the JupyterLite file browser — and this notebook uploads them alongside the\n", + "script. The default example needs the pseudopotential `Si.upf`, which ships with this notebook." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4", + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from datetime import datetime\n", + "\n", + "from mat3ra.ide.compute import QueueName\n", + "\n", + "# 2. Auth and organization parameters\n", + "# Set organization name to use it as the owner, otherwise your personal account is used\n", + "ORGANIZATION_NAME = None\n", + "\n", + "# 3. Material parameters\n", + "FOLDER = \"../uploads\"\n", + "MATERIAL_NAMES = [\"Silicon\"] # One job is created per material\n", + "\n", + "# 4. Script parameters\n", + "USER_ASSET_FILES = [\"Si.upf\"] # Files the script opens, taken from FOLDER\n", + "\n", + "# 5. Workflow parameters\n", + "WORKFLOW_SEARCH_TERM = \"custom_script.json\" # Search term for Workflows Standata\n", + "APPLICATION_NAME = \"shell\"\n", + "MY_WORKFLOW_NAME = \"Custom Shell Calculation\"\n", + "save_to_collection = True\n", + "\n", + "# 6. Compute parameters\n", + "CLUSTER_NAME = None # specify full or partial name i.e. \"cluster-001\" to select\n", + "QUEUE_NAME = QueueName.D\n", + "PPN = 1\n", + "\n", + "# 7. Job parameters\n", + "timestamp = datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n", + "POLL_INTERVAL = 30 # seconds" + ] + }, + { + "cell_type": "markdown", + "id": "5", + "metadata": {}, + "source": [ + "### 1.3. Set the script to run\n", + "\n", + "This is the calculation. It runs on the compute node with `material.json` and your data files\n", + "beside it, and whatever it prints becomes the result. Replace it with your own.\n", + "\n", + "The default example runs a Quantum ESPRESSO band structure with the uploaded pseudopotential: it\n", + "builds the pw.x inputs from `material.json`, points `pseudo_dir` at the working directory so the\n", + "uploaded `Si.upf` is the file the calculation reads, runs an SCF and a bands step, and prints the\n", + "band edges as JSON. The script is uploaded as a file and fetched onto the node, never inlined into\n", + "the workflow, so its contents reach the shell exactly as written." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6", + "metadata": {}, + "outputs": [], + "source": [ + "USER_SCRIPT = r\"\"\"#!/bin/bash\n", + "# Runs a Quantum ESPRESSO band structure for the job's material, reading the\n", + "# pseudopotential uploaded next to this script instead of a platform default.\n", + "\n", + "module add espresso\n", + "\n", + "# Build the pw.x inputs from the job's material.\n", + "python3 - <<'BUILD_INPUTS_EOF'\n", + "import json\n", + "import math\n", + "\n", + "material = json.load(open(\"material.json\"))\n", + "lattice = material[\"lattice\"]\n", + "a, b, c = lattice[\"a\"], lattice[\"b\"], lattice[\"c\"]\n", + "alpha, beta, gamma = (math.radians(lattice[key]) for key in (\"alpha\", \"beta\", \"gamma\"))\n", + "c_x = c * math.cos(beta)\n", + "c_y = c * (math.cos(alpha) - math.cos(beta) * math.cos(gamma)) / math.sin(gamma)\n", + "vectors = [\n", + " [a, 0.0, 0.0],\n", + " [b * math.cos(gamma), b * math.sin(gamma), 0.0],\n", + " [c_x, c_y, math.sqrt(max(c**2 - c_x**2 - c_y**2, 0.0))],\n", + "]\n", + "elements = [element[\"value\"] for element in material[\"basis\"][\"elements\"]]\n", + "coordinates = [point[\"value\"] for point in material[\"basis\"][\"coordinates\"]]\n", + "\n", + "MASSES = {\"Si\": 28.0855}\n", + "PSEUDOS = {\"Si\": \"Si.upf\"} # the uploaded file; pseudo_dir is the working directory\n", + "\n", + "cell = \"\\n\".join(\" \".join(f\"{value:.9f}\" for value in vector) for vector in vectors)\n", + "positions = \"\\n\".join(f\"{el} \" + \" \".join(f\"{x:.9f}\" for x in xyz) for el, xyz in zip(elements, coordinates))\n", + "species = \"\\n\".join(f\"{el} {MASSES[el]} {PSEUDOS[el]}\" for el in sorted(set(elements)))\n", + "\n", + "common = f'''&SYSTEM\n", + " ibrav = 0\n", + " nat = {len(elements)}\n", + " ntyp = {len(set(elements))}\n", + " ecutwfc = 40\n", + " ecutrho = 200\n", + " occupations = 'fixed'\n", + " nbnd = 8\n", + "/\n", + "&ELECTRONS\n", + "/\n", + "ATOMIC_SPECIES\n", + "{species}\n", + "CELL_PARAMETERS angstrom\n", + "{cell}\n", + "ATOMIC_POSITIONS crystal\n", + "{positions}\n", + "'''\n", + "\n", + "with open(\"pw_scf.in\", \"w\") as f:\n", + " f.write(\"&CONTROL\\n calculation = 'scf'\\n pseudo_dir = './'\\n outdir = './outdir'\\n/\\n\")\n", + " f.write(common)\n", + " f.write(\"K_POINTS automatic\\n6 6 6 0 0 0\\n\")\n", + "\n", + "with open(\"pw_bands.in\", \"w\") as f:\n", + " f.write(\"&CONTROL\\n calculation = 'bands'\\n pseudo_dir = './'\\n outdir = './outdir'\\n/\\n\")\n", + " f.write(common)\n", + " f.write(\"K_POINTS crystal_b\\n3\\n0.5 0.5 0.5 20\\n0.0 0.0 0.0 20\\n0.5 0.0 0.5 20\\n\")\n", + "BUILD_INPUTS_EOF\n", + "\n", + "# Run the calculation. EXEC_CMD is (conditionally) set by the module.\n", + "mpirun -np $PBS_NP $EXEC_CMD pw.x -in pw_scf.in > pw_scf.out\n", + "mpirun -np $PBS_NP $EXEC_CMD pw.x -in pw_bands.in > pw_bands.out\n", + "\n", + "# Proof and results: QE's own log names the pseudopotential file it read.\n", + "grep \"read from file\" pw_scf.out\n", + "grep \"highest occupied\" pw_scf.out\n", + "\n", + "# Band edges at Gamma from the bands run, as JSON for the results table.\n", + "python3 - <<'PARSE_BANDS_EOF'\n", + "import json\n", + "import re\n", + "\n", + "text = open(\"pw_bands.out\").read()\n", + "block = text[text.index(\"End of band structure calculation\"):]\n", + "gamma = re.search(r\"k = 0\\.0000 0\\.0000 0\\.0000[^\\n]*\\n\\n([\\s\\S]*?)\\n\\n\", block).group(1)\n", + "energies = sorted(float(value) for value in re.findall(r\"-?\\d+\\.\\d+\", gamma))\n", + "print(json.dumps({\n", + " \"gamma_homo_ev\": energies[3],\n", + " \"gamma_lumo_ev\": energies[4],\n", + " \"gamma_direct_gap_ev\": round(energies[4] - energies[3], 4),\n", + "}))\n", + "PARSE_BANDS_EOF\n", + "\"\"\"" + ] + }, + { + "cell_type": "markdown", + "id": "7", + "metadata": {}, + "source": [ + "## 2. Authenticate and initialize API client\n", + "### 2.1. Authenticate\n", + "Authenticate in the browser and have credentials stored in environment variable \"OIDC_ACCESS_TOKEN\"." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "8", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.auth import authenticate\n", + "\n", + "\n", + "await authenticate()" + ] + }, + { + "cell_type": "markdown", + "id": "9", + "metadata": {}, + "source": [ + "### 2.2. Initialize API Client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "10", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.api_client import APIClient\n", + "\n", + "client = APIClient.authenticate()\n", + "client" + ] + }, + { + "cell_type": "markdown", + "id": "11", + "metadata": {}, + "source": [ + "### 2.3. Select account to work under" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "12", + "metadata": {}, + "outputs": [], + "source": [ + "client.list_accounts()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "13", + "metadata": {}, + "outputs": [], + "source": [ + "selected_account = client.my_account\n", + "\n", + "if ORGANIZATION_NAME:\n", + " selected_account = client.get_account(name=ORGANIZATION_NAME)\n", + "\n", + "ACCOUNT_ID = selected_account.id\n", + "print(f\"Using account: {selected_account.name} ({ACCOUNT_ID})\")" + ] + }, + { + "cell_type": "markdown", + "id": "14", + "metadata": {}, + "source": [ + "### 2.4. Select project" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "15", + "metadata": {}, + "outputs": [], + "source": [ + "projects = client.projects.list({\"isDefault\": True, \"owner._id\": ACCOUNT_ID})\n", + "project_id = projects[0][\"_id\"]\n", + "print(f\"✅ Using project: {projects[0]['name']} ({project_id})\")" + ] + }, + { + "cell_type": "markdown", + "id": "16", + "metadata": {}, + "source": [ + "## 3. Create materials\n", + "### 3.1. Load materials from local files (or Standata)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "17", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.made.material import Material\n", + "from mat3ra.standata.materials import Materials\n", + "from mat3ra.notebooks_utils.ipython.entity.material.visualize import visualize_materials as visualize\n", + "from mat3ra.notebooks_utils.material import load_material_from_folder\n", + "\n", + "materials = [\n", + " load_material_from_folder(FOLDER, name) or Material.create(Materials.get_by_name_first_match(name))\n", + " for name in MATERIAL_NAMES\n", + "]\n", + "visualize([{\"material\": material, \"title\": material.name} for material in materials])" + ] + }, + { + "cell_type": "markdown", + "id": "18", + "metadata": {}, + "source": [ + "### 3.2. Save materials to the platform" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "19", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.core.entity.material.api import get_or_create_material\n", + "\n", + "saved_materials = [\n", + " Material.create(get_or_create_material(client, material, ACCOUNT_ID)) for material in materials\n", + "]" + ] + }, + { + "cell_type": "markdown", + "id": "20", + "metadata": {}, + "source": [ + "## 4. Create workflow and set its parameters\n", + "### 4.1. Upload the script and its data files\n", + "\n", + "The files go to your account's object storage folder (\"Dropbox\"), which the compute node reads them\n", + "from. An upload travels as a string inside the request body, which the API caps at 50 MB, and must\n", + "be UTF-8 text (UPF pseudopotentials qualify — they are XML-like text). A file that is too large or\n", + "not text is uploaded through the Dropbox page in the web interface instead; it is then already in\n", + "the folder, so rather than listing it in `USER_ASSET_FILES`, add its storage record to\n", + "`uploaded_files` after the cell below." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "21", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from mat3ra.notebooks_utils.core.entity.file.api import upload_files\n", + "\n", + "files_to_upload = {\"user_script.sh\": USER_SCRIPT}\n", + "for name in USER_ASSET_FILES:\n", + " if name in files_to_upload:\n", + " raise ValueError(f\"Rename '{name}' in USER_ASSET_FILES: this notebook already uploads a file by that name.\")\n", + " path = os.path.join(FOLDER, name)\n", + " if not os.path.exists(path):\n", + " raise FileNotFoundError(f\"'{name}' is listed in USER_ASSET_FILES but is not in {FOLDER}.\")\n", + " with open(path, \"rb\") as file:\n", + " content = file.read()\n", + " try:\n", + " files_to_upload[name] = content.decode(\"utf-8\")\n", + " except UnicodeDecodeError:\n", + " raise ValueError(\n", + " f\"'{name}' is not UTF-8 text. An upload travels as a JSON string, so binary files go \"\n", + " f\"through the Dropbox page of the web interface instead, into the same folder.\"\n", + " )\n", + "\n", + "uploaded_files = upload_files(client, files_to_upload, ACCOUNT_ID)" + ] + }, + { + "cell_type": "markdown", + "id": "22", + "metadata": {}, + "source": [ + "### 4.2. Create workflow from standard workflows and preview it\n", + "\n", + "The `Custom Shell Script` workflow already carries the unit chain this needs: fetch the uploaded\n", + "files, fetch the material, run the script. Two things are filled in per job — the objects to fetch\n", + "and the runner that hands your script its inputs." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "23", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.standata.applications import ApplicationStandata\n", + "from mat3ra.ade.application import Application\n", + "from mat3ra.standata.workflows import WorkflowStandata\n", + "from mat3ra.wode.workflows import Workflow\n", + "from mat3ra.notebooks_utils.core.entity.file.api import to_object_storage_input\n", + "from mat3ra.notebooks_utils.core.entity.workflow.api import CUSTOM_SCRIPT_RUNNER_SH, set_execution_unit_input\n", + "from mat3ra.notebooks_utils.ipython.entity.workflow.visualize import visualize_workflow\n", + "\n", + "app_config = ApplicationStandata.get_by_name_first_match(APPLICATION_NAME)\n", + "app = Application(**app_config)\n", + "\n", + "workflow_config = WorkflowStandata.filter_by_application(app.name).get_by_name_first_match(WORKFLOW_SEARCH_TERM)\n", + "workflow_config[\"name\"] = MY_WORKFLOW_NAME\n", + "subworkflow = workflow_config[\"subworkflows\"][0]\n", + "subworkflow[\"name\"] = MY_WORKFLOW_NAME\n", + "units = {unit[\"name\"]: unit for unit in subworkflow[\"units\"]}\n", + "\n", + "units[\"io-user-files\"][\"input\"] = [to_object_storage_input(file) for file in uploaded_files]\n", + "\n", + "set_execution_unit_input(units[\"custom_script\"], \"hello_world.sh\", CUSTOM_SCRIPT_RUNNER_SH)\n", + "\n", + "workflow = Workflow.create(workflow_config)\n", + "visualize_workflow(workflow)" + ] + }, + { + "cell_type": "markdown", + "id": "24", + "metadata": {}, + "source": [ + "### 4.3. Save workflow to collection\n", + "\n", + "Saving it makes the workflow reusable: it stays in your collection pointing at the uploaded files,\n", + "so it can be run again — from the UI or another notebook — against any material. Re-running this\n", + "notebook overwrites the uploaded files in place, so a saved workflow picks up an edited script\n", + "without being rebuilt." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "25", + "metadata": {}, + "outputs": [], + "source": [ + "saved_workflow = None\n", + "if save_to_collection:\n", + " saved_workflow = Workflow.create(\n", + " client.workflows.create(workflow.to_dict_without_special_keys(), owner_id=ACCOUNT_ID)\n", + " )\n", + " print(f\"✅ Workflow saved to collection: {saved_workflow.id}\")" + ] + }, + { + "cell_type": "markdown", + "id": "26", + "metadata": {}, + "source": [ + "## 5. Create the compute configuration\n", + "### 5.1. Get list of clusters" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "27", + "metadata": {}, + "outputs": [], + "source": [ + "clusters = client.clusters.list()\n", + "print(f\"Available clusters: {[c['hostname'] for c in clusters]}\")" + ] + }, + { + "cell_type": "markdown", + "id": "28", + "metadata": {}, + "source": [ + "### 5.2. Create compute configuration for the jobs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "29", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.ide.compute import Compute\n", + "\n", + "# Select cluster: use specified name if provided, otherwise use first available\n", + "if CLUSTER_NAME:\n", + " cluster = next((c for c in clusters if CLUSTER_NAME in c[\"hostname\"]), None)\n", + "else:\n", + " cluster = clusters[0]\n", + "\n", + "compute = Compute(\n", + " cluster=cluster,\n", + " queue=QUEUE_NAME,\n", + " ppn=PPN,\n", + ")\n", + "print(f\"Using cluster: {compute.cluster.hostname}, queue: {QUEUE_NAME}, ppn: {PPN}\")" + ] + }, + { + "cell_type": "markdown", + "id": "30", + "metadata": {}, + "source": [ + "## 6. Create the jobs with material and workflow configuration\n", + "### 6.1. Create one job per material" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "31", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.job import create_job\n", + "from mat3ra.notebooks_utils.ui import display_JSON\n", + "\n", + "jobs = []\n", + "for saved_material in saved_materials:\n", + " job_response = create_job(\n", + " api_client=client,\n", + " materials=[saved_material],\n", + " workflow=saved_workflow or workflow,\n", + " project_id=project_id,\n", + " owner_id=ACCOUNT_ID,\n", + " prefix=f\"{MY_WORKFLOW_NAME} {saved_material.formula} {timestamp}\",\n", + " compute=compute.to_dict(),\n", + " )\n", + " jobs.append(job_response if not isinstance(job_response, list) else job_response[0])\n", + "\n", + "job_ids = [job[\"_id\"] for job in jobs]\n", + "print(f\"✅ Created {len(job_ids)} jobs: {job_ids}\")\n", + "display_JSON(jobs[0])" + ] + }, + { + "cell_type": "markdown", + "id": "32", + "metadata": {}, + "source": [ + "## 7. Submit the jobs and monitor the status" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "33", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.api.job import submit_jobs\n", + "\n", + "submit_jobs(client.jobs, job_ids)\n", + "print(f\"✅ Submitted {len(job_ids)} jobs successfully!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "34", + "metadata": {}, + "outputs": [], + "source": [ + "from mat3ra.notebooks_utils.api.job import wait_for_jobs_to_finish_async\n", + "\n", + "await wait_for_jobs_to_finish_async(client.jobs, job_ids, poll_interval=POLL_INTERVAL)" + ] + }, + { + "cell_type": "markdown", + "id": "35", + "metadata": {}, + "source": [ + "## 8. Retrieve results\n", + "\n", + "Each job's standard output is the script's own output. Lines that parse as JSON become table\n", + "columns; everything else is shown as printed." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "36", + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "\n", + "from mat3ra.notebooks_utils.io import read_from_url\n", + "\n", + "\n", + "async def read_job_stdout(job_id):\n", + " \"\"\"Contents of the execution unit's .out file, or why the job produced none.\"\"\"\n", + " files = client.jobs.list_files(job_id)\n", + " stdout_file = next((file for file in files if file[\"key\"].endswith(\".out\")), None)\n", + " if stdout_file is None:\n", + " job = client.jobs.get(job_id)\n", + " errors = job.get(\"compute\", {}).get(\"errors\", [])\n", + " return f\"No output. Job status: {job['status']}. {json.dumps(errors, indent=2)}\"\n", + " return await read_from_url(stdout_file[\"signedUrl\"])\n", + "\n", + "\n", + "results = []\n", + "for saved_material, job in zip(saved_materials, jobs):\n", + " stdout = await read_job_stdout(job[\"_id\"])\n", + " print(f\"--- {job['name']} ---\\n{stdout}\")\n", + " for line in stdout.splitlines():\n", + " try:\n", + " results.append({\"material\": saved_material.name, **json.loads(line)})\n", + " except json.JSONDecodeError:\n", + " continue\n", + "\n", + "pd.DataFrame(results)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.2" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py index 94eabe9e0..50d23deea 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py @@ -7,8 +7,9 @@ # All of these land in the job's working directory after the IO unit has fetched the uploaded # files, so an upload under any of them is overwritten before the user's script runs: the execution # unit renders `script.py` and `requirements.txt`, and the runner it renders writes `material.json`. -# Keep in step with CUSTOM_SCRIPT_RUNNER in ../workflow/api.py. -RESERVED_FILENAMES = ("script.py", "requirements.txt", "material.json") +# Keep in step with CUSTOM_SCRIPT_RUNNER / CUSTOM_SCRIPT_RUNNER_SH in ../workflow/api.py. +# `hello_world.sh` is the on-disk name of the shell runner (the shell flavor's input name). +RESERVED_FILENAMES = ("script.py", "requirements.txt", "material.json", "hello_world.sh") def _files_endpoint(api_client: APIClient) -> BaseEndpoint: diff --git a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py index 3ac427d25..00410da36 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py @@ -52,6 +52,19 @@ def copy_bank_workflow_by_system_name(endpoint: BankWorkflowEndpoints, system_na ''' +# The shell twin of CUSTOM_SCRIPT_RUNNER: rendered as `hello_world.sh` (the shell flavor's input +# name), it writes the job's material next to the uploaded files and hands control to the user's +# script. The script is *sourced*, not run in a subshell, so `module` stays available - rupy runs +# the runner the same way. Also listed in RESERVED_FILENAMES in ../file/api.py. +CUSTOM_SCRIPT_RUNNER_SH = """#!/bin/bash +cat > material.json <<'MATERIAL_JSON_EOF' +{{ MATERIAL | default({}) | tojson }} +MATERIAL_JSON_EOF + +. user_script.sh +""" + + def set_execution_unit_input(unit: dict, template_name: str, content: str) -> None: """ Replaces one input file of an execution unit with fixed content. diff --git a/tests/py/unit/core/entity/test_file_api.py b/tests/py/unit/core/entity/test_file_api.py index 28c96a5e1..dab663cb3 100644 --- a/tests/py/unit/core/entity/test_file_api.py +++ b/tests/py/unit/core/entity/test_file_api.py @@ -81,6 +81,12 @@ def test_object_storage_basename_is_the_name_the_script_opens(): assert to_object_storage_input(nested)["basename"] == "radii.json" +def test_upload_refuses_the_shell_runner_name(): + """hello_world.sh is what the shell workflow's runner lands as - an upload would be clobbered.""" + with pytest.raises(ValueError, match="hello_world.sh"): + upload_files(None, {"hello_world.sh": "echo hi"}, "account") + + def test_object_storage_pathname_targets_a_subdirectory(): """A pathname of "pseudo" fetches the file into /pseudo — where QE's pseudo_dir points.""" item = to_object_storage_input(CLOUD_FILE, pathname="pseudo") From a81589064069cdffd3ffb74cf0b2105ce8f811d5 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 24 Aug 2026 15:02:47 -0700 Subject: [PATCH 16/21] fix: source the user script by explicit path - POSIX dot searches PATH, not cwd First live run of the shell runner died with ". user_script.sh: file not found": the dot builtin looks the operand up in PATH and the working directory is not on it. "./user_script.sh" is unambiguous in both POSIX and bash modes. --- src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py index 00410da36..fc932d520 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/workflow/api.py @@ -61,7 +61,7 @@ def copy_bank_workflow_by_system_name(endpoint: BankWorkflowEndpoints, system_na {{ MATERIAL | default({}) | tojson }} MATERIAL_JSON_EOF -. user_script.sh +. ./user_script.sh """ From 9e46823c41d1b9c10c1d70c63ad8175f935b6058 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Mon, 24 Aug 2026 15:08:38 -0700 Subject: [PATCH 17/21] fix(notebook): include the wrapped filename in the pseudopotential proof line The calculation itself was already correct on the first complete run - modules loaded, both pw.x steps finished, gamma_direct_gap_ev 2.544 eV, the PBE value for Si - but QE 6.3 wraps "PseudoPot. # 1 for Si read from file:" with the filename on the next line, so the bare grep dropped the very token the proof is about. grep -A 1 carries it. --- .../workflows/custom_shell_calculation.ipynb | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/other/materials_designer/workflows/custom_shell_calculation.ipynb b/other/materials_designer/workflows/custom_shell_calculation.ipynb index 05fb9753a..c166eeaf0 100644 --- a/other/materials_designer/workflows/custom_shell_calculation.ipynb +++ b/other/materials_designer/workflows/custom_shell_calculation.ipynb @@ -215,8 +215,9 @@ "mpirun -np $PBS_NP $EXEC_CMD pw.x -in pw_scf.in > pw_scf.out\n", "mpirun -np $PBS_NP $EXEC_CMD pw.x -in pw_bands.in > pw_bands.out\n", "\n", - "# Proof and results: QE's own log names the pseudopotential file it read.\n", - "grep \"read from file\" pw_scf.out\n", + "# Proof and results: QE's own log names the pseudopotential file it read\n", + "# (the filename wraps onto the next line, hence -A 1).\n", + "grep -A 1 \"read from file\" pw_scf.out\n", "grep \"highest occupied\" pw_scf.out\n", "\n", "# Band edges at Gamma from the bands run, as JSON for the results table.\n", From c516dae8c739c396d2b25d2b9b29f6f4177716a6 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Thu, 27 Aug 2026 09:59:41 -0700 Subject: [PATCH 18/21] refactor(notebook): a four-line default script Per Vsevolod: "simplify the python code example ... few lines". The default reads material.json, counts the atoms, lists the elements, prints one JSON line. Verified against Silicon and Graphene. The lattice-volume arithmetic that made the previous default look like homework is gone; the Cypress feature still runs the fuller coordination script through its cell override. --- .../workflows/custom_python_calculation.ipynb | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index 6ed3036da..8bf3236f8 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -146,29 +146,11 @@ "source": [ "USER_SCRIPT = r\"\"\"\n", "import json\n", - "import math\n", "\n", "material = json.load(open(\"material.json\"))\n", - "\n", - "# The platform stores the unit cell as lengths and angles, so the volume comes from those.\n", - "lattice = material[\"lattice\"]\n", - "a, b, c = lattice[\"a\"], lattice[\"b\"], lattice[\"c\"]\n", - "cos_alpha, cos_beta, cos_gamma = (math.cos(math.radians(lattice[key])) for key in (\"alpha\", \"beta\", \"gamma\"))\n", - "volume = (\n", - " a\n", - " * b\n", - " * c\n", - " * math.sqrt(1 - cos_alpha**2 - cos_beta**2 - cos_gamma**2 + 2 * cos_alpha * cos_beta * cos_gamma)\n", - ")\n", - "\n", "elements = [element[\"value\"] for element in material[\"basis\"][\"elements\"]]\n", "\n", - "print(json.dumps({\n", - " \"elements\": sorted(set(elements)),\n", - " \"n_atoms\": len(elements),\n", - " \"volume\": round(volume, 4),\n", - " \"atoms_per_volume\": round(len(elements) / volume, 4),\n", - "}))\n", + "print(json.dumps({\"n_atoms\": len(elements), \"elements\": sorted(set(elements))}))\n", "\"\"\"" ] }, From 64344f1beebdf7f1f62905e008c29f32495e8a34 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Thu, 27 Aug 2026 19:25:35 -0700 Subject: [PATCH 19/21] refactor: nothing shipped in uploads/; the shell default reads the platform's pseudo library Vsevolod: "I never asked for this ... this is not generic". uploads/ is the folder users drop their own files into; example data does not belong in it. radii.json (my own test fixture) and Si.upf are removed. custom_shell_calculation defaults to USER_ASSET_FILES = [] and its script reads the pseudopotential from the platform library, the way the bank's job_espresso_pw_scf.sh does (PSEUDO_DIR / PSEUDO_FILES at the top of the script, exported into the input builder). Using one's own file is the same three-step change the comments describe: drop it in uploads, list it, point PSEUDO_DIR at "./". to_object_storage_input loses the pathname parameter that only the deleted band-structure notebook used. Input building re-verified against the Silicon material; 8/8 unit tests pass. --- other/materials_designer/uploads/Si.upf | 3 -- other/materials_designer/uploads/radii.json | 18 ------------ .../workflows/custom_python_calculation.ipynb | 2 +- .../workflows/custom_shell_calculation.ipynb | 29 ++++++++++++------- .../notebooks_utils/core/entity/file/api.py | 9 ++---- tests/py/unit/core/entity/test_file_api.py | 8 ----- 6 files changed, 22 insertions(+), 47 deletions(-) delete mode 100644 other/materials_designer/uploads/Si.upf delete mode 100644 other/materials_designer/uploads/radii.json diff --git a/other/materials_designer/uploads/Si.upf b/other/materials_designer/uploads/Si.upf deleted file mode 100644 index 4fb6731d6..000000000 --- a/other/materials_designer/uploads/Si.upf +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:39822757f53f36e3bf3bfb779356152a8d3f21199c7db9dd5a931e5d18c45282 -size 225602 diff --git a/other/materials_designer/uploads/radii.json b/other/materials_designer/uploads/radii.json deleted file mode 100644 index 69f11ff0e..000000000 --- a/other/materials_designer/uploads/radii.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "H": 0.31, - "C": 0.76, - "N": 0.71, - "O": 0.66, - "F": 0.57, - "Si": 1.11, - "P": 1.07, - "S": 1.05, - "Cl": 1.02, - "Ge": 1.2, - "As": 1.19, - "Se": 1.2, - "Br": 1.2, - "Ni": 1.24, - "Cu": 1.32, - "Au": 1.36 -} diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index 8bf3236f8..0edb0cb82 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -104,7 +104,7 @@ "\n", "# 4. Script parameters\n", "USER_REQUIREMENTS = [] # e.g. [\"numpy<2\"], installed into a virtual environment on the compute node\n", - "USER_ASSET_FILES = [] # e.g. [\"radii.json\"], data files the script opens, taken from FOLDER\n", + "USER_ASSET_FILES = [] # data files the script opens, taken from FOLDER\n", "\n", "# 5. Workflow parameters\n", "WORKFLOW_SEARCH_TERM = \"custom_script.json\" # Search term for Workflows Standata\n", diff --git a/other/materials_designer/workflows/custom_shell_calculation.ipynb b/other/materials_designer/workflows/custom_shell_calculation.ipynb index c166eeaf0..1e39cbf1d 100644 --- a/other/materials_designer/workflows/custom_shell_calculation.ipynb +++ b/other/materials_designer/workflows/custom_shell_calculation.ipynb @@ -83,7 +83,7 @@ "\n", "`USER_ASSET_FILES` names UTF-8 text files the script opens. Put them in the `../uploads` folder\n", "first — drag them into the JupyterLite file browser — and this notebook uploads them alongside the\n", - "script. The default example needs the pseudopotential `Si.upf`, which ships with this notebook." + "script." ] }, { @@ -107,7 +107,7 @@ "MATERIAL_NAMES = [\"Silicon\"] # One job is created per material\n", "\n", "# 4. Script parameters\n", - "USER_ASSET_FILES = [\"Si.upf\"] # Files the script opens, taken from FOLDER\n", + "USER_ASSET_FILES = [] # data files the script opens, taken from FOLDER, e.g. a pseudopotential\n", "\n", "# 5. Workflow parameters\n", "WORKFLOW_SEARCH_TERM = \"custom_script.json\" # Search term for Workflows Standata\n", @@ -135,10 +135,10 @@ "This is the calculation. It runs on the compute node with `material.json` and your data files\n", "beside it, and whatever it prints becomes the result. Replace it with your own.\n", "\n", - "The default example runs a Quantum ESPRESSO band structure with the uploaded pseudopotential: it\n", - "builds the pw.x inputs from `material.json`, points `pseudo_dir` at the working directory so the\n", - "uploaded `Si.upf` is the file the calculation reads, runs an SCF and a bands step, and prints the\n", - "band edges as JSON. The script is uploaded as a file and fetched onto the node, never inlined into\n", + "The default example runs a Quantum ESPRESSO band structure for silicon: it builds the pw.x inputs\n", + "from `material.json`, runs an SCF and a bands step, and prints the band edges as JSON. It reads the\n", + "pseudopotential from the platform's library; to use your own, put the file in `../uploads`, list it\n", + "in `USER_ASSET_FILES`, and set `PSEUDO_DIR` in the script to the working directory. The script is uploaded as a file and fetched onto the node, never inlined into\n", "the workflow, so its contents reach the shell exactly as written." ] }, @@ -150,12 +150,17 @@ "outputs": [], "source": [ "USER_SCRIPT = r\"\"\"#!/bin/bash\n", - "# Runs a Quantum ESPRESSO band structure for the job's material, reading the\n", - "# pseudopotential uploaded next to this script instead of a platform default.\n", + "# Runs a Quantum ESPRESSO band structure for the job's material.\n", + "#\n", + "# The pseudopotential comes from the platform's library. To use your own instead, put it in\n", + "# uploads, list it in USER_ASSET_FILES, and point PSEUDO_DIR at the working directory (\"./\").\n", + "PSEUDO_DIR=\"/export/share/pseudo/si/gga/pbe/gbrv/1.0/us/\"\n", + "PSEUDO_FILES='{\"Si\": \"si_pbe_gbrv_1.0.upf\"}'\n", "\n", "module add espresso\n", "\n", "# Build the pw.x inputs from the job's material.\n", + "export PSEUDO_DIR PSEUDO_FILES\n", "python3 - <<'BUILD_INPUTS_EOF'\n", "import json\n", "import math\n", @@ -174,8 +179,10 @@ "elements = [element[\"value\"] for element in material[\"basis\"][\"elements\"]]\n", "coordinates = [point[\"value\"] for point in material[\"basis\"][\"coordinates\"]]\n", "\n", + "import os\n", "MASSES = {\"Si\": 28.0855}\n", - "PSEUDOS = {\"Si\": \"Si.upf\"} # the uploaded file; pseudo_dir is the working directory\n", + "PSEUDOS = json.loads(os.environ[\"PSEUDO_FILES\"])\n", + "PSEUDO_DIR = os.environ[\"PSEUDO_DIR\"]\n", "\n", "cell = \"\\n\".join(\" \".join(f\"{value:.9f}\" for value in vector) for vector in vectors)\n", "positions = \"\\n\".join(f\"{el} \" + \" \".join(f\"{x:.9f}\" for x in xyz) for el, xyz in zip(elements, coordinates))\n", @@ -201,12 +208,12 @@ "'''\n", "\n", "with open(\"pw_scf.in\", \"w\") as f:\n", - " f.write(\"&CONTROL\\n calculation = 'scf'\\n pseudo_dir = './'\\n outdir = './outdir'\\n/\\n\")\n", + " f.write(f\"&CONTROL\\n calculation = 'scf'\\n pseudo_dir = '{PSEUDO_DIR}'\\n outdir = './outdir'\\n/\\n\")\n", " f.write(common)\n", " f.write(\"K_POINTS automatic\\n6 6 6 0 0 0\\n\")\n", "\n", "with open(\"pw_bands.in\", \"w\") as f:\n", - " f.write(\"&CONTROL\\n calculation = 'bands'\\n pseudo_dir = './'\\n outdir = './outdir'\\n/\\n\")\n", + " f.write(f\"&CONTROL\\n calculation = 'bands'\\n pseudo_dir = '{PSEUDO_DIR}'\\n outdir = './outdir'\\n/\\n\")\n", " f.write(common)\n", " f.write(\"K_POINTS crystal_b\\n3\\n0.5 0.5 0.5 20\\n0.0 0.0 0.0 20\\n0.5 0.0 0.5 20\\n\")\n", "BUILD_INPUTS_EOF\n", diff --git a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py index 50d23deea..8a6c18f1e 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py @@ -63,24 +63,21 @@ def upload_files(api_client: APIClient, files: Dict[str, str], account_id: str) return uploaded -def to_object_storage_input(cloud_file: dict, pathname: str = "") -> dict: +def to_object_storage_input(cloud_file: dict) -> dict: """ Converts an upload record into an `object_storage` input item for a workflow IO unit. Args: cloud_file (dict): A record returned by `upload_files`. - pathname (str): Subdirectory of the job's working directory to fetch into. The runner - creates it if needed. E.g. "pseudo" places the file where Quantum ESPRESSO's - `pseudo_dir` points. Returns: dict: IO unit input item. The runner fetches it into the job's working directory under - `pathname`/`basename`, which is what makes relative paths resolve. + `basename`, which is what makes the user script's relative paths resolve. """ return { "type": "object_storage", "basename": cloud_file["key"].split("/")[-1], - "pathname": pathname, + "pathname": "", "objectData": { "NAME": cloud_file["key"], "PROVIDER": cloud_file["provider"], diff --git a/tests/py/unit/core/entity/test_file_api.py b/tests/py/unit/core/entity/test_file_api.py index dab663cb3..7b4a105d0 100644 --- a/tests/py/unit/core/entity/test_file_api.py +++ b/tests/py/unit/core/entity/test_file_api.py @@ -85,11 +85,3 @@ def test_upload_refuses_the_shell_runner_name(): """hello_world.sh is what the shell workflow's runner lands as - an upload would be clobbered.""" with pytest.raises(ValueError, match="hello_world.sh"): upload_files(None, {"hello_world.sh": "echo hi"}, "account") - - -def test_object_storage_pathname_targets_a_subdirectory(): - """A pathname of "pseudo" fetches the file into /pseudo — where QE's pseudo_dir points.""" - item = to_object_storage_input(CLOUD_FILE, pathname="pseudo") - - assert item["pathname"] == "pseudo" - assert item["basename"] == "user_script.py" From 680360930a51476981623e09781bf475165dd178 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Thu, 27 Aug 2026 20:39:36 -0700 Subject: [PATCH 20/21] feat: bytes upload through a platform-signed PUT URL MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vsevolod: "ok. this is what needs to be." The platform's file create is text in a JSON body, so a model checkpoint or any binary could not be uploaded at all - not from the notebook, not from the Dropbox page (readAsText). upload_files now takes bytes as well as str: text still goes through POST /files; bytes ask POST /files/signed-urls for a putObject URL (web-app, stacked on #2965) and PUT the content to storage directly - in JupyterLite via the browser's XMLHttpRequest, elsewhere via urllib. The response carries the storage coordinates, so the record feeds to_object_storage_input without a second call. Both notebooks read files as bytes and keep them as bytes when they are not UTF-8 text; the §4.1 limits paragraph goes. 10/10 unit tests. --- .../workflows/custom_python_calculation.ipynb | 21 ++---- .../workflows/custom_shell_calculation.ipynb | 15 ++--- .../notebooks_utils/core/entity/file/api.py | 55 +++++++++++++-- tests/py/unit/core/entity/test_file_api.py | 67 +++++++++++++++++++ 4 files changed, 126 insertions(+), 32 deletions(-) diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index 0edb0cb82..d8fa6b55b 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -310,17 +310,9 @@ "### 4.1. Upload the script and its data files\n", "\n", "The files go to your account's object storage folder (\"Dropbox\"), which the compute node reads them\n", - "from. An upload travels as a string inside the request body, which the API caps at 50 MB, and\n", - "JupyterLite holds the content in memory before sending it.\n", - "\n", - "A file that is too large for that, or that is not UTF-8 text, is uploaded through the Dropbox page\n", - "in the web interface instead. It is then already in the folder, so rather than listing it in\n", - "`USER_ASSET_FILES`, add its storage record to `uploaded_files` after the cell below — reusing the\n", - "record of the script, which sits in the same folder:\n", - "\n", - "```python\n", - "uploaded_files.append({**uploaded_files[0], \"key\": f\"{os.path.dirname(uploaded_files[0]['key'])}/big_file.dat\"})\n", - "```" + "from. Text travels inside the request; anything else — a model checkpoint, an archive, a large file\n", + "— is sent to a URL the platform signs for it, straight to storage, so there is no size or format\n", + "limit beyond what the browser can hold in memory." ] }, { @@ -344,12 +336,9 @@ " with open(path, \"rb\") as file:\n", " content = file.read()\n", " try:\n", - " files_to_upload[name] = content.decode(\"utf-8\")\n", + " files_to_upload[name] = content.decode(\"utf-8\") # text travels in the request body\n", " except UnicodeDecodeError:\n", - " raise ValueError(\n", - " f\"'{name}' is not UTF-8 text. An upload travels as a JSON string, so binary files go \"\n", - " f\"through the Dropbox page of the web interface instead, into the same folder.\"\n", - " )\n", + " files_to_upload[name] = content # anything else goes to storage through a signed URL\n", "\n", "uploaded_files = upload_files(client, files_to_upload, ACCOUNT_ID)" ] diff --git a/other/materials_designer/workflows/custom_shell_calculation.ipynb b/other/materials_designer/workflows/custom_shell_calculation.ipynb index 1e39cbf1d..dc2416af8 100644 --- a/other/materials_designer/workflows/custom_shell_calculation.ipynb +++ b/other/materials_designer/workflows/custom_shell_calculation.ipynb @@ -402,11 +402,9 @@ "### 4.1. Upload the script and its data files\n", "\n", "The files go to your account's object storage folder (\"Dropbox\"), which the compute node reads them\n", - "from. An upload travels as a string inside the request body, which the API caps at 50 MB, and must\n", - "be UTF-8 text (UPF pseudopotentials qualify — they are XML-like text). A file that is too large or\n", - "not text is uploaded through the Dropbox page in the web interface instead; it is then already in\n", - "the folder, so rather than listing it in `USER_ASSET_FILES`, add its storage record to\n", - "`uploaded_files` after the cell below." + "from. Text travels inside the request; anything else — a model checkpoint, an archive, a large file\n", + "— is sent to a URL the platform signs for it, straight to storage, so there is no size or format\n", + "limit beyond what the browser can hold in memory." ] }, { @@ -430,12 +428,9 @@ " with open(path, \"rb\") as file:\n", " content = file.read()\n", " try:\n", - " files_to_upload[name] = content.decode(\"utf-8\")\n", + " files_to_upload[name] = content.decode(\"utf-8\") # text travels in the request body\n", " except UnicodeDecodeError:\n", - " raise ValueError(\n", - " f\"'{name}' is not UTF-8 text. An upload travels as a JSON string, so binary files go \"\n", - " f\"through the Dropbox page of the web interface instead, into the same folder.\"\n", - " )\n", + " files_to_upload[name] = content # anything else goes to storage through a signed URL\n", "\n", "uploaded_files = upload_files(client, files_to_upload, ACCOUNT_ID)" ] diff --git a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py index 8a6c18f1e..b4bc76bad 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py @@ -1,5 +1,5 @@ import json -from typing import Dict, List +from typing import Dict, List, Union from mat3ra.api_client import APIClient from mat3ra.api_client.endpoints import BaseEndpoint @@ -31,13 +31,19 @@ def _files_endpoint(api_client: APIClient) -> BaseEndpoint: ) -def upload_files(api_client: APIClient, files: Dict[str, str], account_id: str) -> List[dict]: +def upload_files(api_client: APIClient, files: Dict[str, Union[str, bytes]], account_id: str) -> List[dict]: """ - Uploads text files to the account's object storage ("Dropbox") folder. + Uploads files to the account's object storage ("Dropbox") folder. + + Text travels inside the request body (`POST /files`). Bytes - a model checkpoint, an archive, + anything that is not UTF-8 text or is large - go through a URL the platform signs for the + purpose (`POST /files/signed-urls`) and are PUT to storage directly, so neither the JSON body + limit nor the text encoding applies. Args: api_client (APIClient): API client instance carrying the authorization context. - files (dict): File name (relative to the account folder) mapped to its text content. + files (dict): File name (relative to the account folder) mapped to its content, `str` for + text or `bytes` for anything else. account_id (str): Account to upload under. Returns: @@ -56,13 +62,50 @@ def upload_files(api_client: APIClient, files: Dict[str, str], account_id: str) uploaded = [] for name, content in files.items(): - payload = {"name": name, "body": content, "accountId": account_id} - record = endpoint.request("POST", "files", data=json.dumps(payload), headers=headers) + if isinstance(content, bytes): + record = _upload_bytes(endpoint, headers, name, content, account_id) + else: + payload = {"name": name, "body": content, "accountId": account_id} + record = endpoint.request("POST", "files", data=json.dumps(payload), headers=headers) print(f"⬆️ Uploaded {record['key']} ({record['size']} bytes)") uploaded.append(record) return uploaded +def _upload_bytes(endpoint: BaseEndpoint, headers: dict, name: str, content: bytes, account_id: str) -> dict: + """Asks the platform for a signed PUT URL for `name`, PUTs the bytes, returns a file record.""" + payload = {"names": [name], "operation": "putObject", "accountId": account_id} + [target] = endpoint.request("POST", "files/signed-urls", data=json.dumps(payload), headers=headers) + _put(target["signedUrl"], content) + return { + "name": name, + "key": target["key"], + "size": len(content), + "bucket": target["bucket"], + "region": target["region"], + "provider": target["provider"], + } + + +def _put(url: str, data: bytes) -> None: + """PUTs bytes to a signed URL - from the browser in JupyterLite, from the process elsewhere.""" + try: + from js import XMLHttpRequest # type: ignore[import-not-found] + from pyodide.ffi import to_js # type: ignore[import-not-found] + except ImportError: + from urllib.request import Request, urlopen + + with urlopen(Request(url, data=data, method="PUT")) as response: + response.read() + return + + request = XMLHttpRequest.new() + request.open("PUT", url, False) + request.send(to_js(data)) + if request.status < 200 or request.status >= 300: + raise RuntimeError(f"PUT to storage failed with HTTP {request.status}: {request.responseText}") + + def to_object_storage_input(cloud_file: dict) -> dict: """ Converts an upload record into an `object_storage` input item for a workflow IO unit. diff --git a/tests/py/unit/core/entity/test_file_api.py b/tests/py/unit/core/entity/test_file_api.py index 7b4a105d0..7637aeec3 100644 --- a/tests/py/unit/core/entity/test_file_api.py +++ b/tests/py/unit/core/entity/test_file_api.py @@ -85,3 +85,70 @@ def test_upload_refuses_the_shell_runner_name(): """hello_world.sh is what the shell workflow's runner lands as - an upload would be clobbered.""" with pytest.raises(ValueError, match="hello_world.sh"): upload_files(None, {"hello_world.sh": "echo hi"}, "account") + + +def test_bytes_go_through_a_signed_put_url(monkeypatch): + """Bytes never enter a JSON body: the helper asks for a putObject URL and PUTs the content.""" + from mat3ra.notebooks_utils.core.entity.file import api as file_api + + calls: list = [] + puts: list = [] + endpoint = MagicMock() + endpoint.get_headers.return_value = {} + + def request(method, path, data=None, headers=None): + calls.append((method, path, json.loads(data))) + return [ + { + "key": "user-abc/model.pt", + "signedUrl": "https://s3/put?sig", + "bucket": "b", + "region": "r", + "provider": "aws", + } + ] + + endpoint.request.side_effect = request + monkeypatch.setattr(file_api, "_files_endpoint", lambda client: endpoint) + monkeypatch.setattr(file_api, "_put", lambda url, data: puts.append((url, data))) + + client = MagicMock() + client.auth.account_id = "acc" + client.auth.auth_token = "tok" + [record] = upload_files(client, {"model.pt": b"\x00\x01binary"}, "acc") + + assert calls == [ + ("POST", "files/signed-urls", {"names": ["model.pt"], "operation": "putObject", "accountId": "acc"}) + ] + assert puts == [("https://s3/put?sig", b"\x00\x01binary")] + assert record == { + "name": "model.pt", + "key": "user-abc/model.pt", + "size": 8, + "bucket": "b", + "region": "r", + "provider": "aws", + } + assert to_object_storage_input(record)["objectData"]["NAME"] == "user-abc/model.pt" + + +def test_text_still_travels_in_the_body(monkeypatch): + from mat3ra.notebooks_utils.core.entity.file import api as file_api + + endpoint = MagicMock() + endpoint.get_headers.return_value = {} + endpoint.request.return_value = { + "key": "user-abc/a.txt", + "size": 2, + "bucket": "b", + "region": "r", + "provider": "aws", + "name": "a.txt", + } + monkeypatch.setattr(file_api, "_files_endpoint", lambda client: endpoint) + client = MagicMock() + client.auth.account_id = "acc" + client.auth.auth_token = "tok" + upload_files(client, {"a.txt": "hi"}, "acc") + method, path = endpoint.request.call_args[0][:2] + assert (method, path) == ("POST", "files") From f68016cd86bfd90e38881b39db2ad29fb9312931 Mon Sep 17 00:00:00 2001 From: VsevolodX Date: Fri, 28 Aug 2026 11:57:18 -0700 Subject: [PATCH 21/21] fix: name the stdout file instead of taking the first .out in the job Review on #360: the results cell took `next(f for f in files if f["key"].endswith(".out"))`. The runner names a unit's stdout after the unit, so that is `custom_script.out` - but the shell notebook's own default writes pw_scf.out and pw_bands.out beside it, and the right file was picked only because "c" sorts before "p". A user script writing bands.out would have made the results table read the calculation's log instead of the script's output. Both notebooks now select by name. Also from the review: _put uses the repo's own is_pyodide_environment() rather than a third detection mechanism whose ImportError fallback would have tried a urllib PUT inside a browser, and it no longer lets urllib stamp a form content-type on an object; set_execution_unit_input gains the unit tests it shipped without (name matching, position independence, unknown-name error); and both notebooks stop telling the reader that asset files must be UTF-8 text, which stopped being true when bytes started going through a signed PUT. --- .../workflows/custom_python_calculation.ipynb | 15 ++++--- .../workflows/custom_shell_calculation.ipynb | 15 ++++--- .../notebooks_utils/core/entity/file/api.py | 13 +++--- .../py/unit/core/entity/test_workflow_api.py | 40 +++++++++++++++++++ 4 files changed, 68 insertions(+), 15 deletions(-) create mode 100644 tests/py/unit/core/entity/test_workflow_api.py diff --git a/other/materials_designer/workflows/custom_python_calculation.ipynb b/other/materials_designer/workflows/custom_python_calculation.ipynb index d8fa6b55b..bd63fe237 100644 --- a/other/materials_designer/workflows/custom_python_calculation.ipynb +++ b/other/materials_designer/workflows/custom_python_calculation.ipynb @@ -77,9 +77,9 @@ "source": [ "### 1.2. Set parameters and configurations for the workflow and job\n", "\n", - "`USER_ASSET_FILES` names UTF-8 text files the script opens. Put them in the `../uploads` folder\n", - "first — drag them into the JupyterLite file browser — and this notebook uploads them alongside the\n", - "script." + "`USER_ASSET_FILES` names data files the script opens, of any type. Put them in the `../uploads`\n", + "folder first — drag them into the JupyterLite file browser — and this notebook uploads them\n", + "alongside the script." ] }, { @@ -560,10 +560,15 @@ "from mat3ra.notebooks_utils.io import read_from_url\n", "\n", "\n", + "# The runner names a unit's stdout after the unit, so this is the execution unit's own output -\n", + "# not whatever .out file the script itself may have written next to it.\n", + "STDOUT_FILENAME = \"custom_script.out\"\n", + "\n", + "\n", "async def read_job_stdout(job_id):\n", - " \"\"\"Contents of the execution unit's .out file, or why the job produced none.\"\"\"\n", + " \"\"\"Contents of the execution unit's stdout, or why the job produced none.\"\"\"\n", " files = client.jobs.list_files(job_id)\n", - " stdout_file = next((file for file in files if file[\"key\"].endswith(\".out\")), None)\n", + " stdout_file = next((file for file in files if file[\"key\"].rsplit(\"/\", 1)[-1] == STDOUT_FILENAME), None)\n", " if stdout_file is None:\n", " job = client.jobs.get(job_id)\n", " errors = job.get(\"compute\", {}).get(\"errors\", [])\n", diff --git a/other/materials_designer/workflows/custom_shell_calculation.ipynb b/other/materials_designer/workflows/custom_shell_calculation.ipynb index dc2416af8..b6377afc5 100644 --- a/other/materials_designer/workflows/custom_shell_calculation.ipynb +++ b/other/materials_designer/workflows/custom_shell_calculation.ipynb @@ -81,9 +81,9 @@ "source": [ "### 1.2. Set parameters and configurations for the workflow and job\n", "\n", - "`USER_ASSET_FILES` names UTF-8 text files the script opens. Put them in the `../uploads` folder\n", - "first — drag them into the JupyterLite file browser — and this notebook uploads them alongside the\n", - "script." + "`USER_ASSET_FILES` names data files the script opens, of any type. Put them in the `../uploads`\n", + "folder first — drag them into the JupyterLite file browser — and this notebook uploads them\n", + "alongside the script." ] }, { @@ -651,10 +651,15 @@ "from mat3ra.notebooks_utils.io import read_from_url\n", "\n", "\n", + "# The runner names a unit's stdout after the unit, so this is the execution unit's own output -\n", + "# not whatever .out file the script itself may have written next to it.\n", + "STDOUT_FILENAME = \"custom_script.out\"\n", + "\n", + "\n", "async def read_job_stdout(job_id):\n", - " \"\"\"Contents of the execution unit's .out file, or why the job produced none.\"\"\"\n", + " \"\"\"Contents of the execution unit's stdout, or why the job produced none.\"\"\"\n", " files = client.jobs.list_files(job_id)\n", - " stdout_file = next((file for file in files if file[\"key\"].endswith(\".out\")), None)\n", + " stdout_file = next((file for file in files if file[\"key\"].rsplit(\"/\", 1)[-1] == STDOUT_FILENAME), None)\n", " if stdout_file is None:\n", " job = client.jobs.get(job_id)\n", " errors = job.get(\"compute\", {}).get(\"errors\", [])\n", diff --git a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py index b4bc76bad..c4847f8bd 100644 --- a/src/py/mat3ra/notebooks_utils/core/entity/file/api.py +++ b/src/py/mat3ra/notebooks_utils/core/entity/file/api.py @@ -3,6 +3,7 @@ from mat3ra.api_client import APIClient from mat3ra.api_client.endpoints import BaseEndpoint +from mat3ra.notebooks_utils.primitive.environment import is_pyodide_environment # All of these land in the job's working directory after the IO unit has fetched the uploaded # files, so an upload under any of them is overwritten before the user's script runs: the execution @@ -89,16 +90,18 @@ def _upload_bytes(endpoint: BaseEndpoint, headers: dict, name: str, content: byt def _put(url: str, data: bytes) -> None: """PUTs bytes to a signed URL - from the browser in JupyterLite, from the process elsewhere.""" - try: - from js import XMLHttpRequest # type: ignore[import-not-found] - from pyodide.ffi import to_js # type: ignore[import-not-found] - except ImportError: + if not is_pyodide_environment(): from urllib.request import Request, urlopen - with urlopen(Request(url, data=data, method="PUT")) as response: + put = Request(url, data=data, method="PUT") + put.add_header("Content-Type", "application/octet-stream") + with urlopen(put) as response: response.read() return + from js import XMLHttpRequest # type: ignore[import-not-found] + from pyodide.ffi import to_js # type: ignore[import-not-found] + request = XMLHttpRequest.new() request.open("PUT", url, False) request.send(to_js(data)) diff --git a/tests/py/unit/core/entity/test_workflow_api.py b/tests/py/unit/core/entity/test_workflow_api.py new file mode 100644 index 000000000..337db1851 --- /dev/null +++ b/tests/py/unit/core/entity/test_workflow_api.py @@ -0,0 +1,40 @@ +import pytest +from mat3ra.notebooks_utils.core.entity.workflow.api import set_execution_unit_input + + +def _unit(): + return { + "name": "custom_script", + "input": [ + {"template": {"name": "script.py", "content": "original"}}, + {"template": {"name": "requirements.txt", "content": ""}}, + ], + } + + +def test_replaces_the_named_input_and_marks_it_manually_changed(): + """isManuallyChanged is what stops the platform re-rendering the runner's placeholders.""" + unit = _unit() + set_execution_unit_input(unit, "requirements.txt", "numpy<2\n") + + changed = unit["input"][1] + assert changed["template"]["content"] == "numpy<2\n" + assert changed["rendered"] == "numpy<2\n" + assert changed["isManuallyChanged"] is True + assert unit["input"][0]["template"]["content"] == "original", "other inputs are untouched" + + +def test_matches_by_name_not_position(): + """A flavor may list its inputs in any order; position would write into the wrong file.""" + unit = _unit() + unit["input"].reverse() + set_execution_unit_input(unit, "script.py", "print(1)") + + by_name = {item["template"]["name"]: item for item in unit["input"]} + assert by_name["script.py"]["rendered"] == "print(1)" + assert "rendered" not in by_name["requirements.txt"] + + +def test_unknown_input_name_is_an_error_naming_what_is_available(): + with pytest.raises(KeyError, match="requirements.txt"): + set_execution_unit_input(_unit(), "nope.txt", "x")