From 4cfbcf97a996981f7c9087ad229d0524f9d9dc5a Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 12 Oct 2026 17:06:42 +0100 Subject: [PATCH 01/13] Add API endpoint to return number of preprocessed and unprocessed movies for a session --- src/murfey/server/api/session_info.py | 39 +++++++++++++++++++++++++++ src/murfey/util/route_manifest.yaml | 7 +++++ 2 files changed, 46 insertions(+) diff --git a/src/murfey/server/api/session_info.py b/src/murfey/server/api/session_info.py index 33c872144..3867d35ae 100644 --- a/src/murfey/server/api/session_info.py +++ b/src/murfey/server/api/session_info.py @@ -7,6 +7,7 @@ from fastapi import APIRouter, Depends, Request from fastapi.responses import FileResponse, JSONResponse from pydantic import BaseModel +from sqlalchemy import func from sqlmodel import Session as SQLModelSession, select import murfey.server @@ -286,6 +287,44 @@ def get_data_collections( return data_collections +@router.get("/sessions/{session_id}/movies/preprocessed") +async def get_preprocessed_movie_count( + session_id: MurfeySessionID, + db: SQLModelSession = murfey_db, +): + """ + Helper function that counts the number of movies for the current session that have + been preprocessed or not, and returns the result as a dictionary. + """ + grouped_movies = db.exec( + select(MurfeyDB.Movie.preprocessed, func.count().label("number")) + .join( + MurfeyDB.DataCollection, + MurfeyDB.Movie.data_collection_id == MurfeyDB.DataCollection.id, + ) + .join( + MurfeyDB.DataCollectionGroup, + MurfeyDB.DataCollection.dcg_id == MurfeyDB.DataCollectionGroup.id, + ) + .join( + MurfeyDB.Session, + MurfeyDB.DataCollectionGroup.session_id == MurfeyDB.Session.id, + ) + .where(MurfeyDB.Session.id == session_id) + .group_by(MurfeyDB.Movie.preprocessed) + ).all() + preprocessed_movies = { + "complete": 0, + "incomplete": 0, + } + for preprocessed, number in grouped_movies: + if preprocessed: + preprocessed_movies["complete"] = number + else: + preprocessed_movies["incomplete"] = number + return preprocessed_movies + + @router.get("/clients") async def get_clients(db: SQLModelSession = murfey_db): clients = db.exec(select(MurfeyDB.ClientEnvironment)).all() diff --git a/src/murfey/util/route_manifest.yaml b/src/murfey/util/route_manifest.yaml index db529858e..0cb6d7352 100644 --- a/src/murfey/util/route_manifest.yaml +++ b/src/murfey/util/route_manifest.yaml @@ -1178,6 +1178,13 @@ murfey.server.api.session_info.router: type: int methods: - GET + - path: /session_info/sessions/{session_id}/movies/preprocessed + function: get_preprocessed_movie_count + path_params: + - name: session_id + type: int + methods: + - GET - path: /session_info/clients function: get_clients path_params: [] From 7a1cabf248103e0aaa877070255f70a3f483a8e5 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 12 Oct 2026 17:24:54 +0100 Subject: [PATCH 02/13] Add test for new endpoint --- tests/server/api/test_session_info.py | 60 +++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) diff --git a/tests/server/api/test_session_info.py b/tests/server/api/test_session_info.py index efeaf268c..3564ee432 100644 --- a/tests/server/api/test_session_info.py +++ b/tests/server/api/test_session_info.py @@ -114,6 +114,66 @@ def mock_get_db_session(): assert murfey_session.visit_end_time is None +async def test_get_preprocessed_movie_count( + tmp_path: Path, + murfey_db_session: SQLModelSession, +): + # Set up parameters + session_id = ExampleVisit.murfey_session_id + + # Create some DataCollectionGroups, DataCollections, and Movies + for i in range(3): + dcg_name = f"dcg_{i}" + dcg_entry = MurfeyDB.DataCollectionGroup( + session_id=session_id, + tag=dcg_name, + ) + murfey_db_session.add(dcg_entry) + + for j in range(3): + dc_entry = MurfeyDB.DataCollection( + dcg_id=dcg_entry.id, + tag=dcg_name, + ) + murfey_db_session.add(dc_entry) + for k in range(4): + movie_entry = MurfeyDB.Movie( + dataCollectionId=dc_entry.id, + path=str(tmp_path / dcg_name / f"movie_{j + k}.mrc"), + tag=dcg_name, + preprocessed=bool(j % 2), + ) + murfey_db_session.add(movie_entry) + # Commit all at once + murfey_db_session.commit() + + # Set up a mock Murfey database session + def mock_get_db_session(): + yield murfey_db_session + + # Set up the backend server + backend_server = set_up_test_backend_client( + router=router, + instrument_name=instrument_name, + session_id=session_id, + mock_db_session=mock_get_db_session, + ) + # Construct the URL path to poke + backend_url_path = url_path_for( + "api.session_info.router", + "get_preprocessed_movie_count", + session_id=session_id, + ) + # Poke the backend + response = backend_server.get( + backend_url_path, + ) + assert response.status_code == 200 + data: dict[str, int] = response.json() + assert data["complete"] == 12 + assert data["incomplete"] == 24 + + @pytest.mark.parametrize( "search_strings", ( From a0978b6b53ded783d26cc9d8952c37091b2b7a4f Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 12 Oct 2026 17:30:07 +0100 Subject: [PATCH 03/13] Test function doesn't need to be async --- tests/server/api/test_session_info.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/server/api/test_session_info.py b/tests/server/api/test_session_info.py index 3564ee432..93d856620 100644 --- a/tests/server/api/test_session_info.py +++ b/tests/server/api/test_session_info.py @@ -114,7 +114,7 @@ def mock_get_db_session(): assert murfey_session.visit_end_time is None -async def test_get_preprocessed_movie_count( +def test_get_preprocessed_movie_count( tmp_path: Path, murfey_db_session: SQLModelSession, ): From 87fd64ed260d18659c5fd981d0ddd341ab3255f0 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 12 Oct 2026 17:38:47 +0100 Subject: [PATCH 04/13] Commit at each level --- tests/server/api/test_session_info.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/server/api/test_session_info.py b/tests/server/api/test_session_info.py index 93d856620..176ffeefb 100644 --- a/tests/server/api/test_session_info.py +++ b/tests/server/api/test_session_info.py @@ -129,13 +129,14 @@ def test_get_preprocessed_movie_count( tag=dcg_name, ) murfey_db_session.add(dcg_entry) - + murfey_db_session.commit() for j in range(3): dc_entry = MurfeyDB.DataCollection( dcg_id=dcg_entry.id, tag=dcg_name, ) murfey_db_session.add(dc_entry) + murfey_db_session.commit() for k in range(4): movie_entry = MurfeyDB.Movie( dataCollectionId=dc_entry.id, From 4bfd10eaa8641de2620cabd511ed33a56a740c40 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 12 Oct 2026 18:07:42 +0100 Subject: [PATCH 05/13] Manually provide row IDs for the test case --- tests/server/api/test_session_info.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/server/api/test_session_info.py b/tests/server/api/test_session_info.py index 176ffeefb..d5d166f6b 100644 --- a/tests/server/api/test_session_info.py +++ b/tests/server/api/test_session_info.py @@ -122,25 +122,31 @@ def test_get_preprocessed_movie_count( session_id = ExampleVisit.murfey_session_id # Create some DataCollectionGroups, DataCollections, and Movies - for i in range(3): - dcg_name = f"dcg_{i}" + num_dcgs = 3 + num_dcs = 3 + num_movies = 4 + for i in range(num_dcgs): + dcg_id = i + dcg_name = f"dcg_{dcg_id}" dcg_entry = MurfeyDB.DataCollectionGroup( + dataCollectionGroupId=dcg_id, session_id=session_id, tag=dcg_name, ) murfey_db_session.add(dcg_entry) - murfey_db_session.commit() - for j in range(3): + for j in range(num_dcs): + dc_id = (i * num_dcs) + j dc_entry = MurfeyDB.DataCollection( - dcg_id=dcg_entry.id, + dataCollectionId=dc_id, + dataCollectionGroupId=dcg_id, tag=dcg_name, ) murfey_db_session.add(dc_entry) - murfey_db_session.commit() - for k in range(4): + for k in range(num_movies): movie_entry = MurfeyDB.Movie( - dataCollectionId=dc_entry.id, - path=str(tmp_path / dcg_name / f"movie_{j + k}.mrc"), + movieId=(i * num_dcs) + (j * num_movies) + k, + dataCollectionId=dc_id, + path=str(tmp_path / dcg_name / f"movie_{(j * num_movies) + k}.mrc"), tag=dcg_name, preprocessed=bool(j % 2), ) From 95ef98d9e17f33ad73818720ea4495dbf06f2c48 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 12 Oct 2026 18:15:17 +0100 Subject: [PATCH 06/13] Populate 'movieNumber' field as well --- tests/server/api/test_session_info.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/server/api/test_session_info.py b/tests/server/api/test_session_info.py index d5d166f6b..9f61d130f 100644 --- a/tests/server/api/test_session_info.py +++ b/tests/server/api/test_session_info.py @@ -143,8 +143,10 @@ def test_get_preprocessed_movie_count( ) murfey_db_session.add(dc_entry) for k in range(num_movies): + movie_id = (i * num_dcs) + (j * num_movies) + k movie_entry = MurfeyDB.Movie( - movieId=(i * num_dcs) + (j * num_movies) + k, + movieId=movie_id, + movieNumber=movie_id, dataCollectionId=dc_id, path=str(tmp_path / dcg_name / f"movie_{(j * num_movies) + k}.mrc"), tag=dcg_name, From 97045432efe1b3108a11f5bb18c7ca5eba101e85 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 12 Oct 2026 18:28:03 +0100 Subject: [PATCH 07/13] More placeholder table entries needed --- tests/server/api/test_session_info.py | 30 ++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/tests/server/api/test_session_info.py b/tests/server/api/test_session_info.py index 9f61d130f..bb7aa760f 100644 --- a/tests/server/api/test_session_info.py +++ b/tests/server/api/test_session_info.py @@ -143,10 +143,34 @@ def test_get_preprocessed_movie_count( ) murfey_db_session.add(dc_entry) for k in range(num_movies): - movie_id = (i * num_dcs) + (j * num_movies) + k + # Add prerequisite table entriees + # ProcessingJob + pj_id = (i * num_dcs) + (j * num_movies) + k + pj_entry = MurfeyDB.ProcessingJob( + processingJobId=pj_id, + recipe="dummy", + dataCollectionId=dc_id, + ) + murfey_db_session.add(pj_entry) + + # AutoProcProgram + app_entry = MurfeyDB.AutoProcProgram( + autoProcProgramId=pj_entry.id, + pj_id=pj_entry.id, + ) + murfey_db_session.add(app_entry) + + # MurfeyLedger + ledger_entry = MurfeyDB.MurfeyLedger( + id=app_entry.id, + app_id=app_entry.id, + ) + murfey_db_session.add(ledger_entry) + + # Add test Movie entry movie_entry = MurfeyDB.Movie( - movieId=movie_id, - movieNumber=movie_id, + movieId=ledger_entry.id, + movieNumber=ledger_entry.id, dataCollectionId=dc_id, path=str(tmp_path / dcg_name / f"movie_{(j * num_movies) + k}.mrc"), tag=dcg_name, From 17b1ed0bb6e306ff96d068baf6712df9e40d17e5 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 12 Oct 2026 19:17:39 +0100 Subject: [PATCH 08/13] Try bulk insert with return --- tests/server/api/test_session_info.py | 144 ++++++++++++++++---------- 1 file changed, 91 insertions(+), 53 deletions(-) diff --git a/tests/server/api/test_session_info.py b/tests/server/api/test_session_info.py index bb7aa760f..a68493a0d 100644 --- a/tests/server/api/test_session_info.py +++ b/tests/server/api/test_session_info.py @@ -1,13 +1,13 @@ from datetime import datetime from pathlib import Path -from typing import Any, Callable +from typing import Any, Callable, Sequence from unittest.mock import MagicMock import pytest from fastapi import APIRouter, FastAPI from fastapi.testclient import TestClient from pytest_mock import MockerFixture -from sqlmodel import Session as SQLModelSession, select +from sqlmodel import Session as SQLModelSession, insert, select import murfey.util.db as MurfeyDB from murfey.server.api.auth import ( @@ -125,60 +125,98 @@ def test_get_preprocessed_movie_count( num_dcgs = 3 num_dcs = 3 num_movies = 4 - for i in range(num_dcgs): - dcg_id = i - dcg_name = f"dcg_{dcg_id}" - dcg_entry = MurfeyDB.DataCollectionGroup( - dataCollectionGroupId=dcg_id, - session_id=session_id, - tag=dcg_name, + + # Insert all DataCollectionGroup rows and return inserted rows + dcgs: Sequence[MurfeyDB.DataCollectionGroup] = murfey_db_session.exec( + insert(MurfeyDB.DataCollectionGroup) + .values( + [ + { + "session_id": session_id, + "tag": f"dcg_{i}", + } + for i in range(num_dcgs) + ] + ) + .returning(MurfeyDB.DataCollectionGroup) + ).all() + + # Insert all DataCollection rows and return inserted rows + dcs: Sequence[MurfeyDB.DataCollection] = murfey_db_session.exec( + insert(MurfeyDB.DataCollection) + .values( + [ + { + "dataCollectionGroupId": dcg.id, + "tag": dcg.tag, + } + for dcg in dcgs + for _ in range(num_dcs) + ] ) - murfey_db_session.add(dcg_entry) - for j in range(num_dcs): - dc_id = (i * num_dcs) + j - dc_entry = MurfeyDB.DataCollection( - dataCollectionId=dc_id, - dataCollectionGroupId=dcg_id, - tag=dcg_name, + .returning(MurfeyDB.DataCollection) + ).all() + + for n, dc in enumerate(dcs): + # Insert all ProcessingJob rows and return inserted rows + pjs: Sequence[MurfeyDB.ProcessingJob] = murfey_db_session.exec( + insert(MurfeyDB.ProcessingJob) + .values( + [ + { + "dataCollectionId": dc.id, + "recipe": "dummy", + } + for _ in range(num_movies) + ] ) - murfey_db_session.add(dc_entry) - for k in range(num_movies): - # Add prerequisite table entriees - # ProcessingJob - pj_id = (i * num_dcs) + (j * num_movies) + k - pj_entry = MurfeyDB.ProcessingJob( - processingJobId=pj_id, - recipe="dummy", - dataCollectionId=dc_id, - ) - murfey_db_session.add(pj_entry) + .returning(MurfeyDB.ProcessingJob) + ).all() - # AutoProcProgram - app_entry = MurfeyDB.AutoProcProgram( - autoProcProgramId=pj_entry.id, - pj_id=pj_entry.id, - ) - murfey_db_session.add(app_entry) + # Insert all ProcessingJob rows and return inserted rows + apps: Sequence[MurfeyDB.AutoProcProgram] = murfey_db_session.exec( + insert(MurfeyDB.AutoProcProgram) + .values( + [ + { + "pj_id": pj.id, + } + for pj in pjs + ] + ) + .returning(MurfeyDB.AutoProcProgram) + ).all() - # MurfeyLedger - ledger_entry = MurfeyDB.MurfeyLedger( - id=app_entry.id, - app_id=app_entry.id, - ) - murfey_db_session.add(ledger_entry) + ledgers: Sequence[MurfeyDB.MurfeyLedger] = murfey_db_session.exec( + insert(MurfeyDB.MurfeyLedger) + .values( + [ + { + "app_id": app.id, + } + for app in apps + ] + ) + .returning(MurfeyDB.MurfeyLedger) + ).all() - # Add test Movie entry - movie_entry = MurfeyDB.Movie( - movieId=ledger_entry.id, - movieNumber=ledger_entry.id, - dataCollectionId=dc_id, - path=str(tmp_path / dcg_name / f"movie_{(j * num_movies) + k}.mrc"), - tag=dcg_name, - preprocessed=bool(j % 2), - ) - murfey_db_session.add(movie_entry) - # Commit all at once - murfey_db_session.commit() + # Insert all Movie rows + murfey_db_session.exec( + insert(MurfeyDB.Movie).values( + [ + { + "movieId": ledger.id, + "dataCollectionId": dc.id, + "path": str(tmp_path) + / dc.tag + / f"movie_{(n * num_movies) + l}.mrc", + "tag": dc.tag, + "preprocessed": bool(n % 2), + } + for l, ledger in enumerate(ledgers) + ] + ) + ) # Set up a mock Murfey database session def mock_get_db_session(): @@ -203,8 +241,8 @@ def mock_get_db_session(): ) assert response.status_code == 200 data: dict[str, int] = response.json() - assert data["complete"] == 12 - assert data["incomplete"] == 24 + assert data["complete"] == 16 + assert data["incomplete"] == 20 @pytest.mark.parametrize( From c0265231512451722e4ad90834918ab192e9a87f Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 12 Oct 2026 21:35:01 +0100 Subject: [PATCH 09/13] Revert "Try bulk insert with return" This reverts commit 17b1ed0bb6e306ff96d068baf6712df9e40d17e5. --- tests/server/api/test_session_info.py | 144 ++++++++++---------------- 1 file changed, 53 insertions(+), 91 deletions(-) diff --git a/tests/server/api/test_session_info.py b/tests/server/api/test_session_info.py index a68493a0d..bb7aa760f 100644 --- a/tests/server/api/test_session_info.py +++ b/tests/server/api/test_session_info.py @@ -1,13 +1,13 @@ from datetime import datetime from pathlib import Path -from typing import Any, Callable, Sequence +from typing import Any, Callable from unittest.mock import MagicMock import pytest from fastapi import APIRouter, FastAPI from fastapi.testclient import TestClient from pytest_mock import MockerFixture -from sqlmodel import Session as SQLModelSession, insert, select +from sqlmodel import Session as SQLModelSession, select import murfey.util.db as MurfeyDB from murfey.server.api.auth import ( @@ -125,98 +125,60 @@ def test_get_preprocessed_movie_count( num_dcgs = 3 num_dcs = 3 num_movies = 4 - - # Insert all DataCollectionGroup rows and return inserted rows - dcgs: Sequence[MurfeyDB.DataCollectionGroup] = murfey_db_session.exec( - insert(MurfeyDB.DataCollectionGroup) - .values( - [ - { - "session_id": session_id, - "tag": f"dcg_{i}", - } - for i in range(num_dcgs) - ] - ) - .returning(MurfeyDB.DataCollectionGroup) - ).all() - - # Insert all DataCollection rows and return inserted rows - dcs: Sequence[MurfeyDB.DataCollection] = murfey_db_session.exec( - insert(MurfeyDB.DataCollection) - .values( - [ - { - "dataCollectionGroupId": dcg.id, - "tag": dcg.tag, - } - for dcg in dcgs - for _ in range(num_dcs) - ] + for i in range(num_dcgs): + dcg_id = i + dcg_name = f"dcg_{dcg_id}" + dcg_entry = MurfeyDB.DataCollectionGroup( + dataCollectionGroupId=dcg_id, + session_id=session_id, + tag=dcg_name, ) - .returning(MurfeyDB.DataCollection) - ).all() - - for n, dc in enumerate(dcs): - # Insert all ProcessingJob rows and return inserted rows - pjs: Sequence[MurfeyDB.ProcessingJob] = murfey_db_session.exec( - insert(MurfeyDB.ProcessingJob) - .values( - [ - { - "dataCollectionId": dc.id, - "recipe": "dummy", - } - for _ in range(num_movies) - ] + murfey_db_session.add(dcg_entry) + for j in range(num_dcs): + dc_id = (i * num_dcs) + j + dc_entry = MurfeyDB.DataCollection( + dataCollectionId=dc_id, + dataCollectionGroupId=dcg_id, + tag=dcg_name, ) - .returning(MurfeyDB.ProcessingJob) - ).all() + murfey_db_session.add(dc_entry) + for k in range(num_movies): + # Add prerequisite table entriees + # ProcessingJob + pj_id = (i * num_dcs) + (j * num_movies) + k + pj_entry = MurfeyDB.ProcessingJob( + processingJobId=pj_id, + recipe="dummy", + dataCollectionId=dc_id, + ) + murfey_db_session.add(pj_entry) - # Insert all ProcessingJob rows and return inserted rows - apps: Sequence[MurfeyDB.AutoProcProgram] = murfey_db_session.exec( - insert(MurfeyDB.AutoProcProgram) - .values( - [ - { - "pj_id": pj.id, - } - for pj in pjs - ] - ) - .returning(MurfeyDB.AutoProcProgram) - ).all() + # AutoProcProgram + app_entry = MurfeyDB.AutoProcProgram( + autoProcProgramId=pj_entry.id, + pj_id=pj_entry.id, + ) + murfey_db_session.add(app_entry) - ledgers: Sequence[MurfeyDB.MurfeyLedger] = murfey_db_session.exec( - insert(MurfeyDB.MurfeyLedger) - .values( - [ - { - "app_id": app.id, - } - for app in apps - ] - ) - .returning(MurfeyDB.MurfeyLedger) - ).all() + # MurfeyLedger + ledger_entry = MurfeyDB.MurfeyLedger( + id=app_entry.id, + app_id=app_entry.id, + ) + murfey_db_session.add(ledger_entry) - # Insert all Movie rows - murfey_db_session.exec( - insert(MurfeyDB.Movie).values( - [ - { - "movieId": ledger.id, - "dataCollectionId": dc.id, - "path": str(tmp_path) - / dc.tag - / f"movie_{(n * num_movies) + l}.mrc", - "tag": dc.tag, - "preprocessed": bool(n % 2), - } - for l, ledger in enumerate(ledgers) - ] - ) - ) + # Add test Movie entry + movie_entry = MurfeyDB.Movie( + movieId=ledger_entry.id, + movieNumber=ledger_entry.id, + dataCollectionId=dc_id, + path=str(tmp_path / dcg_name / f"movie_{(j * num_movies) + k}.mrc"), + tag=dcg_name, + preprocessed=bool(j % 2), + ) + murfey_db_session.add(movie_entry) + # Commit all at once + murfey_db_session.commit() # Set up a mock Murfey database session def mock_get_db_session(): @@ -241,8 +203,8 @@ def mock_get_db_session(): ) assert response.status_code == 200 data: dict[str, int] = response.json() - assert data["complete"] == 16 - assert data["incomplete"] == 20 + assert data["complete"] == 12 + assert data["incomplete"] == 24 @pytest.mark.parametrize( From ce8a801d4394ee02ed051eb62e6c3c48b2884806 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 12 Oct 2026 21:41:31 +0100 Subject: [PATCH 10/13] Mistake when generating incremental IDs --- tests/server/api/test_session_info.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/server/api/test_session_info.py b/tests/server/api/test_session_info.py index bb7aa760f..5938db5e8 100644 --- a/tests/server/api/test_session_info.py +++ b/tests/server/api/test_session_info.py @@ -126,7 +126,7 @@ def test_get_preprocessed_movie_count( num_dcs = 3 num_movies = 4 for i in range(num_dcgs): - dcg_id = i + dcg_id = i + 1 dcg_name = f"dcg_{dcg_id}" dcg_entry = MurfeyDB.DataCollectionGroup( dataCollectionGroupId=dcg_id, @@ -145,7 +145,7 @@ def test_get_preprocessed_movie_count( for k in range(num_movies): # Add prerequisite table entriees # ProcessingJob - pj_id = (i * num_dcs) + (j * num_movies) + k + pj_id = (i * num_dcs * num_movies) + (j * num_movies) + k pj_entry = MurfeyDB.ProcessingJob( processingJobId=pj_id, recipe="dummy", From b893531f18aab0d4091305e503a3e195d8fed63c Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 12 Oct 2026 21:46:58 +0100 Subject: [PATCH 11/13] Start IDs from 1 --- tests/server/api/test_session_info.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/server/api/test_session_info.py b/tests/server/api/test_session_info.py index 5938db5e8..56f43b271 100644 --- a/tests/server/api/test_session_info.py +++ b/tests/server/api/test_session_info.py @@ -135,7 +135,7 @@ def test_get_preprocessed_movie_count( ) murfey_db_session.add(dcg_entry) for j in range(num_dcs): - dc_id = (i * num_dcs) + j + dc_id = (i * num_dcs) + j + 1 dc_entry = MurfeyDB.DataCollection( dataCollectionId=dc_id, dataCollectionGroupId=dcg_id, @@ -145,7 +145,7 @@ def test_get_preprocessed_movie_count( for k in range(num_movies): # Add prerequisite table entriees # ProcessingJob - pj_id = (i * num_dcs * num_movies) + (j * num_movies) + k + pj_id = (i * num_dcs * num_movies) + (j * num_movies) + k + 1 pj_entry = MurfeyDB.ProcessingJob( processingJobId=pj_id, recipe="dummy", @@ -172,7 +172,9 @@ def test_get_preprocessed_movie_count( movieId=ledger_entry.id, movieNumber=ledger_entry.id, dataCollectionId=dc_id, - path=str(tmp_path / dcg_name / f"movie_{(j * num_movies) + k}.mrc"), + path=str( + tmp_path / dcg_name / f"movie_{(j * num_movies) + k + 1}.mrc" + ), tag=dcg_name, preprocessed=bool(j % 2), ) From a6dca2fb4bbe051e44bdf618c05ff7e75854dcd4 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 12 Oct 2026 21:57:52 +0100 Subject: [PATCH 12/13] Add logic to 'motion_corrected' block in 'feedback_callback' to mark movies related to the current tilt series as having been preprocessed --- src/murfey/server/feedback.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/murfey/server/feedback.py b/src/murfey/server/feedback.py index f4c4ddd98..e89730985 100644 --- a/src/murfey/server/feedback.py +++ b/src/murfey/server/feedback.py @@ -2118,6 +2118,14 @@ def feedback_callback( relevant_tilt_series.processing_requested = True _db.add(relevant_tilt_series) + # Update all related Movie rows and mark as preprocessed + relevant_movies = _db.exec( + select(db.Movie).where(db.Movie.path == message["movie"]) + ).all() + for movie in relevant_movies: + movie.preprocessed = True + _db.add(movie) + prom.preprocessed_movies.labels( processing_job=collected_ids[2].id ).inc() From 42377dbae950df471c739813ca0b515260194f95 Mon Sep 17 00:00:00 2001 From: Eu Pin Tien Date: Mon, 12 Oct 2026 22:41:57 +0100 Subject: [PATCH 13/13] Movie update occurred in wrong position --- src/murfey/server/feedback.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/murfey/server/feedback.py b/src/murfey/server/feedback.py index e89730985..666ac66d6 100644 --- a/src/murfey/server/feedback.py +++ b/src/murfey/server/feedback.py @@ -2051,6 +2051,7 @@ def feedback_callback( .where(db.ProcessingJob.recipe == "em-tomo-align") ).one() + # Update the tilt series and its corresponding tilts relevant_tilt_and_series = _db.exec( select(db.Tilt, db.TiltSeries) .where(db.Tilt.movie_path == message.get("movie")) @@ -2061,7 +2062,17 @@ def feedback_callback( relevant_tilt_series = relevant_tilt_and_series[1] relevant_tilt.motion_corrected = True _db.add(relevant_tilt) + + # Update all related Movie rows and mark as preprocessed + relevant_movies = _db.exec( + select(db.Movie).where(db.Movie.path == message["movie"]) + ).all() + for movie in relevant_movies: + movie.preprocessed = True + _db.add(movie) + _db.commit() + if ( check_tilt_series_mc(relevant_tilt_series.id, _db) and not relevant_tilt_series.processing_requested @@ -2118,14 +2129,6 @@ def feedback_callback( relevant_tilt_series.processing_requested = True _db.add(relevant_tilt_series) - # Update all related Movie rows and mark as preprocessed - relevant_movies = _db.exec( - select(db.Movie).where(db.Movie.path == message["movie"]) - ).all() - for movie in relevant_movies: - movie.preprocessed = True - _db.add(movie) - prom.preprocessed_movies.labels( processing_job=collected_ids[2].id ).inc()