diff --git a/.github/actions/start-services/action.yaml b/.github/actions/start-services/action.yaml new file mode 100644 index 0000000..1e8aad5 --- /dev/null +++ b/.github/actions/start-services/action.yaml @@ -0,0 +1,35 @@ +name: Start Services +description: Start Docker Compose services + +inputs: + profiles: + description: "Comma-separated list of profiles to start (e.g. 'all' or 'rest-api,minio')" + required: false + default: "all" + services: + description: "Comma-separated list of specific services to start (e.g. 'database,elasticsearch')" + required: false + default: "" + +runs: + using: composite + steps: + - name: Set permission for mount paths + shell: bash + run: | + chmod -R 777 ./data/php + chmod -R 777 ./logs + - name: Start services + shell: bash + run: | + PROFILE_FLAGS="" + for profile in $(echo "${{ inputs.profiles }}" | tr ',' ' '); do + PROFILE_FLAGS="$PROFILE_FLAGS --profile $profile" + done + SERVICES=$(echo "${{ inputs.services }}" | tr ',' ' ') + docker compose $PROFILE_FLAGS up -d --build --wait $SERVICES + + - name: Show service status + if: always() + shell: bash + run: docker compose ps diff --git a/.github/workflows/integration-test.yaml b/.github/workflows/integration-test.yaml new file mode 100644 index 0000000..a01771b --- /dev/null +++ b/.github/workflows/integration-test.yaml @@ -0,0 +1,90 @@ +name: Integration Test + +on: + pull_request: + paths: + - "docker-compose.yaml" + - "scripts/**" + - "config/**" + +jobs: + dataset-processing: + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Start services + uses: ./.github/actions/start-services + with: + profiles: "rest-api,evaluation-engine" + + - name: Create dataset + id: dataset + run: | + DATASET_ID=$(bash scripts/upload_dataset.sh) + echo "Dataset ID: $DATASET_ID" + echo "id=$DATASET_ID" >> "$GITHUB_OUTPUT" + + - name: Dataset Processing + run: bash scripts/wait_for_dataset.sh "${{ steps.dataset.outputs.id }}" 180 + + - name: Show logs on failure + if: failure() + run: docker compose logs --tail=50 + + task-processing: + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Start services + uses: ./.github/actions/start-services + with: + profiles: "rest-api,evaluation-engine" + + - name: Create task + id: task + run: | + TASK_ID=$(bash scripts/create_task.sh 129 1) + echo "Task ID: $TASK_ID" + echo "id=$TASK_ID" >> "$GITHUB_OUTPUT" + + - name: Task Processing + run: bash scripts/wait_for_task.sh "${{ steps.task.outputs.id }}" 180 + + - name: Show logs on failure + if: failure() + run: docker compose logs --tail=50 + + run-processing: + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Start services + uses: ./.github/actions/start-services + with: + profiles: "rest-api,evaluation-engine" + + - name: Upload run + id: run + run: | + RUN_ID=$(bash scripts/upload_run.sh) + echo "Run ID: $RUN_ID" + echo "id=$RUN_ID" >> "$GITHUB_OUTPUT" + + - name: Run Processing + run: bash scripts/wait_for_run.sh "${{ steps.run.outputs.id }}" 180 + + - name: Show logs on failure + if: failure() + run: docker compose logs --tail=50 diff --git a/README.md b/README.md index 9d76501..f18ab25 100644 --- a/README.md +++ b/README.md @@ -114,6 +114,20 @@ be slightly altered to use them in the test setup: change https to http. ## Development +### Services +When working on the `services` repository itself, the `scripts` directory provides a small number of integration tests to catch the most glaring issues. They are currently limited to REST API and evaluation engine functionality and also run automatically in CI. + +For example, after starting services, to test dataset upload run: +```bash +./scripts/upload_dataset.sh +``` +If successful, this will also print the dataset id assigned to the new dataset. +To test if dataset processing works, we can then run: +```bash +./scripts/wait_for_dataset.sh DATASET_ID +``` +where `DATASET_ID` is replaced by the identifier printed in the previous step. + ### PHP, Parquet and Croissant converter If you want to do local development on containers that are part of the docker-compose, you want those containers to change based on your code. You should have the relevant code somewhere on your system. You will need to tell the docker-compose where to find it. You can do so by setting environment variables. diff --git a/scripts/create_task.sh b/scripts/create_task.sh new file mode 100755 index 0000000..d7bc9ac --- /dev/null +++ b/scripts/create_task.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Creates a task for a given dataset and estimation procedure, prints the task ID to stdout. +# Usage: create_task.sh DATASET_ID ESTIMATION_PROCEDURE_ID [TARGET_FEATURE] +set -euo pipefail + +DATASET_ID="${1:?Usage: create_task.sh DATASET_ID ESTIMATION_PROCEDURE_ID [TARGET_FEATURE]}" +ESTIMATION_PROCEDURE_ID="${2:?Usage: create_task.sh DATASET_ID ESTIMATION_PROCEDURE_ID [TARGET_FEATURE]}" +TARGET_FEATURE="${3:-class}" + +TMPFILE=$(mktemp /tmp/task_desc.XXXXXX.xml) +trap 'rm -f "$TMPFILE"' EXIT + +cat > "$TMPFILE" << EOF + + 1 + ${DATASET_ID} + ${ESTIMATION_PROCEDURE_ID} + ${TARGET_FEATURE} + +EOF + +RESPONSE=$(curl -sS -X POST "http://localhost:8000/api/v1/xml/task?api_key=normaluser" \ + -F "description=@${TMPFILE};type=text/xml") + +TASK_ID=$(echo "$RESPONSE" | sed -n 's/.*\([0-9]*\)<\/oml:id>.*/\1/p') + +if [ -z "$TASK_ID" ]; then + echo "Server response:" >&2 + echo "$RESPONSE" >&2 + exit 1 +fi + +echo "$TASK_ID" diff --git a/scripts/data/run/description.xml b/scripts/data/run/description.xml new file mode 100644 index 0000000..032178e --- /dev/null +++ b/scripts/data/run/description.xml @@ -0,0 +1,305 @@ + + 1196 + 1 + Python_3.10.16. Sklearn_1.6.1. NumPy_2.2.2. SciPy_1.15.1. + + -do-not-check-capabilities + + + + batch-size + 100 + + + num-decimal-places + 2 + + + output-debug-info + + + openml-python + Sklearn_1.6.1. + + + usercpu_time_millis_training + 11.353665999999762 + + + usercpu_time_millis_training + 1.2970839999999484 + + + usercpu_time_millis_training + 1.1723329999995258 + + + usercpu_time_millis_training + 1.1706670000002362 + + + usercpu_time_millis_training + 1.2168329999999727 + + + usercpu_time_millis_training + 1.226291000000046 + + + usercpu_time_millis_training + 1.2974170000008556 + + + usercpu_time_millis_training + 1.2798339999999797 + + + usercpu_time_millis_training + 1.1370409999997833 + + + usercpu_time_millis_training + 1.3173340000003364 + + + wall_clock_time_millis_training + 11.349678039550781 + + + wall_clock_time_millis_training + 1.294851303100586 + + + wall_clock_time_millis_training + 1.1701583862304688 + + + wall_clock_time_millis_training + 1.1692047119140625 + + + wall_clock_time_millis_training + 1.2137889862060547 + + + wall_clock_time_millis_training + 1.2259483337402344 + + + wall_clock_time_millis_training + 1.294851303100586 + + + wall_clock_time_millis_training + 1.2731552124023438 + + + wall_clock_time_millis_training + 1.1365413665771484 + + + wall_clock_time_millis_training + 1.318216323852539 + + + usercpu_time_millis_testing + 0.8559159999999011 + + + usercpu_time_millis_testing + 0.4579580000001471 + + + usercpu_time_millis_testing + 0.45920799999965567 + + + usercpu_time_millis_testing + 0.449209000000117 + + + usercpu_time_millis_testing + 0.452333000000138 + + + usercpu_time_millis_testing + 0.4404999999998438 + + + usercpu_time_millis_testing + 0.46437499999996135 + + + usercpu_time_millis_testing + 0.43887499999950563 + + + usercpu_time_millis_testing + 0.4358330000000521 + + + usercpu_time_millis_testing + 0.4521660000005312 + + + usercpu_time_millis + 12.209581999999664 + + + usercpu_time_millis + 1.7550420000000955 + + + usercpu_time_millis + 1.6315409999991815 + + + usercpu_time_millis + 1.6198760000003531 + + + usercpu_time_millis + 1.6691660000001107 + + + usercpu_time_millis + 1.6667909999998898 + + + usercpu_time_millis + 1.761792000000817 + + + usercpu_time_millis + 1.7187089999994853 + + + usercpu_time_millis + 1.5728739999998353 + + + usercpu_time_millis + 1.7695000000008676 + + + wall_clock_time_millis_testing + 0.8571147918701172 + + + wall_clock_time_millis_testing + 0.45943260192871094 + + + wall_clock_time_millis_testing + 0.4608631134033203 + + + wall_clock_time_millis_testing + 0.4506111145019531 + + + wall_clock_time_millis_testing + 0.45418739318847656 + + + wall_clock_time_millis_testing + 0.44226646423339844 + + + wall_clock_time_millis_testing + 0.46944618225097656 + + + wall_clock_time_millis_testing + 0.44035911560058594 + + + wall_clock_time_millis_testing + 0.4374980926513672 + + + wall_clock_time_millis_testing + 0.45371055603027344 + + + wall_clock_time_millis + 12.206792831420898 + + + wall_clock_time_millis + 1.7542839050292969 + + + wall_clock_time_millis + 1.631021499633789 + + + wall_clock_time_millis + 1.6198158264160156 + + + wall_clock_time_millis + 1.6679763793945312 + + + wall_clock_time_millis + 1.6682147979736328 + + + wall_clock_time_millis + 1.7642974853515625 + + + wall_clock_time_millis + 1.7135143280029297 + + + wall_clock_time_millis + 1.5740394592285156 + + + wall_clock_time_millis + 1.7719268798828125 + + + predictive_accuracy + 0.9333333333333333 + + + predictive_accuracy + 1.0 + + + predictive_accuracy + 1.0 + + + predictive_accuracy + 0.9333333333333333 + + + predictive_accuracy + 1.0 + + + predictive_accuracy + 0.9333333333333333 + + + predictive_accuracy + 1.0 + + + predictive_accuracy + 0.8666666666666667 + + + predictive_accuracy + 0.8666666666666667 + + + predictive_accuracy + 1.0 + + + diff --git a/scripts/data/run/predictions.arff b/scripts/data/run/predictions.arff new file mode 100644 index 0000000..31a0eb3 --- /dev/null +++ b/scripts/data/run/predictions.arff @@ -0,0 +1,170 @@ +% Python_3.10.16. +% Sklearn_1.6.1. +% NumPy_2.2.2. +% SciPy_1.15.1. +% Thu Oct 8 10:27:27 2026 +% Created by run_flow_on_task +@RELATION openml_task_1196_predictions + +@ATTRIBUTE repeat NUMERIC +@ATTRIBUTE fold NUMERIC +@ATTRIBUTE sample NUMERIC +@ATTRIBUTE row_id NUMERIC +@ATTRIBUTE prediction {Iris-setosa, Iris-versicolor, Iris-virginica} +@ATTRIBUTE correct {Iris-setosa, Iris-versicolor, Iris-virginica} +@ATTRIBUTE confidence.Iris-setosa NUMERIC +@ATTRIBUTE confidence.Iris-versicolor NUMERIC +@ATTRIBUTE confidence.Iris-virginica NUMERIC + +@DATA +0,0,0,43,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,0,0,14,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,0,0,37,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,0,0,23,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,0,0,10,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,0,0,99,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,0,0,87,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,0,0,97,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,0,0,62,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,0,0,92,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,0,0,119,Iris-versicolor,Iris-virginica,0.0,1.0,0.0 +0,0,0,111,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,0,0,144,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,0,0,116,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,0,0,125,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,1,0,49,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,1,0,15,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,1,0,47,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,1,0,0,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,1,0,44,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,1,0,76,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,1,0,89,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,1,0,58,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,1,0,54,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,1,0,53,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,1,0,149,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,1,0,114,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,1,0,112,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,1,0,115,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,1,0,139,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,2,0,42,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,2,0,41,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,2,0,31,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,2,0,48,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,2,0,7,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,2,0,94,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,2,0,93,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,2,0,95,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,2,0,51,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,2,0,72,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,2,0,135,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,2,0,124,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,2,0,141,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,2,0,105,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,2,0,143,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,3,0,34,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,3,0,22,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,3,0,45,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,3,0,3,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,3,0,16,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,3,0,79,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,3,0,55,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,3,0,98,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,3,0,80,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,3,0,77,Iris-virginica,Iris-versicolor,0.0,0.0,1.0 +0,3,0,120,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,3,0,122,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,3,0,145,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,3,0,100,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,3,0,107,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,4,0,4,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,4,0,27,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,4,0,38,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,4,0,19,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,4,0,33,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,4,0,96,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,4,0,50,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,4,0,61,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,4,0,65,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,4,0,73,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,4,0,109,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,4,0,126,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,4,0,110,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,4,0,148,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,4,0,103,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,5,0,46,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,5,0,6,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,5,0,18,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,5,0,36,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,5,0,24,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,5,0,83,Iris-virginica,Iris-versicolor,0.0,0.0,1.0 +0,5,0,90,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,5,0,68,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,5,0,52,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,5,0,75,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,5,0,134,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,5,0,104,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,5,0,117,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,5,0,127,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,5,0,121,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,6,0,26,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,6,0,13,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,6,0,32,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,6,0,20,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,6,0,17,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,6,0,63,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,6,0,88,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,6,0,64,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,6,0,91,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,6,0,81,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,6,0,137,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,6,0,101,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,6,0,130,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,6,0,118,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,6,0,108,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,7,0,29,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,7,0,40,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,7,0,12,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,7,0,35,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,7,0,8,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,7,0,86,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,7,0,82,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,7,0,84,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,7,0,74,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,7,0,57,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,7,0,140,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,7,0,133,Iris-versicolor,Iris-virginica,0.0,1.0,0.0 +0,7,0,106,Iris-versicolor,Iris-virginica,0.0,1.0,0.0 +0,7,0,128,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,7,0,102,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,8,0,25,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,8,0,1,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,8,0,21,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,8,0,30,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,8,0,28,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,8,0,67,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,8,0,78,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,8,0,66,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,8,0,70,Iris-virginica,Iris-versicolor,0.0,0.0,1.0 +0,8,0,59,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,8,0,136,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,8,0,138,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,8,0,129,Iris-versicolor,Iris-virginica,0.0,1.0,0.0 +0,8,0,113,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,8,0,132,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,9,0,5,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,9,0,11,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,9,0,9,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,9,0,39,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,9,0,2,Iris-setosa,Iris-setosa,1.0,0.0,0.0 +0,9,0,85,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,9,0,56,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,9,0,69,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,9,0,71,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,9,0,60,Iris-versicolor,Iris-versicolor,0.0,1.0,0.0 +0,9,0,142,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,9,0,123,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,9,0,131,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,9,0,147,Iris-virginica,Iris-virginica,0.0,0.0,1.0 +0,9,0,146,Iris-virginica,Iris-virginica,0.0,0.0,1.0 + diff --git a/scripts/data/test.arff b/scripts/data/test.arff new file mode 100644 index 0000000..51ab11c --- /dev/null +++ b/scripts/data/test.arff @@ -0,0 +1,14 @@ +@RELATION test-dataset +@ATTRIBUTE foo NUMERIC +@ATTRIBUTE bar NUMERIC +@ATTRIBUTE bazz NUMERIC +@ATTRIBUTE class {'a', 'b'} +@DATA +0,0,0,'a' +0,0,1,'a' +0,1,0,'a' +0,1,1,'a' +1,0,0,'b' +1,0,1,'b' +1,1,0,'b' +1,1,1,'b' diff --git a/scripts/upload_dataset.sh b/scripts/upload_dataset.sh new file mode 100755 index 0000000..669ceb1 --- /dev/null +++ b/scripts/upload_dataset.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Uploads a dataset to the REST API and prints the dataset ID to stdout. +set -euo pipefail + +XML_DESCRIPTION=' + test_dataset + test + arff + I + now + en + BSD (from scikit-learn) + class + test + citation + https://www4.stat.ncsu.edu/~boos/var.select/diabetes.html + url +' + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +TMPFILE=$(mktemp /tmp/dataset_desc.XXXXXX.xml) +trap 'rm -f "$TMPFILE"' EXIT +echo "$XML_DESCRIPTION" > "$TMPFILE" + +RESPONSE=$(curl -sS -X POST "http://localhost:8000/api/v1/xml/data?api_key=normaluser" \ + -F "description=@${TMPFILE};type=text/xml" \ + -F "dataset=@${SCRIPT_DIR}/data/test.arff") + +DATASET_ID=$(echo "$RESPONSE" | sed -n 's/.*\([0-9]*\)<\/oml:id>.*/\1/p') + +if [ -z "$DATASET_ID" ]; then + echo "ERROR: Failed to parse dataset ID from response:" >&2 + echo "$RESPONSE" >&2 + exit 1 +fi + +echo "$DATASET_ID" diff --git a/scripts/upload_run.sh b/scripts/upload_run.sh new file mode 100755 index 0000000..912b072 --- /dev/null +++ b/scripts/upload_run.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Uploads a run to the REST API and prints the run ID to stdout. +# The predictions and description files are fixed in scripts/data/run/. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +RESPONSE=$(curl -sS -X POST "http://localhost:8000/api/v1/xml/run?api_key=normaluser" \ + -F "description=@${SCRIPT_DIR}/data/run/description.xml;type=text/xml" \ + -F "predictions=@${SCRIPT_DIR}/data/run/predictions.arff") + +RUN_ID=$(echo "$RESPONSE" | sed -n 's/.*\([0-9]*\)<\/oml:run_id>.*/\1/p') + +if [ -z "$RUN_ID" ]; then + echo "Server response:" >&2 + echo "$RESPONSE" >&2 + exit 1 +fi + +echo "$RUN_ID" diff --git a/scripts/wait_for_dataset.sh b/scripts/wait_for_dataset.sh new file mode 100755 index 0000000..f7c1c52 --- /dev/null +++ b/scripts/wait_for_dataset.sh @@ -0,0 +1,27 @@ +#!/bin/bash +# Waits for a dataset to become active. +# Usage: wait_for_dataset.sh DATASET_ID [TIMEOUT] +set -euo pipefail + +DATASET_ID="${1:?Usage: wait_for_dataset.sh DATASET_ID [TIMEOUT]}" +TIMEOUT="${2:-120}" +INTERVAL=5 +ELAPSED=0 + +while [ "$ELAPSED" -lt "$TIMEOUT" ]; do + STATUS_RESPONSE=$(curl -sS "http://localhost:8000/api/v1/xml/data/${DATASET_ID}") + STATUS=$(echo "$STATUS_RESPONSE" | sed -n 's/.*\([a-zA-Z_]*\)<\/oml:status>.*/\1/p') + + echo "Status after ${ELAPSED}s: ${STATUS:-unknown}" + + if [ "$STATUS" = "active" ]; then + echo "Dataset $DATASET_ID is active." + exit 0 + fi + + sleep "$INTERVAL" + ELAPSED=$((ELAPSED + INTERVAL)) +done + +echo "ERROR: Dataset $DATASET_ID did not become active within ${TIMEOUT} seconds." +exit 1 diff --git a/scripts/wait_for_run.sh b/scripts/wait_for_run.sh new file mode 100755 index 0000000..9cecbd4 --- /dev/null +++ b/scripts/wait_for_run.sh @@ -0,0 +1,26 @@ +#!/bin/bash +# Waits until a run has evaluations computed by the evaluation engine. +# Usage: wait_for_run.sh RUN_ID [TIMEOUT] +set -euo pipefail + +RUN_ID="${1:?Usage: wait_for_run.sh RUN_ID [TIMEOUT]}" +TIMEOUT="${2:-120}" +INTERVAL=5 +ELAPSED=0 + +while [ "$ELAPSED" -lt "$TIMEOUT" ]; do + RESPONSE=$(curl -sS "http://localhost:8000/api/v1/xml/run/${RUN_ID}") + + if echo "$RESPONSE" | grep -q "area_under_roc_curve"; then + echo "Run $RUN_ID has evaluations." + exit 0 + fi + + echo "No evaluations yet after ${ELAPSED}s" + + sleep "$INTERVAL" + ELAPSED=$((ELAPSED + INTERVAL)) +done + +echo "ERROR: Run $RUN_ID did not get evaluations within ${TIMEOUT} seconds." +exit 1 diff --git a/scripts/wait_for_task.sh b/scripts/wait_for_task.sh new file mode 100755 index 0000000..b77aece --- /dev/null +++ b/scripts/wait_for_task.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Waits until a task has associated split files. +# Usage: wait_for_task.sh TASK_ID [TIMEOUT] +set -euo pipefail + +TASK_ID="${1:?Usage: wait_for_task.sh TASK_ID [TIMEOUT]}" +TIMEOUT="${2:-120}" +INTERVAL=5 +ELAPSED=0 + +while [ "$ELAPSED" -lt "$TIMEOUT" ]; do + RESPONSE=$(curl -sS "http://localhost:8000/api/v1/xml/task/${TASK_ID}") + SPLITS_URL=$(echo "$RESPONSE" | sed -n 's/.*\(.*\)<\/oml:data_splits_url>.*/\1/p') + + if [ -n "$SPLITS_URL" ]; then + SPLITS_RESPONSE=$(curl -sS -w "\n%{http_code}" "$SPLITS_URL") + STATUS_CODE=$(echo "$SPLITS_RESPONSE" | tail -n1) + SPLITS_BODY=$(echo "$SPLITS_RESPONSE" | sed '$d') + if [ "$STATUS_CODE" = "200" ]; then + echo "Task $TASK_ID has split files available." + exit 0 + fi + echo "Splits URL exists but returned HTTP $STATUS_CODE after ${ELAPSED}s" + echo "$SPLITS_BODY" + else + echo "No splits URL yet after ${ELAPSED}s" + fi + + sleep "$INTERVAL" + ELAPSED=$((ELAPSED + INTERVAL)) +done + +echo "ERROR: Task $TASK_ID did not get split files within ${TIMEOUT} seconds." +exit 1