Skip to content

Commit e39aa01

Browse files
rashiguptrashigupt
authored
Merge branch 'master' into rashigupta
2 parents 30e5bd0 + c91c959 commit e39aa01

803 files changed

Lines changed: 50904 additions & 5938 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/sim-tests.yml‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
---
2+
name: Simulation tests
3+
4+
'on':
5+
pull_request: {}
6+
workflow_dispatch:
7+
inputs:
8+
tag:
9+
description: 'O2PDPSuite tag to test against (default: newest daily)'
10+
type: string
11+
required: false
12+
13+
permissions: {}
14+
15+
concurrency:
16+
group: sim-tests-${{ github.event.pull_request.number || github.ref }}
17+
cancel-in-progress: true
18+
19+
jobs:
20+
sim-tests:
21+
name: Simulation tests against CVMFS
22+
runs-on: [self-hosted, cvmfs]
23+
timeout-minutes: 180
24+
25+
steps:
26+
- name: Checkout code
27+
uses: actions/checkout@v4
28+
with:
29+
# The changed-file logic diffs against the merge base, so the full
30+
# history is needed, not a shallow clone.
31+
fetch-depth: 0
32+
33+
- name: Resolve the diff base
34+
id: base
35+
if: github.event_name == 'pull_request'
36+
env:
37+
BASE_SHA: ${{ github.event.pull_request.base.sha }}
38+
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
39+
run: |
40+
set -eu
41+
git rev-parse --verify "$BASE_SHA^{commit}" >/dev/null || {
42+
echo "::error title=Cannot resolve diff base::pull request base commit $BASE_SHA does not resolve in this checkout"
43+
exit 1
44+
}
45+
git rev-parse --verify "$HEAD_SHA^{commit}" >/dev/null || {
46+
echo "::error title=Cannot resolve diff head::pull request head commit $HEAD_SHA does not resolve in this checkout"
47+
exit 1
48+
}
49+
merge_base=$(git merge-base "$BASE_SHA" "$HEAD_SHA") || {
50+
echo "::error title=Cannot compute diff base::git merge-base failed"
51+
exit 1
52+
}
53+
[ -n "$merge_base" ] || {
54+
echo "::error title=Cannot compute diff base::merge base is empty"
55+
exit 1
56+
}
57+
echo "sha=$merge_base" >> "$GITHUB_OUTPUT"
58+
59+
- name: Skip when not relevant or opted into the source build
60+
id: gate
61+
if: github.event_name == 'pull_request'
62+
env:
63+
BASE_SHA: ${{ steps.base.outputs.sha }}
64+
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
65+
run: |
66+
set -eu
67+
changed=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA")
68+
# Opting in means *editing* the existing sentinel, so only count it
69+
# when it is Modified. A pull request that Adds it — which is what the
70+
# pull request introducing the sentinel does — would otherwise trip its
71+
# own opt-out and pass without testing anything.
72+
opted=$(git diff --name-only --diff-filter=M "$BASE_SHA" "$HEAD_SHA")
73+
if grep -qx 'test/needs-o2-dev' <<< "$opted" ; then
74+
echo "::notice title=Skipped::this pull request touches test/needs-o2-dev, so it is tested by build/O2DPG/sim/o2dev against O2 dev instead"
75+
echo "skip=true" >> "$GITHUB_OUTPUT"
76+
elif ! grep -qE '^(DATA/|MC/|test/|RelVal/)' <<< "$changed" ; then
77+
echo "::notice title=Skipped::no changed file matches DATA/, MC/, test/ or RelVal/"
78+
echo "skip=true" >> "$GITHUB_OUTPUT"
79+
else
80+
echo "skip=false" >> "$GITHUB_OUTPUT"
81+
fi
82+
83+
- name: Check the CVMFS environment
84+
if: steps.gate.outputs.skip != 'true'
85+
run: |
86+
set -eu
87+
test -d /cvmfs/alice.cern.ch || {
88+
echo "::error title=CVMFS unavailable::/cvmfs/alice.cern.ch is not mounted on this runner"
89+
exit 1
90+
}
91+
test -x /cvmfs/alice.cern.ch/bin/alienv || {
92+
echo "::error title=CVMFS unavailable::/cvmfs/alice.cern.ch/bin/alienv is missing"
93+
exit 1
94+
}
95+
96+
- name: Check the AliEn token
97+
if: steps.gate.outputs.skip != 'true'
98+
env:
99+
JALIEN_TOKEN_CERT: /run/alien-ci/tokencert.pem
100+
JALIEN_TOKEN_KEY: /run/alien-ci/tokenkey.pem
101+
run: |
102+
set -eu
103+
# DPL's CCDB backend refuses to talk to alice-ccdb.cern.ch without a
104+
# token and aborts the whole device, so check up front rather than
105+
# letting it surface as a confusing task crash deep in a workflow.
106+
for f in "$JALIEN_TOKEN_CERT" "$JALIEN_TOKEN_KEY" ; do
107+
test -r "$f" || {
108+
echo "::error title=No AliEn token::$f is missing or unreadable. On the runner host: systemctl start alien-ci-token.service"
109+
exit 1
110+
}
111+
done
112+
if ! openssl x509 -in "$JALIEN_TOKEN_CERT" -noout -checkend 3600 >/dev/null 2>&1 ; then
113+
echo "::error title=AliEn token expiring::the token expires within the hour. On the runner host: systemctl start alien-ci-token.service"
114+
exit 1
115+
fi
116+
echo "AliEn token valid until $(openssl x509 -in "$JALIEN_TOKEN_CERT" -noout -enddate | cut -d= -f2)"
117+
118+
- name: Resolve the O2PDPSuite tag
119+
id: tag
120+
if: steps.gate.outputs.skip != 'true'
121+
env:
122+
REQUESTED_TAG: ${{ inputs.tag }}
123+
PR_BODY: ${{ github.event.pull_request.body }}
124+
run: |
125+
set -eu
126+
# shellcheck source=test/ci/resolve_tag.sh
127+
. test/ci/resolve_tag.sh
128+
moduledir=/cvmfs/alice.cern.ch/el9-x86_64/Modules/modulefiles/O2PDPSuite
129+
requested=$REQUESTED_TAG
130+
if [ -z "$requested" ]; then
131+
# A PR can pin the release with a line "sim-tests-tag: <tag>".
132+
requested=$(printf '%s\n' "$PR_BODY" |
133+
sed -n 's/^[[:space:]]*sim-tests-tag:[[:space:]]*//p' | head -n 1 |
134+
tr -d '[:space:]')
135+
fi
136+
tag=$(resolve_o2pdpsuite_tag "$moduledir" "$requested") || {
137+
echo "::error title=No usable O2PDPSuite release::see the message above"
138+
exit 1
139+
}
140+
echo "Testing against O2PDPSuite::$tag"
141+
echo "tag=$tag" >> "$GITHUB_OUTPUT"
142+
143+
- name: Run the O2DPG tests
144+
if: steps.gate.outputs.skip != 'true'
145+
env:
146+
O2PDPSUITE_TAG: ${{ steps.tag.outputs.tag }}
147+
O2DPG_TEST_HASH_BASE: ${{ steps.base.outputs.sha }}
148+
O2DPG_TEST_HASH_HEAD: ${{ github.event.pull_request.head.sha }}
149+
JOBS: 8
150+
# o2dpg_sim_workflow.py calls JAlien(['whoami']) purely to fill the
151+
# AOD's --created-by field, and only when JALIEN_USER is unset. Setting
152+
# it keeps the tests from needing a GRID credential at all, which
153+
# matters here: anything readable by this account is readable by the
154+
# fork-pull-request code that runs as it.
155+
JALIEN_USER: alien-ci
156+
# DPL's CCDB backend needs a GRID token for alice-ccdb.cern.ch. These
157+
# are *paths* to a short-lived token minted from a service certificate
158+
# by a root-owned timer on the runner host; the certificate itself is
159+
# never readable by this account. Verified that paths work — the same
160+
# variables also accept PEM content, which is what ali-bot passes.
161+
JALIEN_TOKEN_CERT: /run/alien-ci/tokencert.pem
162+
JALIEN_TOKEN_KEY: /run/alien-ci/tokenkey.pem
163+
run: |
164+
set -eu
165+
# Everything after "-c" is joined into one string and re-evaluated
166+
# by the CVMFS alienv via "bash -c \"$*\"". Quoting here is applied
167+
# once then discarded, so it is safe for the runner's workspace path,
168+
# but a path with a space or "$" would break or double-evaluate.
169+
/cvmfs/alice.cern.ch/bin/alienv setenv "O2PDPSuite/$O2PDPSUITE_TAG" -c \
170+
env O2DPG_ROOT="$PWD" O2DPG_MC_CONFIG_ROOT="$PWD" \
171+
O2DPG_TEST_REPO_DIR="$PWD" \
172+
O2DPG_TEST_HASH_BASE="$O2DPG_TEST_HASH_BASE" \
173+
O2DPG_TEST_HASH_HEAD="$O2DPG_TEST_HASH_HEAD" \
174+
JOBS="$JOBS" \
175+
bash test/run_tests.sh
176+
177+
- name: Upload logs
178+
if: always()
179+
uses: actions/upload-artifact@v4
180+
with:
181+
name: o2dpg-test-logs
182+
path: |
183+
o2dpg_tests/**/*.log
184+
o2dpg_tests/**/*serverlog*
185+
o2dpg_tests/**/*workerlog*
186+
o2dpg_tests/**/*mergerlog*
187+
if-no-files-found: ignore
188+
retention-days: 14
189+
190+
- name: Prune test artifacts
191+
if: always()
192+
run: find o2dpg_tests -type f ! -name '*.log' ! -name '*serverlog*' ! -name '*workerlog*' ! -name '*mergerlog*' -delete || true

‎.github/workflows/syntax-checks.yml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -105,6 +105,53 @@ jobs:
105105
done
106106
exit "$error"
107107
108+
runner-tests:
109+
name: Workflow-runner unit tests
110+
runs-on: ubuntu-latest
111+
112+
steps:
113+
- name: Checkout code
114+
uses: actions/checkout@v4
115+
116+
- name: Install prerequisites
117+
run: pip install pytest psutil
118+
119+
- name: Run the o2dpg_runner test suite
120+
working-directory: MC/workflow_runner
121+
run: pytest o2dpg_runner/tests -q
122+
123+
filegraph-tests:
124+
name: File-IO-graph unit tests
125+
runs-on: ubuntu-latest
126+
127+
steps:
128+
- name: Checkout code
129+
uses: actions/checkout@v4
130+
131+
- name: Install prerequisites
132+
run: pip install psutil
133+
134+
- name: Run the FileIOGraph test suite
135+
run: python3 -m unittest discover -s UTILS/FileIOGraph/tests -t UTILS/FileIOGraph/tests
136+
137+
bash-harnesses:
138+
name: Test-harness unit tests
139+
runs-on: ubuntu-latest
140+
141+
steps:
142+
- name: Checkout code
143+
uses: actions/checkout@v4
144+
145+
- name: Run the offline test harnesses
146+
run: |
147+
error=0
148+
for t in test/tests/*.sh ; do
149+
echo "::group::$t"
150+
bash "$t" || error=1
151+
echo "::endgroup::"
152+
done
153+
exit "$error"
154+
108155
pylint:
109156
name: Pylint
110157
runs-on: ubuntu-latest

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@
22
.vscode
33
*.pyc
44
o2dpg_tests/**
5+
__pycache__/

‎CODEOWNERS‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
/DATA/testing/detectors/ZDC
1919

2020
/MC @sawenzel @jackal1-66
21+
/GRID @sawenzel
2122

2223
# PWG experts
2324
/MC/*/ALICE3 @njacazio

‎DATA/common/gen_topo_helper_functions.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,11 @@ has_secvtx_source()
5959
[[ $SVERTEXING_SOURCES =~ (^|,)"ALL"(,|$) ]] || [[ $SVERTEXING_SOURCES =~ (^|,)"$1"(,|$) ]]
6060
}
6161

62+
has_detector_in_secvtx_sources()
63+
{
64+
[[ $SVERTEXING_SOURCES =~ (^|,)"ALL"(,|$) ]] || [[ $SVERTEXING_SOURCES =~ (^|,|-)"$1"(-|,|$) ]]
65+
}
66+
6267
has_detector_qc()
6368
{
6469
has_detector $1 && [[ $WORKFLOW_DETECTORS_QC =~ (^|,)"$1"(,|$) ]]

‎DATA/common/getCommonArgs.sh‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,18 @@ if [[ -z $SEVERITY || -z $NUMAID || -z $SHMSIZE || -z $FILEWORKDIR || -z $EPNSYN
1010
fi
1111

1212
ARGS_ALL="--session ${OVERRIDE_SESSION:-default} --severity $SEVERITY --shm-segment-id ${O2JOBSHMID:-$NUMAID} --shm-segment-size $SHMSIZE ${ARGS_ALL_EXTRA:-} --early-forward-policy noraw"
13-
ARGS_ALL_CONFIG="keyval.input_dir=$FILEWORKDIR;keyval.output_dir=/dev/null;${ALL_EXTRA_CONFIG:-}"
13+
ARGS_ALL_CONFIG="keyval.input_dir=$FILEWORKDIR;"
1414
if [[ $EPNSYNCMODE == 1 ]]; then
1515
ARGS_ALL+=" --infologger-severity $INFOLOGGER_SEVERITY"
1616
ARGS_ALL+=" --monitoring-backend influxdb-unix:///tmp/telegraf.sock --resources-monitoring ${GEN_TOPO_RESOURCE_MONITORING:-15} ${GEN_TOPO_RESOURCE_MONITORING:+--dpl-stats-min-online-publishing-interval $GEN_TOPO_RESOURCE_MONITORING}"
17-
ARGS_ALL_CONFIG+="NameConf.mCCDBServer=$GEN_TOPO_EPN_CCDB_SERVER;"
17+
ARGS_ALL_CONFIG+="keyval.output_dir=/dev/null;NameConf.mCCDBServer=$GEN_TOPO_EPN_CCDB_SERVER;"
1818
export DPL_CONDITION_BACKEND=$GEN_TOPO_EPN_CCDB_SERVER
1919
[[ -z ${O2_DPL_DEPLOYMENT_MODE:-} ]] && O2_DPL_DEPLOYMENT_MODE=OnlineECS
2020
elif [[ "${ENABLE_METRICS:-}" != "1" ]]; then
2121
ARGS_ALL+=" --monitoring-backend no-op://"
2222
fi
23+
ARGS_ALL_CONFIG+="${ALL_EXTRA_CONFIG:-}"
24+
2325
[[ $SHMTHROW == 0 ]] && ARGS_ALL+=" --bad-alloc-max-attempts 60 --bad-alloc-attempt-interval 1000"
2426
[[ ! -z ${SHM_MANAGER_SHMID:-} && ${GEN_TOPO_CALIB_WORKFLOW:-} != 1 ]] && ARGS_ALL+=" --no-cleanup --shm-no-cleanup on --shmid $SHM_MANAGER_SHMID"
2527
[[ $NORATELOG == 1 ]] && ARGS_ALL+=" --fairmq-rate-logging 0"

‎DATA/common/setenv.sh‎

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -164,8 +164,8 @@ DISABLE_ROOT_INPUT="--disable-root-input"
164164
# Special detector related settings
165165
if [[ -z "${TPC_CORR_SCALING:-}" ]]; then # TPC corr.map lumi scaling options, any combination of --lumi-type <0,1,2> --corrmap-lumi-mode <0,1> and TPCCorrMap... configurable param
166166
TPC_CORR_SCALING=
167-
if [[ $BEAMTYPE == "pp" || $LIGHTNUCLEI == "1" ]] && has_detector CTP; then TPC_CORR_SCALING+="--lumi-type 1"; fi
168-
if [[ $BEAMTYPE == "PbPb" ]] && has_detector CTP; then TPC_CORR_SCALING+="--lumi-type 1 TPCCorrMap.lumiInstFactor=2.414"; fi
167+
if [[ $BEAMTYPE == "pp" || $LIGHTNUCLEI == "1" ]] && has_detector CTP; then TPC_CORR_SCALING+="--lumi-type 1 --corrmap-lumi-mode 1"; fi
168+
if [[ $BEAMTYPE == "PbPb" ]] && has_detector CTP; then TPC_CORR_SCALING+="--lumi-type 1 --corrmap-lumi-mode 1 TPCCorrMap.lumiInstFactor=2.414"; fi
169169
if [[ $BEAMTYPE == "cosmic" ]]; then TPC_CORR_SCALING=" TPCCorrMap.lumiMean=-1;"; fi # for COSMICS we disable all corrections
170170
export TPC_CORR_SCALING=$TPC_CORR_SCALING
171171
fi
@@ -195,6 +195,7 @@ TRD_SOURCES=
195195
TOF_SOURCES=
196196
HMP_SOURCES=
197197
TRACK_SOURCES=
198+
: ${TRACK_SOURCES_GLO:=}
198199
has_detectors_reco ITS TPC && has_detector_matching ITSTPC && add_comma_separated TRACK_SOURCES "ITS-TPC"
199200
has_detectors_reco TPC TRD && has_detector_matching TPCTRD && { add_comma_separated TRD_SOURCES TPC; add_comma_separated TRACK_SOURCES "TPC-TRD"; }
200201
has_detectors_reco ITS TPC TRD && has_detector_matching ITSTPC && has_detector_matching ITSTPCTRD && { add_comma_separated TRD_SOURCES ITS-TPC; add_comma_separated TRACK_SOURCES "ITS-TPC-TRD"; }
@@ -211,6 +212,8 @@ has_detectors_reco HMP TPC TOF && has_detector_matching TPCTOF && add_comma_sepa
211212
has_detectors_reco HMP TPC TRD TOF && has_detector_matching TPCTRD && has_detector_matching TPCTRDTOF && add_comma_separated HMP_SOURCES "TPC-TRD-TOF"
212213
has_detectors_reco MFT MCH && has_detector_matching MFTMCH && add_comma_separated TRACK_SOURCES "MFT-MCH"
213214
has_detectors_reco MCH MID && has_detector_matching MCHMID && add_comma_separated TRACK_SOURCES "MCH-MID"
215+
[[ "0$TRACK_SOURCES_GLO" == "0" ]] && TRACK_SOURCES_GLO=$TRACK_SOURCES
216+
214217
for det in `echo $LIST_OF_DETECTORS | sed "s/,/ /g"`; do
215218
if [[ $LIST_OF_ASYNC_RECO_STEPS =~ (^| )${det}( |$) ]]; then
216219
has_detector ${det} && has_processing_step ${det}_RECO && add_comma_separated TRACK_SOURCES "$det"
@@ -241,12 +244,6 @@ fi
241244
[[ ! -z $VERTEXING_SOURCES ]] && PVERTEX_CONFIG+=" --vertexing-sources $VERTEXING_SOURCES"
242245
[[ ! -z $VERTEX_TRACK_MATCHING_SOURCES ]] && PVERTEX_CONFIG+=" --vertex-track-matching-sources $VERTEX_TRACK_MATCHING_SOURCES"
243246

244-
if [[ -z ${SVERTEXING_SOURCES:-} ]]; then
245-
SVERTEXING_SOURCES="$VERTEXING_SOURCES"
246-
elif [[ "${SVERTEXING_SOURCES^^}" == "NONE" ]]; then
247-
SVERTEXING_SOURCES=
248-
fi
249-
250247
# this option requires well calibrated timing beween different detectors, at the moment suppress it
251248
#has_detector_reco FT0 && PVERTEX_CONFIG+=" --validate-with-ft0"
252249

0 commit comments

Comments
 (0)