From 784a424b03e5b086ae63d5c3664b6c5eefd927a8 Mon Sep 17 00:00:00 2001 From: Matthias Kleiner Date: Fri, 31 Jul 2026 08:14:56 +0200 Subject: [PATCH] MapCreation: avoid copying same file twice to the output folder - use 2 replicas instead of 1 - use proper O2 daily tag - store output alien path of stage 2 --- .../configurations/mapCreationGrid/README.md | 7 +++- .../mapCreationGrid/mapCreationJob.sh | 2 +- .../mapCreationGrid/postProcessJob.sh | 7 +++- .../mapCreationGrid/submitMapCreation.sh | 2 +- .../mapCreationGrid/submitPostProcess.sh | 32 +++++++++++++++++-- 5 files changed, 43 insertions(+), 7 deletions(-) diff --git a/DATA/production/configurations/mapCreationGrid/README.md b/DATA/production/configurations/mapCreationGrid/README.md index d022fe8f5..31dd71544 100644 --- a/DATA/production/configurations/mapCreationGrid/README.md +++ b/DATA/production/configurations/mapCreationGrid/README.md @@ -353,7 +353,12 @@ Then: This downloads the *existing* manifest stage 1 already uploaded (does not rebuild it — see the comment in the script for why that matters), re-stages the three macros if `MACRO_SOURCE=alien`, and submits one -production, same slot granularity as stage 1. +production, same slot granularity as stage 1. It also captures and uploads its own workdir pointer, +`stage2Workdir..txt`, to `MACRO_ALIEN_DIR` — the same mechanism stage 1 uses for its own output +(see Step 3), so an external consumer that needs to locate a specific slot's final `FT_*.root`/smoothed +map (e.g. a downstream pipeline using this stage's output as an input) can reconstruct the path the same +way stage 2 itself reconstructs stage 1's — read `stage2Workdir..txt` for the base, recompute +`COUNTERWIDTH` from this submission's manifest line count, join with the zero-padded subjob index. **Local single-slot test:** diff --git a/DATA/production/configurations/mapCreationGrid/mapCreationJob.sh b/DATA/production/configurations/mapCreationGrid/mapCreationJob.sh index 0e0d2c65c..e5bae7188 100755 --- a/DATA/production/configurations/mapCreationGrid/mapCreationJob.sh +++ b/DATA/production/configurations/mapCreationGrid/mapCreationJob.sh @@ -55,7 +55,7 @@ MACRO_ALIEN_DIR_BARE=${MACRO_ALIEN_DIR_BARE:-} # expect to change it in the wrapper when the data changes -- not here. Empty ("") is a supported value # meaning "skip bad-ranges filtering entirely for this submission" -- see BAD_RANGES_ARG below. BAD_RANGES_NAME=${BAD_RANGES_NAME:-Merge_BadIntervals_LHC26_PbPb_DCA-0P300.txt} -#JDL_OUTPUT=*.log@disk=1,*.txt@disk=1,*.info@disk=1 +#JDL_OUTPUT=*.log@disk=2,*.txt@disk=2,*.info@disk=2 #JDL_REQUIRE={member(other.GridPartitions,"multicore_8") && (other.CE != "ALICE::NIHAM::PBS64") && (other.CE != "ALICE::KISTI_GSDC::LCG") && (other.CE != "ALICE::SaoPaulo::LCG")}; # Site blacklist: these three CEs excluded from this pipeline's jobs. Setting #JDL_REQUIRE overrides # grid_submit.sh's own default Requirements entirely (it's read verbatim, not merged -- see diff --git a/DATA/production/configurations/mapCreationGrid/postProcessJob.sh b/DATA/production/configurations/mapCreationGrid/postProcessJob.sh index 676bcf459..e555d09cc 100755 --- a/DATA/production/configurations/mapCreationGrid/postProcessJob.sh +++ b/DATA/production/configurations/mapCreationGrid/postProcessJob.sh @@ -44,7 +44,7 @@ STAGE1_OUTPUTDIR_BASE= COUNTERWIDTH=3 MACRO_SOURCE=official MACRO_ALIEN_DIR_BARE=${MACRO_ALIEN_DIR_BARE:-} -#JDL_OUTPUT=*.log@disk=1,*.txt@disk=1,*.png@disk=1 +#JDL_OUTPUT=*.log@disk=2,*.txt@disk=2,*.png@disk=2 #JDL_REQUIRE={member(other.GridPartitions,"multicore_8") && (other.CE != "ALICE::NIHAM::PBS64") && (other.CE != "ALICE::KISTI_GSDC::LCG") && (other.CE != "ALICE::SaoPaulo::LCG")}; # Site blacklist -- same three CEs excluded as stage 1, see that script's comment for why the # GridPartitions clause has to be repeated here (grid_submit.sh replaces its whole default Requirements, @@ -215,6 +215,11 @@ echo "${STAGE1_OUTPUT_LOCATION}/${STAGE1_OUTPUT_FILE}" > "${STAGE1_SOURCE_FILE}" if ! alien.py cp -f "file:${STAGE1_SOURCE_FILE}" "${ALIEN_OUTPUT_DIR}/${STAGE1_SOURCE_FILE}"; then echo "WARNING: failed to stage ${STAGE1_SOURCE_FILE} -- provenance pointer only, not fatal" >&2 fi +# Remove the local copy now that it's staged: it matches the #JDL_OUTPUT *.txt glob above, and the +# JobAgent's own passive upload of that glob (run after this script exits) does a plain, non-forced +# cp -- it would otherwise collide with the LFN this explicit -f upload just created and put the +# whole subjob into ERROR_SV, even though the actual deliverable upload above succeeded. +rm -f "${STAGE1_SOURCE_FILE}" # --- QA plots (voxResQA.C) -- runs on just this one slot's smoothed map, same per-slot granularity as # everything else in this pipeline. Non-fatal on failure: QA is a diagnostic nice-to-have, not a blocking diff --git a/DATA/production/configurations/mapCreationGrid/submitMapCreation.sh b/DATA/production/configurations/mapCreationGrid/submitMapCreation.sh index 72d80cc6b..f63b5cd7b 100755 --- a/DATA/production/configurations/mapCreationGrid/submitMapCreation.sh +++ b/DATA/production/configurations/mapCreationGrid/submitMapCreation.sh @@ -65,7 +65,7 @@ TOPWORKDIR="MapCreation" # replaces grid_submit.sh's generic default "selfjobs # like the real input data's own convention (.../2026/LHC26ak/572244/...) -- # that needs a real post-processing/organize step, deliberately deferred to a # later iteration of this pipeline. -PACKAGESPEC="O2::daily-20260729-0000-1" +PACKAGESPEC="O2::daily-20260730-0000-1" ASUSER="pwg_pp" # your own account = personal quota. "pwg_pp" = charged to that working group's quota AND # moves MY_JOBWORKDIR/OutputDir to pwg_pp's own space automatically -- see the # long comment above for why this one value now does everything. NOTE support diff --git a/DATA/production/configurations/mapCreationGrid/submitPostProcess.sh b/DATA/production/configurations/mapCreationGrid/submitPostProcess.sh index dce0500b7..4f34df062 100755 --- a/DATA/production/configurations/mapCreationGrid/submitPostProcess.sh +++ b/DATA/production/configurations/mapCreationGrid/submitPostProcess.sh @@ -32,6 +32,13 @@ # script doesn't need its own OutputDir logic, it just needs to read/write consistently with whatever # stage 1 did. # +# This wrapper also captures and uploads ITS OWN workdir pointer (stage2Workdir..txt), the same +# way stage 1 does for its own output -- nothing inside this pipeline needed that before (stage 2 is the +# end of the line here), but an external consumer that needs to locate a specific slot's final +# FT_*.root/smoothed map (e.g. a downstream pipeline using it as an input) has no other way to find it, +# since it also lands in an ephemeral ${ALIEN_JDL_OUTPUTDIR}, not a fixed path. Purely additive: this +# stage's own control flow and outputs are unchanged, it just also records where they went. +# # Usage: # ./submitPostProcess.sh @@ -49,7 +56,7 @@ STAGE1_JOBNAME_TAG="mapcreation-v1" # must equal the JOBNAME stage 1 (sub # actually submitted with -- that's what determined where # stage 1 staged its map. TOPWORKDIR="MapCreation" -PACKAGESPEC="O2::daily-20260729-0000-1" +PACKAGESPEC="O2::daily-20260730-0000-1" ASUSER="pwg_pp" # see the long comment above -- your own account for personal quota, "pwg_pp" for group # quota (this stage's own submission; independent of what stage 1 used). MACRO_SOURCE="official" # same meaning/default as stage 1 (submitMapCreation.sh) -- applies @@ -104,7 +111,8 @@ COUNTERWIDTH=${#N} TMP_SCRIPT="$(mktemp)" TMP_WORKDIR_POINTER="$(mktemp)" -trap 'rm -f "${TMP_MANIFEST}" "${TMP_SCRIPT:-}" "${TMP_WORKDIR_POINTER}"' EXIT +TMP_STAGE2_WORKDIR_POINTER="$(mktemp)" +trap 'rm -f "${TMP_MANIFEST}" "${TMP_SCRIPT:-}" "${TMP_WORKDIR_POINTER}" "${TMP_STAGE2_WORKDIR_POINTER}"' EXIT # ONE submission, mirroring stage 1: the windows live in the manifest, so a subjob index identifies a # (run, window) pair by itself and there is nothing to loop over. @@ -142,4 +150,22 @@ GRID_SUBMIT_ARGS=( ) echo "== Submitting ${JOBNAME} (prodsplit=${N}, asuser=${ASUSER}) ==" -"${O2DPG_ROOT}/GRID/utils/grid_submit.sh" "${GRID_SUBMIT_ARGS[@]}" +# Capture grid_submit.sh's own stderr (its "pok" status messages go there) while still showing it live, +# so we can pull out the one line that tells us where this submission's data will actually land -- same +# mechanism submitMapCreation.sh uses for its own stage1Workdir pointer. Nothing downstream needed +# stage 2's own output location before now; this exists so a future consumer (e.g. a pipeline that reads +# stage 2's FT_*.root as an input) can find it without guessing. +SUBMIT_LOG=$("${O2DPG_ROOT}/GRID/utils/grid_submit.sh" "${GRID_SUBMIT_ARGS[@]}" 2>&1 | tee /dev/stderr) + +# Strip ANSI color codes (pok() wraps its output in \033[32m...\033[m) before parsing. +MY_JOBWORKDIR=$(echo "${SUBMIT_LOG}" | sed -E 's/\x1b\[[0-9;]*m//g' | sed -n "s/^Your job's working directory will be //p" | tail -1) +if [[ -z "${MY_JOBWORKDIR}" ]]; then + echo "ERROR: could not find \"Your job's working directory will be ...\" in grid_submit.sh's output -- a future consumer of this stage's output won't be able to find it. Aborting." >&2 + exit 1 +fi +echo "== MY_JOBWORKDIR (stage 2's own) = ${MY_JOBWORKDIR} ==" + +STAGE2_WORKDIR_POINTER_NAME="stage2Workdir.${JOBNAME}.txt" +echo "${MY_JOBWORKDIR}" > "${TMP_STAGE2_WORKDIR_POINTER}" +alien.py cp -f "file:${TMP_STAGE2_WORKDIR_POINTER}" "${MACRO_ALIEN_DIR}/${STAGE2_WORKDIR_POINTER_NAME}" +echo "== Uploaded ${STAGE2_WORKDIR_POINTER_NAME} =="