GitHub Issue #1130: Metrics to track R & Python package usage

api/src/org/labkey/api/reports/ExternalScriptEngine.java

@@ -23,6 +23,7 @@
 import org.labkey.api.miniprofiler.MiniProfiler;
 import org.labkey.api.pipeline.PipelineJobService;
 import org.labkey.api.reader.Readers;
+import org.labkey.api.reports.report.ScriptPackageUsageTracker;
 import org.labkey.api.reports.report.r.ParamReplacementSvc;
 import org.labkey.api.util.ExceptionUtil;
 import org.labkey.api.util.LabKeyProcessBuilder;
@@ -113,15 +114,127 @@ public boolean isBinary(FileLike file)
     public Object eval(String script, ScriptContext context) throws ScriptException
     {
         List<String> extensions = getFactory().getExtensions();
+        if (extensions.isEmpty())
+            throw new ScriptException("There are no file name extensions registered for this ScriptEngine : " + getFactory().getLanguageName());
+
+        String epilog = getPackageCaptureEpilog(context);
+        if (epilog != null)
+            script = script + "\n" + epilog;
 
-        if (!extensions.isEmpty())
+        FileLike scriptFile = prepareScriptFile(script, context, extensions);
+        try
         {
-            // write out the script file to disk using the first extension as the default
-            FileLike scriptFile = writeScriptFile(script, context, extensions);
-            return eval(scriptFile, context);
+            Object result = eval(scriptFile, context);
+
+            // Metric tracking must never affect script execution, so swallow any failure here
+            try
+            {
+                recordSuccessfulRun(context);
+            }
+            catch (Exception e)
+            {
+                LOG.warn("Failed to record successful script run", e);
+            }
+
+            return result;
+        }
+        finally
+        {
+            // Guard so a metric failure can't mask the script's result or a real exception
+            try
+            {
+                recordPackageUsage(context);
+            }
+            catch (Exception e)
+            {
+                LOG.warn("Failed to record script package usage", e);
+            }
+        }
+    }
+
+    /**
+     * Prepare the on-disk script file that will be executed. The default writes the script as-is; subclasses (e.g. the
+     * R engine's knitr handling) may wrap it in a different driver script.
+     */
+    protected FileLike prepareScriptFile(String script, ScriptContext context, List<String> extensions)
+    {
+        return writeScriptFile(script, context, extensions);
+    }
+
+    /**
+     * GitHub Issue #1130
+     * Script appended to the end of the user script (in the same process) that captures the loaded packages/modules and
+     * writes them, one per line, to a sidecar file in the working directory for {@link #recordPackageUsage} to read
+     * back. The default returns null (no capture); language-specific engines (e.g. R, Python) override this. Wrapped so
+     * a capture failure can never break the script run.
+     */
+    protected @Nullable String getPackageCaptureEpilog(ScriptContext context)
+    {
+        return null;
+    }
+
+    /**
+     * GitHub Issue #1130
+     * After a script runs, read back and record the packages loaded by the script. The default does nothing;
+     * language-specific engines override this, typically delegating to {@link #readPackageSidecar}. Never throws:
+     * package tracking must not affect script execution.
+     */
+    protected void recordPackageUsage(ScriptContext context)
+    {
+    }
+
+    /**
+     * GitHub Issue #1130
+     * Called after a script has run successfully (eval returned without throwing). The default does nothing;
+     * language-specific engines may override to record success metrics.
+     */
+    protected void recordSuccessfulRun(ScriptContext context)
+    {
+    }
+
+    /**
+     * GitHub Issue #1130
+     * Read a sidecar file of package names (one per line) from the working directory and record each under the given
+     * language in {@link ScriptPackageUsageTracker}. Never throws. A missing file means the script errored before the
+     * capture epilog ran (or capture was skipped) - nothing to do. The file is deleted after reading.
+     */
+    protected void readPackageSidecar(ScriptContext context, String fileName, String language)
+    {
+        FileLike packagesFile;
+        try
+        {
+            packagesFile = getWorkingDir(context).resolveChild(fileName);
+        }
+        catch (Exception e)
+        {
+            LOG.warn("Failed to locate " + language + " package usage sidecar", e);
+            return;
+        }
+
+        if (!packagesFile.exists())
+            return;
+
+        try (BufferedReader reader = Readers.getReader(packagesFile.openInputStream()))
+        {
+            String packageName;
+            while ((packageName = reader.readLine()) != null)
+                ScriptPackageUsageTracker.record(language, packageName);
+        }
+        catch (Exception e)
+        {
+            LOG.warn("Failed to record " + language + " package usage", e);
+        }
+        finally
+        {
+            try
+            {
+                packagesFile.delete();
+            }
+            catch (Exception e)
+            {
+                LOG.warn("Failed to delete " + language + " package usage sidecar", e);
+            }
         }
-        else
-            throw new ScriptException("There are no file name extensions registered for this ScriptEngine : " + getFactory().getLanguageName());
     }
 
     protected Object eval(FileLike scriptFile, ScriptContext context) throws ScriptException

api/src/org/labkey/api/reports/report/ScriptPackageUsageTracker.java

@@ -0,0 +1,68 @@
+/*
+ * Copyright (c) 2026 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.labkey.api.reports.report;
+
+import org.labkey.api.reports.ExternalScriptEngine;
+import org.labkey.api.usageMetrics.SimpleMetricsService;
+
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Tracks which packages/modules are loaded by scripts run on this server (R reports, assay transform scripts, Python
+ * scripts, and anything else that runs through {@link ExternalScriptEngine}). Populated by a language-specific epilog
+ * appended to each script that captures the loaded packages and writes them to a sidecar file, which the engine reads
+ * back after the script runs. Usage is tracked per language (e.g. "r", "python").
+ *
+ * Each load is recorded via {@link SimpleMetricsService}, which persists a cumulative per-package load count across
+ * restarts and reports it to mothership under "simpleMetricCounts". The package name is the metric name and the feature
+ * area is "<language>PackageUsage".
+ */
+public class ScriptPackageUsageTracker
+{
+    private static final String MODULE_NAME = "API";
+    private static final String FEATURE_AREA_SUFFIX = "PackageUsage";
+
+    /**
+     * Packages that ship with a given language's runtime and are always present, so aren't interesting as "library
+     * usage". R's base packages are filtered here; Python's standard library is filtered in the capture epilog itself
+     * (via sys.stdlib_module_names), so no Python entry is needed.
+     */
+    private static final Map<String, Set<String>> BASE_PACKAGES = Map.of(
+        "r", Set.of("base", "compiler", "datasets", "graphics", "grDevices", "methods", "stats", "utils")
+    );
+
+    private ScriptPackageUsageTracker()
+    {
+    }
+
+    private static boolean isBasePackage(String language, String packageName)
+    {
+        return BASE_PACKAGES.getOrDefault(language, Set.of()).contains(packageName);
+    }
+
+    /**
+     * Record that the given package was loaded by a script run in the given language (e.g. "r", "python"). Safe to call
+     * repeatedly; base packages and blank names are ignored.
+     */
+    public static void record(String language, String packageName)
+    {
+        if (packageName == null || packageName.isBlank() || isBasePackage(language, packageName))
+            return;
+
+        SimpleMetricsService.get().increment(MODULE_NAME, language + FEATURE_AREA_SUFFIX, packageName);
+    }
+}

api/src/org/labkey/api/reports/report/python/PythonScriptEngine.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2026 LabKey Corporation
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.labkey.api.reports.report.python;
+
+import org.jetbrains.annotations.Nullable;
+import org.labkey.api.reports.ExternalScriptEngine;
+import org.labkey.api.reports.ExternalScriptEngineDefinition;
+
+import javax.script.ScriptContext;
+
+/**
+ * Script engine for locally-executed Python scripts (Python assay transform scripts configured as an
+ * external ".py" engine). Behaves like the base {@link ExternalScriptEngine} except that it appends a capture epilog to
+ * track which Python modules each script loads; see
+ * {@link org.labkey.api.reports.report.ScriptPackageUsageTracker}.
+ */
+public class PythonScriptEngine extends ExternalScriptEngine
+{
+    private static final String PACKAGES_FILE = "labkeyPythonPackages.txt";
+
+    // Python appended to the end of a user script to capture the loaded modules (top-level names, excluding the standard
+    // library). Wrapped in try/except so a capture failure can never break the script run.
+    private static final String PACKAGE_CAPTURE_EPILOG = """
+            # --- LabKey Python package usage capture ---
+            try:
+                import sys as _lk_sys
+                _lk_stdlib = set(getattr(_lk_sys, 'stdlib_module_names', ()))
+                _lk_mods = sorted({m.split('.')[0] for m in list(_lk_sys.modules)} - _lk_stdlib)
+                with open('%s', 'w') as _lk_f:
+                    _lk_f.write('\\n'.join(m for m in _lk_mods if m and not m.startswith('_')))
+            except Exception:
+                pass
+            """.formatted(PACKAGES_FILE);
+
+    public PythonScriptEngine(ExternalScriptEngineDefinition def)
+    {
+        super(def);
+    }
+
+    @Override
+    protected @Nullable String getPackageCaptureEpilog(ScriptContext context)
+    {
+        return PACKAGE_CAPTURE_EPILOG;
+    }
+
+    @Override
+    protected void recordPackageUsage(ScriptContext context)
+    {
+        readPackageSidecar(context, PACKAGES_FILE, "python");
+    }
+}

api/src/org/labkey/api/reports/report/r/RScriptEngine.java

@@ -16,15 +16,16 @@
 package org.labkey.api.reports.report.r;
 
 import org.apache.commons.lang3.StringUtils;
+import org.jetbrains.annotations.Nullable;
 import org.labkey.api.data.JdbcType;
 import org.labkey.api.reports.ExternalScriptEngine;
 import org.labkey.api.reports.ExternalScriptEngineDefinition;
+import org.labkey.api.reports.report.ScriptPackageUsageTracker;
 import org.labkey.vfs.FileLike;
 
 import javax.script.Bindings;
 import javax.script.ScriptContext;
 import javax.script.ScriptEngineFactory;
-import javax.script.ScriptException;
 import java.util.Arrays;
 import java.util.List;
 
@@ -35,6 +36,7 @@
 */
 public class RScriptEngine extends ExternalScriptEngine
 {
+    private static final String PACKAGES_FILE = "labkeyRPackages.txt";
     public static final String KNITR_FORMAT = "r.script.engine.knitrFormat";
     public static final String KNITR_OUTPUT = "r.script.engine.knitrOutput";
     public static final String PANDOC_USE_DEFAULT_OUTPUT_FORMAT = "r.script.engine.pandocUseDefaultOutputFormat";
@@ -59,6 +61,7 @@ public ScriptEngineFactory getFactory()
         return new RScriptEngineFactory(_def);
     }
 
+    @Override
     protected FileLike prepareScriptFile(String script, ScriptContext context, List<String> extensions)
     {
         FileLike scriptFile;
@@ -86,18 +89,48 @@ protected FileLike prepareScriptFile(String script, ScriptContext context, List<
         return scriptFile;
     }
 
+    /**
+     * R appended to the end of the user script (in the same R session) that captures the set of loaded packages,
+     * writing them (one per line) to a sidecar file in the working directory for {@link #recordPackageUsage} to read
+     * back. Wrapped in tryCatch so a capture failure can never break the report or transform run.
+     */
     @Override
-    public Object eval(String script, ScriptContext context) throws ScriptException
+    protected @Nullable String getPackageCaptureEpilog(ScriptContext context)
     {
-        List<String> extensions = getFactory().getExtensions();
+        // For knitr the executed file is a generated wrapper (createKnitrScript), not the user's R, so appending R here
+        // wouldn't run. We instead record the wrapper's known libraries in recordSuccessfulRun().
+        if (getKnitrFormat(context) != RReportDescriptor.KnitrFormat.None)
+            return null;
 
-        if (!extensions.isEmpty())
-        {
-            FileLike scriptFile = prepareScriptFile(script, context, extensions);
-            return eval(scriptFile, context);
-        }
-        else
-            throw new ScriptException("There are no file name extensions registered for this ScriptEngine : " + getFactory().getLanguageName());
+        return """
+                # --- LabKey R package usage capture ---
+                tryCatch({
+                    writeLines(sort(loadedNamespaces()), "%s")
+                }, error = function(e) invisible(NULL))
+                """.formatted(PACKAGES_FILE);
+    }
+
+    @Override
+    protected void recordPackageUsage(ScriptContext context)
+    {
+        readPackageSidecar(context, PACKAGES_FILE, "r");
+    }
+
+    @Override
+    protected void recordSuccessfulRun(ScriptContext context)
+    {
+        // Non-knitr R runs report their loaded packages via the capture epilog (see getPackageCaptureEpilog).
+        if (getKnitrFormat(context) == RReportDescriptor.KnitrFormat.None)
+            return;
+
+        // For knitr, the generated wrapper (createKnitrScript) always loads knitr and, for the markdown+pandoc path,
+        // rmarkdown; record those as R package usage so they show up alongside other packages.
+        // NOTE: this only captures the wrapper's libraries, not packages loaded inside the report's own R chunks (e.g.
+        // ggplot2 used within an .rmd). Fully capturing those would require injecting a loadedNamespaces() write into
+        // createKnitrScript after the knit()/render() call.
+        ScriptPackageUsageTracker.record("r", "knitr");
+        if (getKnitrFormat(context) == RReportDescriptor.KnitrFormat.Markdown && isPandocEnabled())
+            ScriptPackageUsageTracker.record("r", "rmarkdown");
     }
 
     private boolean isPandocEnabled()

core/src/org/labkey/core/reports/ScriptEngineManagerImpl.java

@@ -47,6 +47,7 @@
 import org.labkey.api.reports.ExternalScriptEngineDefinition;
 import org.labkey.api.reports.ExternalScriptEngineFactory;
 import org.labkey.api.reports.LabKeyScriptEngineManager;
+import org.labkey.api.reports.report.python.PythonScriptEngine;
 import org.labkey.api.reports.report.r.RDockerScriptEngineFactory;
 import org.labkey.api.reports.report.r.RScriptEngineFactory;
 import org.labkey.api.reports.report.r.RemoteRNotEnabledException;
@@ -348,12 +349,21 @@ else if (def.getType().equals(ExternalScriptEngineDefinition.Type.Jupyter) && !P
                 LOG.error("Jupyter Report engine [{}] requested, but premium module not available/enabled.", def.getName());
                 throw new PremiumFeatureNotEnabledException("Jupyter Reports are not available. Please talk to your account representative for additional information.");
             }
+            else if (isPythonEngine(def))
+                return new PythonScriptEngine(def);
             else
                 return new ExternalScriptEngineFactory(def).getScriptEngine();
         }
         return null;
     }
 
+    // A Python engine is configured as a generic external engine (there is no dedicated Type.Python), so identify it by
+    // its ".py" file extension. R and Jupyter (.ipynb) are handled by earlier branches, so they won't match here.
+    private static boolean isPythonEngine(ExternalScriptEngineDefinition def)
+    {
+        return Arrays.stream(def.getExtensions()).anyMatch("py"::equalsIgnoreCase);
+    }
+
     // Locates any specific engines scoped at either the container or project level for an engine context
     @Override
     @Nullable