DRAFT: Add Zstd GPU CI test infrastructure (Phase 1)

zstd/scripts/generate_histogram.py

@@ -0,0 +1,115 @@
+#!/usr/bin/env python3
+"""
+Generate histogram PNGs from zstdgpu_demo performance CSV output.
+
+Usage:
+    python generate_histogram.py --input <csv_file> --output <png_file> [--title <title>]
+
+Consumes the wide-format CSV emitted by zstdgpu_demo's --out-csv flag:
+    RunIdx, Stage 0 (us), Stage 0 :: <scope> (us), ..., Readback 0 (us),
+    Stage 1 (us), ..., Stage 2 (us), Bandwidth (GB/s)
+
+Plots a histogram of the 'Bandwidth (GB/s)' column. Per-stage timing columns
+are preserved in the CSV but not plotted here (the histogram is the summary
+view; users wanting per-stage detail can read the CSV directly).
+"""
+
+import argparse
+import csv
+import math
+import sys
+
+
+def try_import_matplotlib():
+    try:
+        import matplotlib
+        matplotlib.use("Agg")  # Non-interactive backend for CI
+        import matplotlib.pyplot as plt
+        return plt
+    except ImportError:
+        print(
+            "WARNING: matplotlib not installed. Install with: pip install matplotlib",
+            file=sys.stderr,
+        )
+        return None
+
+
+# Match Pavel's --out-csv column header verbatim ("Bandwidth (GB/s)").
+# Kept case-insensitive and whitespace-tolerant in case the schema spelling
+# drifts upstream — the eyeballed match is "bandwidth".
+def _is_bandwidth_column(col_name: str) -> bool:
+    return col_name is not None and "bandwidth" in col_name.strip().lower()
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Generate histogram from zstdgpu perf CSV")
+    parser.add_argument("--input", required=True, help="Path to input CSV file")
+    parser.add_argument("--output", required=True, help="Path to output PNG file")
+    parser.add_argument("--title", default="Bandwidth", help="Chart title")
+    args = parser.parse_args()
+
+    plt = try_import_matplotlib()
+    if plt is None:
+        return 1
+
+    data = []
+    skipped_non_finite = 0
+    bandwidth_col = None
+    with open(args.input, newline="") as f:
+        reader = csv.DictReader(f)
+        if reader.fieldnames is None:
+            print(f"CSV has no header row: {args.input}", file=sys.stderr)
+            return 1
+        # Pick the bandwidth column by name (resilient to header drift).
+        for col in reader.fieldnames:
+            if _is_bandwidth_column(col):
+                bandwidth_col = col
+                break
+        if bandwidth_col is None:
+            print(
+                f"No Bandwidth column found in {args.input} "
+                f"(headers: {reader.fieldnames})",
+                file=sys.stderr,
+            )
+            return 1
+
+        for row in reader:
+            raw = row.get(bandwidth_col, "")
+            if raw is None or raw == "":
+                continue
+            try:
+                val = float(raw)
+            except ValueError:
+                # Pavel may write empty strings for skipped iterations; ignore.
+                continue
+            if math.isfinite(val):
+                data.append(val)
+            else:
+                skipped_non_finite += 1
+
+    if skipped_non_finite > 0:
+        print(
+            f"WARNING: Skipped {skipped_non_finite} non-finite (Inf/NaN) value(s) from {args.input}",
+            file=sys.stderr,
+        )
+
+    if not data:
+        print(
+            f"No finite Bandwidth (GB/s) data found in {args.input}",
+            file=sys.stderr,
+        )
+        return 1
+
+    plt.figure()
+    plt.hist(data, bins=20)
+    plt.xlabel("Bandwidth (GB/s)")
+    plt.ylabel("Count")
+    plt.title(args.title)
+    plt.savefig(args.output)
+    plt.close()
+    print(f"Generated: {args.output}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

zstd/zstd.sln

@@ -11,6 +11,8 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdgpu_tests", "zstdgpu_te
 EndProject
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "googletest_static", "ThirdParty\googletest_static.vcxproj", "{49811F10-3D14-403E-859D-40DFCBB35C7B}"
 EndProject
+Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "zstdgpu_ci_tests", "zstdgpu_ci_tests\zstdgpu_ci_tests.vcxproj", "{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|ARM64 = Debug|ARM64
@@ -69,6 +71,18 @@ Global
 		{49811F10-3D14-403E-859D-40DFCBB35C7B}.Release|x64.Build.0 = Release|x64
 		{49811F10-3D14-403E-859D-40DFCBB35C7B}.Release|x86.ActiveCfg = Release|Win32
 		{49811F10-3D14-403E-859D-40DFCBB35C7B}.Release|x86.Build.0 = Release|Win32
+		{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Debug|ARM64.ActiveCfg = Debug|ARM64
+		{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Debug|ARM64.Build.0 = Debug|ARM64
+		{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Debug|x64.ActiveCfg = Debug|x64
+		{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Debug|x64.Build.0 = Debug|x64
+		{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Debug|x86.ActiveCfg = Debug|Win32
+		{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Debug|x86.Build.0 = Debug|Win32
+		{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Release|ARM64.ActiveCfg = Release|ARM64
+		{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Release|ARM64.Build.0 = Release|ARM64
+		{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Release|x64.ActiveCfg = Release|x64
+		{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Release|x64.Build.0 = Release|x64
+		{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Release|x86.ActiveCfg = Release|Win32
+		{A1B2C3D4-E5F6-7890-ABCD-EF1234567890}.Release|x86.Build.0 = Release|Win32
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

zstd/zstdgpu_ci_tests/main.cpp

@@ -0,0 +1,182 @@
+/**
+ * Copyright (c) Microsoft. All rights reserved.
+ * This code is licensed under the MIT License (MIT).
+ * THIS CODE IS PROVIDED *AS IS* WITHOUT WARRANTY OF
+ * ANY KIND, EITHER EXPRESS OR IMPLIED, INCLUDING ANY
+ * IMPLIED WARRANTIES OF FITNESS FOR A PARTICULAR
+ * PURPOSE, MERCHANTABILITY, OR NON-INFRINGEMENT.
+ */
+
+// Entry point for the Zstd GPU CI tests. This is a thin GTest wrapper
+// that shells out to zstdgpu_demo.exe to validate Zstd GPU decompression shaders.
+//
+//   - parses custom CLI flags (--content-path, --demo-path, etc.), resolves the
+// demo executable, then hands off to GTest which runs parameterized tests defined
+// in zstdgpu_ci_tests.cpp. Each test spawns the demo as a child process.
+//
+// If no .zst content files are found, zero tests are instantiated and the test
+// binary exits 0 (success). If the demo exe is missing, tests are skipped (not failed).
+//
+// This file also implements the TestConfig singleton and file discovery helpers
+// declared in zstdgpu_ci_tests.h.
+
+#include "zstdgpu_ci_tests.h"
+#include <gtest/gtest.h>
+#include <algorithm>
+#include <cstring>
+#include <filesystem>
+#include <iostream>
+#include <string>
+
+// TestConfig singleton
+// Implementation of the singleton declared in zstdgpu_ci_tests.h.
+
+static TestConfig g_testConfig;
+
+const TestConfig& GetTestConfig()
+{
+    return g_testConfig;
+}
+
+void SetTestConfig(TestConfig config)
+{
+    g_testConfig = std::move(config);
+}
+
+// File discovery
+
+std::vector<std::string> DiscoverZstFiles(const std::string& contentPath)
+{
+    std::vector<std::string> files;
+
+    if (contentPath.empty() || !std::filesystem::exists(contentPath) || !std::filesystem::is_directory(contentPath))
+    {
+        return files;
+    }
+
+    for (const auto& entry : std::filesystem::recursive_directory_iterator(contentPath))
+    {
+        if (entry.is_regular_file() && entry.path().extension() == ".zst")
+        {
+            files.push_back(entry.path().string());
+        }
+    }
+
+    std::sort(files.begin(), files.end());
+    return files;
+}
+
+// CLI and entry point
+
+// QOL for diagnostics. For running manually
+// Activate with --help-ci to avoid conflicting with GTest's own --help output.
+static void PrintUsage(const char* exe)
+{
+    std::cout << "Usage: " << exe << " [gtest_options] [options]\n"
+              << "\n"
+              << "Options:\n"
+              << "  --content-path <dir>    Directory containing .zst test files\n"
+              << "  --demo-path <path>      Path to zstdgpu_demo.exe\n"
+              << "  --log-dir <dir>         Directory for logs and CSV output\n"
+              << "  --log-file <path>       Consolidated text log file\n"
+              << "  --run-count <N>         Perf test iteration count (default: 40)\n"
+              << "  --timeout <seconds>     Per-test process timeout (default: no timeout)\n"
+              << std::endl;
+}
+
+int main(int argc, char** argv)
+{
+    // Parse custom flags before handing off to GTest. GTest's InitGoogleTest()
+    // is called later and will consume its own flags (e.g. --gtest_filter).
+    TestConfig config;
+
+    for (int i = 1; i < argc; ++i)
+    {
+        if (std::strcmp(argv[i], "--content-path") == 0 && i + 1 < argc)
+        {
+            config.contentPath = argv[++i];
+        }
+        else if (std::strcmp(argv[i], "--demo-path") == 0 && i + 1 < argc)
+        {
+            config.demoPath = argv[++i];
+        }
+        else if (std::strcmp(argv[i], "--log-dir") == 0 && i + 1 < argc)
+        {
+            config.logDir = argv[++i];
+        }
+        else if (std::strcmp(argv[i], "--log-file") == 0 && i + 1 < argc)
+        {
+            config.logFile = argv[++i];
+        }
+        else if (std::strcmp(argv[i], "--run-count") == 0 && i + 1 < argc)
+        {
+            config.runCount = std::atoi(argv[++i]);
+            if (config.runCount <= 0)
+                config.runCount = 40;
+        }
+        else if (std::strcmp(argv[i], "--timeout") == 0 && i + 1 < argc)
+        {
+            config.timeoutSeconds = std::atoi(argv[++i]);
+            if (config.timeoutSeconds < 0)
+                config.timeoutSeconds = 0;
+        }
+        else if (std::strcmp(argv[i], "--help-ci") == 0)
+        {
+            PrintUsage(argv[0]);
+            return 0;
+        }
+    }
+
+    if (config.demoPath.empty())
+    {
+        std::cerr << "Warning: --demo-path not set. Tests will skip.\n";
+    }
+
+    if (config.contentPath.empty())
+    {
+        std::cerr << "Warning: --content-path not set. Zero tests will be discovered "
+                     "(gtest will print 'This test program does NOT link in any test case').\n";
+    }
+    else if (!std::filesystem::exists(config.contentPath))
+    {
+        std::cerr << "Warning: --content-path '" << config.contentPath
+                  << "' does not exist. Zero tests will be discovered.\n";
+    }
+    else if (!std::filesystem::is_directory(config.contentPath))
+    {
+        std::cerr << "Warning: --content-path '" << config.contentPath
+                  << "' is not a directory. Zero tests will be discovered.\n";
+    }
+    else
+    {
+        const size_t fileCount = DiscoverZstFiles(config.contentPath).size();
+        if (fileCount == 0)
+        {
+            std::cerr << "Warning: --content-path '" << config.contentPath
+                      << "' contains no .zst files. Zero tests will be discovered.\n";
+        }
+        else
+        {
+            std::cout << "Discovered " << fileCount << " .zst file(s) at '"
+                      << config.contentPath << "'.\n";
+        }
+    }
+
+    // Default log dir to current directory.
+    if (config.logDir.empty())
+    {
+        config.logDir = std::filesystem::current_path().string();
+    }
+
+    // Ensure log directory exists.
+    if (!std::filesystem::exists(config.logDir))
+    {
+        std::filesystem::create_directories(config.logDir);
+    }
+
+    SetTestConfig(std::move(config));
+
+    testing::InitGoogleTest(&argc, argv);
+    testing::GTEST_FLAG(catch_exceptions) = false;
+    return RUN_ALL_TESTS();
+}