Make visuals more responsive at lower frame rates - block-peak fold p…

cleave/projectm.py

@@ -363,6 +363,11 @@ def _get_lib() -> ctypes.CDLL:
     )
 
 
+def pcm_max_samples_per_channel() -> int:
+    """Per-channel sample count libprojectM uses for one PCM feed window."""
+    return int(_get_lib().projectm_pcm_get_max_samples())
+
+
 class ProjectM:
     """Context-manager-friendly wrapper around a libprojectM instance."""
 

cleave/stem_pcm.py

@@ -64,6 +64,70 @@ def samples_for_dt(dt_sec: float) -> int:
     return max(1, round(dt_sec * SAMPLE_RATE_HZ))
 
 
+def _fold_channel_block_peak(channel: np.ndarray, target: int) -> np.ndarray:
+    """Fold one channel from *len(channel)* samples to *target* via block-peak."""
+    n = int(channel.shape[0])
+    if n <= target:
+        return np.ascontiguousarray(channel, dtype=np.float32)
+    out = np.zeros(target, dtype=np.float32)
+    for i in range(target):
+        start = i * n // target
+        end = (i + 1) * n // target
+        block = channel[start:end]
+        if block.size == 0:
+            continue
+        out[i] = block[int(np.argmax(np.abs(block)))]
+    return out
+
+
+def fold_pcm_to_max_samples(
+    samples: np.ndarray,
+    *,
+    channels: int,
+    max_samples: int,
+) -> np.ndarray:
+    """Compress per-channel PCM to at most *max_samples* via block-peak aggregation.
+
+    When the frame timeslice exceeds libprojectM's PCM window, feeding the raw
+    slice leaves only the tail in the ring buffer. Block-peak folding maps energy
+    from the full interval into the buffer while preserving percussive transients.
+    """
+    if max_samples <= 0:
+        return samples
+
+    if isinstance(samples, np.ndarray):
+        if samples.size == 0:
+            return samples
+        arr = np.ascontiguousarray(samples, dtype=np.float32).ravel()
+    else:
+        try:
+            if len(samples) == 0:
+                return samples
+        except TypeError:
+            pass
+        arr = np.ascontiguousarray(np.asarray(samples, dtype=np.float32)).ravel()
+    if arr.size == 0:
+        return arr
+    if channels == 2:
+        n_frames = arr.size // 2
+        arr = arr[: n_frames * 2]
+        if n_frames <= max_samples:
+            return arr
+        frames = arr.reshape(n_frames, 2)
+        folded = np.column_stack(
+            (
+                _fold_channel_block_peak(frames[:, 0], max_samples),
+                _fold_channel_block_peak(frames[:, 1], max_samples),
+            )
+        )
+        return np.ascontiguousarray(folded.ravel(), dtype=np.float32)
+
+    n_frames = arr.size
+    if n_frames <= max_samples:
+        return arr
+    return _fold_channel_block_peak(arr, max_samples)
+
+
 def load_stem_pcm(project_dir: Path) -> StemPcmBank:
     """Load five audio sources from *project_dir* into memory."""
     project_dir = project_dir.resolve()

cleave/viz/layer_pipeline.py

@@ -11,13 +11,13 @@
 from cleave.gl_compositor import GlCompositor
 from cleave.gl_post_process import GlPostProcess
 from cleave.preset_playlist import PresetPlaylist
-from cleave.projectm import ProjectM
+from cleave.projectm import ProjectM, pcm_max_samples_per_channel
 from cleave.projectm_health import (
     drain_projectm_log_notifications,
     drain_stem_layers_preset_failures,
 )
 from cleave.signals import Signals
-from cleave.stem_pcm import StemPcmBank
+from cleave.stem_pcm import StemPcmBank, fold_pcm_to_max_samples
 from cleave.viz.layer import StemLayer
 from cleave.viz.layer_preview_resolution import (
     preview_layer_size,
@@ -396,7 +396,13 @@ def render_frame(
                     continue
                 stem = session.layers[layer.slot].stem
                 pcm = pcm_bank.slice_pcm(stem, t_sec, n_pcm)
-                layer.pm.feed_pcm(pcm, channels=pcm_bank.channels(stem))
+                ch = pcm_bank.channels(stem)
+                max_pcm = pcm_max_samples_per_channel()
+                if max_pcm > 0:
+                    pcm = fold_pcm_to_max_samples(
+                        pcm, channels=ch, max_samples=max_pcm
+                    )
+                layer.pm.feed_pcm(pcm, channels=ch)
                 layer.pm.set_frame_time(pm_time_sec)
 
         apply_effect_modifiers(

docs/projectm-api-coverage.md

@@ -20,7 +20,7 @@ Related: [legacy-plans/presets-scan-plan.md](legacy-plans/presets-scan-plan.md)
 | `projectm_pcm_add_float` | bound, used | Stem PCM feed |
 | `projectm_pcm_add_int16` | ignored | Cleave normalizes to float32 |
 | `projectm_pcm_add_uint8` | ignored | Cleave normalizes to float32 |
-| `projectm_pcm_get_max_samples` | bound, used | Chunk sizing |
+| `projectm_pcm_get_max_samples` | bound, used | Chunk sizing; also target for block-peak PCM fold |
 | `projectm_opengl_render_frame` | ignored | Cleave renders to layer FBOs |
 | `projectm_opengl_render_frame_fbo` | bound, used | Per-layer render |
 | `projectm_opengl_burn_texture` | ignored | No burn pass in Cleave |
@@ -102,6 +102,10 @@ Related: [legacy-plans/presets-scan-plan.md](legacy-plans/presets-scan-plan.md)
 - [cleave/projectm_health.py](../cleave/projectm_health.py) drains queues each frame before layer render; rate-limited skip notifications and rotation-stall message via panel notification sink.
 - Warning+ libprojectM log lines are queued in [cleave/projectm.py](../cleave/projectm.py) and drained to panel toasts (`projectM: ...`) once per unique message per session.
 
+## PCM feeding
+
+libprojectM 4.x keeps a **576**-sample circular buffer per channel; presets read **480** waveform samples and a **512**-bin spectrum from it. Cleave slices each frame's song playhead interval (`samples_per_frame` offline, `samples_for_dt` live) and, when that interval exceeds `projectm_pcm_get_max_samples`, folds it with block-peak aggregation in [cleave/stem_pcm.py](../cleave/stem_pcm.py) `fold_pcm_to_max_samples` before [cleave/viz/layer_pipeline.py](../cleave/viz/layer_pipeline.py) calls `feed_pcm`. Without folding, low-FPS frames only retain the tail of the timeslice in the ring buffer.
+
 ## Environment
 
 | Variable | Effect |

docs/roadmap.md

@@ -15,10 +15,6 @@ Extend cues beyond visibility toggles so timeline events can control more layer
 
 Emit MIDI notes or CC from drum onsets (and other signals in `signals.json`) to drive hardware lighting, drum pads, or synths during playback or export.
 
-## projectM PCM feeding
-
-Investigate PCM feeding strategy for the projectM buffer. Only the last ~11 ms (480 samples at 44.1 kHz) is actively used per render; at low FPS Cleave feeds a much longer timeslice per frame. Explore whether sampling or aggregating across the full frame timeslice (not just the tail) improves visual matchup with audio. Uncertain payoff; worth exploring.
-
 ## projectM beat sensitivity
 
 Cleave multiplies PCM by beat sensitivity in [cleave/projectm.py](../cleave/projectm.py) `feed_pcm` (default 2.0). That is intentional: after projectM's 2023 audio rewrite ([69d2134](https://github.com/projectM-visualizer/projectm/commit/69d2134fa2c39901eb354eac546c09e1be5c794b)), `projectm_set_beat_sensitivity` became a store-only stub. Older projectM applied sensitivity as a PCM scale via `BeatDetect::GetPCMScale()` (see [issue #161](https://github.com/projectM-visualizer/projectm/issues/161)); Cleave recreates that outside the library so presets stay reactive.

tests/cleave/test_stem_pcm.py

@@ -12,7 +12,12 @@
 from cleave.extract import STEM_NAMES, stems_dir
 from cleave.pcm_io import SAMPLE_RATE_HZ, _to_mono_float32
 from cleave.project import write_manifest
-from cleave.stem_pcm import StemPcmBank, load_stem_pcm, samples_per_frame
+from cleave.stem_pcm import (
+    StemPcmBank,
+    fold_pcm_to_max_samples,
+    load_stem_pcm,
+    samples_per_frame,
+)
 
 
 def _make_bank(*, duration_samples: int = 4410) -> StemPcmBank:
@@ -189,3 +194,67 @@ def test_load_stem_pcm_stereo_preserves_channels(tmp_path: Path) -> None:
         bank.pcm("drums"),
         np.array([1.0, 2.0, 3.0, 4.0], dtype=np.float32),
     )
+
+
+def test_fold_pcm_to_max_samples_identity_when_short() -> None:
+    pcm = np.linspace(0.0, 1.0, 100, dtype=np.float32)
+    out = fold_pcm_to_max_samples(pcm, channels=1, max_samples=480)
+    np.testing.assert_array_equal(out, pcm)
+
+
+def test_fold_pcm_to_max_samples_mono_30fps_length() -> None:
+    n_pcm = samples_per_frame(fps=30)
+    assert n_pcm == 1470
+    pcm = np.zeros(n_pcm, dtype=np.float32)
+    out = fold_pcm_to_max_samples(pcm, channels=1, max_samples=480)
+    assert out.shape == (480,)
+    assert out.dtype == np.float32
+
+
+def test_fold_pcm_to_max_samples_stereo_interleaved_length() -> None:
+    n_pcm = samples_per_frame(fps=30)
+    pcm = np.zeros(n_pcm * 2, dtype=np.float32)
+    out = fold_pcm_to_max_samples(pcm, channels=2, max_samples=480)
+    assert out.shape == (480 * 2,)
+    assert out.dtype == np.float32
+
+
+def test_fold_pcm_to_max_samples_preserves_early_spike() -> None:
+    n_pcm = samples_per_frame(fps=30)
+    pcm = np.zeros(n_pcm, dtype=np.float32)
+    spike_idx = n_pcm // 6
+    pcm[spike_idx] = 1.0
+    out = fold_pcm_to_max_samples(pcm, channels=1, max_samples=480)
+    assert np.max(np.abs(out)) == 1.0
+    assert np.any(out == 1.0)
+
+
+def test_fold_pcm_to_max_samples_stereo_early_spike_per_channel() -> None:
+    n_pcm = samples_per_frame(fps=30)
+    pcm = np.zeros(n_pcm * 2, dtype=np.float32)
+    pcm[0] = 0.9
+    pcm[1] = -0.8
+    out = fold_pcm_to_max_samples(pcm, channels=2, max_samples=480)
+    assert out[0] == 0.9
+    assert out[1] == -0.8
+
+
+def test_fold_pcm_to_max_samples_noop_when_max_non_positive() -> None:
+    pcm = np.arange(1000, dtype=np.float32)
+    out = fold_pcm_to_max_samples(pcm, channels=1, max_samples=0)
+    np.testing.assert_array_equal(out, pcm)
+
+
+def test_fold_pcm_captures_early_spike_lost_by_tail_only_feed() -> None:
+    """At 30 fps only the last 480 samples reach projectM without folding."""
+    n_pcm = samples_per_frame(fps=30)
+    max_pcm = 480
+    pcm = np.zeros(n_pcm, dtype=np.float32)
+    spike_idx = n_pcm // 6
+    pcm[spike_idx] = 1.0
+
+    tail_only = pcm[-max_pcm:]
+    assert np.max(np.abs(tail_only)) == 0.0
+
+    folded = fold_pcm_to_max_samples(pcm, channels=1, max_samples=max_pcm)
+    assert np.max(np.abs(folded)) == 1.0