fix(psd): skip empty layers when reading PSD files

src/cmake/testing.cmake

@@ -545,6 +545,7 @@ macro (oiio_add_all_tests)
     oiio_add_tests (psd psd-colormodes
                     ENABLEVAR ENABLE_PSD
                     IMAGEDIR oiio-images/psd)
+    oiio_add_tests (psd-empty-layers ENABLEVAR ENABLE_PSD)
     oiio_add_tests (ptex
                     FOUNDVAR Ptex_FOUND ENABLEVAR ENABLE_PTEX)
     oiio_add_tests (raw raw-thumbnail

src/psd.imageio/psdinput.cpp

@@ -147,6 +147,8 @@ class PSDInput final : public ImageInput {
         uint32_t width, height;
         uint16_t channel_count;
 
+        bool empty() const { return width == 0 || height == 0; }
+
         std::vector<ChannelInfo> channel_info;
         std::map<int16_t, ChannelInfo*> channel_id_map;
 
@@ -195,7 +197,7 @@ class PSDInput final : public ImageInput {
     std::string m_filename;
     //Current subimage
     int m_subimage;
-    //Subimage count (1 + layer count)
+    //Subimage count (composite plus nonempty layers)
     int m_subimage_count;
     std::vector<ImageSpec> m_specs;
     static const ResourceLoader resource_loaders[];
@@ -651,8 +653,6 @@ PSDInput::open(const std::string& name, ImageSpec& newspec)
         return false;
     }
 
-    // Layer count + 1 for merged composite (Image Data Section)
-    m_subimage_count = m_layers.size() + 1;
     // Setup ImageSpecs and m_channels
     bool ok = setup();
 
@@ -1632,7 +1632,9 @@ PSDInput::load_layer(Layer& layer)
     int64_t h    = int64_t(layer.bottom) - int64_t(layer.top);
     layer.width  = uint32_t(w < 0 ? -w : w);
     layer.height = uint32_t(h < 0 ? -h : h);
-    if (!validate_resolution("Layer Record", layer.width, layer.height))
+    // Tolerate empty rectangles, but still enforce upper size limits.
+    if (!validate_resolution("Layer Record", std::max(layer.width, 1u),
+                             std::max(layer.height, 1u)))
         return false;
     layer.channel_info.resize(layer.channel_count);
     for (uint16_t channel = 0; channel < layer.channel_count; channel++) {
@@ -1756,6 +1758,22 @@ PSDInput::load_layer_channel(Layer& layer, ChannelInfo& channel_info)
     if (channel_info.data_length <= 2)
         return true;
 
+    // A channel's stored data, including the 2 byte compression tag already
+    // read, cannot extend past the end of the file. Compare unsigned: for PSB
+    // the length is 64 bits and a corrupt value would wrap negative if cast.
+    int64_t file_size = ioproxy() ? ioproxy()->size() : 0;
+    int64_t remaining = file_size - start_pos;
+    if (remaining < 0 || channel_info.data_length > uint64_t(remaining)) {
+        errorfmt("[Layer Channel] data length {} exceeds remaining file size",
+                 channel_info.data_length);
+        return false;
+    }
+
+    // Empty layers are not exposed as images, but their channels may still
+    // contain RLE row counts or ZIP data. Masks have independent dimensions.
+    if (layer.empty() && channel_info.channel_id != ChannelID_LayerMask)
+        return ioseek(start_pos + channel_info.data_length);
+
     // Use mask_data size when channel_id is -2
     uint32_t width, height;
     if (channel_info.channel_id == ChannelID_LayerMask) {
@@ -1780,17 +1798,6 @@ PSDInput::load_layer_channel(Layer& layer, ChannelInfo& channel_info)
 
     channel_info.data_pos = iotell();
 
-    // A channel's stored data, including the 2 byte compression tag already
-    // read, cannot extend past the end of the file. Compare unsigned: for PSB
-    // the length is 64 bits and a corrupt value would wrap negative if cast.
-    int64_t file_size = ioproxy() ? ioproxy()->size() : 0;
-    int64_t remaining = file_size - start_pos;
-    if (remaining < 0 || channel_info.data_length > uint64_t(remaining)) {
-        errorfmt("[Layer Channel] data length {} exceeds remaining file size",
-                 channel_info.data_length);
-        return false;
-    }
-
     channel_info.row_pos.resize(height);
     channel_info.row_length = (width * m_header.depth + 7) / 8;
 
@@ -2158,13 +2165,15 @@ PSDInput::setup()
             spec.attribute("oiio:UnassociatedAlpha", 1);
 
     // Composite channels
-    m_channels.reserve(m_subimage_count);
+    m_channels.reserve(m_layers.size() + 1);
     m_channels.resize(1);
     m_channels[0].reserve(raw_channel_count);
     for (int i = 0; i < raw_channel_count; ++i)
         m_channels[0].push_back(&m_image_data.channel_info[i]);
 
     for (Layer& layer : m_layers) {
+        if (layer.empty())
+            continue;
         spec_channel_count = m_WantRaw ? mode_channel_count[m_header.color_mode]
                                        : 3;
         raw_channel_count  = mode_channel_count[m_header.color_mode];
@@ -2209,6 +2218,7 @@ PSDInput::setup()
             spec.attribute("oiio:subimagename", layer.name);
     }
 
+    m_subimage_count = int(m_specs.size());
     return true;
 }
 

testsuite/psd-empty-layers/ref/out.txt

@@ -0,0 +1 @@
+Empty-layer subimages, populated pixels, raw mode and size limits verified.

testsuite/psd-empty-layers/run.py

@@ -0,0 +1,9 @@
+#!/usr/bin/env python
+
+# Copyright Contributors to the OpenImageIO project.
+# SPDX-License-Identifier: Apache-2.0
+# https://github.com/AcademySoftwareFoundation/OpenImageIO
+
+redirect = ' >> out.txt 2>&1 '
+command += run_app(f'"{pythonbin}" src/check-empty-layers.py '
+                   f'"{oiio_app("oiiotool").strip()}" src')

testsuite/psd-empty-layers/src/check-empty-layers.py

@@ -0,0 +1,68 @@
+#!/usr/bin/env python
+
+# Copyright Contributors to the OpenImageIO project.
+# SPDX-License-Identifier: Apache-2.0
+# https://github.com/AcademySoftwareFoundation/OpenImageIO
+
+import subprocess
+import sys
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+
+tool, root = sys.argv[1], Path(sys.argv[2])
+composite = (2, 2, "", "6585CA1F6B702CF27768484219F6F9200F4E9192")
+before = (1, 1, "before", "E3FF44389E80210E4545A907BE55980CEB6EA821")
+after = (1, 1, "after", "5818BEB65A1338D7C80684EEC2EA2ECB0DB183D0")
+populated = (1, 1, "empty", before[3])
+cases = []
+for compression in (0, 1):
+    for width, height in ((0, 0), (0, 2), (2, 0)):
+        cases.append((f"empty-layer-{width}x{height}-{compression}.psd",
+                      [composite, before, after]))
+    cases.append((f"hidden-empty-layer-{compression}.psd", [composite, before, after]))
+cases += [
+    ("hidden-populated-layer.psd", [composite, before, populated, after]),
+    ("all-empty.psd", [composite]),
+    ("trailing-empty.psd", [composite, before]),
+    ("empty-layer-with-mask.psd", [composite, before, after]),
+    ("empty-layer-zip.psd", [composite, before, after]),
+    ("empty-layer-0x300000.psb", [composite, before, after]),
+    ("empty-layer-300000x0.psb", [composite, before, after]),
+]
+
+for raw in (0, 1):
+    for filename, expected in cases:
+        result = subprocess.run(
+            [tool, "--iconfig", "psd:RawData", str(raw),
+             "--info:format=xml:verbose=1", "-a", "--hash", str(root / filename)],
+            stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
+        )
+        assert result.returncode == 0, (filename, raw, result.stderr)
+        specs = ET.fromstring("<images>" + result.stdout + "</images>")
+        actual = [
+            (int(spec.findtext("width")), int(spec.findtext("height")),
+             spec.findtext("attrib[@name='oiio:subimagename']", ""),
+             spec.findtext("SHA1"))
+            for spec in specs
+        ]
+        assert actual == expected, (filename, raw, actual, expected)
+        assert all(int(spec.findtext("subimages", "1")) == len(expected)
+                   for spec in specs), (filename, raw, "incorrect subimage count")
+
+for filename, diagnostic in (
+    ("empty-layer-invalid-canvas.psd", "[Header]"),
+    ("empty-layer-oversized.psd", "[Layer Record]"),
+    ("empty-layer-oversized-width.psd", "[Layer Record]"),
+    ("oversized-layer-0x300001.psb", "[Layer Record]"),
+    ("oversized-layer-300001x0.psb", "[Layer Record]"),
+    ("empty-layer-truncated-channel.psd", "[Layer Channel]"),
+):
+    result = subprocess.run(
+        [tool, "--info", str(root / filename)],
+        stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True,
+    )
+    assert result.returncode != 0 and diagnostic in result.stderr, (
+        filename, result.returncode, result.stderr)
+
+print("Empty-layer subimages, populated pixels, raw mode and size limits verified.")

testsuite/psd-empty-layers/src/make-empty-layer-psds.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+
+# Copyright Contributors to the OpenImageIO project.
+# SPDX-License-Identifier: Apache-2.0
+# https://github.com/AcademySoftwareFoundation/OpenImageIO
+
+import struct
+import zlib
+from pathlib import Path
+
+
+def layer(width, height, compression, name, values=(32, 96, 160), hidden=False,
+          mask=False, version=1, data_length_extra=0):
+    encoded_name = name.encode("ascii")
+    pascal_name = bytes([len(encoded_name)]) + encoded_name
+    pascal_name += bytes(-len(pascal_name) % 4)
+    mask_data = struct.pack(">iiiiBB2x", 0, 0, 2, 2, 255, 0) if mask else b""
+    extra = struct.pack(">I", len(mask_data)) + mask_data
+    extra += struct.pack(">I", 0) + pascal_name
+    channels = []
+    for value in values:
+        if not width or not height:
+            data = struct.pack(">H", compression)
+            if compression == 1:
+                data += bytes(height * (2 if version == 1 else 4))
+            elif compression == 2:
+                data += zlib.compress(b"")
+        else:
+            data = struct.pack(">H", 0) + bytes([value]) * width * height
+        channels.append((len(channels), data))
+    if mask:
+        channels.append((-2, struct.pack(">H", 0) + bytes([255]) * 4))
+    record = struct.pack(">iiiiH", 0, 0, height, width, len(channels))
+    record += b"".join(
+        struct.pack(">hI" if version == 1 else ">hQ", channel,
+                    len(data) + data_length_extra)
+        for channel, data in channels
+    )
+    record += b"8BIMnorm" + bytes([255, 0, 2 if hidden else 0, 0])
+    record += struct.pack(">I", len(extra)) + extra
+    return record, b"".join(data for channel, data in channels)
+
+
+def write_psd(filename, width, height, compression, canvas_width=2,
+              hidden=False, records=None, version=1):
+    header = b"8BPS" + struct.pack(">H6sHIIHH", version, bytes(6), 3,
+                                 2, canvas_width, 8, 3)
+    if records is None:
+        records = [layer(1, 1, 0, "before", version=version),
+                   layer(width, height, compression, "empty", hidden=hidden,
+                         version=version),
+                   layer(1, 1, 0, "after", values=(160, 96, 32), version=version)]
+    layer_info = struct.pack(">h", len(records))
+    layer_info += b"".join(record for record, data in records)
+    layer_info += b"".join(data for record, data in records)
+    layer_info += bytes(len(layer_info) % 2)
+    length_format = ">I" if version == 1 else ">Q"
+    layer_mask = struct.pack(length_format, len(layer_info)) + layer_info
+    layer_mask += struct.pack(">I", 0)
+    composite = struct.pack(">H", 0)
+    composite += b"".join(bytes([v]) * 4 for v in (32, 96, 160))
+    Path(filename).write_bytes(
+        header + struct.pack(">II", 0, 0)
+        + struct.pack(length_format, len(layer_mask)) + layer_mask + composite
+    )
+
+
+for compression in (0, 1):
+    for width, height in ((0, 0), (0, 2), (2, 0)):
+        write_psd(f"empty-layer-{width}x{height}-{compression}.psd",
+                  width, height, compression)
+
+for compression in (0, 1):
+    write_psd(f"hidden-empty-layer-{compression}.psd", 0, 0, compression,
+              hidden=True)
+write_psd("hidden-populated-layer.psd", 1, 1, 0, hidden=True)
+
+# An empty layer must not bypass the existing document and layer size limits.
+write_psd("empty-layer-invalid-canvas.psd", 0, 0, 0, canvas_width=0)
+write_psd("empty-layer-oversized.psd", 0, 30001, 0)
+write_psd("empty-layer-oversized-width.psd", 30001, 0, 0)
+write_psd("all-empty.psd", 0, 0, 0,
+          records=[layer(0, 0, 0, "empty"), layer(0, 2, 1, "empty2")])
+write_psd("trailing-empty.psd", 0, 0, 0,
+          records=[layer(1, 1, 0, "before"), layer(0, 0, 0, "empty")])
+write_psd("empty-layer-with-mask.psd", 0, 0, 0,
+          records=[layer(1, 1, 0, "before"), layer(0, 0, 0, "empty", mask=True),
+                   layer(1, 1, 0, "after", values=(160, 96, 32))])
+write_psd("empty-layer-zip.psd", 0, 2, 2)
+write_psd("empty-layer-truncated-channel.psd", 0, 2, 1,
+          records=[layer(0, 2, 1, "empty", data_length_extra=1000000)])
+for width, height in ((0, 300000), (300000, 0)):
+    write_psd(f"empty-layer-{width}x{height}.psb", width, height, 0, version=2)
+for width, height in ((0, 300001), (300001, 0)):
+    write_psd(f"oversized-layer-{width}x{height}.psb", width, height, 0, version=2)