diff --git a/Library/DiscUtils.SquashFs/ExtendedSymlinkInode.cs b/Library/DiscUtils.SquashFs/ExtendedSymlinkInode.cs new file mode 100644 index 000000000..526f37d34 --- /dev/null +++ b/Library/DiscUtils.SquashFs/ExtendedSymlinkInode.cs @@ -0,0 +1,36 @@ +// +// Copyright (c) 2008-2011, Kenneth Bell +// +// Permission is hereby granted, free of charge, to any person obtaining a +// copy of this software and associated documentation files (the "Software"), +// to deal in the Software without restriction, including without limitation +// the rights to use, copy, modify, merge, publish, distribute, sublicense, +// and/or sell copies of the Software, and to permit persons to whom the +// Software is furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +// FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +// DEALINGS IN THE SOFTWARE. +// + + +namespace DiscUtils.SquashFs; + +/// +/// An extended symbolic link inode (type 10). Its fixed part is the basic symlink inode's: the common header, the +/// link count and the target length. The target path follows, as in the basic form, and then a 32-bit index into +/// the xattr table, which the basic form does not have. mksquashfs writes this form for a symlink that carries +/// extended attributes. +/// +internal sealed class ExtendedSymlinkInode : SymlinkInode +{ + /// The xattr table index stored after the target path (0xFFFFFFFF for none); read along with the target. + public uint XattrIndex { get; internal set; } = uint.MaxValue; +} diff --git a/Library/DiscUtils.SquashFs/File.cs b/Library/DiscUtils.SquashFs/File.cs index 4414ca573..e25d28df8 100644 --- a/Library/DiscUtils.SquashFs/File.cs +++ b/Library/DiscUtils.SquashFs/File.cs @@ -43,6 +43,9 @@ public File(Context context, Inode inode, MetadataRef inodeRef) protected Context Context { get; } + /// Where the inode sits in the inode table; what follows the inode (a symlink's target, a file's block sizes) is read from here. + protected MetadataRef InodeRef => _inodeRef; + internal Inode Inode { get; } public DateTime LastAccessTimeUtc diff --git a/Library/DiscUtils.SquashFs/FileContentBuffer.cs b/Library/DiscUtils.SquashFs/FileContentBuffer.cs index 8acc197a8..7d2d8b1ef 100644 --- a/Library/DiscUtils.SquashFs/FileContentBuffer.cs +++ b/Library/DiscUtils.SquashFs/FileContentBuffer.cs @@ -35,6 +35,7 @@ internal class FileContentBuffer : Streams.Buffer private const uint InvalidFragmentKey = 0xFFFFFFFF; private readonly int[] _blockLengths; + private readonly long[] _blockStarts; private readonly Context _context; private readonly RegularInode _inode; @@ -64,6 +65,17 @@ public FileContentBuffer(Context context, RegularInode inode, MetadataRef inodeR _blockLengths[i] = EndianUtilities.ToInt32LittleEndian(lengthData.Slice(i * 4, 4)); } } + + // Where each block starts on disk: the blocks lie one after another from the inode's start, each as long + // as the low 24 bits of its size field say (a sparse block takes no room). Computed once, so a read at any + // position finds its block without walking the sizes from the first block. + _blockStarts = new long[numBlocks]; + var start = _inode.StartBlock; + for (var i = 0; i < numBlocks; ++i) + { + _blockStarts[i] = start; + start += _blockLengths[i] & 0x00FFFFFF; + } } public override bool CanRead => true; @@ -86,8 +98,6 @@ public override int Read(long pos, byte[] buffer, int offset, int count) var currentPos = pos; var totalRead = 0; var totalToRead = (int)Math.Min(_inode.FileSize - pos, count); - var currentBlock = 0; - long currentBlockDiskStart = _inode.StartBlock; while (totalRead < totalToRead) { if (currentPos >= startOfFragment) @@ -97,13 +107,7 @@ public override int Read(long pos, byte[] buffer, int offset, int count) return totalRead + read; } - var targetBlock = (int)(currentPos / _context.SuperBlock.BlockSize); - while (currentBlock < targetBlock) - { - currentBlockDiskStart += _blockLengths[currentBlock] & 0x7FFFFF; - ++currentBlock; - } - + var currentBlock = (int)(currentPos / _context.SuperBlock.BlockSize); var blockOffset = (int)(currentPos % _context.SuperBlock.BlockSize); if ((_blockLengths[currentBlock] & 0x00FFFFFF) == 0) { @@ -115,7 +119,7 @@ public override int Read(long pos, byte[] buffer, int offset, int count) continue; } - var block = _context.ReadBlock(currentBlockDiskStart, _blockLengths[currentBlock]); + var block = _context.ReadBlock(_blockStarts[currentBlock], _blockLengths[currentBlock]); var toCopy = Math.Min(block.Available - blockOffset, totalToRead - totalRead); System.Buffer.BlockCopy(block.Data, blockOffset, buffer, offset + totalRead, toCopy); @@ -148,8 +152,6 @@ public override int Read(long pos, Span buffer) var currentPos = pos; var totalRead = 0; var totalToRead = (int)Math.Min(_inode.FileSize - pos, buffer.Length); - var currentBlock = 0; - long currentBlockDiskStart = _inode.StartBlock; while (totalRead < totalToRead) { if (currentPos >= startOfFragment) @@ -158,13 +160,7 @@ public override int Read(long pos, Span buffer) return totalRead + read; } - var targetBlock = (int)(currentPos / _context.SuperBlock.BlockSize); - while (currentBlock < targetBlock) - { - currentBlockDiskStart += _blockLengths[currentBlock] & 0x7FFFFF; - ++currentBlock; - } - + var currentBlock = (int)(currentPos / _context.SuperBlock.BlockSize); var blockOffset = (int)(currentPos % _context.SuperBlock.BlockSize); if ((_blockLengths[currentBlock] & 0x00FFFFFF) == 0) { @@ -176,7 +172,7 @@ public override int Read(long pos, Span buffer) continue; } - var block = _context.ReadBlock(currentBlockDiskStart, _blockLengths[currentBlock]); + var block = _context.ReadBlock(_blockStarts[currentBlock], _blockLengths[currentBlock]); var toCopy = Math.Min(block.Available - blockOffset, totalToRead - totalRead); block.Data.AsSpan(blockOffset, toCopy).CopyTo(buffer[totalRead..]); diff --git a/Library/DiscUtils.SquashFs/Inode.cs b/Library/DiscUtils.SquashFs/Inode.cs index 9ceee27d0..42708bfe9 100644 --- a/Library/DiscUtils.SquashFs/Inode.cs +++ b/Library/DiscUtils.SquashFs/Inode.cs @@ -117,6 +117,7 @@ private static Inode InstantiateType(InodeType type) InodeType.File => new RegularInode(), InodeType.ExtendedFile => new ExtendedFileInode(), InodeType.Symlink => new SymlinkInode(), + InodeType.ExtendedSymlink => new ExtendedSymlinkInode(), InodeType.CharacterDevice or InodeType.BlockDevice => new DeviceInode(), _ => throw new NotImplementedException($"Inode type not implemented: {type}"), }; diff --git a/Library/DiscUtils.SquashFs/MetablockReader.cs b/Library/DiscUtils.SquashFs/MetablockReader.cs index dd697e980..86844f4d7 100644 --- a/Library/DiscUtils.SquashFs/MetablockReader.cs +++ b/Library/DiscUtils.SquashFs/MetablockReader.cs @@ -39,6 +39,9 @@ public MetablockReader(Context context, long start) _start = start; } + /// Where the metadata block the reader is in starts in the image, in bytes. + public long CurrentBlockStart => _start + _currentBlockStart; + public void SetPosition(MetadataRef position) { SetPosition(position.Block, position.Offset); diff --git a/Library/DiscUtils.SquashFs/Symlink.cs b/Library/DiscUtils.SquashFs/Symlink.cs index 6ca251540..33f7863ef 100644 --- a/Library/DiscUtils.SquashFs/Symlink.cs +++ b/Library/DiscUtils.SquashFs/Symlink.cs @@ -20,15 +20,64 @@ // DEALINGS IN THE SOFTWARE. // -using System; + +using System.IO; +using System.Text; using DiscUtils.Vfs; namespace DiscUtils.SquashFs; internal class Symlink : File, IVfsSymlink { + /// + /// The format's maximum target length: SQUASHFS_SYMLINK_MAX in squashfs_fs.h (squashfs-tools and the Linux + /// driver), 65535 bytes. mksquashfs enforces it when it reads a link (a 65536-byte readlink buffer, and a + /// result that fills it is refused), so no well-formed image stores a longer target. + /// + public const int MaxTargetLength = 65535; + + private string _targetPath; + public Symlink(Context context, Inode inode, MetadataRef inodeRef) : base(context, inode, inodeRef) { } - public string TargetPath => throw new NotImplementedException(); -} \ No newline at end of file + /// + /// The link's target as stored: the UTF-8 path that follows the inode in the inode table, relative to the + /// link's directory unless it starts with a slash. The length comes from the image, so it is checked against + /// before anything is allocated, and the bytes must lie within the inode table, + /// which the directory table follows directly. An extended inode keeps its xattr index after the target. + /// + public string TargetPath + { + get + { + if (_targetPath is null) + { + var inode = (SymlinkInode)Inode; + if (inode.SymlinkSize > MaxTargetLength) + { + throw new IOException($"Corrupt symlink inode: target length {inode.SymlinkSize} exceeds {MaxTargetLength}"); + } + + var reader = Context.InodeReader; + reader.SetPosition(InodeRef); + reader.Skip(inode.Size); + var target = new byte[inode.SymlinkSize]; + var read = reader.Read(target); + if (inode is ExtendedSymlinkInode extended) + { + extended.XattrIndex = reader.ReadUInt(); + } + + if (read != target.Length || reader.CurrentBlockStart >= Context.SuperBlock.DirectoryTableStart) + { + throw new IOException("Corrupt symlink inode: target runs past the inode table"); + } + + _targetPath = Encoding.UTF8.GetString(target); + } + + return _targetPath; + } + } +} diff --git a/Library/DiscUtils.SquashFs/VfsSquashFileSystemReader.cs b/Library/DiscUtils.SquashFs/VfsSquashFileSystemReader.cs index aac64b615..64b6aea60 100644 --- a/Library/DiscUtils.SquashFs/VfsSquashFileSystemReader.cs +++ b/Library/DiscUtils.SquashFs/VfsSquashFileSystemReader.cs @@ -85,10 +85,8 @@ public VfsSquashFileSystemReader(Stream stream, SquashFileSystemReaderOptions op throw new IOException($"Unsupported compression {_context.SuperBlock.Compression} used"); } - if (_context.SuperBlock.ExtendedAttrsTableStart != -1) - { - throw new IOException("Unsupported extended attributes present"); - } + // An extended attribute table is left alone: the reader exposes no attributes, and the inodes of the files + // that carry them (the extended forms) are read like the others, their attribute index unused. if (_context.SuperBlock.MajorVersion != 4) { @@ -122,7 +120,7 @@ public VfsSquashFileSystemReader(Stream stream, SquashFileSystemReaderOptions op // Bootstrap the root directory _context.InodeReader.SetPosition(_context.SuperBlock.RootInode); - var dirInode = (DirectoryInode)Inode.Read(_context.InodeReader); + var dirInode = Inode.Read(_context.InodeReader); RootDirectory = new Directory(_context, dirInode, _context.SuperBlock.RootInode); } diff --git a/Tests/LibraryTests/LibraryTests.csproj b/Tests/LibraryTests/LibraryTests.csproj index 622ba8b08..014633ba8 100644 --- a/Tests/LibraryTests/LibraryTests.csproj +++ b/Tests/LibraryTests/LibraryTests.csproj @@ -25,6 +25,8 @@ + + diff --git a/Tests/LibraryTests/SquashFs/SquashFileSystemReaderTest.cs b/Tests/LibraryTests/SquashFs/SquashFileSystemReaderTest.cs index 8c4a28cb0..0dc1676e9 100644 --- a/Tests/LibraryTests/SquashFs/SquashFileSystemReaderTest.cs +++ b/Tests/LibraryTests/SquashFs/SquashFileSystemReaderTest.cs @@ -22,6 +22,7 @@ using System; using System.IO; +using System.Linq; using System.Text; using DiscUtils.SquashFs; @@ -132,6 +133,92 @@ public void SparseFileLargerThan4GiB() Assert.Equal(0, stream.Read(tail, 0, tail.Length)); } + /// + /// mksquashfs stores a directory whose table exceeds one metadata block as an extended directory inode (with an + /// index), the root included; the reader used to cast the root to the basic inode type. big-root.sqsh holds + /// 1,000 files and a symlink in its root. + /// + [Fact] + public void RootStoredAsExtendedDirectoryInode() + { + using var fs = OpenImage("big-root.sqsh"); + Assert.Equal(1001, fs.GetFileSystemEntries("").Count()); + Assert.True(fs.FileExists("entry-0999")); + Assert.Equal(0, fs.GetFileLength("entry-0999")); + } + + /// + /// A symbolic link's target is the path stored after its inode; a lookup through the link reaches the target + /// (the reader used to throw NotImplementedException from any path that landed on a symlink). + /// + [Fact] + public void SymlinkTargetIsRead() + { + using var fs = OpenImage("big-root.sqsh"); + Assert.True(fs.FileExists("link")); + Assert.Equal("first", ReadText(fs, "link")); + Assert.Equal(5, fs.GetFileLength("link")); + } + + /// + /// A symlink that carries an extended attribute is stored as an extended symlink inode: the fixed part and the + /// target path of the basic form, then a 32-bit xattr index after the path. xattr-symlink.sqsh was made with + /// -xattrs-add, which puts an attribute on every inode, so its link is of that kind, and the image has an xattr + /// table (checked here from the superblock), which the reader used to refuse outright. + /// + [Fact] + public void ExtendedSymlinkInodeOfALinkWithAttributes() + { + using var image = SquashFixtures.Open("xattr-symlink.sqsh"); + var superblock = new byte[96]; + Assert.Equal(superblock.Length, image.Read(superblock, 0, superblock.Length)); + Assert.NotEqual(-1L, BitConverter.ToInt64(superblock, 56)); + image.Position = 0; + + using var fs = new SquashFileSystemReader(image); + Assert.Equal("hello", ReadText(fs, "target.txt")); + Assert.True(fs.FileExists("link")); + Assert.Equal("hello", ReadText(fs, "link")); + } + + /// + /// mixed.bin has, in order, a compressed block, a block stored as is (incompressible), a sparse block (all + /// zeros, taking no room on disk), another compressed block and a 100-byte tail in a fragment. Each block's + /// start on disk is the sum of the stored sizes before it, a sparse block counting nothing, so a read across + /// every boundary and a read of the whole file must come back exact. + /// + [Fact] + public void BlocksCompressedStoredSparseAndFragmentInOneFile() + { + var expected = SquashFixtures.Mixed; + using var fs = OpenImage("extended-inodes.sqsh"); + Assert.Equal(expected.Length, fs.GetFileLength("mixed.bin")); + using var stream = fs.OpenFile("mixed.bin", FileMode.Open, FileAccess.Read); + + const int block = 128 * 1024; + foreach (var offset in new[] { block - 50, 2 * block - 50, 3 * block - 50, 4 * block - 50 }) + { + var piece = new byte[100]; + stream.Position = offset; + Assert.Equal(piece.Length, stream.Read(piece, 0, piece.Length)); + var want = new byte[piece.Length]; + Array.Copy(expected, offset, want, 0, want.Length); + Assert.Equal(want, piece); + } + + stream.Position = 0; + var all = new byte[expected.Length]; + var total = 0; + while (total < all.Length) + { + var read = stream.Read(all, total, all.Length - total); + Assert.NotEqual(0, read); + total += read; + } + + Assert.Equal(expected, all); + } + private static SquashFileSystemReader OpenImage(string name) => new(SquashFixtures.Open(name)); private static string ReadText(SquashFileSystemReader fs, string path) diff --git a/Tests/LibraryTests/SquashFs/SquashFixtures.cs b/Tests/LibraryTests/SquashFs/SquashFixtures.cs index 3e464b5af..7ff6f2cb9 100644 --- a/Tests/LibraryTests/SquashFs/SquashFixtures.cs +++ b/Tests/LibraryTests/SquashFs/SquashFixtures.cs @@ -56,6 +56,38 @@ public static string Text } } + /// + /// The 524,388 bytes of mixed.bin: with 128 KB blocks, a block of text (compressed), a block of pseudo-random + /// bytes (incompressible, so mksquashfs stores it as is), a block of zeros (sparse), another block of text and + /// a 100-byte tail (a fragment). + /// + public static byte[] Mixed + { + get + { + const int block = 128 * 1024; + var bytes = new byte[4 * block + 100]; + var text = Encoding.ASCII.GetBytes(Text); + Array.Copy(text, 0, bytes, 0, block); + var x = 0x9E3779B9u; // xorshift32: the same bytes on every machine + for (var i = block; i < 2 * block; i++) + { + x ^= x << 13; + x ^= x >> 17; + x ^= x << 5; + bytes[i] = (byte)x; + } + + Array.Copy(text, 0, bytes, 3 * block, block); + for (var i = 4 * block; i < bytes.Length; i++) + { + bytes[i] = (byte)('a' + i % 26); + } + + return bytes; + } + } + /// A seekable copy of an image: freshly built when mksquashfs is available, else the embedded one. public static Stream Open(string name) { @@ -100,9 +132,15 @@ public static Stream Open(string name) /// /// The recipe. extended-inodes.sqsh: 128 KB blocks; small.txt ("hello", a fragment), text.bin (300,000 bytes, /// two stored blocks and a fragment), sparse.bin (2 MiB of zeros then "end": sparse blocks, an extended inode), - /// hard.txt and dir/hard2.txt (one file, two names: an extended inode). huge-sparse.sqsh: 1 MiB blocks; + /// hard.txt and dir/hard2.txt (one file, two names: an extended inode), mixed.bin (: a + /// compressed, a stored, a sparse and a compressed block, then a fragment). huge-sparse.sqsh: 1 MiB blocks; /// huge.bin, 4 GiB + 4 bytes, "mid" at 2 GiB + 5 and "end!" past the 4 GiB mark, nearly all of it sparse. - /// Both gzip at level 6, so the superblock carries compressor options. + /// Both gzip at level 6, so the superblock carries compressor options. big-root.sqsh: 1,000 files entry-0000 + /// ("first") to entry-0999 (empty) in the root, whose directory table then exceeds one metadata block, so + /// mksquashfs stores the root as an extended directory inode with an index; and link, a symbolic link to + /// entry-0000. xattr-symlink.sqsh: target.txt ("hello") and link, a symbolic link to it, built with + /// -xattrs-add, which puts an attribute on every inode: the link is an extended symlink inode and the image + /// has an xattr table. The other three are built with -no-xattrs. /// private static void Build(string work) { @@ -118,6 +156,7 @@ private static void Build(string work) } File.WriteAllText(Path.Combine(ext, "hard.txt"), "twice", Encoding.ASCII); Run("ln", Path.Combine(ext, "hard.txt"), Path.Combine(ext, "dir", "hard2.txt")); + File.WriteAllBytes(Path.Combine(ext, "mixed.bin"), Mixed); var huge = Path.Combine(work, "huge"); Directory.CreateDirectory(huge); @@ -130,11 +169,29 @@ private static void Build(string work) Write(file, "end!"); } - var flags = new[] { "-noappend", "-no-xattrs", "-no-progress", "-quiet", "-mkfs-time", "0", "-all-time", "0", "-force-uid", "0", "-force-gid", "0" }; - Run("mksquashfs", new[] { ext, Path.Combine(work, "extended-inodes.sqsh"), "-comp", "gzip", "-Xcompression-level", "6", "-b", "128K" }.Concat(flags).ToArray()); - Run("mksquashfs", new[] { huge, Path.Combine(work, "huge-sparse.sqsh"), "-comp", "gzip", "-Xcompression-level", "6", "-b", "1M" }.Concat(flags).ToArray()); + var bigRoot = Path.Combine(work, "big-root"); + Directory.CreateDirectory(bigRoot); + for (var i = 0; i < 1000; i++) + { + File.WriteAllText(Path.Combine(bigRoot, $"entry-{i:D4}"), i == 0 ? "first" : "", Encoding.ASCII); + } + Run("ln", "-s", "entry-0000", Path.Combine(bigRoot, "link")); + + var xattrSymlink = Path.Combine(work, "xattr-symlink"); + Directory.CreateDirectory(xattrSymlink); + File.WriteAllText(Path.Combine(xattrSymlink, "target.txt"), "hello", Encoding.ASCII); + Run("ln", "-s", "target.txt", Path.Combine(xattrSymlink, "link")); + + var flags = new[] { "-noappend", "-no-progress", "-quiet", "-mkfs-time", "0", "-all-time", "0", "-force-uid", "0", "-force-gid", "0" }; + var noXattrs = new[] { "-no-xattrs" }.Concat(flags).ToArray(); + Run("mksquashfs", new[] { ext, Path.Combine(work, "extended-inodes.sqsh"), "-comp", "gzip", "-Xcompression-level", "6", "-b", "128K" }.Concat(noXattrs).ToArray()); + Run("mksquashfs", new[] { huge, Path.Combine(work, "huge-sparse.sqsh"), "-comp", "gzip", "-Xcompression-level", "6", "-b", "1M" }.Concat(noXattrs).ToArray()); + Run("mksquashfs", new[] { bigRoot, Path.Combine(work, "big-root.sqsh"), "-comp", "gzip", "-b", "128K" }.Concat(noXattrs).ToArray()); + Run("mksquashfs", new[] { xattrSymlink, Path.Combine(work, "xattr-symlink.sqsh"), "-comp", "gzip", "-b", "128K", "-xattrs-add", "trusted.test=1" }.Concat(flags).ToArray()); Directory.Delete(ext, recursive: true); Directory.Delete(huge, recursive: true); + Directory.Delete(bigRoot, recursive: true); + Directory.Delete(xattrSymlink, recursive: true); } private static void Write(Stream stream, string ascii) diff --git a/Tests/LibraryTests/SquashFs/big-root.sqsh b/Tests/LibraryTests/SquashFs/big-root.sqsh new file mode 100644 index 000000000..25b9bc75a Binary files /dev/null and b/Tests/LibraryTests/SquashFs/big-root.sqsh differ diff --git a/Tests/LibraryTests/SquashFs/extended-inodes.sqsh b/Tests/LibraryTests/SquashFs/extended-inodes.sqsh index 728e52450..cc107953c 100644 Binary files a/Tests/LibraryTests/SquashFs/extended-inodes.sqsh and b/Tests/LibraryTests/SquashFs/extended-inodes.sqsh differ diff --git a/Tests/LibraryTests/SquashFs/xattr-symlink.sqsh b/Tests/LibraryTests/SquashFs/xattr-symlink.sqsh new file mode 100644 index 000000000..2320e26d4 Binary files /dev/null and b/Tests/LibraryTests/SquashFs/xattr-symlink.sqsh differ