create: archive recursion roots that are hard links of an earlier root, fixes #10388

src/borg/archiver/create_cmd.py

@@ -62,6 +62,21 @@ def stat_root(path):
         raise BackupBrokenSymlinkError("stat", "broken symlink, skipping it") from None
 
 
+def skip_key(path, st):
+    """
+    Return the key that identifies the fs object at *path* (with stat result *st*) in the skip_inodes set.
+
+    A directory is identified by its inode alone: borg does not support hard links to directories, so an
+    identical directory inode is the same directory, even if it is reached via a different path.
+
+    Any other fs object is identified by its inode and its path: hard links are different paths pointing
+    to the same inode and each of them must be archived, only the very same path must not be archived twice.
+    """
+    if stat.S_ISDIR(st.st_mode):
+        return st.st_ino, st.st_dev
+    return st.st_ino, st.st_dev, path
+
+
 class CreateMixIn:
     @with_repository()
     def do_create(self, args, repository, manifest):
@@ -284,8 +299,9 @@ def create_inner(archive, cache, fso):
                             follow_symlink=followed,
                         )
                         # if we get back here, we've finished recursing into <path>,
-                        # we do not ever want to get back in there (even if path is given twice as recursion root)
-                        skip_inodes.add((st.st_ino, st.st_dev))
+                        # we do not ever want to get back in there (even if path is given twice as recursion root).
+                        # other hard links of a non-directory <path> are not skipped, see skip_key.
+                        skip_inodes.add(skip_key(path, st))
                     except BackupError as e:
                         # this comes from os.stat, self._rec_walk has own exception handler
                         self.print_warning_instance(BackupWarning(path, e))
@@ -624,7 +640,7 @@ def _rec_walk(
                 if not stat.S_ISDIR(st.st_mode):
                     return
 
-            if (st.st_ino, st.st_dev) in skip_inodes:
+            if skip_key(path, st) in skip_inodes:
                 return
             # if restrict_dev is given, we do not want to recurse into a new filesystem,
             # but we WILL save the mountpoint directory (or more precise: the root
@@ -772,8 +788,11 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser):
         but let borg find it while recursing (symlinks found that way are never followed).
         A recursion root that is a symlink with a non-existing target is skipped with a warning.
 
-        If you give both a symlink and its target as recursion roots, borg archives the fs
-        objects only once, under the path given first (like for any other root given twice).
+        If you give the same recursion root twice, borg archives it only once. That also
+        applies if you give both a symlink to a directory and that directory as recursion
+        roots: borg archives the fs objects only once, under the path given first.
+        Different paths pointing to the same non-directory fs object (hard links, or a symlink
+        and the file it points to) are all archived if you give them as recursion roots.
 
         When specifying '-' as a path, borg will read data from standard input and create a
         file named 'stdin' in the created archive from that data. In some cases, it is more

src/borg/testsuite/archiver/create_cmd_test.py

@@ -216,6 +216,65 @@ def test_create_duplicate_root(archivers, request):
     assert sorted(paths) == ["input", "input/a", "input/a/hardlink", "input/b", "input/b/hardlink"]
 
 
+def _create_hardlinked_files(archiver):
+    create_regular_file(archiver.input_path, "file1", contents=b"123456")
+    for name in "file2", "file3":
+        os.link(os.path.join(archiver.input_path, "file1"), os.path.join(archiver.input_path, name))
+
+
+def _list_items(archiver, name):
+    archive_list = cmd(archiver, "list", name, "--json-lines", "--format={path}{hlid}")
+    return [json.loads(line) for line in archive_list.split("\n") if line]
+
+
+@requires_hardlinks
+def test_create_hardlinked_roots(archivers, request):
+    # recursion roots that are hard links of each other are different paths and must all be archived,
+    # they are not the same root given twice (#5603).
+    archiver = request.getfixturevalue(archivers)
+    _create_hardlinked_files(archiver)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "test", "input/file1", "input/file2", "input/file3")
+    items = _list_items(archiver, "test")
+    assert [item["path"] for item in items] == ["input/file1", "input/file2", "input/file3"]
+    hlids = {item["hlid"] for item in items}
+    assert len(hlids) == 1 and hlids != {""}  # one hard link group
+    with changedir("output"):
+        cmd(archiver, "extract", "test")
+        sts = [os.stat(f"input/{name}") for name in ("file1", "file2", "file3")]
+        assert {st.st_ino for st in sts} == {sts[0].st_ino}
+        assert all(st.st_nlink == 3 for st in sts)
+        with open("input/file3", "rb") as f:
+            assert f.read() == b"123456"
+
+
+@requires_hardlinks
+def test_create_hardlinked_root_and_parent_dir(archivers, request):
+    # a file root given before its parent directory must not hide its hard links when recursing into
+    # the directory, and the file itself must be archived only once.
+    archiver = request.getfixturevalue(archivers)
+    _create_hardlinked_files(archiver)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "test", "input/file1", "input")
+    paths = [item["path"] for item in _list_items(archiver, "test")]
+    assert sorted(paths) == ["input", "input/file1", "input/file2", "input/file3"]
+    with changedir("output"):
+        cmd(archiver, "extract", "test")
+        assert all(os.stat(f"input/{name}").st_nlink == 3 for name in ("file1", "file2", "file3"))
+
+
+@requires_hardlinks
+def test_create_duplicate_file_root(archivers, request):
+    # the very same file given twice as a recursion root (also with a different spelling of the path)
+    # is archived only once, like a directory given twice (#5603).
+    archiver = request.getfixturevalue(archivers)
+    _create_hardlinked_files(archiver)
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "test", "input/file1", "input/file1", "./input/../input/file1", "input/file2")
+    paths = [item["path"] for item in _list_items(archiver, "test")]
+    assert paths == ["input/file1", "input/file2"]
+
+
 def test_create_unreadable_parent(archiver):
     parent_dir = os.path.join(archiver.input_path, "parent")
     root_dir = os.path.join(archiver.input_path, "parent", "root")
@@ -1283,8 +1342,8 @@ def test_create_symlink_below_root_not_followed(archivers, request):
 
 @pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported")
 def test_create_symlink_root_and_target(archivers, request):
-    # a followed symlink root and its target are the same fs objects, so they are archived
-    # only once, under the path given first (like any other recursion root given twice).
+    # a followed symlink root and its target directory are the same directory, so its contents are
+    # archived only once, under the path given first (like any other directory given twice).
     archiver = request.getfixturevalue(archivers)
     create_regular_file(archiver.input_path, "target/file", contents=b"content")
     os.symlink("target", os.path.join(archiver.input_path, "link"))
@@ -1295,6 +1354,19 @@ def test_create_symlink_root_and_target(archivers, request):
     assert "input/target/file" not in output
 
 
+@pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported")
+def test_create_symlink_root_and_target_file(archivers, request):
+    # a followed symlink root and its target file are different paths, so both are archived.
+    archiver = request.getfixturevalue(archivers)
+    create_regular_file(archiver.input_path, "target", contents=b"content")
+    os.symlink("target", os.path.join(archiver.input_path, "link"))
+    cmd(archiver, "repo-create", RK_ENCRYPTION)
+    cmd(archiver, "create", "test", "input/link", "input/target")
+    archive_list = cmd(archiver, "list", "test", "--json-lines")
+    items = [json.loads(line) for line in archive_list.split("\n") if line]
+    assert [(item["path"], item["type"]) for item in items] == [("input/link", "-"), ("input/target", "-")]
+
+
 @pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported")
 def test_create_symlink_root_broken(archivers, request):
     # a recursion root that is a symlink with a non-existing target is skipped with a warning