diff --git a/src/borg/archiver/create_cmd.py b/src/borg/archiver/create_cmd.py index 03e1874570..c6570a5fe4 100644 --- a/src/borg/archiver/create_cmd.py +++ b/src/borg/archiver/create_cmd.py @@ -62,6 +62,21 @@ def stat_root(path): raise BackupBrokenSymlinkError("stat", "broken symlink, skipping it") from None +def skip_key(path, st): + """ + Return the key that identifies the fs object at *path* (with stat result *st*) in the skip_inodes set. + + A directory is identified by its inode alone: borg does not support hard links to directories, so an + identical directory inode is the same directory, even if it is reached via a different path. + + Any other fs object is identified by its inode and its path: hard links are different paths pointing + to the same inode and each of them must be archived, only the very same path must not be archived twice. + """ + if stat.S_ISDIR(st.st_mode): + return st.st_ino, st.st_dev + return st.st_ino, st.st_dev, path + + class CreateMixIn: @with_repository() def do_create(self, args, repository, manifest): @@ -284,8 +299,9 @@ def create_inner(archive, cache, fso): follow_symlink=followed, ) # if we get back here, we've finished recursing into , - # we do not ever want to get back in there (even if path is given twice as recursion root) - skip_inodes.add((st.st_ino, st.st_dev)) + # we do not ever want to get back in there (even if path is given twice as recursion root). + # other hard links of a non-directory are not skipped, see skip_key. + skip_inodes.add(skip_key(path, st)) except BackupError as e: # this comes from os.stat, self._rec_walk has own exception handler self.print_warning_instance(BackupWarning(path, e)) @@ -624,7 +640,7 @@ def _rec_walk( if not stat.S_ISDIR(st.st_mode): return - if (st.st_ino, st.st_dev) in skip_inodes: + if skip_key(path, st) in skip_inodes: return # if restrict_dev is given, we do not want to recurse into a new filesystem, # but we WILL save the mountpoint directory (or more precise: the root @@ -772,8 +788,11 @@ def build_parser_create(self, subparsers, common_parser, mid_common_parser): but let borg find it while recursing (symlinks found that way are never followed). A recursion root that is a symlink with a non-existing target is skipped with a warning. - If you give both a symlink and its target as recursion roots, borg archives the fs - objects only once, under the path given first (like for any other root given twice). + If you give the same recursion root twice, borg archives it only once. That also + applies if you give both a symlink to a directory and that directory as recursion + roots: borg archives the fs objects only once, under the path given first. + Different paths pointing to the same non-directory fs object (hard links, or a symlink + and the file it points to) are all archived if you give them as recursion roots. When specifying '-' as a path, borg will read data from standard input and create a file named 'stdin' in the created archive from that data. In some cases, it is more diff --git a/src/borg/testsuite/archiver/create_cmd_test.py b/src/borg/testsuite/archiver/create_cmd_test.py index 5e99a8c753..55d06a2958 100644 --- a/src/borg/testsuite/archiver/create_cmd_test.py +++ b/src/borg/testsuite/archiver/create_cmd_test.py @@ -216,6 +216,65 @@ def test_create_duplicate_root(archivers, request): assert sorted(paths) == ["input", "input/a", "input/a/hardlink", "input/b", "input/b/hardlink"] +def _create_hardlinked_files(archiver): + create_regular_file(archiver.input_path, "file1", contents=b"123456") + for name in "file2", "file3": + os.link(os.path.join(archiver.input_path, "file1"), os.path.join(archiver.input_path, name)) + + +def _list_items(archiver, name): + archive_list = cmd(archiver, "list", name, "--json-lines", "--format={path}{hlid}") + return [json.loads(line) for line in archive_list.split("\n") if line] + + +@requires_hardlinks +def test_create_hardlinked_roots(archivers, request): + # recursion roots that are hard links of each other are different paths and must all be archived, + # they are not the same root given twice (#5603). + archiver = request.getfixturevalue(archivers) + _create_hardlinked_files(archiver) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test", "input/file1", "input/file2", "input/file3") + items = _list_items(archiver, "test") + assert [item["path"] for item in items] == ["input/file1", "input/file2", "input/file3"] + hlids = {item["hlid"] for item in items} + assert len(hlids) == 1 and hlids != {""} # one hard link group + with changedir("output"): + cmd(archiver, "extract", "test") + sts = [os.stat(f"input/{name}") for name in ("file1", "file2", "file3")] + assert {st.st_ino for st in sts} == {sts[0].st_ino} + assert all(st.st_nlink == 3 for st in sts) + with open("input/file3", "rb") as f: + assert f.read() == b"123456" + + +@requires_hardlinks +def test_create_hardlinked_root_and_parent_dir(archivers, request): + # a file root given before its parent directory must not hide its hard links when recursing into + # the directory, and the file itself must be archived only once. + archiver = request.getfixturevalue(archivers) + _create_hardlinked_files(archiver) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test", "input/file1", "input") + paths = [item["path"] for item in _list_items(archiver, "test")] + assert sorted(paths) == ["input", "input/file1", "input/file2", "input/file3"] + with changedir("output"): + cmd(archiver, "extract", "test") + assert all(os.stat(f"input/{name}").st_nlink == 3 for name in ("file1", "file2", "file3")) + + +@requires_hardlinks +def test_create_duplicate_file_root(archivers, request): + # the very same file given twice as a recursion root (also with a different spelling of the path) + # is archived only once, like a directory given twice (#5603). + archiver = request.getfixturevalue(archivers) + _create_hardlinked_files(archiver) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test", "input/file1", "input/file1", "./input/../input/file1", "input/file2") + paths = [item["path"] for item in _list_items(archiver, "test")] + assert paths == ["input/file1", "input/file2"] + + def test_create_unreadable_parent(archiver): parent_dir = os.path.join(archiver.input_path, "parent") root_dir = os.path.join(archiver.input_path, "parent", "root") @@ -1283,8 +1342,8 @@ def test_create_symlink_below_root_not_followed(archivers, request): @pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported") def test_create_symlink_root_and_target(archivers, request): - # a followed symlink root and its target are the same fs objects, so they are archived - # only once, under the path given first (like any other recursion root given twice). + # a followed symlink root and its target directory are the same directory, so its contents are + # archived only once, under the path given first (like any other directory given twice). archiver = request.getfixturevalue(archivers) create_regular_file(archiver.input_path, "target/file", contents=b"content") os.symlink("target", os.path.join(archiver.input_path, "link")) @@ -1295,6 +1354,19 @@ def test_create_symlink_root_and_target(archivers, request): assert "input/target/file" not in output +@pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported") +def test_create_symlink_root_and_target_file(archivers, request): + # a followed symlink root and its target file are different paths, so both are archived. + archiver = request.getfixturevalue(archivers) + create_regular_file(archiver.input_path, "target", contents=b"content") + os.symlink("target", os.path.join(archiver.input_path, "link")) + cmd(archiver, "repo-create", RK_ENCRYPTION) + cmd(archiver, "create", "test", "input/link", "input/target") + archive_list = cmd(archiver, "list", "test", "--json-lines") + items = [json.loads(line) for line in archive_list.split("\n") if line] + assert [(item["path"], item["type"]) for item in items] == [("input/link", "-"), ("input/target", "-")] + + @pytest.mark.skipif(not are_symlinks_supported(), reason="symlinks not supported") def test_create_symlink_root_broken(archivers, request): # a recursion root that is a symlink with a non-existing target is skipped with a warning