From 48feb28e1194e2962cb45413649ddc67775facf5 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 29 Oct 2023 01:01:17 +0200 Subject: [PATCH 1/2] check --repair: test if shadow index is recreated still failing here, because it is not. --- src/borg/testsuite/repository.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/borg/testsuite/repository.py b/src/borg/testsuite/repository.py index 927e926c52..39017d72a9 100644 --- a/src/borg/testsuite/repository.py +++ b/src/borg/testsuite/repository.py @@ -826,6 +826,15 @@ def test_hints_persistence(self): self.assert_equal(compact_expected, self.repository.compact) del self.repository.segments[2] # ignore the segment created by put(H(42), ...) self.assert_equal(segments_expected, self.repository.segments) + self.reopen() + self.assert_equal(self.repository.check(repair=True), True) + self.reopen() + self.repository.put(H(42), b'foobar') # this will call prepare_txn() and load the hints data + self.assert_equal(shadow_index_expected, self.repository.shadow_index) + # sizes do not match, with vs. without header? + # self.assert_equal(compact_expected, self.repository.compact) + del self.repository.segments[2] # ignore the segment created by put(H(42), ...) + self.assert_equal(segments_expected, self.repository.segments) def test_hints_behaviour(self): self.repository.put(H(0), b'data') From a7809429b3ab6e714385880756d5baaa1ca4ffe6 Mon Sep 17 00:00:00 2001 From: Thomas Waldmann Date: Sun, 29 Oct 2023 01:07:16 +0200 Subject: [PATCH 2/2] check --repair: recreate shadow index, see #6687 before this fix, borg check --repair just created an empty shadow index, which can lead to incomplete entries if entries are added later. and such incomplete (but present) entries can lead to compact_segments() resurrecting old PUTs by accidentally dropping related DELs. --- src/borg/repository.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/borg/repository.py b/src/borg/repository.py index 9c0284831a..f02224e869 100644 --- a/src/borg/repository.py +++ b/src/borg/repository.py @@ -917,6 +917,7 @@ def _update_index(self, segment, objects, report=None): s, _ = self.index[key] self.compact[s] += size self.segments[s] -= 1 + self.shadow_index.setdefault(key, []).append(s) except KeyError: pass self.index[key] = segment, offset @@ -935,6 +936,7 @@ def _update_index(self, segment, objects, report=None): self.segments[s] -= 1 size = self.io.read(s, offset, key, read_data=False) self.compact[s] += size + self.shadow_index.setdefault(key, []).append(s) elif tag == TAG_COMMIT: continue else: