Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fix file not found for GC #19850

Open
wants to merge 5 commits into
base: 2.0-dev
Choose a base branch
from
+25 −6
Open

Fix file not found for GC #19850

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7e42a12
Fix file not found for GC
LeftHandCold Nov 7, 2024
daf94cb
Merge branch '2.0-dev' into fix_notfound_2.0
mergify[bot] Nov 7, 2024
54b759b
remove panic
LeftHandCold Nov 7, 2024
ec91723
Update code
LeftHandCold Nov 7, 2024
275a529
Merge branch '2.0-dev' into fix_notfound_2.0
LeftHandCold Nov 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 15 additions & 1 deletion pkg/vm/engine/tae/db/gc/v3/exec_v1.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -342,11 +342,25 @@ func MakeFinalCanGCSinker(
ctx context.Context, bat *batch.Batch,
) error {
clear(buffer)
var dropTSs []types.TS
var tableIDs []uint64
if bat.Vecs[0].Length() > 0 {
dropTSs = vector.MustFixedColNoTypeCheck[types.TS](bat.Vecs[2])
tableIDs = vector.MustFixedColNoTypeCheck[uint64](bat.Vecs[4])
}
for i := 0; i < bat.Vecs[0].Length(); i++ {
buf := bat.Vecs[0].GetRawBytesAt(i)
stats := (*objectio.ObjectStats)(unsafe.Pointer(&buf[0]))
name := stats.ObjectName().String()
buffer[name] = struct{}{}
dropTS := dropTSs[i]
tableID := tableIDs[i]
if !dropTS.IsEmpty() {
buffer[name] = struct{}{}
continue
}
if !logtail.IsMoTable(tableID) {
buffer[name] = struct{}{}
}
}
for name := range buffer {
*filesToGC = append(*filesToGC, name)
Expand Down
15 changes: 10 additions & 5 deletions pkg/vm/engine/tae/logtail/snapshot.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -325,7 +325,7 @@ func (sm *SnapshotMeta) copyTablesLocked() map[uint32]map[uint64]*tableInfo {
return tables
}

func isMoTable(tid uint64) bool {
func IsMoTable(tid uint64) bool {
return tid == catalog2.MO_TABLES_ID
}

Expand Down Expand Up @@ -353,7 +353,7 @@ func (sm *SnapshotMeta) updateTableInfo(
stats objectio.ObjectStats,
createTS types.TS, deleteTS types.TS,
) {
if !isMoTable(tid) {
if !IsMoTable(tid) {
return
}
if !stats.GetAppendable() {
Expand Down Expand Up @@ -546,7 +546,10 @@ func (sm *SnapshotMeta) updateTableInfo(

for pk, tables := range sm.tablePKIndex {
if len(tables) > 1 {
panic(fmt.Sprintf("table %v has more than one entry, tables len %d", pk, len(tables)))
logutil.Warn("UpdateSnapTable-Error",
zap.String("table", pk),
zap.Int("len", len(tables)),
)
}
if len(tables) == 0 {
continue
Expand Down Expand Up @@ -1140,7 +1143,9 @@ func (sm *SnapshotMeta) RebuildTableInfo(ins *containers.Batch) {
continue
}
if len(sm.tablePKIndex[pk]) > 0 {
panic(fmt.Sprintf("pk %s already exists, table: %d", pk, tid))
logutil.Warn("RebuildTableInfo-PK-Exists",
zap.String("pk", pk),
zap.Uint64("table", tid))
}
sm.tablePKIndex[pk] = make([]*tableInfo, 1)
sm.tablePKIndex[pk][0] = table
Expand Down Expand Up @@ -1197,7 +1202,7 @@ func (sm *SnapshotMeta) RebuildAObjectDel(ins *containers.Batch) {
for i := 0; i < ins.Length(); i++ {
commitTs := commitTsVec[i]
if _, ok := sm.aobjDelTsMap[commitTs]; ok {
panic(fmt.Sprintf("commitTs %v already exists", commitTs.ToString()))
logutil.Warn("RebuildAObjectDel-Exists", zap.Any("commitTs", commitTs))
}
sm.aobjDelTsMap[commitTs] = struct{}{}
}
Expand Down
Loading