From f204705b86423ad450e94a78edd2d5f359b21cde Mon Sep 17 00:00:00 2001 From: "GUY.MOLINARI" Date: Fri, 9 Aug 2024 18:22:36 +0000 Subject: [PATCH 1/3] Fix bug where truncate does not delete index files properly. --- server/bitmap.go | 6 ++++++ server/kvstore.go | 46 +++++++++++++++++++++++++++++++++------------- version.txt | 2 +- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/server/bitmap.go b/server/bitmap.go index aef7c7d..87065ff 100644 --- a/server/bitmap.go +++ b/server/bitmap.go @@ -1386,6 +1386,9 @@ func (m *BitmapIndex) PartitionInfo(ctx context.Context, if req.Index != "" && p.Index != req.Index { return nil } + if p.TQType == "" { + return nil + } bsi := p.Shard.(*BSIBitmap) r := &pb.PartitionInfoResult{Time: p.Time.UnixNano(), Index: p.Index, Field: p.Field, RowIdOrValue: p.RowIDOrBits * -1, ModTime: bsi.ModTime.UnixNano(), TqType: p.TQType} @@ -1407,6 +1410,9 @@ func (m *BitmapIndex) PartitionInfo(ctx context.Context, if req.Index != "" && p.Index != req.Index { return nil } + if p.TQType == "" { + return nil + } bitmap := p.Shard.(*StandardBitmap) r := &pb.PartitionInfoResult{Time: p.Time.UnixNano(), Index: p.Index, Field: p.Field, RowIdOrValue: p.RowIDOrBits, ModTime: bitmap.ModTime.UnixNano(), TqType: p.TQType} diff --git a/server/kvstore.go b/server/kvstore.go index 6c7ad3e..b25c854 100644 --- a/server/kvstore.go +++ b/server/kvstore.go @@ -475,26 +475,46 @@ func (m *KVStore) DeleteIndicesWithPrefix(ctx context.Context, if req.Prefix == "" { return &empty.Empty{}, fmt.Errorf("Index prefix must be specified") } + + u.Infof("Deleting index files for prefix %v, retain enums = %v", req.Prefix, req.RetainEnums) + + // Interate over storeCache and close everything currently open for this prefix m.storeCacheLock.Lock() - defer m.storeCacheLock.Unlock() for k, v := range m.storeCache { - if strings.HasPrefix(k, req.Prefix) { + if strings.HasPrefix(k, req.Prefix + sep) { v.db.Sync() v.db.Close() delete(m.storeCache, k) - if req.RetainEnums && strings.HasSuffix(k, "StringEnum") { - u.Infof("Sync and close [%s]", k) - continue - } - if err := os.RemoveAll(m.Node.dataDir + sep + "index" + sep + k); err != nil { - return &empty.Empty{}, fmt.Errorf("DeleteIndicesWithPrefix error [%v]", err) - } - u.Infof("Sync, close, and delete [%s]", k) + u.Infof("Sync and close [%s]", k) } } - if !req.RetainEnums { - if err := os.RemoveAll(m.Node.dataDir + sep + "index" + sep + req.Prefix); err != nil { - return &empty.Empty{}, fmt.Errorf("DeleteIndicesWithPrefix error [%v]", err) + m.storeCacheLock.Unlock() + + // retrieve list of indices matching the prefix from the filesystem + baseDir := m.Node.dataDir + sep + "index" + sep + req.Prefix + files, err := os.ReadDir(baseDir) + if err != nil { + return &empty.Empty{}, fmt.Errorf("DeleteIndicesWithPrefix: %v", err) + } + for _, file := range files { + if !file.IsDir() { + continue + } + k := baseDir + sep + file.Name() + if !req.RetainEnums { + if err := os.RemoveAll(baseDir); err != nil { + return &empty.Empty{}, fmt.Errorf("DeleteIndicesWithPrefix retain is false: error [%v]", err) + } else { + u.Infof("Deleted [%s]") + } + } else { + if !strings.HasSuffix(k, "StringEnum") { + if err := os.RemoveAll(k); err != nil { + return &empty.Empty{}, fmt.Errorf("DeleteIndicesWithPrefix retain is true:error [%v]", err) + } else { + u.Infof("Deleted [%s]") + } + } } } return &empty.Empty{}, nil diff --git a/version.txt b/version.txt index 831b1b2..5c2f0a0 100644 --- a/version.txt +++ b/version.txt @@ -1,2 +1,2 @@ -quanta 0.9.14-rc-2 +quanta 0.9.14-rc-3 From e182eb311032870063235d65b77ff54671b8b04d Mon Sep 17 00:00:00 2001 From: "GUY.MOLINARI" Date: Fri, 9 Aug 2024 18:59:30 +0000 Subject: [PATCH 2/3] Fix minor bug with INFO message. --- server/kvstore.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/kvstore.go b/server/kvstore.go index b25c854..6a03f16 100644 --- a/server/kvstore.go +++ b/server/kvstore.go @@ -505,14 +505,14 @@ func (m *KVStore) DeleteIndicesWithPrefix(ctx context.Context, if err := os.RemoveAll(baseDir); err != nil { return &empty.Empty{}, fmt.Errorf("DeleteIndicesWithPrefix retain is false: error [%v]", err) } else { - u.Infof("Deleted [%s]") + u.Infof("Deleted [%s]", k) } } else { if !strings.HasSuffix(k, "StringEnum") { if err := os.RemoveAll(k); err != nil { return &empty.Empty{}, fmt.Errorf("DeleteIndicesWithPrefix retain is true:error [%v]", err) } else { - u.Infof("Deleted [%s]") + u.Infof("Deleted [%s]", k) } } } From b13b66ec3e6f5522e280d5fd7218afee7cd74883 Mon Sep 17 00:00:00 2001 From: "GUY.MOLINARI" Date: Fri, 9 Aug 2024 19:24:11 +0000 Subject: [PATCH 3/3] Minor typo fix. --- test-integration/sqlrunner_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test-integration/sqlrunner_test.go b/test-integration/sqlrunner_test.go index 905d708..f28096d 100644 --- a/test-integration/sqlrunner_test.go +++ b/test-integration/sqlrunner_test.go @@ -88,7 +88,7 @@ func TestMutate(t *testing.T) { // ensure we have a cluster on localhost, start one if necessary state := test.Ensure_cluster(3) - fmt.Println("TestBasic") + fmt.Println("TestMutate") currentDir, err := os.Getwd() check(err) err = os.Chdir("../sqlrunner") // these run from the sqlrunner/ directory