From f84f37b3518cfba03ce7e9c858a9cb469d4b584d Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 26 Dec 2024 17:52:49 +0530 Subject: [PATCH 1/5] views: default on for VTGate Signed-off-by: Harshit Gangal --- go/vt/vtgate/vtgate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/vtgate/vtgate.go b/go/vt/vtgate/vtgate.go index 8bab05479dd..3b4a77c88f9 100644 --- a/go/vt/vtgate/vtgate.go +++ b/go/vt/vtgate/vtgate.go @@ -116,7 +116,7 @@ var ( // schema tracking flags enableSchemaChangeSignal = true - enableViews bool + enableViews = true enableUdfs bool // vtgate views flags From c2ac290b22266b3d6f419ec5b996f60b26444845 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 26 Dec 2024 19:40:44 +0530 Subject: [PATCH 2/5] unify drop view logic Signed-off-by: Harshit Gangal --- go/test/vschemawrapper/vschema_wrapper.go | 11 +++++++ go/vt/vtgate/executor_test.go | 24 ++++++++------- go/vt/vtgate/executorcontext/vcursor_impl.go | 29 +++++++++++++++---- go/vt/vtgate/planbuilder/ddl.go | 2 +- .../plancontext/planning_context_test.go | 5 ++++ .../vtgate/planbuilder/plancontext/vschema.go | 2 ++ 6 files changed, 55 insertions(+), 18 deletions(-) diff --git a/go/test/vschemawrapper/vschema_wrapper.go b/go/test/vschemawrapper/vschema_wrapper.go index 3f9f072afc6..dd0a704b7d0 100644 --- a/go/test/vschemawrapper/vschema_wrapper.go +++ b/go/test/vschemawrapper/vschema_wrapper.go @@ -271,6 +271,17 @@ func (vw *VSchemaWrapper) FindView(tab sqlparser.TableName) sqlparser.SelectStat return vw.V.FindView(destKeyspace, tab.Name.String()) } +func (vw *VSchemaWrapper) FindViewTarget(name sqlparser.TableName) (*vindexes.Keyspace, error) { + destKeyspace, _, _, err := topoproto.ParseDestination(name.Qualifier.String(), topodatapb.TabletType_PRIMARY) + if err != nil { + return nil, err + } + if ks, ok := vw.V.Keyspaces[destKeyspace]; ok { + return ks.Keyspace, nil + } + return nil, nil +} + func (vw *VSchemaWrapper) FindTableOrVindex(tab sqlparser.TableName) (*vindexes.Table, vindexes.Vindex, string, topodatapb.TabletType, key.Destination, error) { return vw.Vcursor.FindTableOrVindex(tab) } diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index 74bfb710582..f75a4f045a7 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -1277,17 +1277,19 @@ func TestExecutorDDL(t *testing.T) { } for _, stmt := range stmts2 { - sbc1.ExecCount.Store(0) - sbc2.ExecCount.Store(0) - sbclookup.ExecCount.Store(0) - _, err := executor.Execute(ctx, nil, "TestExecute", econtext.NewSafeSession(&vtgatepb.Session{TargetString: ""}), stmt.input, nil) - if stmt.hasErr { - require.EqualError(t, err, econtext.ErrNoKeyspace.Error(), "expect query to fail") - testQueryLog(t, executor, logChan, "TestExecute", "", stmt.input, 0) - } else { - require.NoError(t, err) - testQueryLog(t, executor, logChan, "TestExecute", "DDL", stmt.input, 8) - } + t.Run(stmt.input, func(t *testing.T) { + sbc1.ExecCount.Store(0) + sbc2.ExecCount.Store(0) + sbclookup.ExecCount.Store(0) + _, err := executor.Execute(ctx, nil, "TestExecute", econtext.NewSafeSession(&vtgatepb.Session{TargetString: ""}), stmt.input, nil) + if stmt.hasErr { + assert.EqualError(t, err, econtext.ErrNoKeyspace.Error(), "expect query to fail") + testQueryLog(t, executor, logChan, "TestExecute", "", stmt.input, 0) + } else { + assert.NoError(t, err) + testQueryLog(t, executor, logChan, "TestExecute", "DDL", stmt.input, 8) + } + }) } } diff --git a/go/vt/vtgate/executorcontext/vcursor_impl.go b/go/vt/vtgate/executorcontext/vcursor_impl.go index 40317f5103a..55f84e04f51 100644 --- a/go/vt/vtgate/executorcontext/vcursor_impl.go +++ b/go/vt/vtgate/executorcontext/vcursor_impl.go @@ -389,7 +389,7 @@ func (vc *VCursorImpl) StartPrimitiveTrace() func() engine.Stats { // FindTable finds the specified table. If the keyspace what specified in the input, it gets used as qualifier. // Otherwise, the keyspace from the request is used, if one was provided. func (vc *VCursorImpl) FindTable(name sqlparser.TableName) (*vindexes.Table, string, topodatapb.TabletType, key.Destination, error) { - destKeyspace, destTabletType, dest, err := vc.ParseDestinationTarget(name.Qualifier.String()) + destKeyspace, destTabletType, dest, err := vc.parseDestinationTarget(name.Qualifier.String()) if err != nil { return nil, "", destTabletType, nil, err } @@ -404,7 +404,7 @@ func (vc *VCursorImpl) FindTable(name sqlparser.TableName) (*vindexes.Table, str } func (vc *VCursorImpl) FindView(name sqlparser.TableName) sqlparser.SelectStatement { - ks, _, _, err := vc.ParseDestinationTarget(name.Qualifier.String()) + ks, _, _, err := vc.parseDestinationTarget(name.Qualifier.String()) if err != nil { return nil } @@ -415,7 +415,7 @@ func (vc *VCursorImpl) FindView(name sqlparser.TableName) sqlparser.SelectStatem } func (vc *VCursorImpl) FindRoutedTable(name sqlparser.TableName) (*vindexes.Table, error) { - destKeyspace, destTabletType, _, err := vc.ParseDestinationTarget(name.Qualifier.String()) + destKeyspace, destTabletType, _, err := vc.parseDestinationTarget(name.Qualifier.String()) if err != nil { return nil, err } @@ -439,7 +439,7 @@ func (vc *VCursorImpl) FindTableOrVindex(name sqlparser.TableName) (*vindexes.Ta return vc.getDualTable() } - destKeyspace, destTabletType, dest, err := ParseDestinationTarget(name.Qualifier.String(), vc.tabletType, vc.vschema) + destKeyspace, destTabletType, dest, err := vc.parseDestinationTarget(name.Qualifier.String()) if err != nil { return nil, nil, "", destTabletType, nil, err } @@ -453,7 +453,24 @@ func (vc *VCursorImpl) FindTableOrVindex(name sqlparser.TableName) (*vindexes.Ta return table, vindex, destKeyspace, destTabletType, dest, nil } -func (vc *VCursorImpl) ParseDestinationTarget(targetString string) (string, topodatapb.TabletType, key.Destination, error) { +// FindViewTarget finds the specified view's target keyspace. +func (vc *VCursorImpl) FindViewTarget(name sqlparser.TableName) (*vindexes.Keyspace, error) { + destKeyspace, _, _, err := vc.parseDestinationTarget(name.Qualifier.String()) + if err != nil { + return nil, err + } + if destKeyspace != "" { + return vc.FindKeyspace(destKeyspace) + } + + tbl, err := vc.vschema.FindRoutedTable("", name.Name.String(), vc.tabletType) + if err != nil || tbl == nil { + return nil, err + } + return tbl.Keyspace, nil +} + +func (vc *VCursorImpl) parseDestinationTarget(targetString string) (string, topodatapb.TabletType, key.Destination, error) { return ParseDestinationTarget(targetString, vc.tabletType, vc.vschema) } @@ -1308,7 +1325,7 @@ func (vc *VCursorImpl) GetAggregateUDFs() []string { // FindMirrorRule finds the mirror rule for the requested table name and // VSchema tablet type. func (vc *VCursorImpl) FindMirrorRule(name sqlparser.TableName) (*vindexes.MirrorRule, error) { - destKeyspace, destTabletType, _, err := vc.ParseDestinationTarget(name.Qualifier.String()) + destKeyspace, destTabletType, _, err := vc.parseDestinationTarget(name.Qualifier.String()) if err != nil { return nil, err } diff --git a/go/vt/vtgate/planbuilder/ddl.go b/go/vt/vtgate/planbuilder/ddl.go index a0045cec060..959151074cc 100644 --- a/go/vt/vtgate/planbuilder/ddl.go +++ b/go/vt/vtgate/planbuilder/ddl.go @@ -248,7 +248,7 @@ func buildDropView(vschema plancontext.VSchema, ddlStatement sqlparser.DDLStatem var ks *vindexes.Keyspace viewMap := make(map[string]any) for _, tbl := range ddlStatement.GetFromTables() { - _, ksForView, _, err := vschema.TargetDestination(tbl.Qualifier.String()) + ksForView, err := vschema.FindViewTarget(tbl) if err != nil { return nil, nil, err } diff --git a/go/vt/vtgate/planbuilder/plancontext/planning_context_test.go b/go/vt/vtgate/planbuilder/plancontext/planning_context_test.go index e5e96b0a4be..63999bb5ce7 100644 --- a/go/vt/vtgate/planbuilder/plancontext/planning_context_test.go +++ b/go/vt/vtgate/planbuilder/plancontext/planning_context_test.go @@ -186,6 +186,11 @@ func createPlanContext(st *semantics.SemTable) *PlanningContext { type vschema struct{} +func (v *vschema) FindViewTarget(name sqlparser.TableName) (*vindexes.Keyspace, error) { + // TODO implement me + panic("implement me") +} + func (v *vschema) FindTable(tablename sqlparser.TableName) (*vindexes.Table, string, topodatapb.TabletType, key.Destination, error) { // TODO implement me panic("implement me") diff --git a/go/vt/vtgate/planbuilder/plancontext/vschema.go b/go/vt/vtgate/planbuilder/plancontext/vschema.go index b4560424718..4a2af6e3fff 100644 --- a/go/vt/vtgate/planbuilder/plancontext/vschema.go +++ b/go/vt/vtgate/planbuilder/plancontext/vschema.go @@ -26,6 +26,8 @@ type PlannerVersion = querypb.ExecuteOptions_PlannerVersion type VSchema interface { FindTable(tablename sqlparser.TableName) (*vindexes.Table, string, topodatapb.TabletType, key.Destination, error) FindView(name sqlparser.TableName) sqlparser.SelectStatement + // FindViewTarget finds the target keyspace for the view table provided. + FindViewTarget(name sqlparser.TableName) (*vindexes.Keyspace, error) FindTableOrVindex(tablename sqlparser.TableName) (*vindexes.Table, vindexes.Vindex, string, topodatapb.TabletType, key.Destination, error) // SelectedKeyspace returns the current keyspace if set, otherwise returns an error From 60c843212b05ee252ef793b968b8468fd7094b6d Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Thu, 26 Dec 2024 20:08:05 +0530 Subject: [PATCH 3/5] test: update help output Signed-off-by: Harshit Gangal --- go/flags/endtoend/vtcombo.txt | 2 +- go/flags/endtoend/vtgate.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go/flags/endtoend/vtcombo.txt b/go/flags/endtoend/vtcombo.txt index 1c57dd0c08e..84cc9f6e53c 100644 --- a/go/flags/endtoend/vtcombo.txt +++ b/go/flags/endtoend/vtcombo.txt @@ -108,7 +108,7 @@ Flags: --enable-partial-keyspace-migration (Experimental) Follow shard routing rules: enable only while migrating a keyspace shard by shard. See documentation on Partial MoveTables for more. (default false) --enable-per-workload-table-metrics If true, query counts and query error metrics include a label that identifies the workload --enable-tx-throttler Synonym to -enable_tx_throttler - --enable-views Enable views support in vtgate. + --enable-views Enable views support in vtgate. (default true) --enable_buffer Enable buffering (stalling) of primary traffic during failovers. --enable_buffer_dry_run Detect and log failover events, but do not actually buffer requests. --enable_consolidator This option enables the query consolidator. (default true) diff --git a/go/flags/endtoend/vtgate.txt b/go/flags/endtoend/vtgate.txt index fde17f89c49..eebb2954e05 100644 --- a/go/flags/endtoend/vtgate.txt +++ b/go/flags/endtoend/vtgate.txt @@ -57,7 +57,7 @@ Flags: --emit_stats If set, emit stats to push-based monitoring and stats backends --enable-balancer Enable the tablet balancer to evenly spread query load for a given tablet type --enable-partial-keyspace-migration (Experimental) Follow shard routing rules: enable only while migrating a keyspace shard by shard. See documentation on Partial MoveTables for more. (default false) - --enable-views Enable views support in vtgate. + --enable-views Enable views support in vtgate. (default true) --enable_buffer Enable buffering (stalling) of primary traffic during failovers. --enable_buffer_dry_run Detect and log failover events, but do not actually buffer requests. --enable_direct_ddl Allow users to submit direct DDL statements (default true) From 43c85191dba85df7aa4eefcea4521eefee6480fe Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Fri, 27 Dec 2024 10:53:36 +0530 Subject: [PATCH 4/5] create view to lookup for keyspaces after semantic analysis, planning not required Signed-off-by: Harshit Gangal --- go/vt/vtgate/planbuilder/ddl.go | 56 +++++++++++++++++++++++++++++++-- 1 file changed, 53 insertions(+), 3 deletions(-) diff --git a/go/vt/vtgate/planbuilder/ddl.go b/go/vt/vtgate/planbuilder/ddl.go index 959151074cc..d4d1d771378 100644 --- a/go/vt/vtgate/planbuilder/ddl.go +++ b/go/vt/vtgate/planbuilder/ddl.go @@ -199,6 +199,10 @@ func buildCreateViewCommon( ddlSelect sqlparser.SelectStatement, ddl sqlparser.DDLStatement, ) (key.Destination, *vindexes.Keyspace, error) { + if vschema.IsViewsEnabled() { + return createViewEnabled(vschema, reservedVars, ddlSelect, ddl) + } + // For Create View, we require that the keyspace exist and the select query can be satisfied within the keyspace itself // We should remove the keyspace name from the table name, as the database name in MySQL might be different than the keyspace name destination, keyspace, err := findTableDestinationAndKeyspace(vschema, ddl) @@ -228,9 +232,6 @@ func buildCreateViewCommon( sqlparser.RemoveKeyspace(ddl) - if vschema.IsViewsEnabled() { - return destination, keyspace, nil - } isRoutePlan, opCode := tryToGetRoutePlan(selectPlan.primitive) if !isRoutePlan { return nil, nil, vterrors.VT12001(ViewComplex) @@ -241,6 +242,55 @@ func buildCreateViewCommon( return destination, keyspace, nil } +func createViewEnabled(vschema plancontext.VSchema, reservedVars *sqlparser.ReservedVars, ddlSelect sqlparser.SelectStatement, ddl sqlparser.DDLStatement) (key.Destination, *vindexes.Keyspace, error) { + // For Create View, we require that the keyspace exist and the select query can be satisfied within the keyspace itself + // We should remove the keyspace name from the table name, as the database name in MySQL might be different than the keyspace name + destination, keyspace, err := findTableDestinationAndKeyspace(vschema, ddl) + if err != nil { + return nil, nil, err + } + + // views definition with `select *` should not be expanded as schema tracker might not be up-to-date + // We copy the expressions and restore them after the planning context is created + var expressions []sqlparser.SelectExprs + _ = sqlparser.VisitAllSelects(ddlSelect, func(p *sqlparser.Select, idx int) error { + expressions = append(expressions, sqlparser.Clone(p.SelectExprs)) + return nil + }) + + pCtx, err := plancontext.CreatePlanningContext(ddlSelect, reservedVars, vschema, Gen4) + if err != nil { + return nil, nil, err + } + + var tblKs string + for _, tbl := range pCtx.SemTable.Tables { + vTbl := tbl.GetVindexTable() + if vTbl == nil { + continue + } + if tblKs == "" { + tblKs = vTbl.Keyspace.Name + } + if tblKs != vTbl.Keyspace.Name { + return nil, nil, vterrors.VT12001(ViewComplex) + } + } + + if tblKs != keyspace.Name { + return nil, nil, vterrors.VT12001(ViewDifferentKeyspace) + } + + _ = sqlparser.VisitAllSelects(ddlSelect, func(p *sqlparser.Select, idx int) error { + p.SelectExprs = expressions[idx] + return nil + }) + + sqlparser.RemoveKeyspace(ddl) + + return destination, keyspace, nil +} + func buildDropView(vschema plancontext.VSchema, ddlStatement sqlparser.DDLStatement) (key.Destination, *vindexes.Keyspace, error) { if !vschema.IsViewsEnabled() { return buildDropTable(vschema, ddlStatement) From 43cc0f5107833210d2902a98deb753c175c1e972 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Fri, 27 Dec 2024 12:11:53 +0530 Subject: [PATCH 5/5] add view as table to vschema, if exists then update the type to view Signed-off-by: Harshit Gangal --- go/vt/vtgate/vindexes/vschema.go | 3 ++- go/vt/vtgate/vschema_manager.go | 23 ++++++++++++++++++----- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/go/vt/vtgate/vindexes/vschema.go b/go/vt/vtgate/vindexes/vschema.go index 3852bbfcde3..a9580bdcfb2 100644 --- a/go/vt/vtgate/vindexes/vschema.go +++ b/go/vt/vtgate/vindexes/vschema.go @@ -58,6 +58,7 @@ const ( TypeTable = "" TypeSequence = "sequence" TypeReference = "reference" + TypeView = "view" ) // VSchema represents the denormalized version of SrvVSchema, @@ -440,7 +441,7 @@ func (vschema *VSchema) AddView(ksname, viewName, query string, parser *sqlparse } ks.Views[viewName] = selectStmt t := &Table{ - Type: "View", + Type: TypeView, Name: sqlparser.NewIdentifierCS(viewName), Keyspace: ks.Keyspace, ColumnListAuthoritative: true, diff --git a/go/vt/vtgate/vschema_manager.go b/go/vt/vtgate/vschema_manager.go index 62ea2cd3455..9ae7164eb99 100644 --- a/go/vt/vtgate/vschema_manager.go +++ b/go/vt/vtgate/vschema_manager.go @@ -200,21 +200,34 @@ func (vm *VSchemaManager) buildAndEnhanceVSchema(v *vschemapb.SrvVSchema) *vinde func (vm *VSchemaManager) updateFromSchema(vschema *vindexes.VSchema) { for ksName, ks := range vschema.Keyspaces { - vm.updateTableInfo(vschema, ks, ksName) vm.updateViewInfo(ks, ksName) + vm.updateTableInfo(vschema, ks, ksName) vm.updateUDFsInfo(ks, ksName) } } func (vm *VSchemaManager) updateViewInfo(ks *vindexes.KeyspaceSchema, ksName string) { views := vm.schema.Views(ksName) - if views != nil { - ks.Views = make(map[string]sqlparser.SelectStatement, len(views)) - for name, def := range views { - ks.Views[name] = sqlparser.Clone(def) + if views == nil { + return + } + ks.Views = make(map[string]sqlparser.SelectStatement, len(views)) + for name, def := range views { + ks.Views[name] = sqlparser.Clone(def) + vTbl, ok := ks.Tables[name] + if ok { + vTbl.Type = vindexes.TypeView + } else { + // Adding view to the VSchema as a table. + ks.Tables[name] = &vindexes.Table{ + Type: vindexes.TypeView, + Name: sqlparser.NewIdentifierCS(name), + Keyspace: ks.Keyspace, + } } } } + func (vm *VSchemaManager) updateTableInfo(vschema *vindexes.VSchema, ks *vindexes.KeyspaceSchema, ksName string) { m := vm.schema.Tables(ksName) // Before we add the foreign key definitions in the tables, we need to make sure that all the tables