From 3eb20d7c35dd7b2906cafaa43006e60bca5b0f0c Mon Sep 17 00:00:00 2001 From: daviszhen Date: Fri, 15 Sep 2023 17:16:44 +0800 Subject: [PATCH 01/15] add mo_configurations --- pkg/cnservice/options.go | 13 + pkg/cnservice/server.go | 6 + pkg/cnservice/server_heartbeat.go | 3 + pkg/cnservice/types.go | 167 ++ pkg/cnservice/types_test.go | 10 + pkg/frontend/mysql_cmd_executor.go | 2 + pkg/frontend/session.go | 1 + pkg/hakeeper/rsm.go | 3 + pkg/logservice/config.go | 7 + pkg/logservice/option.go | 11 + pkg/logservice/service.go | 6 +- pkg/logservice/service_commands.go | 3 + pkg/pb/logservice/logservice.go | 3 + pkg/pb/logservice/logservice.pb.go | 1696 ++++++++++++++--- pkg/pb/plan/plan.pb.go | 34 +- pkg/proxy/config.go | 8 + pkg/proxy/server.go | 1 + pkg/sql/colexec/lockop/lock_op_test.go | 1 + pkg/sql/colexec/table_function/system_view.go | 132 ++ .../colexec/table_function/table_function.go | 4 + pkg/sql/compile/scopeRemoteRun.go | 4 +- pkg/sql/compile/scopeRemoteRunTypes.go | 5 + pkg/sql/compile/sql_executor.go | 5 + pkg/sql/plan/function/ctl/cmd_ping_test.go | 8 +- pkg/sql/plan/query_builder.go | 2 + pkg/sql/plan/system_view.go | 77 + pkg/testutil/util_new.go | 1 + pkg/tnservice/cfg.go | 108 ++ pkg/tnservice/cfg_test.go | 5 + pkg/tnservice/store.go | 14 + pkg/tnservice/store_heartbeat.go | 3 + pkg/util/dump_config.go | 213 +++ pkg/util/dump_config_test.go | 55 + pkg/vm/engine/disttae/engine.go | 17 +- pkg/vm/engine/disttae/types.go | 2 + pkg/vm/process/process.go | 4 + pkg/vm/process/types.go | 3 + proto/logservice.proto | 27 + 38 files changed, 2380 insertions(+), 284 deletions(-) create mode 100644 pkg/cnservice/types_test.go create mode 100644 pkg/sql/colexec/table_function/system_view.go create mode 100644 pkg/sql/plan/system_view.go create mode 100644 pkg/util/dump_config.go create mode 100644 pkg/util/dump_config_test.go diff --git a/pkg/cnservice/options.go b/pkg/cnservice/options.go index 1890b79dc863..2efa4547cdc4 100644 --- a/pkg/cnservice/options.go +++ b/pkg/cnservice/options.go @@ -16,6 +16,8 @@ package cnservice import ( "context" + "github.com/matrixorigin/matrixone/pkg/logservice" + logservicepb "github.com/matrixorigin/matrixone/pkg/pb/logservice" "github.com/matrixorigin/matrixone/pkg/common/morpc" "github.com/matrixorigin/matrixone/pkg/defines" @@ -54,6 +56,7 @@ func WithMessageHandle(f func(ctx context.Context, fs fileservice.FileService, lockService lockservice.LockService, queryService queryservice.QueryService, + hakeeper logservice.CNHAKeeperClient, cli client.TxnClient, aicm *defines.AutoIncrCacheManager, mAcquirer func() morpc.Message) error) Option { @@ -61,3 +64,13 @@ func WithMessageHandle(f func(ctx context.Context, s.requestHandler = f } } + +// WithConfigData saves the data from the config file +func WithConfigData(data map[string]*logservicepb.ConfigItem) Option { + return func(s *service) { + s.configData = make(map[string]*logservicepb.ConfigItem, len(data)) + for k, v := range data { + s.configData[k] = v + } + } +} diff --git a/pkg/cnservice/server.go b/pkg/cnservice/server.go index 9e8e261daea5..edc153f023d8 100644 --- a/pkg/cnservice/server.go +++ b/pkg/cnservice/server.go @@ -63,6 +63,9 @@ func NewService( return nil, err } + configKVMap, _ := dumpCnConfig(*cfg) + options = append(options, WithConfigData(configKVMap)) + // get metadata fs metadataFS, err := fileservice.Get[fileservice.ReplaceableFileService](fileService, defines.LocalFileServiceName) if err != nil { @@ -166,6 +169,7 @@ func NewService( fService fileservice.FileService, lockService lockservice.LockService, queryService queryservice.QueryService, + hakeeper logservice.CNHAKeeperClient, cli client.TxnClient, aicm *defines.AutoIncrCacheManager, messageAcquirer func() morpc.Message) error { @@ -332,6 +336,7 @@ func (s *service) handleRequest( s.fileService, s.lockService, s.queryService, + s._hakeeperClient, s._txnClient, s.aicm, s.acquireMessage) @@ -630,6 +635,7 @@ func (s *service) initInternalSQlExecutor(mp *mpool.MPool) { s._txnClient, s.fileService, s.queryService, + s._hakeeperClient, s.aicm) runtime.ProcessLevelRuntime().SetGlobalVariables(runtime.InternalSQLExecutor, exec) } diff --git a/pkg/cnservice/server_heartbeat.go b/pkg/cnservice/server_heartbeat.go index 4c4e5f01c696..9b2d1bc96810 100644 --- a/pkg/cnservice/server_heartbeat.go +++ b/pkg/cnservice/server_heartbeat.go @@ -74,6 +74,9 @@ func (s *service) heartbeat(ctx context.Context) { TaskServiceCreated: s.GetTaskRunner() != nil, QueryAddress: s.queryServiceServiceAddr(), InitWorkState: s.cfg.InitWorkState, + ConfigData: &logservicepb.ConfigData{ + Content: s.configData, + }, } cb, err := s._hakeeperClient.SendCNHeartbeat(ctx2, hb) if err != nil { diff --git a/pkg/cnservice/types.go b/pkg/cnservice/types.go index cde93e3fa757..b7ec9c5f382f 100644 --- a/pkg/cnservice/types.go +++ b/pkg/cnservice/types.go @@ -16,7 +16,9 @@ package cnservice import ( "context" + logservicepb "github.com/matrixorigin/matrixone/pkg/pb/logservice" moconnector "github.com/matrixorigin/matrixone/pkg/stream/connector" + "github.com/matrixorigin/matrixone/pkg/util" "runtime" "strings" "sync" @@ -239,6 +241,27 @@ type Config struct { InitWorkState string `toml:"init-work-state"` } +type abc struct { + A string + B string +} + +type kkk struct { + kk string + jj string +} + +type def struct { + DDD []kkk + ABC []abc `toml:"abc"` + EEE [3]abc + MMM map[string]string + NNN map[string]abc + PPP *abc + QQQ *abc + RRR map[string]string +} + func (c *Config) Validate() error { foundMachineHost := "" if c.UUID == "" { @@ -387,6 +410,140 @@ func (c *Config) Validate() error { return nil } +func (c *Config) SetDefaultValue() { + foundMachineHost := "" + if c.ListenAddress == "" { + c.ListenAddress = defaultListenAddress + } + if c.ServiceAddress == "" { + c.ServiceAddress = c.ListenAddress + } else { + foundMachineHost = strings.Split(c.ServiceAddress, ":")[0] + } + if c.Role == "" { + c.Role = metadata.CNRole_TP.String() + } + if c.HAKeeper.DiscoveryTimeout.Duration == 0 { + c.HAKeeper.DiscoveryTimeout.Duration = time.Second * 30 + } + if c.HAKeeper.HeatbeatInterval.Duration == 0 { + c.HAKeeper.HeatbeatInterval.Duration = time.Second + } + if c.HAKeeper.HeatbeatTimeout.Duration == 0 { + c.HAKeeper.HeatbeatTimeout.Duration = time.Second * 3 + } + if c.TaskRunner.Parallelism == 0 { + c.TaskRunner.Parallelism = runtime.NumCPU() / 16 + if c.TaskRunner.Parallelism <= ReservedTasks { + c.TaskRunner.Parallelism = 1 + ReservedTasks + } + } + if c.TaskRunner.FetchInterval.Duration == 0 { + c.TaskRunner.FetchInterval.Duration = time.Second * 10 + } + if c.TaskRunner.FetchTimeout.Duration == 0 { + c.TaskRunner.FetchTimeout.Duration = time.Second * 10 + } + if c.TaskRunner.HeartbeatInterval.Duration == 0 { + c.TaskRunner.HeartbeatInterval.Duration = time.Second * 5 + } + if c.TaskRunner.MaxWaitTasks == 0 { + c.TaskRunner.MaxWaitTasks = 256 + } + if c.TaskRunner.QueryLimit == 0 { + c.TaskRunner.QueryLimit = c.TaskRunner.Parallelism + } + if c.TaskRunner.RetryInterval.Duration == 0 { + c.TaskRunner.RetryInterval.Duration = time.Second + } + if c.Engine.Type == "" { + c.Engine.Type = EngineDistributedTAE + } + if c.Engine.Logstore == "" { + c.Engine.Logstore = options.LogstoreLogservice + } + if c.Cluster.RefreshInterval.Duration == 0 { + c.Cluster.RefreshInterval.Duration = time.Second * 10 + } + + if c.Txn.Mode == "" { + c.Txn.Mode = defaultTxnMode.String() + } + + if c.Txn.Isolation == "" { + if txn.GetTxnMode(c.Txn.Mode) == txn.TxnMode_Pessimistic { + c.Txn.Isolation = txn.TxnIsolation_RC.String() + } else { + c.Txn.Isolation = txn.TxnIsolation_SI.String() + } + } + // Fix txn mode various config, simply override + if txn.GetTxnMode(c.Txn.Mode) == txn.TxnMode_Pessimistic { + if c.Txn.EnableSacrificingFreshness == 0 { + c.Txn.EnableSacrificingFreshness = 1 + } + if c.Txn.EnableCNBasedConsistency == 0 { + c.Txn.EnableCNBasedConsistency = -1 + } + // We don't support the following now, so always disable + c.Txn.EnableRefreshExpression = -1 + if c.Txn.EnableLeakCheck == 0 { + c.Txn.EnableLeakCheck = -1 + } + } else { + if c.Txn.EnableSacrificingFreshness == 0 { + c.Txn.EnableSacrificingFreshness = 1 + } + if c.Txn.EnableCNBasedConsistency == 0 { + c.Txn.EnableCNBasedConsistency = 1 + } + // We don't support the following now, so always disable + c.Txn.EnableRefreshExpression = -1 + if c.Txn.EnableLeakCheck == 0 { + c.Txn.EnableLeakCheck = -1 + } + } + + if c.Txn.MaxActiveAges.Duration == 0 { + c.Txn.MaxActiveAges.Duration = time.Minute * 2 + } + if c.Txn.MaxActive == 0 { + c.Txn.MaxActive = runtime.NumCPU() * 4 + } + c.Ctl.Adjust(foundMachineHost, defaultCtlListenAddress) + c.LockService.ServiceID = "temp" + c.LockService.Validate() + c.LockService.ServiceID = c.UUID + + // pessimistic mode implies primary key check + if txn.GetTxnMode(c.Txn.Mode) == txn.TxnMode_Pessimistic || c.PrimaryKeyCheck { + plan.CNPrimaryCheck = true + } else { + plan.CNPrimaryCheck = false + } + + if c.MaxPreparedStmtCount > 0 { + if c.MaxPreparedStmtCount > maxForMaxPreparedStmtCount { + frontend.MaxPrepareNumberInOneSession = maxForMaxPreparedStmtCount + } else { + frontend.MaxPrepareNumberInOneSession = c.MaxPreparedStmtCount + } + } else { + frontend.MaxPrepareNumberInOneSession = 100000 + } + c.QueryServiceConfig.Adjust(foundMachineHost, defaultQueryServiceListenAddress) + + if c.PortBase != 0 { + if c.ServiceHost == "" { + c.ServiceHost = defaultServiceHost + } + } + + if !metadata.ValidStateString(c.InitWorkState) { + c.InitWorkState = metadata.WorkState_Working.String() + } +} + func (s *service) getLockServiceConfig() lockservice.Config { s.cfg.LockService.ServiceID = s.cfg.UUID s.cfg.LockService.RPC = s.cfg.RPC @@ -408,6 +565,7 @@ type service struct { fService fileservice.FileService, lockService lockservice.LockService, queryService queryservice.QueryService, + hakeeper logservice.CNHAKeeperClient, cli client.TxnClient, aicm *defines.AutoIncrCacheManager, messageAcquirer func() morpc.Message) error @@ -444,4 +602,13 @@ type service struct { addressMgr address.AddressManager connectorMgr moconnector.ConnectorManagerInterface + + //data from cn config file + configData map[string]*logservicepb.ConfigItem +} + +func dumpCnConfig(cfg Config) (map[string]*logservicepb.ConfigItem, error) { + defCfg := Config{} + defCfg.SetDefaultValue() + return util.DumpConfig(cfg, defCfg) } diff --git a/pkg/cnservice/types_test.go b/pkg/cnservice/types_test.go new file mode 100644 index 000000000000..7be59f188d62 --- /dev/null +++ b/pkg/cnservice/types_test.go @@ -0,0 +1,10 @@ +package cnservice + +import ( + "testing" +) + +func TestDefaultCnConfig(t *testing.T) { + cfg := Config{} + cfg.SetDefaultValue() +} diff --git a/pkg/frontend/mysql_cmd_executor.go b/pkg/frontend/mysql_cmd_executor.go index 376955450f66..7414b49b7921 100644 --- a/pkg/frontend/mysql_cmd_executor.go +++ b/pkg/frontend/mysql_cmd_executor.go @@ -3476,6 +3476,7 @@ func (mce *MysqlCmdExecutor) doComQuery(requestCtx context.Context, input *UserI pu.FileService, pu.LockService, pu.QueryService, + pu.HAKeeperClient, ses.GetAutoIncrCacheManager()) proc.CopyVectorPool(ses.proc) proc.CopyValueScanBatch(ses.proc) @@ -3652,6 +3653,7 @@ func (mce *MysqlCmdExecutor) doComQueryInProgress(requestCtx context.Context, in pu.FileService, pu.LockService, pu.QueryService, + pu.HAKeeperClient, ses.GetAutoIncrCacheManager()) proc.CopyVectorPool(ses.proc) proc.CopyValueScanBatch(ses.proc) diff --git a/pkg/frontend/session.go b/pkg/frontend/session.go index 1195aeb8caa6..dacb8098676f 100644 --- a/pkg/frontend/session.go +++ b/pkg/frontend/session.go @@ -456,6 +456,7 @@ func NewSession(proto Protocol, mp *mpool.MPool, pu *config.ParameterUnit, pu.FileService, pu.LockService, pu.QueryService, + pu.HAKeeperClient, ses.GetAutoIncrCacheManager()) runtime.SetFinalizer(ses, func(ss *Session) { diff --git a/pkg/hakeeper/rsm.go b/pkg/hakeeper/rsm.go index 2f11bce73e1b..188d01254fde 100644 --- a/pkg/hakeeper/rsm.go +++ b/pkg/hakeeper/rsm.go @@ -675,6 +675,7 @@ func (s *stateMachine) handleClusterDetailsQuery(cfg Config) *pb.ClusterDetails WorkState: info.WorkState, Labels: info.Labels, QueryAddress: info.QueryAddress, + ConfigData: info.ConfigData, } cd.CNStores = append(cd.CNStores, n) } @@ -692,6 +693,7 @@ func (s *stateMachine) handleClusterDetailsQuery(cfg Config) *pb.ClusterDetails LogtailServerAddress: info.LogtailServerAddress, LockServiceAddress: info.LockServiceAddress, CtlAddress: info.CtlAddress, + ConfigData: info.ConfigData, } cd.TNStores = append(cd.TNStores, n) } @@ -706,6 +708,7 @@ func (s *stateMachine) handleClusterDetailsQuery(cfg Config) *pb.ClusterDetails State: state, ServiceAddress: info.ServiceAddress, Replicas: info.Replicas, + ConfigData: info.ConfigData, } cd.LogStores = append(cd.LogStores, n) } diff --git a/pkg/logservice/config.go b/pkg/logservice/config.go index 96b808682e8f..407f2c7f9286 100644 --- a/pkg/logservice/config.go +++ b/pkg/logservice/config.go @@ -16,6 +16,8 @@ package logservice import ( "fmt" + logservicepb "github.com/matrixorigin/matrixone/pkg/pb/logservice" + "github.com/matrixorigin/matrixone/pkg/util" "strconv" "strings" "time" @@ -600,3 +602,8 @@ func (c *Config) GossipServiceAddr() string { } return c.GossipAddress } + +func dumpLogConfig(cfg Config) (map[string]*logservicepb.ConfigItem, error) { + defCfg := Config{} + return util.DumpConfig(cfg, defCfg) +} \ No newline at end of file diff --git a/pkg/logservice/option.go b/pkg/logservice/option.go index 27cf6511a4c6..9cff66b31b5e 100644 --- a/pkg/logservice/option.go +++ b/pkg/logservice/option.go @@ -16,6 +16,7 @@ package logservice import ( "context" + pb "github.com/matrixorigin/matrixone/pkg/pb/logservice" "github.com/matrixorigin/matrixone/pkg/common/morpc" "github.com/matrixorigin/matrixone/pkg/common/runtime" @@ -46,6 +47,16 @@ func WithTaskStorageFactory(factory taskservice.TaskStorageFactory) Option { } } +// WithConfigData saves the data from the config file +func WithConfigData(data map[string]*pb.ConfigItem) Option { + return func(s *Service) { + s.configData = make(map[string]*pb.ConfigItem, len(data)) + for k, v := range data { + s.configData[k] = v + } + } +} + type ContextKey string const ( diff --git a/pkg/logservice/service.go b/pkg/logservice/service.go index d9583889d192..227cf9892cc1 100644 --- a/pkg/logservice/service.go +++ b/pkg/logservice/service.go @@ -83,6 +83,8 @@ type Service struct { holder taskservice.TaskServiceHolder storageFactory taskservice.TaskStorageFactory } + + configData map[string]*pb.ConfigItem } func NewService( @@ -95,7 +97,9 @@ func NewService( if err := cfg.Validate(); err != nil { return nil, err } - + configKVMap, _ := dumpLogConfig(cfg) + opts = append(opts, WithConfigData(configKVMap)) + service := &Service{ cfg: cfg, stopper: stopper.NewStopper("log-service"), diff --git a/pkg/logservice/service_commands.go b/pkg/logservice/service_commands.go index 6994b42d0bf0..48707cdfa627 100644 --- a/pkg/logservice/service_commands.go +++ b/pkg/logservice/service_commands.go @@ -164,6 +164,9 @@ func (s *Service) heartbeat(ctx context.Context) { hb := s.store.getHeartbeatMessage() hb.TaskServiceCreated = s.taskServiceCreated() + hb.ConfigData = &pb.ConfigData{ + Content: s.configData, + } cb, err := s.haClient.SendLogHeartbeat(ctx2, hb) if err != nil { s.runtime.Logger().Error("failed to send log service heartbeat", zap.Error(err)) diff --git a/pkg/pb/logservice/logservice.go b/pkg/pb/logservice/logservice.go index cbc673d05eb1..4d32a7fdf80c 100644 --- a/pkg/pb/logservice/logservice.go +++ b/pkg/pb/logservice/logservice.go @@ -91,6 +91,7 @@ func (s *CNState) Update(hb CNStoreHeartbeat, tick uint64) { storeInfo.Role = hb.Role storeInfo.TaskServiceCreated = hb.TaskServiceCreated storeInfo.QueryAddress = hb.QueryAddress + storeInfo.ConfigData = hb.ConfigData s.Stores[hb.UUID] = storeInfo } @@ -160,6 +161,7 @@ func (s *TNState) Update(hb TNStoreHeartbeat, tick uint64) { storeInfo.LockServiceAddress = hb.LockServiceAddress storeInfo.CtlAddress = hb.CtlAddress storeInfo.TaskServiceCreated = hb.TaskServiceCreated + storeInfo.ConfigData = hb.ConfigData s.Stores[hb.UUID] = storeInfo } @@ -189,6 +191,7 @@ func (s *LogState) updateStores(hb LogStoreHeartbeat, tick uint64) { storeInfo.GossipAddress = hb.GossipAddress storeInfo.Replicas = hb.Replicas storeInfo.TaskServiceCreated = hb.TaskServiceCreated + storeInfo.ConfigData = hb.ConfigData s.Stores[hb.UUID] = storeInfo } diff --git a/pkg/pb/logservice/logservice.pb.go b/pkg/pb/logservice/logservice.pb.go index d6911ee520c9..87b4cbf4991a 100644 --- a/pkg/pb/logservice/logservice.pb.go +++ b/pkg/pb/logservice/logservice.pb.go @@ -441,6 +441,7 @@ type CNStore struct { Labels map[string]metadata.LabelList `protobuf:"bytes,9,rep,name=Labels,proto3" json:"Labels" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` WorkState metadata.WorkState `protobuf:"varint,10,opt,name=WorkState,proto3,enum=metadata.WorkState" json:"WorkState,omitempty"` QueryAddress string `protobuf:"bytes,11,opt,name=QueryAddress,proto3" json:"QueryAddress,omitempty"` + ConfigData *ConfigData `protobuf:"bytes,12,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -556,6 +557,13 @@ func (m *CNStore) GetQueryAddress() string { return "" } +func (m *CNStore) GetConfigData() *ConfigData { + if m != nil { + return m.ConfigData + } + return nil +} + type TNStore struct { UUID string `protobuf:"bytes,1,opt,name=UUID,proto3" json:"UUID,omitempty"` ServiceAddress string `protobuf:"bytes,2,opt,name=ServiceAddress,proto3" json:"ServiceAddress,omitempty"` @@ -565,11 +573,12 @@ type TNStore struct { // Server address for logtail push model LogtailServerAddress string `protobuf:"bytes,6,opt,name=LogtailServerAddress,proto3" json:"LogtailServerAddress,omitempty"` // LockServiceAddress lock service address for lock table allocator - LockServiceAddress string `protobuf:"bytes,7,opt,name=LockServiceAddress,proto3" json:"LockServiceAddress,omitempty"` - CtlAddress string `protobuf:"bytes,8,opt,name=CtlAddress,proto3" json:"CtlAddress,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + LockServiceAddress string `protobuf:"bytes,7,opt,name=LockServiceAddress,proto3" json:"LockServiceAddress,omitempty"` + CtlAddress string `protobuf:"bytes,8,opt,name=CtlAddress,proto3" json:"CtlAddress,omitempty"` + ConfigData *ConfigData `protobuf:"bytes,9,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *TNStore) Reset() { *m = TNStore{} } @@ -661,12 +670,20 @@ func (m *TNStore) GetCtlAddress() string { return "" } +func (m *TNStore) GetConfigData() *ConfigData { + if m != nil { + return m.ConfigData + } + return nil +} + type LogStore struct { UUID string `protobuf:"bytes,1,opt,name=UUID,proto3" json:"UUID,omitempty"` ServiceAddress string `protobuf:"bytes,2,opt,name=ServiceAddress,proto3" json:"ServiceAddress,omitempty"` Tick uint64 `protobuf:"varint,3,opt,name=Tick,proto3" json:"Tick,omitempty"` State NodeState `protobuf:"varint,4,opt,name=State,proto3,enum=logservice.NodeState" json:"State,omitempty"` Replicas []LogReplicaInfo `protobuf:"bytes,5,rep,name=Replicas,proto3" json:"Replicas"` + ConfigData *ConfigData `protobuf:"bytes,6,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -740,6 +757,13 @@ func (m *LogStore) GetReplicas() []LogReplicaInfo { return nil } +func (m *LogStore) GetConfigData() *ConfigData { + if m != nil { + return m.ConfigData + } + return nil +} + // LogShardInfo contains information a log shard. type LogShardInfo struct { // ShardID is the ID of a Log shard. @@ -889,6 +913,7 @@ type CNStoreHeartbeat struct { TaskServiceCreated bool `protobuf:"varint,7,opt,name=TaskServiceCreated,proto3" json:"TaskServiceCreated,omitempty"` QueryAddress string `protobuf:"bytes,8,opt,name=QueryAddress,proto3" json:"QueryAddress,omitempty"` InitWorkState string `protobuf:"bytes,9,opt,name=InitWorkState,proto3" json:"InitWorkState,omitempty"` + ConfigData *ConfigData `protobuf:"bytes,10,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -990,6 +1015,13 @@ func (m *CNStoreHeartbeat) GetInitWorkState() string { return "" } +func (m *CNStoreHeartbeat) GetConfigData() *ConfigData { + if m != nil { + return m.ConfigData + } + return nil +} + // CNAllocateID is the periodic message sent tp the HAKeeper by CN stores. type CNAllocateID struct { Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` @@ -1059,10 +1091,11 @@ type LogStoreHeartbeat struct { // update to date due to various reasons. Replicas []LogReplicaInfo `protobuf:"bytes,5,rep,name=Replicas,proto3" json:"Replicas"` // TaskServiceCreated task service is created at the current log node - TaskServiceCreated bool `protobuf:"varint,6,opt,name=TaskServiceCreated,proto3" json:"TaskServiceCreated,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + TaskServiceCreated bool `protobuf:"varint,6,opt,name=TaskServiceCreated,proto3" json:"TaskServiceCreated,omitempty"` + ConfigData *ConfigData `protobuf:"bytes,7,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *LogStoreHeartbeat) Reset() { *m = LogStoreHeartbeat{} } @@ -1140,6 +1173,13 @@ func (m *LogStoreHeartbeat) GetTaskServiceCreated() bool { return false } +func (m *LogStoreHeartbeat) GetConfigData() *ConfigData { + if m != nil { + return m.ConfigData + } + return nil +} + // TNShardInfo contains information of a launched TN shard. type TNShardInfo struct { // ShardID uniquely identifies a TN shard. Each TN shard manages a Primary @@ -1213,11 +1253,12 @@ type TNStoreHeartbeat struct { // Server address for logtail push model LogtailServerAddress string `protobuf:"bytes,5,opt,name=LogtailServerAddress,proto3" json:"LogtailServerAddress,omitempty"` // LockServiceAddress lock service address for lock table allocator - LockServiceAddress string `protobuf:"bytes,6,opt,name=LockServiceAddress,proto3" json:"LockServiceAddress,omitempty"` - CtlAddress string `protobuf:"bytes,7,opt,name=CtlAddress,proto3" json:"CtlAddress,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + LockServiceAddress string `protobuf:"bytes,6,opt,name=LockServiceAddress,proto3" json:"LockServiceAddress,omitempty"` + CtlAddress string `protobuf:"bytes,7,opt,name=CtlAddress,proto3" json:"CtlAddress,omitempty"` + ConfigData *ConfigData `protobuf:"bytes,8,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *TNStoreHeartbeat) Reset() { *m = TNStoreHeartbeat{} } @@ -1302,6 +1343,13 @@ func (m *TNStoreHeartbeat) GetCtlAddress() string { return "" } +func (m *TNStoreHeartbeat) GetConfigData() *ConfigData { + if m != nil { + return m.ConfigData + } + return nil +} + type RSMState struct { Tso uint64 `protobuf:"varint,1,opt,name=Tso,proto3" json:"Tso,omitempty"` Index uint64 `protobuf:"varint,2,opt,name=Index,proto3" json:"Index,omitempty"` @@ -2765,6 +2813,7 @@ type CNStoreInfo struct { Labels map[string]metadata.LabelList `protobuf:"bytes,8,rep,name=Labels,proto3" json:"Labels" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` WorkState metadata.WorkState `protobuf:"varint,9,opt,name=WorkState,proto3,enum=metadata.WorkState" json:"WorkState,omitempty"` QueryAddress string `protobuf:"bytes,10,opt,name=QueryAddress,proto3" json:"QueryAddress,omitempty"` + ConfigData *ConfigData `protobuf:"bytes,11,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -2873,6 +2922,13 @@ func (m *CNStoreInfo) GetQueryAddress() string { return "" } +func (m *CNStoreInfo) GetConfigData() *ConfigData { + if m != nil { + return m.ConfigData + } + return nil +} + // CNState contains all CN details known to the HAKeeper. type CNState struct { // Stores is keyed by CN store UUID. @@ -2931,11 +2987,12 @@ type TNStoreInfo struct { // Server address for logtail push model LogtailServerAddress string `protobuf:"bytes,5,opt,name=LogtailServerAddress,proto3" json:"LogtailServerAddress,omitempty"` // LockServiceAddress lock service address for lock table allocator - LockServiceAddress string `protobuf:"bytes,6,opt,name=LockServiceAddress,proto3" json:"LockServiceAddress,omitempty"` - CtlAddress string `protobuf:"bytes,7,opt,name=CtlAddress,proto3" json:"CtlAddress,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + LockServiceAddress string `protobuf:"bytes,6,opt,name=LockServiceAddress,proto3" json:"LockServiceAddress,omitempty"` + CtlAddress string `protobuf:"bytes,7,opt,name=CtlAddress,proto3" json:"CtlAddress,omitempty"` + ConfigData *ConfigData `protobuf:"bytes,8,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *TNStoreInfo) Reset() { *m = TNStoreInfo{} } @@ -3020,6 +3077,13 @@ func (m *TNStoreInfo) GetCtlAddress() string { return "" } +func (m *TNStoreInfo) GetConfigData() *ConfigData { + if m != nil { + return m.ConfigData + } + return nil +} + // TNState contains all TN details known to the HAKeeper. type TNState struct { // Stores is keyed by TN store UUID. @@ -3277,6 +3341,7 @@ type LogStoreInfo struct { GossipAddress string `protobuf:"bytes,4,opt,name=GossipAddress,proto3" json:"GossipAddress,omitempty"` Replicas []LogReplicaInfo `protobuf:"bytes,5,rep,name=Replicas,proto3" json:"Replicas"` TaskServiceCreated bool `protobuf:"varint,6,opt,name=TaskServiceCreated,proto3" json:"TaskServiceCreated,omitempty"` + ConfigData *ConfigData `protobuf:"bytes,7,opt,name=ConfigData,proto3" json:"ConfigData,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -3357,6 +3422,13 @@ func (m *LogStoreInfo) GetTaskServiceCreated() bool { return false } +func (m *LogStoreInfo) GetConfigData() *ConfigData { + if m != nil { + return m.ConfigData + } + return nil +} + type LogState struct { // Shards is keyed by ShardID, it contains details aggregated from all Log // stores. Each pb.LogShardInfo here contains data aggregated from @@ -3889,6 +3961,130 @@ func (m *BackupData) GetNextIDByKey() map[string]uint64 { return nil } +type ConfigItem struct { + // Name is the name of config item + Name string `protobuf:"bytes,1,opt,name=Name,proto3" json:"Name,omitempty"` + // CurrentValue is the current value of config item + CurrentValue string `protobuf:"bytes,2,opt,name=CurrentValue,proto3" json:"CurrentValue,omitempty"` + // DefaultValue is the default value of config item + DefaultValue string `protobuf:"bytes,3,opt,name=DefaultValue,proto3" json:"DefaultValue,omitempty"` + // Internal denotes the config item is internal or not + Internal string `protobuf:"bytes,4,opt,name=Internal,proto3" json:"Internal,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ConfigItem) Reset() { *m = ConfigItem{} } +func (m *ConfigItem) String() string { return proto.CompactTextString(m) } +func (*ConfigItem) ProtoMessage() {} +func (*ConfigItem) Descriptor() ([]byte, []int) { + return fileDescriptor_fd1040c5381ab5a7, []int{45} +} +func (m *ConfigItem) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ConfigItem) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ConfigItem.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ConfigItem) XXX_Merge(src proto.Message) { + xxx_messageInfo_ConfigItem.Merge(m, src) +} +func (m *ConfigItem) XXX_Size() int { + return m.Size() +} +func (m *ConfigItem) XXX_DiscardUnknown() { + xxx_messageInfo_ConfigItem.DiscardUnknown(m) +} + +var xxx_messageInfo_ConfigItem proto.InternalMessageInfo + +func (m *ConfigItem) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *ConfigItem) GetCurrentValue() string { + if m != nil { + return m.CurrentValue + } + return "" +} + +func (m *ConfigItem) GetDefaultValue() string { + if m != nil { + return m.DefaultValue + } + return "" +} + +func (m *ConfigItem) GetInternal() string { + if m != nil { + return m.Internal + } + return "" +} + +// ConfigData is the content of config on cn,tn,log and proxy +type ConfigData struct { + // Content is the key value of the config + Content map[string]*ConfigItem `protobuf:"bytes,1,rep,name=Content,proto3" json:"Content,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ConfigData) Reset() { *m = ConfigData{} } +func (m *ConfigData) String() string { return proto.CompactTextString(m) } +func (*ConfigData) ProtoMessage() {} +func (*ConfigData) Descriptor() ([]byte, []int) { + return fileDescriptor_fd1040c5381ab5a7, []int{46} +} +func (m *ConfigData) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ConfigData) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ConfigData.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ConfigData) XXX_Merge(src proto.Message) { + xxx_messageInfo_ConfigData.Merge(m, src) +} +func (m *ConfigData) XXX_Size() int { + return m.Size() +} +func (m *ConfigData) XXX_DiscardUnknown() { + xxx_messageInfo_ConfigData.DiscardUnknown(m) +} + +var xxx_messageInfo_ConfigData proto.InternalMessageInfo + +func (m *ConfigData) GetContent() map[string]*ConfigItem { + if m != nil { + return m.Content + } + return nil +} + func init() { proto.RegisterEnum("logservice.UpdateType", UpdateType_name, UpdateType_value) proto.RegisterEnum("logservice.NodeState", NodeState_name, NodeState_value) @@ -3963,216 +4159,228 @@ func init() { proto.RegisterMapType((map[uint64]ReplicaInfo)(nil), "logservice.ShardInfoQueryResult.ReplicasEntry") proto.RegisterType((*BackupData)(nil), "logservice.BackupData") proto.RegisterMapType((map[string]uint64)(nil), "logservice.BackupData.NextIDByKeyEntry") + proto.RegisterType((*ConfigItem)(nil), "logservice.ConfigItem") + proto.RegisterType((*ConfigData)(nil), "logservice.ConfigData") + proto.RegisterMapType((map[string]*ConfigItem)(nil), "logservice.ConfigData.ContentEntry") } func init() { proto.RegisterFile("logservice.proto", fileDescriptor_fd1040c5381ab5a7) } var fileDescriptor_fd1040c5381ab5a7 = []byte{ - // 3259 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe4, 0x5b, 0x4f, 0x6f, 0x1b, 0xc7, - 0x15, 0xd7, 0x2e, 0x29, 0x91, 0x7c, 0x14, 0xe5, 0xd5, 0x48, 0xb6, 0x19, 0x25, 0x95, 0x55, 0xc6, - 0x4d, 0x6d, 0xa5, 0xa1, 0x51, 0x09, 0x09, 0x92, 0x46, 0xb1, 0x41, 0x91, 0xb4, 0xc5, 0x88, 0xa6, - 0x94, 0xe1, 0xaa, 0x01, 0x02, 0x04, 0xea, 0x8a, 0x1c, 0x4b, 0xac, 0x48, 0x2e, 0xbb, 0xbb, 0x74, - 0xec, 0x9e, 0x8a, 0x02, 0x2d, 0x50, 0xf4, 0xd8, 0x4b, 0x50, 0x14, 0xfd, 0x0a, 0xbd, 0xe4, 0xd0, - 0x9e, 0x8a, 0x1e, 0x5a, 0xe4, 0xe8, 0x4f, 0x10, 0x34, 0x39, 0xb6, 0x87, 0x02, 0x39, 0x14, 0xe8, - 0xad, 0x98, 0x7f, 0xbb, 0x33, 0xe4, 0x92, 0x92, 0x12, 0x3b, 0x2d, 0x92, 0x93, 0x38, 0xef, 0xdf, - 0xcc, 0xbc, 0x79, 0xef, 0x37, 0x6f, 0x66, 0x56, 0x60, 0x75, 0xdd, 0x63, 0x9f, 0x78, 0x0f, 0x3b, - 0x2d, 0x52, 0x1c, 0x78, 0x6e, 0xe0, 0x22, 0x88, 0x28, 0x2b, 0xaf, 0x1c, 0x77, 0x82, 0x93, 0xe1, - 0x51, 0xb1, 0xe5, 0xf6, 0x6e, 0x1d, 0xbb, 0xc7, 0xee, 0x2d, 0x26, 0x72, 0x34, 0x7c, 0xc0, 0x5a, - 0xac, 0xc1, 0x7e, 0x71, 0xd5, 0x95, 0x85, 0x1e, 0x09, 0x9c, 0xb6, 0x13, 0x38, 0xbc, 0x5d, 0xf8, - 0x79, 0x12, 0x52, 0xe5, 0x46, 0x33, 0x70, 0x3d, 0x82, 0x10, 0x24, 0x0f, 0x0e, 0x6a, 0x95, 0xbc, - 0xb1, 0x66, 0xdc, 0xc8, 0x60, 0xf6, 0x1b, 0xbd, 0x04, 0x0b, 0x4d, 0xde, 0x53, 0xa9, 0xdd, 0xf6, - 0x88, 0xef, 0xe7, 0x4d, 0xc6, 0x1d, 0xa1, 0xa2, 0x55, 0x80, 0xe6, 0x3b, 0x75, 0x29, 0x93, 0x60, - 0x32, 0x0a, 0x05, 0x15, 0x01, 0xd5, 0xdd, 0xd6, 0xe9, 0x88, 0xad, 0x24, 0x93, 0x8b, 0xe1, 0x50, - 0x7b, 0xe5, 0xa0, 0x2b, 0xe5, 0x66, 0xb9, 0xbd, 0x88, 0x82, 0xae, 0x43, 0x12, 0xbb, 0x5d, 0x92, - 0x9f, 0x5b, 0x33, 0x6e, 0x2c, 0x6c, 0x58, 0xc5, 0x70, 0x5a, 0xe5, 0x06, 0xa5, 0x63, 0xc6, 0xa5, - 0x33, 0xb2, 0x3b, 0xad, 0xd3, 0x7c, 0x6a, 0xcd, 0xb8, 0x91, 0xc4, 0xec, 0x37, 0x7a, 0x19, 0x66, - 0x9b, 0x81, 0x13, 0x90, 0x7c, 0x9a, 0xa9, 0x5e, 0x2e, 0x2a, 0xee, 0x6d, 0xb8, 0x6d, 0xc2, 0x98, - 0x98, 0xcb, 0xa0, 0xb7, 0x60, 0xae, 0xee, 0x1c, 0x91, 0xae, 0x9f, 0xcf, 0xac, 0x25, 0x6e, 0x64, - 0x37, 0xae, 0xa9, 0xd2, 0xc2, 0x6f, 0x45, 0x2e, 0x51, 0xed, 0x07, 0xde, 0xe3, 0xed, 0xe4, 0xc7, - 0x9f, 0x5c, 0x9b, 0xc1, 0x42, 0x09, 0x7d, 0x1f, 0x32, 0xef, 0xba, 0xde, 0x29, 0xef, 0x0f, 0x58, - 0x7f, 0x4b, 0xd1, 0x50, 0x43, 0x16, 0x8e, 0xa4, 0x50, 0x01, 0xe6, 0xdf, 0x19, 0x12, 0xef, 0xb1, - 0x9c, 0x7a, 0x96, 0x4d, 0x5d, 0xa3, 0xad, 0x34, 0x20, 0xab, 0xf4, 0x89, 0x2c, 0x48, 0x9c, 0x92, - 0xc7, 0x62, 0xd9, 0xe8, 0x4f, 0x74, 0x13, 0x66, 0x1f, 0x3a, 0xdd, 0x21, 0x61, 0x8b, 0x95, 0x55, - 0xfb, 0x64, 0x7a, 0xf5, 0x8e, 0x1f, 0x60, 0x2e, 0xf1, 0x03, 0xf3, 0x75, 0xa3, 0xf0, 0x17, 0x13, - 0x52, 0xf6, 0x53, 0x08, 0x02, 0xe9, 0xee, 0x44, 0x9c, 0xbb, 0x93, 0xe7, 0x70, 0xf7, 0xab, 0x30, - 0xd7, 0x3c, 0x71, 0xbc, 0x36, 0x5d, 0x71, 0xea, 0xee, 0xab, 0xaa, 0xb4, 0xdd, 0x60, 0xbc, 0x5a, - 0xff, 0x81, 0x2b, 0xdd, 0xcc, 0x85, 0xd1, 0x06, 0x2c, 0xd7, 0xdd, 0xe3, 0xc0, 0xe9, 0x74, 0xe9, - 0x80, 0x88, 0x27, 0x47, 0x39, 0xc7, 0x46, 0x19, 0xcb, 0x9b, 0x10, 0x90, 0xa9, 0x73, 0x06, 0x64, - 0x7a, 0x34, 0x20, 0x0b, 0x7f, 0x35, 0x20, 0x5d, 0x77, 0x8f, 0xff, 0x0f, 0x9c, 0xb8, 0x05, 0x69, - 0x4c, 0x06, 0xdd, 0x4e, 0xcb, 0x91, 0x6e, 0x5c, 0x51, 0xe5, 0xeb, 0xee, 0xb1, 0x60, 0x2b, 0x9e, - 0x0c, 0x35, 0x0a, 0xff, 0x32, 0x60, 0x9e, 0xce, 0x43, 0xba, 0x1a, 0xe5, 0x21, 0xc5, 0x1b, 0x7c, - 0x3a, 0x49, 0x2c, 0x9b, 0x68, 0x5b, 0xe9, 0xc8, 0x64, 0x1d, 0xbd, 0x34, 0xd2, 0x51, 0x68, 0xa5, - 0x28, 0x05, 0x59, 0xc4, 0x46, 0xdd, 0xa1, 0x65, 0x98, 0xad, 0x0e, 0xdc, 0xd6, 0x89, 0x98, 0x2e, - 0x6f, 0xa0, 0x15, 0x48, 0xd7, 0x89, 0xd3, 0x26, 0x5e, 0xad, 0xc2, 0xa6, 0x9c, 0xc4, 0x61, 0x9b, - 0xf9, 0x87, 0x78, 0x3d, 0x86, 0x09, 0xd4, 0x3f, 0xc4, 0xeb, 0xad, 0xbc, 0x09, 0x39, 0xad, 0x03, - 0x35, 0x25, 0x92, 0x3c, 0x25, 0x96, 0xd5, 0x94, 0xc8, 0xa8, 0xd1, 0xff, 0x10, 0x16, 0x74, 0x9f, - 0xa0, 0xbb, 0xba, 0x0b, 0x98, 0x99, 0xec, 0x46, 0x7e, 0xd2, 0xe4, 0xb6, 0xd3, 0xd4, 0x87, 0x4f, - 0x3e, 0xb9, 0x66, 0x60, 0xdd, 0x75, 0x2f, 0x40, 0x46, 0x9a, 0xad, 0xb0, 0x7e, 0x93, 0x38, 0x22, - 0x14, 0x3e, 0x31, 0xc1, 0x12, 0x10, 0xb2, 0x43, 0x1c, 0x2f, 0x38, 0x22, 0x4e, 0xf0, 0x35, 0xc4, - 0xe0, 0x22, 0x20, 0xdb, 0xf1, 0xa5, 0xed, 0xb2, 0x47, 0x9c, 0x80, 0xb4, 0x59, 0xa2, 0xa5, 0x71, - 0x0c, 0x67, 0x0c, 0x00, 0xd3, 0xe3, 0x00, 0x88, 0xae, 0x43, 0xae, 0xd6, 0xef, 0x04, 0x11, 0xb6, - 0x66, 0x98, 0x90, 0x4e, 0x2c, 0xbc, 0x06, 0xf3, 0xe5, 0x46, 0xa9, 0xdb, 0x75, 0x5b, 0x4e, 0x40, - 0x6a, 0x95, 0x18, 0x9c, 0x5c, 0x86, 0xd9, 0x6d, 0x27, 0x68, 0x9d, 0x88, 0xc5, 0xe1, 0x8d, 0xc2, - 0x2f, 0x4c, 0x58, 0x94, 0xa9, 0x3c, 0x7d, 0x65, 0xd6, 0x20, 0x8b, 0x9d, 0x07, 0x81, 0xbe, 0x2c, - 0x2a, 0x29, 0x66, 0xed, 0x12, 0xb1, 0x6b, 0x77, 0x1d, 0x72, 0xf7, 0x5c, 0xdf, 0xef, 0x0c, 0xf4, - 0x65, 0xd1, 0x89, 0x5f, 0x2e, 0xb5, 0x27, 0xac, 0xc4, 0xdc, 0xa4, 0x95, 0x28, 0x54, 0x21, 0xab, - 0x60, 0xee, 0x14, 0x20, 0x98, 0x1e, 0xe7, 0x7f, 0x32, 0xc1, 0xb2, 0x9f, 0x66, 0x9c, 0x47, 0xbb, - 0x44, 0xe2, 0x22, 0xbb, 0x44, 0xfc, 0xf4, 0x93, 0x13, 0x03, 0x71, 0xd2, 0xae, 0x32, 0x7b, 0xe1, - 0x5d, 0x65, 0xee, 0x9c, 0x29, 0x96, 0x1a, 0xdb, 0x55, 0x7e, 0x65, 0x42, 0x1a, 0x37, 0xef, 0x73, - 0x60, 0xb7, 0x20, 0x61, 0xfb, 0xae, 0x04, 0x35, 0xdb, 0x77, 0x69, 0xfc, 0xd6, 0xfa, 0x6d, 0xf2, - 0x48, 0xc6, 0x2f, 0x6b, 0xd0, 0x58, 0xaa, 0x13, 0xc7, 0x27, 0x3b, 0x6e, 0x97, 0x43, 0x28, 0xc7, - 0x56, 0x9d, 0x48, 0xf3, 0xcc, 0xf6, 0x86, 0x7d, 0x9a, 0x1b, 0xed, 0xba, 0xdf, 0x17, 0x38, 0xab, - 0xd1, 0xd0, 0xdb, 0x30, 0xcf, 0x95, 0x3a, 0x7e, 0xe0, 0x7a, 0x8f, 0x45, 0xcc, 0x69, 0x28, 0x2f, - 0x47, 0x57, 0x54, 0x05, 0x39, 0xca, 0x6b, 0xba, 0x2b, 0x77, 0x60, 0x71, 0x4c, 0xe4, 0x2c, 0x9c, - 0x4e, 0xaa, 0x38, 0xfd, 0x3e, 0x64, 0x58, 0x80, 0xb7, 0x5c, 0xaf, 0x4d, 0x15, 0xe9, 0xa0, 0x85, - 0x22, 0x1d, 0xeb, 0x3a, 0x24, 0xed, 0xc7, 0x03, 0xae, 0xb7, 0xb0, 0x71, 0x45, 0x1b, 0x23, 0xd3, - 0xa1, 0x5c, 0xcc, 0x64, 0x68, 0xf4, 0x55, 0x9c, 0xc0, 0x61, 0x8e, 0x99, 0xc7, 0xec, 0x77, 0xe1, - 0x43, 0x03, 0x80, 0xd9, 0xff, 0xc9, 0x90, 0xf8, 0x2c, 0x40, 0x1b, 0x4e, 0x8f, 0xc8, 0x00, 0xa5, - 0xbf, 0xd5, 0x0c, 0x30, 0xf5, 0x0c, 0x10, 0xc3, 0x49, 0x44, 0xc3, 0xc9, 0x43, 0xea, 0xbe, 0xf3, - 0xa8, 0xd9, 0xf9, 0x29, 0x11, 0x9e, 0x95, 0x4d, 0x9a, 0x2d, 0x32, 0x48, 0x2b, 0x62, 0x17, 0x8b, - 0x08, 0x6c, 0x7b, 0x6b, 0xd4, 0x2a, 0x2c, 0x66, 0xe8, 0xf6, 0xd6, 0xa8, 0x55, 0x0a, 0x05, 0x00, - 0xdb, 0x77, 0xe5, 0xc8, 0x96, 0x61, 0xb6, 0xec, 0x0e, 0xfb, 0x81, 0x98, 0x3c, 0x6f, 0x14, 0xfe, - 0x69, 0x50, 0xb4, 0x63, 0x59, 0xc6, 0x6a, 0xbc, 0xd8, 0x0c, 0xdb, 0x84, 0xcc, 0xde, 0x80, 0x78, - 0x4e, 0xd0, 0x71, 0xfb, 0xc2, 0x51, 0x97, 0xf5, 0x8a, 0x96, 0xe9, 0xee, 0x0d, 0x70, 0x24, 0x87, - 0xb6, 0xc3, 0x1a, 0x98, 0xa7, 0xdb, 0xf5, 0x98, 0x1a, 0x98, 0x09, 0x4c, 0x2e, 0x84, 0x9f, 0x7a, - 0xc5, 0x5a, 0x87, 0x6c, 0xb9, 0x11, 0x15, 0xcd, 0x71, 0x73, 0xbd, 0x29, 0x6b, 0x26, 0x73, 0x72, - 0xdd, 0xcd, 0x25, 0x0a, 0x9f, 0x0a, 0xdf, 0x39, 0xc1, 0x14, 0xdf, 0x9d, 0xdf, 0xde, 0xd9, 0x1e, - 0x93, 0x1d, 0x7d, 0x85, 0x1e, 0xfb, 0xdb, 0x2c, 0xa4, 0x64, 0x04, 0x31, 0xbc, 0x66, 0x3f, 0x43, - 0x2c, 0x8f, 0x08, 0xa8, 0x08, 0x73, 0xf7, 0x49, 0x70, 0xe2, 0xb6, 0xe3, 0x52, 0x89, 0x73, 0x58, - 0x2a, 0x09, 0x29, 0xb4, 0xa5, 0xe6, 0x0d, 0x4b, 0x81, 0xac, 0xae, 0x13, 0x71, 0xc5, 0x1c, 0xd5, - 0x3c, 0x2b, 0xb1, 0x5a, 0x2b, 0xdc, 0x18, 0x58, 0xb2, 0x64, 0x37, 0xbe, 0x35, 0x5a, 0x6b, 0x69, - 0xbb, 0x07, 0xd6, 0x54, 0xd0, 0x6d, 0x1a, 0x0c, 0x91, 0x85, 0x59, 0x66, 0xe1, 0x85, 0x98, 0x28, - 0x8d, 0x0c, 0xa8, 0x0a, 0x54, 0xdf, 0x56, 0xf4, 0xe7, 0xc6, 0xf5, 0xed, 0x31, 0x7d, 0x45, 0x01, - 0xbd, 0xa6, 0xa6, 0x27, 0x03, 0xf1, 0x11, 0x07, 0x44, 0x5c, 0xac, 0x26, 0xf2, 0x96, 0x5e, 0x9f, - 0xb0, 0x4a, 0x67, 0xa4, 0xcc, 0x54, 0xf9, 0x58, 0xaf, 0x66, 0xb6, 0xf4, 0x7c, 0x67, 0x25, 0xd0, - 0x98, 0x76, 0xc4, 0xc7, 0x3a, 0x3a, 0xbc, 0xa1, 0x25, 0x10, 0x3b, 0x9b, 0x8e, 0x6c, 0xa4, 0x0a, - 0x1b, 0x6b, 0xc9, 0xb6, 0xa5, 0x27, 0x0b, 0x3b, 0xa1, 0xc6, 0x74, 0x2c, 0xf9, 0x58, 0x4f, 0xad, - 0x3b, 0x90, 0xab, 0x90, 0x2e, 0x09, 0x88, 0x18, 0x4e, 0x7e, 0x9e, 0xa9, 0x3f, 0xa7, 0xaa, 0x6b, - 0x02, 0x58, 0x97, 0x2f, 0x34, 0x21, 0xcb, 0xc2, 0xc7, 0x1f, 0xb8, 0x7d, 0x9f, 0x4c, 0xa9, 0x4a, - 0x04, 0x26, 0x9b, 0x1a, 0x26, 0xd7, 0x1d, 0x3f, 0x88, 0x90, 0x5a, 0x36, 0x0b, 0x45, 0x40, 0x8a, - 0xa3, 0x15, 0xdb, 0x77, 0x3b, 0x9e, 0x92, 0x25, 0xb2, 0x59, 0xf8, 0x77, 0x92, 0x56, 0x62, 0x42, - 0xec, 0xe9, 0xa6, 0xd3, 0x0b, 0x90, 0xa9, 0x7a, 0x9e, 0xeb, 0x95, 0xdd, 0x36, 0x61, 0xc3, 0xcc, - 0xe1, 0x88, 0x40, 0x77, 0x6d, 0xd6, 0xb8, 0x4f, 0x7c, 0xdf, 0x39, 0x26, 0xa2, 0x4c, 0xd4, 0x68, - 0xb4, 0xa8, 0xa8, 0xf9, 0x3b, 0xa5, 0x5d, 0x42, 0x06, 0xc4, 0x63, 0xe9, 0x90, 0xc6, 0x0a, 0x05, - 0xdd, 0xd1, 0x3c, 0x28, 0xe2, 0xfd, 0xea, 0x58, 0xc6, 0x72, 0xb6, 0x48, 0x59, 0xcd, 0xe7, 0x34, - 0x02, 0xdc, 0x5e, 0xcf, 0xe9, 0xb7, 0x79, 0xf5, 0x9c, 0x8a, 0x89, 0x00, 0x85, 0x8f, 0x35, 0x69, - 0x1a, 0x7a, 0x2c, 0x09, 0x44, 0xf7, 0xe9, 0xf1, 0xee, 0x15, 0x36, 0x56, 0x65, 0xd1, 0x36, 0x2c, - 0x94, 0xbb, 0x43, 0x3f, 0x20, 0x5e, 0x85, 0xd0, 0xe2, 0xcb, 0x17, 0x51, 0xaf, 0x55, 0xc1, 0xba, - 0x04, 0x1e, 0xd1, 0x40, 0xb7, 0x21, 0x13, 0x9d, 0xec, 0x78, 0xdc, 0xaf, 0xa9, 0xea, 0x21, 0x93, - 0x9d, 0x38, 0x30, 0xf1, 0x87, 0xdd, 0x00, 0x47, 0x2a, 0xe8, 0x36, 0x80, 0x92, 0xb3, 0x3c, 0xf8, - 0x57, 0x55, 0x03, 0xe3, 0x81, 0x84, 0x61, 0x24, 0x6f, 0x4f, 0x48, 0xeb, 0x94, 0x78, 0x3c, 0xf5, - 0xe6, 0x63, 0x9c, 0xa7, 0xf0, 0xb1, 0x26, 0x5d, 0x78, 0x9b, 0x1d, 0x4d, 0x78, 0x41, 0x13, 0xba, - 0xe5, 0x55, 0x0a, 0xed, 0x94, 0xe2, 0xe7, 0x0d, 0xb6, 0xe1, 0x5c, 0x1e, 0x5b, 0x4c, 0xca, 0x15, - 0x4b, 0x29, 0x65, 0x0b, 0x2f, 0x6a, 0x0b, 0x41, 0xeb, 0x8a, 0x1f, 0xb2, 0x0d, 0x45, 0xd4, 0x15, - 0xac, 0x51, 0xb8, 0x07, 0x39, 0x5a, 0x1b, 0xdb, 0xce, 0x51, 0x97, 0x1c, 0xf8, 0xc4, 0xa3, 0x67, - 0x73, 0xfa, 0xb7, 0x1f, 0x15, 0x47, 0x61, 0x9b, 0xf2, 0xf6, 0x1d, 0xdf, 0xff, 0xc0, 0xf5, 0xda, - 0xa2, 0x76, 0x0f, 0xdb, 0x85, 0x5f, 0x1b, 0x74, 0x94, 0xec, 0x50, 0x10, 0xbb, 0xbf, 0x4e, 0x2e, - 0xae, 0xb4, 0xe3, 0x45, 0x62, 0xe4, 0x78, 0x11, 0xdd, 0x20, 0x24, 0xd5, 0x1b, 0x84, 0x55, 0xb6, - 0x29, 0xe9, 0x55, 0x96, 0x42, 0x29, 0xfc, 0xd6, 0xa4, 0x31, 0xdc, 0x7f, 0xd0, 0x39, 0x2e, 0x9f, - 0x38, 0xfd, 0x63, 0x82, 0x36, 0xc3, 0xd1, 0x89, 0xe3, 0xfe, 0x92, 0x5e, 0x41, 0x32, 0x56, 0xe4, - 0x41, 0x3e, 0x8f, 0x2d, 0x00, 0xae, 0xae, 0x54, 0x9e, 0xfa, 0xc6, 0xa3, 0x74, 0xc1, 0xb2, 0x5c, - 0x91, 0x47, 0x36, 0x2c, 0xd0, 0x03, 0x6b, 0xc7, 0xe9, 0xde, 0x27, 0xbd, 0x23, 0xe2, 0xc9, 0x72, - 0xe1, 0x7b, 0x93, 0x2c, 0x14, 0x75, 0x71, 0x5e, 0x65, 0x8f, 0xd8, 0x58, 0x29, 0xc1, 0x52, 0x8c, - 0xd8, 0x85, 0x6e, 0x44, 0x6e, 0x42, 0xae, 0x79, 0x32, 0x0c, 0xda, 0xee, 0x07, 0x7d, 0x7e, 0x9f, - 0x45, 0xd7, 0x86, 0xfe, 0x08, 0x97, 0x4c, 0x36, 0x0b, 0xbf, 0x49, 0xc0, 0xa5, 0x66, 0xeb, 0x84, - 0xb4, 0x87, 0x5d, 0x22, 0xb2, 0x3c, 0x76, 0x75, 0xaf, 0x43, 0x6e, 0xdb, 0x75, 0x03, 0x3f, 0xf0, - 0x9c, 0xc1, 0xa0, 0xd3, 0x3f, 0x66, 0x9d, 0xa6, 0xb1, 0x4e, 0xa4, 0xd0, 0x20, 0x0e, 0x48, 0xcc, - 0xa1, 0x09, 0xe6, 0x50, 0x0d, 0x1a, 0x14, 0x36, 0x56, 0x65, 0x39, 0x26, 0x45, 0xae, 0x12, 0x75, - 0x44, 0x7e, 0x92, 0x2b, 0xb1, 0xbe, 0xfa, 0x77, 0x46, 0x66, 0x2c, 0x8a, 0x88, 0xe7, 0x74, 0x60, - 0x50, 0x04, 0xf0, 0x88, 0x87, 0x76, 0x61, 0x91, 0x9f, 0x1b, 0x95, 0x83, 0xa4, 0x40, 0x56, 0xad, - 0x96, 0x19, 0x13, 0xc2, 0xe3, 0x7a, 0xe3, 0x7b, 0x64, 0xea, 0x82, 0x7b, 0x64, 0x37, 0x66, 0x34, - 0x68, 0x13, 0x92, 0x34, 0x51, 0x45, 0x78, 0x6b, 0xc6, 0xb4, 0x0c, 0x17, 0x41, 0xce, 0x84, 0xd9, - 0x29, 0xd1, 0xf1, 0x4f, 0xe9, 0x09, 0xe9, 0xc8, 0xf1, 0x65, 0xac, 0x68, 0x34, 0x1a, 0x2e, 0x5a, - 0xf7, 0x53, 0xc2, 0xc5, 0xd1, 0x77, 0x8e, 0xf0, 0x32, 0xcf, 0x88, 0x2e, 0xf3, 0xd0, 0x5b, 0x90, - 0x16, 0x32, 0xf2, 0x5a, 0xf1, 0x79, 0x6d, 0x19, 0xf4, 0x68, 0x93, 0xb7, 0x1c, 0x52, 0xa5, 0xf0, - 0x9f, 0x04, 0x2d, 0x6d, 0x78, 0x87, 0x14, 0xaf, 0xe5, 0x7d, 0xaa, 0xa1, 0xdc, 0xa7, 0x7e, 0xb3, - 0x6e, 0xd4, 0x4a, 0xe1, 0x71, 0x24, 0xcd, 0xdc, 0xf9, 0x62, 0x4c, 0x8d, 0xc8, 0x2e, 0x69, 0xcf, - 0xf9, 0x90, 0x91, 0xf9, 0x42, 0x0f, 0x19, 0xf0, 0x15, 0x3c, 0x64, 0xfc, 0xce, 0xe0, 0xaf, 0x59, - 0xe2, 0xe9, 0x86, 0xcd, 0x4c, 0xee, 0x89, 0xd7, 0x62, 0x0a, 0xd4, 0x22, 0x97, 0xd0, 0x66, 0xcc, - 0x49, 0x2b, 0x18, 0xb2, 0x0a, 0x33, 0x66, 0x68, 0xaf, 0xe8, 0x43, 0xbb, 0x3a, 0xc1, 0xa9, 0xea, - 0xf0, 0x3e, 0x32, 0xd9, 0x8d, 0xda, 0x53, 0x09, 0xcd, 0x6f, 0xd0, 0x25, 0x18, 0x5d, 0x55, 0xfb, - 0x3c, 0xab, 0x6a, 0x3f, 0xdb, 0x55, 0xb5, 0xe3, 0x57, 0xf5, 0x8f, 0xc6, 0x68, 0x55, 0x8a, 0x5e, - 0x85, 0xb4, 0x10, 0x96, 0xe3, 0x5c, 0x8a, 0x31, 0x24, 0xa1, 0x4b, 0x8a, 0x52, 0xb5, 0xb2, 0x54, - 0x33, 0xc7, 0xd5, 0xca, 0xba, 0x9a, 0x14, 0x45, 0xaf, 0xb3, 0x8b, 0x31, 0xa1, 0xc7, 0xa3, 0x61, - 0x39, 0xee, 0xfc, 0x2c, 0x14, 0x23, 0xe1, 0xc2, 0x2f, 0x0d, 0xc8, 0x8a, 0xa1, 0xb3, 0x80, 0x7c, - 0x83, 0x8d, 0x9b, 0x87, 0x95, 0x21, 0xc2, 0x2a, 0xcc, 0x38, 0xc1, 0xd1, 0x6a, 0xc9, 0x50, 0x1c, - 0x6d, 0xf1, 0x41, 0x70, 0x5d, 0x3e, 0xf8, 0xbc, 0x92, 0xad, 0x82, 0xa5, 0x29, 0x47, 0x0a, 0x85, - 0x3f, 0x9b, 0x70, 0x59, 0x54, 0x2d, 0x62, 0x3c, 0xf2, 0x90, 0xfc, 0x12, 0x2c, 0x34, 0x86, 0xbd, - 0xbd, 0x07, 0x91, 0x71, 0x9e, 0x2d, 0x23, 0x54, 0x5a, 0x60, 0x30, 0x4a, 0x38, 0x7e, 0x5e, 0x44, - 0xea, 0x44, 0xb4, 0x0e, 0x96, 0xd4, 0x0b, 0x2f, 0xd2, 0x79, 0x45, 0x39, 0x46, 0x47, 0x57, 0x60, - 0xae, 0x41, 0x1e, 0x05, 0xe1, 0x13, 0x94, 0x68, 0x21, 0x1b, 0xb2, 0xfc, 0xd7, 0xf6, 0xe3, 0x5d, - 0x22, 0xef, 0x44, 0x37, 0x54, 0x87, 0xc7, 0xce, 0xa4, 0xa8, 0x28, 0xf1, 0xca, 0x4d, 0x35, 0xb3, - 0x72, 0x1b, 0xac, 0x51, 0x81, 0xf8, 0x07, 0x8b, 0x09, 0xb7, 0xa3, 0x3f, 0x33, 0xf9, 0xa3, 0xd5, - 0x54, 0x70, 0xf9, 0x7a, 0xbf, 0x57, 0x7c, 0x64, 0x8a, 0x27, 0x58, 0x0a, 0x14, 0xb7, 0x43, 0x7c, - 0xe4, 0x81, 0xbc, 0x36, 0x96, 0x11, 0x0c, 0x29, 0x98, 0x88, 0x8e, 0x14, 0x3c, 0x52, 0x6e, 0x87, - 0x40, 0x63, 0x4e, 0xd3, 0x9f, 0x88, 0x34, 0x4d, 0xc8, 0x2a, 0xc6, 0x63, 0xca, 0xef, 0xa2, 0x8e, - 0x34, 0x13, 0x5f, 0x17, 0x95, 0x45, 0x66, 0x46, 0xa7, 0xc2, 0xd7, 0x59, 0x46, 0xe3, 0xf0, 0xeb, - 0xf3, 0xa4, 0x7e, 0x22, 0x8d, 0x8d, 0x9c, 0x3b, 0x1a, 0x50, 0xc4, 0xee, 0x79, 0x11, 0x5b, 0xde, - 0x19, 0xa8, 0xd0, 0xb2, 0x19, 0x62, 0xb8, 0xb8, 0x22, 0x5c, 0x8a, 0x41, 0x6e, 0x79, 0xbe, 0x92, - 0x68, 0xff, 0x5a, 0xb4, 0xa0, 0xa2, 0xa0, 0x5f, 0x8e, 0x5b, 0x06, 0x19, 0x39, 0xe1, 0xe2, 0x6f, - 0x86, 0x65, 0x80, 0x28, 0xe4, 0x97, 0x62, 0x36, 0x7f, 0xd9, 0x99, 0x2c, 0x18, 0x6e, 0xc9, 0x0b, - 0x5e, 0x5e, 0x7d, 0x69, 0x05, 0xb2, 0xbc, 0x3b, 0xd1, 0xae, 0x79, 0x1b, 0x22, 0x3e, 0x45, 0x41, - 0x2a, 0xce, 0xf3, 0x29, 0xa6, 0xbd, 0x3a, 0x5a, 0x5e, 0xeb, 0x52, 0x38, 0x46, 0x13, 0x55, 0x47, - 0x8e, 0xda, 0xe2, 0x6a, 0xe4, 0xcc, 0x4a, 0x7d, 0xe4, 0x80, 0x2e, 0x71, 0xab, 0xcd, 0x0a, 0x35, - 0x89, 0x5b, 0x6d, 0xb4, 0xab, 0xe3, 0x16, 0xb0, 0xb0, 0xbe, 0x39, 0xe9, 0xde, 0xe1, 0x19, 0xc3, - 0xd5, 0xe7, 0x29, 0xb0, 0xa4, 0x53, 0xc3, 0x07, 0xae, 0xf0, 0x39, 0xcb, 0x50, 0x9f, 0xb3, 0x64, - 0x38, 0x9a, 0x4a, 0x38, 0x46, 0xd8, 0x9c, 0xd0, 0xb0, 0x79, 0x4f, 0x9f, 0x63, 0x92, 0xcd, 0xf1, - 0x95, 0xb8, 0x95, 0x0c, 0xdf, 0xad, 0xa6, 0xce, 0x33, 0xee, 0x6b, 0x83, 0xff, 0x7d, 0xa0, 0xb4, - 0xc1, 0x1a, 0x39, 0x05, 0xc9, 0xd2, 0x7e, 0x63, 0xea, 0x54, 0x47, 0x95, 0x54, 0xdc, 0x1a, 0xb3, - 0x88, 0x6a, 0xea, 0x8e, 0xce, 0x3f, 0x7f, 0x7a, 0x79, 0xaa, 0xf9, 0x50, 0x9a, 0xfb, 0x31, 0xd2, - 0x56, 0xf3, 0x11, 0xce, 0x9d, 0x8f, 0x0a, 0x62, 0x64, 0xbf, 0x10, 0x62, 0xcc, 0x5f, 0x00, 0x31, - 0x46, 0xf0, 0x2d, 0x77, 0x61, 0x7c, 0x1b, 0x4b, 0xde, 0x85, 0x2f, 0x92, 0xbc, 0x5f, 0x36, 0xaf, - 0x56, 0xde, 0x87, 0xcb, 0xb1, 0xab, 0x7c, 0xc1, 0xbd, 0x42, 0xbb, 0xbe, 0x55, 0xcc, 0x6f, 0xb1, - 0x6f, 0x65, 0x26, 0x6c, 0x6c, 0x67, 0x26, 0x7d, 0x0d, 0xb2, 0xea, 0x67, 0x36, 0x5f, 0xe2, 0x1b, - 0x80, 0xc2, 0xef, 0x4d, 0x58, 0x8e, 0xbb, 0xa9, 0x9d, 0xf2, 0x1e, 0xb0, 0x3f, 0xf6, 0xb9, 0x52, - 0xf1, 0xac, 0x7b, 0x5f, 0xfd, 0xb3, 0xa5, 0xb1, 0x02, 0xe5, 0xe9, 0x7c, 0xbc, 0x64, 0x9f, 0xfd, - 0xf1, 0xd2, 0xb4, 0x53, 0x89, 0xe2, 0x51, 0xd5, 0xd7, 0x7f, 0x30, 0x00, 0xb6, 0x9d, 0xd6, 0xe9, - 0x70, 0x50, 0x71, 0x02, 0x47, 0x01, 0x4c, 0x43, 0x03, 0xcc, 0x9a, 0x0e, 0x98, 0xdc, 0x2f, 0xdf, - 0x55, 0xed, 0x47, 0x46, 0x9e, 0xed, 0x96, 0xb0, 0xfe, 0x23, 0x80, 0x83, 0x41, 0xdb, 0x09, 0xf8, - 0x7d, 0xde, 0x55, 0x58, 0xd2, 0xbe, 0x57, 0xe0, 0x2c, 0x6b, 0x06, 0x5d, 0x86, 0x45, 0xf9, 0x8d, - 0x42, 0xbd, 0xd9, 0x10, 0x64, 0x03, 0x2d, 0xc1, 0x25, 0x9a, 0x40, 0xac, 0x5b, 0x41, 0x34, 0x51, - 0x0e, 0x32, 0x76, 0x73, 0x4f, 0x34, 0x13, 0xeb, 0x45, 0xc8, 0x84, 0x5f, 0xcb, 0xa1, 0x4b, 0x90, - 0x6d, 0xb8, 0x5e, 0xcf, 0xe9, 0xb2, 0xa6, 0x35, 0x83, 0x2c, 0x98, 0xb7, 0x3b, 0x3d, 0xe2, 0x0e, - 0x03, 0x4e, 0x31, 0xd6, 0xff, 0x61, 0x02, 0x44, 0x2f, 0x34, 0x68, 0x01, 0xc0, 0x6e, 0xee, 0x1d, - 0x1e, 0xec, 0x57, 0x4a, 0x76, 0xd5, 0x9a, 0x41, 0x00, 0x73, 0xa5, 0xfd, 0xfd, 0x6a, 0xa3, 0x62, - 0x19, 0x28, 0x0d, 0x49, 0x5c, 0x2d, 0x55, 0x2c, 0x13, 0xcd, 0x43, 0xda, 0xc6, 0x07, 0x8d, 0x32, - 0x95, 0x49, 0x50, 0xa3, 0xf7, 0xaa, 0xf6, 0x61, 0x48, 0x49, 0xa2, 0x2c, 0xa4, 0xca, 0x7b, 0x8d, - 0x46, 0xb5, 0x6c, 0x5b, 0xb3, 0xd4, 0xa4, 0x68, 0x1c, 0xe2, 0x3d, 0x6b, 0x0e, 0x2d, 0x42, 0xae, - 0xbe, 0x77, 0xef, 0x70, 0xa7, 0x5a, 0xc2, 0xf6, 0x76, 0xb5, 0x64, 0x5b, 0x29, 0x6a, 0xa1, 0xdc, - 0x50, 0x28, 0x69, 0x36, 0x50, 0x95, 0x92, 0x41, 0x08, 0x16, 0xca, 0x3b, 0xd5, 0xf2, 0xee, 0xe1, - 0x4e, 0x69, 0xb7, 0x5a, 0xdd, 0xaf, 0x62, 0x0b, 0xa8, 0x03, 0x69, 0xcf, 0xe5, 0xfa, 0x41, 0xd3, - 0xae, 0xe2, 0xc3, 0x4a, 0xd5, 0x2e, 0xd5, 0xea, 0x4d, 0x2b, 0x4b, 0x85, 0x29, 0xa3, 0xb9, 0x53, - 0xc2, 0x95, 0xc3, 0x5a, 0xe3, 0xee, 0x9e, 0x35, 0xcf, 0x0c, 0x34, 0x0e, 0x4b, 0xf5, 0xfa, 0x1e, - 0x1d, 0xe5, 0x61, 0xad, 0x62, 0xe5, 0xa8, 0xa3, 0x55, 0x03, 0x4d, 0x9b, 0x8e, 0x7f, 0x81, 0x39, - 0x9a, 0x79, 0xe0, 0xb0, 0xdc, 0x38, 0xac, 0x97, 0xb6, 0xab, 0x75, 0xeb, 0x12, 0xca, 0xc3, 0x72, - 0x44, 0x7c, 0x77, 0x0f, 0xef, 0x0a, 0x71, 0x8b, 0x5a, 0xde, 0x2f, 0xd9, 0xe5, 0x1d, 0xca, 0x68, - 0xda, 0x7b, 0xb8, 0x6a, 0x2d, 0x52, 0x13, 0x95, 0x6a, 0xbd, 0xca, 0xa5, 0x39, 0x11, 0xad, 0x37, - 0x00, 0xa2, 0xef, 0x34, 0xa8, 0x63, 0xe8, 0x72, 0x72, 0x8a, 0x35, 0x43, 0xbd, 0x5a, 0xeb, 0x07, - 0xc4, 0xeb, 0x3b, 0x5d, 0xcb, 0xa0, 0x6b, 0xc7, 0x82, 0x23, 0x5c, 0xe8, 0x45, 0xf1, 0xc9, 0x0b, - 0x26, 0x3f, 0x26, 0xad, 0x80, 0xb4, 0xad, 0xc4, 0xfa, 0x3a, 0x64, 0xc2, 0xcf, 0x19, 0xa8, 0x7a, - 0x93, 0x04, 0xac, 0x65, 0xcd, 0x50, 0x75, 0x7e, 0x61, 0xc9, 0x09, 0xc6, 0xfa, 0x13, 0x13, 0x90, - 0xdc, 0xce, 0x94, 0x18, 0xa4, 0x0b, 0xde, 0x69, 0x9d, 0xaa, 0xa1, 0xa7, 0xbc, 0x1b, 0x87, 0xa1, - 0x47, 0x23, 0x72, 0x8c, 0x6c, 0xa2, 0x2b, 0x80, 0xd4, 0x67, 0x6a, 0x19, 0x85, 0xb4, 0xf7, 0x7b, - 0x24, 0x08, 0x23, 0x3a, 0x89, 0x9e, 0x1b, 0xc3, 0x6c, 0xc1, 0x9a, 0xa5, 0xde, 0x6b, 0x12, 0x1e, - 0x8f, 0x82, 0x36, 0x47, 0x7d, 0xad, 0x1f, 0x30, 0x05, 0x27, 0x85, 0xae, 0xc1, 0xf3, 0x4d, 0x12, - 0x8c, 0x17, 0x0c, 0x42, 0x20, 0x8d, 0x56, 0xe0, 0x8a, 0x10, 0x08, 0x77, 0x1c, 0xc1, 0xcb, 0x50, - 0x17, 0xf2, 0xdf, 0xc2, 0x6b, 0x16, 0xd0, 0x89, 0x49, 0x52, 0x78, 0xaf, 0x67, 0x65, 0x69, 0xfc, - 0xed, 0xd3, 0x8d, 0x41, 0x5c, 0x40, 0x58, 0xf3, 0x54, 0x17, 0x93, 0x9e, 0xfb, 0x50, 0xde, 0x00, - 0x5b, 0xb9, 0xf5, 0x0f, 0x0d, 0xc8, 0x69, 0xc5, 0x10, 0x5d, 0x75, 0x49, 0x10, 0x27, 0x36, 0x6b, - 0x86, 0xce, 0x5d, 0x12, 0xb5, 0xa7, 0x00, 0xcb, 0x40, 0xdf, 0x81, 0x6f, 0x8f, 0xb1, 0xe4, 0x9e, - 0x86, 0x49, 0x8b, 0x74, 0x1e, 0x92, 0xb6, 0x65, 0xa2, 0xe7, 0xe1, 0xea, 0x98, 0xd8, 0x5d, 0xa7, - 0xd3, 0xa5, 0x41, 0xa0, 0xf6, 0x89, 0x87, 0xfd, 0x3e, 0x35, 0x9c, 0x5c, 0x3f, 0x8a, 0x2b, 0xc7, - 0xa8, 0x5b, 0x35, 0x6a, 0x34, 0xc6, 0x51, 0x8e, 0xb4, 0x64, 0x8c, 0x71, 0x9a, 0x81, 0x3b, 0x18, - 0xd0, 0x51, 0xad, 0x9f, 0x80, 0x35, 0xfa, 0xf6, 0x43, 0xc3, 0xa9, 0xd4, 0x6e, 0x0b, 0xbc, 0xb6, - 0x66, 0x22, 0xaf, 0x49, 0x92, 0x41, 0x5d, 0xdb, 0x0c, 0x1c, 0x2f, 0x90, 0x14, 0x93, 0x46, 0x0b, - 0xb5, 0x2a, 0x09, 0x09, 0x6a, 0x65, 0xb7, 0xd3, 0xed, 0xbe, 0xe7, 0xf6, 0x8e, 0x3a, 0xc4, 0x4a, - 0xae, 0xbf, 0xa9, 0xbd, 0x99, 0x50, 0x36, 0xdd, 0xa1, 0x39, 0xc5, 0x9a, 0x61, 0x10, 0xd8, 0x90, - 0x4d, 0x83, 0x36, 0xcb, 0x61, 0xd3, 0xdc, 0xae, 0x3e, 0xf9, 0x74, 0x75, 0xe6, 0xe3, 0xcf, 0x56, - 0x8d, 0x27, 0x9f, 0xad, 0x1a, 0x7f, 0xff, 0x6c, 0xd5, 0x78, 0x6f, 0x53, 0xf9, 0x7f, 0x82, 0x9e, - 0x13, 0x78, 0x9d, 0x47, 0xae, 0xd7, 0x39, 0xee, 0xf4, 0x65, 0xa3, 0x4f, 0x6e, 0x0d, 0x4e, 0x8f, - 0x6f, 0x0d, 0x8e, 0x6e, 0x45, 0x7b, 0xc4, 0xd1, 0x1c, 0xfb, 0x67, 0x82, 0xcd, 0xff, 0x06, 0x00, - 0x00, 0xff, 0xff, 0x2f, 0x36, 0xcd, 0xac, 0xab, 0x30, 0x00, 0x00, + // 3404 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x3b, 0xcd, 0x6f, 0x1b, 0xc7, + 0xf5, 0x5a, 0x92, 0xe2, 0xc7, 0xa3, 0x28, 0xaf, 0x47, 0xb2, 0xcd, 0x30, 0xfe, 0xc9, 0xfa, 0x31, + 0x6e, 0x6a, 0x2b, 0x09, 0x8d, 0x4a, 0x48, 0x90, 0x34, 0x8a, 0x0d, 0x8a, 0xa4, 0x2d, 0x46, 0x34, + 0xa5, 0x0c, 0x57, 0x09, 0x10, 0x20, 0x50, 0x57, 0xe4, 0x58, 0x62, 0x45, 0x72, 0xd9, 0xdd, 0xa5, + 0x63, 0xf7, 0x5c, 0x14, 0x28, 0x7a, 0x2c, 0x50, 0x04, 0x45, 0xd1, 0xfe, 0x09, 0x45, 0x81, 0x1e, + 0x82, 0x02, 0x45, 0x6f, 0x45, 0x8e, 0xfe, 0x0b, 0x82, 0x26, 0xc7, 0xb6, 0x40, 0xd1, 0x1c, 0x0a, + 0xf4, 0x50, 0xa0, 0x98, 0xaf, 0xdd, 0x19, 0x72, 0x49, 0x49, 0x89, 0xd3, 0x14, 0x6d, 0x4e, 0xe2, + 0xbc, 0x8f, 0xd9, 0xb7, 0xef, 0x7b, 0xde, 0xac, 0xc0, 0xec, 0x39, 0x47, 0x1e, 0x71, 0x1f, 0x76, + 0xdb, 0xa4, 0x34, 0x74, 0x1d, 0xdf, 0x41, 0x10, 0x42, 0x0a, 0x2f, 0x1d, 0x75, 0xfd, 0xe3, 0xd1, + 0x61, 0xa9, 0xed, 0xf4, 0x6f, 0x1d, 0x39, 0x47, 0xce, 0x2d, 0x46, 0x72, 0x38, 0x7a, 0xc0, 0x56, + 0x6c, 0xc1, 0x7e, 0x71, 0xd6, 0xc2, 0x62, 0x9f, 0xf8, 0x76, 0xc7, 0xf6, 0x6d, 0xbe, 0x2e, 0xfe, + 0x36, 0x01, 0xa9, 0x4a, 0xb3, 0xe5, 0x3b, 0x2e, 0x41, 0x08, 0x12, 0xfb, 0xfb, 0xf5, 0x6a, 0xde, + 0x58, 0x35, 0x6e, 0x64, 0x30, 0xfb, 0x8d, 0x9e, 0x87, 0xc5, 0x16, 0x7f, 0x52, 0xb9, 0xd3, 0x71, + 0x89, 0xe7, 0xe5, 0x63, 0x0c, 0x3b, 0x06, 0x45, 0x2b, 0x00, 0xad, 0xb7, 0x1a, 0x92, 0x26, 0xce, + 0x68, 0x14, 0x08, 0x2a, 0x01, 0x6a, 0x38, 0xed, 0x93, 0xb1, 0xbd, 0x12, 0x8c, 0x2e, 0x02, 0x43, + 0xf7, 0xab, 0xf8, 0x3d, 0x49, 0x37, 0xcf, 0xf7, 0x0b, 0x21, 0xe8, 0x3a, 0x24, 0xb0, 0xd3, 0x23, + 0xf9, 0xe4, 0xaa, 0x71, 0x63, 0x71, 0xdd, 0x2c, 0x05, 0xaf, 0x55, 0x69, 0x52, 0x38, 0x66, 0x58, + 0xfa, 0x46, 0x56, 0xb7, 0x7d, 0x92, 0x4f, 0xad, 0x1a, 0x37, 0x12, 0x98, 0xfd, 0x46, 0x2f, 0xc0, + 0x7c, 0xcb, 0xb7, 0x7d, 0x92, 0x4f, 0x33, 0xd6, 0x4b, 0x25, 0x45, 0xbd, 0x4d, 0xa7, 0x43, 0x18, + 0x12, 0x73, 0x1a, 0xf4, 0x06, 0x24, 0x1b, 0xf6, 0x21, 0xe9, 0x79, 0xf9, 0xcc, 0x6a, 0xfc, 0x46, + 0x76, 0xfd, 0x9a, 0x4a, 0x2d, 0xf4, 0x56, 0xe2, 0x14, 0xb5, 0x81, 0xef, 0x3e, 0xde, 0x4a, 0x7c, + 0xf4, 0xf1, 0xb5, 0x39, 0x2c, 0x98, 0xd0, 0xb7, 0x20, 0xf3, 0x8e, 0xe3, 0x9e, 0xf0, 0xe7, 0x01, + 0x7b, 0xde, 0x52, 0x28, 0x6a, 0x80, 0xc2, 0x21, 0x15, 0x2a, 0xc2, 0xc2, 0x5b, 0x23, 0xe2, 0x3e, + 0x96, 0xaf, 0x9e, 0x65, 0xaf, 0xae, 0xc1, 0xd0, 0x2b, 0x00, 0x15, 0x67, 0xf0, 0xa0, 0x7b, 0x54, + 0xb5, 0x7d, 0x3b, 0xbf, 0xb0, 0x6a, 0xdc, 0xc8, 0xae, 0x5f, 0xd6, 0x24, 0x0b, 0xb0, 0x58, 0xa1, + 0x2c, 0x34, 0x21, 0xab, 0xc8, 0x8a, 0x4c, 0x88, 0x9f, 0x90, 0xc7, 0xc2, 0xdc, 0xf4, 0x27, 0xba, + 0x09, 0xf3, 0x0f, 0xed, 0xde, 0x88, 0x30, 0x23, 0x67, 0x55, 0x59, 0x19, 0x5f, 0xa3, 0xeb, 0xf9, + 0x98, 0x53, 0x7c, 0x3b, 0xf6, 0xaa, 0x51, 0xfc, 0x47, 0x0c, 0x52, 0xd6, 0x53, 0x70, 0x1e, 0x69, + 0xa6, 0x78, 0x94, 0x99, 0x12, 0x67, 0x30, 0xd3, 0xcb, 0x90, 0x6c, 0x1d, 0xdb, 0x6e, 0x87, 0x7a, + 0x0a, 0x35, 0xd3, 0x15, 0x95, 0xda, 0x6a, 0x32, 0x5c, 0x7d, 0xf0, 0xc0, 0x91, 0xe6, 0xe1, 0xc4, + 0x68, 0x1d, 0x96, 0x1b, 0xce, 0x91, 0x6f, 0x77, 0x7b, 0x54, 0x20, 0xe2, 0x4a, 0x29, 0x93, 0x4c, + 0xca, 0x48, 0xdc, 0x14, 0x47, 0x4e, 0x9d, 0xd1, 0x91, 0xd3, 0x13, 0x8e, 0xac, 0xdb, 0x32, 0x73, + 0x56, 0x5b, 0x16, 0xff, 0x69, 0x40, 0xba, 0xe1, 0x1c, 0xfd, 0x07, 0x28, 0x7f, 0x13, 0xd2, 0x98, + 0x0c, 0x7b, 0xdd, 0xb6, 0x2d, 0xd5, 0x5f, 0x50, 0xe9, 0x1b, 0xce, 0x91, 0x40, 0x2b, 0x16, 0x08, + 0x38, 0xc6, 0xde, 0x3f, 0x79, 0xe6, 0xf7, 0xff, 0xab, 0x01, 0x0b, 0xf4, 0xfd, 0xa5, 0x69, 0x51, + 0x1e, 0x52, 0x7c, 0xc1, 0xd5, 0x90, 0xc0, 0x72, 0x89, 0xb6, 0x14, 0x01, 0x63, 0x4c, 0xc0, 0xe7, + 0xc7, 0x04, 0x0c, 0x76, 0x29, 0x49, 0x42, 0x16, 0x21, 0x8a, 0x98, 0xcb, 0x30, 0x5f, 0x1b, 0x3a, + 0xed, 0x63, 0xa1, 0x26, 0xbe, 0x40, 0x05, 0x48, 0x37, 0x88, 0xdd, 0x21, 0x6e, 0xbd, 0xca, 0x54, + 0x95, 0xc0, 0xc1, 0x9a, 0xe9, 0x95, 0xb8, 0x7d, 0x96, 0xbb, 0xa8, 0x5e, 0x89, 0xdb, 0x2f, 0xbc, + 0x0e, 0x39, 0xed, 0x01, 0x6a, 0x08, 0x26, 0x78, 0x08, 0x2e, 0xab, 0x21, 0x98, 0x51, 0xa3, 0xed, + 0x21, 0x2c, 0xea, 0xba, 0x44, 0x77, 0x75, 0x15, 0xb0, 0x6d, 0xb2, 0xeb, 0xf9, 0x69, 0x2f, 0xb7, + 0x95, 0xa6, 0xba, 0x7f, 0xf2, 0xf1, 0x35, 0x03, 0xeb, 0xaa, 0xbb, 0x0a, 0x19, 0xb9, 0x6d, 0x95, + 0x3d, 0x37, 0x81, 0x43, 0x40, 0xf1, 0xa7, 0x71, 0x30, 0x45, 0xaa, 0xdb, 0x26, 0xb6, 0xeb, 0x1f, + 0x12, 0xdb, 0xff, 0x2f, 0xac, 0x15, 0x25, 0x40, 0x96, 0xed, 0xc9, 0xbd, 0x2b, 0x2e, 0xb1, 0x7d, + 0xd2, 0x61, 0x81, 0x9d, 0xc6, 0x11, 0x98, 0x89, 0x44, 0x9d, 0x8e, 0x48, 0xd4, 0xd7, 0x21, 0x57, + 0x1f, 0x74, 0xfd, 0xb0, 0x06, 0x64, 0x18, 0x91, 0x0e, 0x1c, 0x0b, 0x01, 0x38, 0x73, 0x08, 0xbc, + 0x02, 0x0b, 0x95, 0x66, 0xb9, 0xd7, 0x73, 0xda, 0xb6, 0x4f, 0xea, 0xd5, 0x88, 0x7c, 0xbe, 0x0c, + 0xf3, 0x5b, 0xb6, 0xdf, 0x3e, 0x16, 0x46, 0xe5, 0x8b, 0xe2, 0xef, 0x62, 0x70, 0x51, 0xa6, 0x8e, + 0xd9, 0x16, 0x5d, 0x85, 0x2c, 0xb6, 0x1f, 0xf8, 0xba, 0x39, 0x55, 0x50, 0x84, 0xcd, 0xe3, 0x91, + 0x36, 0xbf, 0x0e, 0xb9, 0x7b, 0x8e, 0xe7, 0x75, 0x87, 0xba, 0x39, 0x75, 0xe0, 0x17, 0x4c, 0x25, + 0xd1, 0x16, 0x4c, 0x4e, 0xb5, 0xa0, 0xae, 0xf7, 0xd4, 0x99, 0xf5, 0x5e, 0x83, 0xac, 0x52, 0x53, + 0x66, 0x24, 0x9e, 0xd9, 0x71, 0xf5, 0xb7, 0x18, 0x98, 0xd6, 0xd3, 0x8c, 0xab, 0xb0, 0x0a, 0xc6, + 0xcf, 0x53, 0x05, 0xa3, 0xd5, 0x96, 0x98, 0xaa, 0xb6, 0x69, 0x55, 0x73, 0xfe, 0xdc, 0x55, 0x33, + 0x79, 0xc6, 0x90, 0x4e, 0x9d, 0x52, 0x35, 0xd3, 0x67, 0x36, 0xdd, 0x8f, 0x62, 0x90, 0xc6, 0xad, + 0xfb, 0x3c, 0xee, 0x4c, 0x88, 0x5b, 0x9e, 0x23, 0x93, 0xaf, 0xe5, 0x39, 0x34, 0x5e, 0xea, 0x83, + 0x0e, 0x79, 0x24, 0xe3, 0x85, 0x2d, 0xa8, 0xef, 0x36, 0x88, 0xed, 0x91, 0x6d, 0xa7, 0xc7, 0x53, + 0x3d, 0xaf, 0x01, 0x3a, 0x90, 0xe6, 0x03, 0xcb, 0x1d, 0x0d, 0x68, 0x2c, 0x76, 0x1a, 0xde, 0x40, + 0xd4, 0x03, 0x0d, 0x86, 0xde, 0x84, 0x05, 0xce, 0xd4, 0xf5, 0x7c, 0xc7, 0x7d, 0x2c, 0x7c, 0x5c, + 0xab, 0x46, 0x52, 0xba, 0x92, 0x4a, 0xc8, 0xab, 0x91, 0xc6, 0x5b, 0xb8, 0x03, 0x17, 0x27, 0x48, + 0x4e, 0xab, 0x27, 0x09, 0xb5, 0x9e, 0xbc, 0x07, 0x19, 0x16, 0x50, 0x6d, 0xc7, 0xed, 0x50, 0x46, + 0x2a, 0xb4, 0x60, 0xa4, 0xb2, 0xae, 0x41, 0xc2, 0x7a, 0x3c, 0xe4, 0x7c, 0x8b, 0xba, 0x72, 0x39, + 0x0f, 0xc5, 0x62, 0x46, 0x43, 0xbd, 0x96, 0x19, 0x82, 0x2a, 0x66, 0x01, 0xb3, 0xdf, 0xc5, 0x0f, + 0x0c, 0x00, 0xb6, 0xff, 0xf7, 0x46, 0xc4, 0x63, 0x8e, 0xdd, 0xb4, 0xfb, 0x44, 0x3a, 0x36, 0xfd, + 0xad, 0x46, 0x4e, 0x4c, 0x8f, 0x1c, 0x21, 0x4e, 0x3c, 0x14, 0x27, 0x0f, 0xa9, 0xfb, 0xf6, 0xa3, + 0x56, 0xf7, 0xfb, 0x44, 0x68, 0x56, 0x2e, 0x69, 0x94, 0x49, 0xe7, 0xae, 0x8a, 0x6a, 0x1b, 0x02, + 0x58, 0x19, 0x6e, 0xd6, 0xab, 0xcc, 0xd7, 0x68, 0x19, 0x6e, 0xd6, 0xab, 0xc5, 0x22, 0x80, 0xe5, + 0x39, 0x52, 0xb2, 0x65, 0x98, 0xaf, 0x38, 0xa3, 0x81, 0x2f, 0x5e, 0x9e, 0x2f, 0x8a, 0x7f, 0x36, + 0x68, 0x76, 0x65, 0xd1, 0xc9, 0x7a, 0xdf, 0xc8, 0xc8, 0xdc, 0x80, 0xcc, 0xee, 0x90, 0xb8, 0xb6, + 0xdf, 0x75, 0x06, 0x42, 0x51, 0x97, 0xf4, 0x13, 0x02, 0xe3, 0xdd, 0x1d, 0xe2, 0x90, 0x0e, 0x6d, + 0x05, 0x67, 0x0a, 0x1e, 0xa6, 0xd7, 0x23, 0xce, 0x14, 0x8c, 0x60, 0xfa, 0xc1, 0xe2, 0xa9, 0x77, + 0xf2, 0x0d, 0xc8, 0x56, 0x9a, 0x61, 0x45, 0x8a, 0x7a, 0xd7, 0x9b, 0xb2, 0x27, 0x8c, 0x4d, 0x3f, + 0xc7, 0x70, 0x8a, 0xe2, 0x27, 0x42, 0x77, 0xb6, 0x3f, 0x43, 0x77, 0x67, 0xdf, 0xef, 0x74, 0x8d, + 0xc9, 0x07, 0xfd, 0x1b, 0x35, 0xf6, 0x87, 0x79, 0x48, 0x49, 0x0f, 0x62, 0x79, 0x9e, 0xfd, 0x0c, + 0x6a, 0x40, 0x08, 0x40, 0x25, 0x48, 0xde, 0x27, 0xfe, 0xb1, 0xd3, 0x89, 0x0a, 0x25, 0x8e, 0x61, + 0xa1, 0x24, 0xa8, 0xd0, 0xa6, 0x1a, 0x37, 0x2c, 0x04, 0xc6, 0x72, 0x5b, 0x88, 0x15, 0xef, 0xa8, + 0xc6, 0x59, 0x99, 0xf5, 0x84, 0x41, 0x41, 0x61, 0xc1, 0x92, 0x5d, 0xff, 0xbf, 0xf1, 0x9e, 0x50, + 0xab, 0x3a, 0x58, 0x63, 0x41, 0xb7, 0xa9, 0x33, 0x84, 0x3b, 0xcc, 0xb3, 0x1d, 0xae, 0x46, 0x78, + 0x69, 0xb8, 0x81, 0xca, 0x40, 0xf9, 0x2d, 0x85, 0x3f, 0x39, 0xc9, 0x6f, 0x4d, 0xf0, 0x2b, 0x0c, + 0x34, 0xb9, 0x87, 0xe1, 0x19, 0x55, 0x97, 0x43, 0x2c, 0x56, 0x03, 0x79, 0x53, 0xef, 0x87, 0x44, + 0x59, 0xc8, 0xeb, 0x82, 0x87, 0x78, 0xac, 0x77, 0x4f, 0x9b, 0x7a, 0xbc, 0x8b, 0xa3, 0x58, 0x7e, + 0x5a, 0x70, 0x62, 0x3d, 0x3b, 0xbc, 0xa6, 0x05, 0x90, 0x68, 0xe2, 0xae, 0xe8, 0xcc, 0xa1, 0x5f, + 0x6b, 0xc1, 0xb6, 0xa9, 0x07, 0x0b, 0x3b, 0xf1, 0x47, 0x3c, 0x58, 0xe2, 0xb1, 0x1e, 0x5a, 0x77, + 0x20, 0x57, 0x25, 0x3d, 0xe2, 0x13, 0x21, 0x8e, 0x18, 0x07, 0x3c, 0xa3, 0xb2, 0x6b, 0x04, 0x58, + 0xa7, 0x2f, 0xb6, 0x20, 0xcb, 0xdc, 0xc7, 0x1b, 0x3a, 0x03, 0x8f, 0xcc, 0xe8, 0x66, 0x44, 0x4e, + 0x8e, 0x69, 0x39, 0xb9, 0x61, 0x7b, 0x7e, 0x98, 0xa9, 0xe5, 0xb2, 0x58, 0x02, 0xa4, 0x28, 0x5a, + 0xd9, 0xfb, 0x6e, 0xd7, 0x55, 0xa2, 0x44, 0x2e, 0x8b, 0x7f, 0x4f, 0xd0, 0xce, 0x4f, 0x90, 0x3d, + 0xdd, 0x70, 0xba, 0x0a, 0x99, 0x9a, 0xeb, 0x3a, 0x6e, 0xc5, 0xe9, 0x10, 0x26, 0x66, 0x0e, 0x87, + 0x00, 0x5a, 0xb5, 0xd9, 0xe2, 0x3e, 0xf1, 0x3c, 0xfb, 0x88, 0x88, 0xb6, 0x54, 0x83, 0xd1, 0x66, + 0xa4, 0xee, 0x6d, 0x97, 0x77, 0x08, 0x19, 0x12, 0x97, 0x85, 0x43, 0x1a, 0x2b, 0x10, 0x74, 0x47, + 0xd3, 0xa0, 0xf0, 0xf7, 0x2b, 0x13, 0x11, 0xcb, 0xd1, 0x22, 0x64, 0x35, 0x9d, 0x53, 0x0f, 0x70, + 0xfa, 0x7d, 0x7b, 0xd0, 0xe1, 0xdd, 0x7a, 0x2a, 0xc2, 0x03, 0x14, 0x3c, 0xd6, 0xa8, 0xa9, 0xeb, + 0xb1, 0x20, 0x10, 0x8f, 0x4f, 0x4f, 0x3e, 0x5e, 0x41, 0x63, 0x95, 0x16, 0x6d, 0xc1, 0x62, 0xa5, + 0x37, 0xf2, 0x7c, 0xe2, 0x56, 0x09, 0x6d, 0xda, 0x3c, 0xe1, 0xf5, 0x5a, 0xd7, 0xad, 0x53, 0xe0, + 0x31, 0x0e, 0x74, 0x1b, 0x32, 0xe1, 0x09, 0x94, 0xfb, 0xfd, 0xaa, 0xca, 0x1e, 0x20, 0xd9, 0xc9, + 0x08, 0x13, 0x6f, 0xd4, 0xf3, 0x71, 0xc8, 0x82, 0x6e, 0x03, 0x28, 0x31, 0xcb, 0x9d, 0x7f, 0x45, + 0xdd, 0x60, 0xd2, 0x91, 0x30, 0x8c, 0xc5, 0xed, 0x31, 0x69, 0x9f, 0x10, 0x97, 0x87, 0xde, 0x42, + 0x84, 0xf2, 0x14, 0x3c, 0xd6, 0xa8, 0x8b, 0x6f, 0xb2, 0xa3, 0x10, 0x6f, 0x68, 0x02, 0xb5, 0xbc, + 0x4c, 0x53, 0x3b, 0x85, 0x78, 0x79, 0x83, 0x15, 0x9c, 0x4b, 0x13, 0xc6, 0xa4, 0x58, 0x61, 0x4a, + 0x49, 0x5b, 0x7c, 0x4e, 0x33, 0x04, 0xed, 0x2b, 0xde, 0x66, 0x05, 0x45, 0xf4, 0x15, 0x6c, 0x51, + 0xbc, 0x07, 0x39, 0xda, 0x53, 0x5b, 0xf6, 0x61, 0x8f, 0xec, 0x7b, 0xc4, 0x45, 0x05, 0x48, 0xd3, + 0xbf, 0x83, 0xb0, 0x39, 0x0a, 0xd6, 0x14, 0xb7, 0x67, 0x7b, 0xde, 0xfb, 0x8e, 0xdb, 0x11, 0x3d, + 0x7f, 0xb0, 0x2e, 0xfe, 0xd8, 0xa0, 0x52, 0xb2, 0xc3, 0x44, 0x64, 0x7d, 0x9d, 0xde, 0x5c, 0x69, + 0xc7, 0x92, 0xf8, 0xd8, 0xb1, 0x24, 0x9c, 0x74, 0x24, 0xd4, 0x49, 0xc7, 0x0a, 0x2b, 0x4a, 0x7a, + 0x97, 0xa5, 0x40, 0x8a, 0x3f, 0x8b, 0x51, 0x1f, 0xa6, 0x7d, 0x76, 0xe5, 0xd8, 0x1e, 0x1c, 0x11, + 0xb4, 0x11, 0x48, 0x27, 0xc6, 0x12, 0x4b, 0x7a, 0x07, 0xc9, 0x50, 0xa1, 0x06, 0xf9, 0x7b, 0x6c, + 0x02, 0x70, 0x76, 0xa5, 0xf3, 0xbc, 0x3a, 0xd9, 0xd6, 0x87, 0x34, 0x58, 0xa1, 0x47, 0x16, 0x2c, + 0xd2, 0x83, 0x75, 0xd7, 0xee, 0xdd, 0x27, 0xfd, 0x43, 0xe2, 0xca, 0x76, 0xe1, 0xc5, 0x69, 0x3b, + 0x94, 0x74, 0x72, 0xde, 0x65, 0x8f, 0xed, 0x51, 0x28, 0xc3, 0x52, 0x04, 0xd9, 0xb9, 0x26, 0x37, + 0x37, 0x21, 0xd7, 0x3a, 0x1e, 0xf9, 0x1d, 0xe7, 0xfd, 0x01, 0x9f, 0xd7, 0x51, 0xdb, 0xd0, 0x1f, + 0x81, 0xc9, 0xe4, 0xb2, 0xf8, 0x93, 0x38, 0x5c, 0x68, 0xb5, 0x8f, 0x49, 0x67, 0xd4, 0x23, 0x22, + 0xca, 0x23, 0xad, 0x7b, 0x1d, 0x72, 0x5b, 0x8e, 0xe3, 0x7b, 0xbe, 0x6b, 0x0f, 0x87, 0xdd, 0xc1, + 0x11, 0x7b, 0x68, 0x1a, 0xeb, 0x40, 0x9a, 0x1a, 0xc4, 0xc1, 0x8a, 0x29, 0x34, 0xce, 0x14, 0xaa, + 0xa5, 0x06, 0x05, 0x8d, 0x55, 0x5a, 0x9e, 0x93, 0x42, 0x55, 0x89, 0x3e, 0x22, 0x3f, 0x4d, 0x95, + 0x58, 0xb7, 0xfe, 0x9d, 0xb1, 0x37, 0x16, 0x4d, 0xc4, 0x33, 0x7a, 0x62, 0x50, 0x08, 0xf0, 0x98, + 0x86, 0x76, 0xe0, 0x22, 0x3f, 0x6f, 0x2a, 0x07, 0x50, 0x91, 0x59, 0xb5, 0x5e, 0x66, 0x82, 0x08, + 0x4f, 0xf2, 0x4d, 0xd6, 0xc8, 0xd4, 0x39, 0x6b, 0x64, 0x2f, 0x42, 0x1a, 0xb4, 0x01, 0x09, 0x1a, + 0xa8, 0xc2, 0xbd, 0xb5, 0xcd, 0xb4, 0x08, 0x17, 0x4e, 0xce, 0x88, 0xd9, 0x29, 0xd1, 0xf6, 0x4e, + 0xe8, 0x09, 0xe9, 0xd0, 0xf6, 0xa4, 0xaf, 0x68, 0x30, 0xea, 0x2e, 0xda, 0xe3, 0x67, 0xb8, 0x8b, + 0xad, 0x57, 0x8e, 0x60, 0xe8, 0x68, 0x84, 0x43, 0x47, 0xf4, 0x06, 0xa4, 0x05, 0x8d, 0x1c, 0x7f, + 0x3e, 0xab, 0x99, 0x41, 0xf7, 0x36, 0x39, 0x55, 0x91, 0x2c, 0xc5, 0x5f, 0x27, 0x68, 0x6b, 0xc3, + 0x1f, 0x48, 0xf3, 0xb5, 0x9c, 0x17, 0x1b, 0xca, 0xbc, 0xf8, 0x7f, 0x6b, 0xf2, 0x57, 0x0e, 0x8e, + 0x23, 0x69, 0xa6, 0xce, 0xe7, 0x22, 0x7a, 0x44, 0x36, 0x4c, 0x3e, 0xe3, 0xc5, 0x50, 0xe6, 0x73, + 0x5d, 0x0c, 0xc1, 0xa9, 0x17, 0x43, 0xd9, 0xaf, 0xec, 0x62, 0xe8, 0xe7, 0x06, 0xbf, 0x55, 0x14, + 0x57, 0x68, 0x4c, 0x23, 0xb2, 0x96, 0x5e, 0x8b, 0x68, 0x6c, 0x4b, 0x9c, 0x42, 0xd3, 0x14, 0x07, + 0x15, 0x30, 0x64, 0x15, 0x64, 0x84, 0x68, 0x2f, 0xe9, 0xa2, 0x5d, 0x99, 0x62, 0x0c, 0x55, 0xbc, + 0xbf, 0xc4, 0xd8, 0x04, 0xef, 0xa9, 0xb8, 0xf4, 0xd7, 0x43, 0xb7, 0x53, 0x87, 0x6e, 0xd4, 0x1b, + 0xac, 0xb3, 0x78, 0x83, 0xf5, 0xe5, 0x7a, 0x83, 0x15, 0xed, 0x0d, 0x1f, 0x1a, 0xe3, 0x5d, 0x30, + 0x7a, 0x19, 0xd2, 0x82, 0x58, 0xca, 0xb9, 0x14, 0xb1, 0x91, 0x4c, 0x95, 0x92, 0x94, 0xb2, 0x55, + 0x24, 0x5b, 0x6c, 0x92, 0xad, 0xa2, 0xb3, 0x49, 0x52, 0xf4, 0x2a, 0x1b, 0xc4, 0x09, 0x3e, 0xee, + 0x45, 0xcb, 0x51, 0xe7, 0x75, 0xc1, 0x18, 0x12, 0x17, 0x7f, 0x68, 0x40, 0x56, 0x88, 0xce, 0x1c, + 0xf9, 0x35, 0x26, 0x37, 0x77, 0x47, 0x43, 0xb8, 0x63, 0x10, 0xa9, 0x02, 0xa3, 0xf5, 0xae, 0x01, + 0x39, 0xda, 0xe4, 0x42, 0x70, 0x5e, 0x2e, 0x7c, 0x5e, 0x89, 0x72, 0x81, 0xd2, 0x98, 0x43, 0x86, + 0xe2, 0xef, 0x63, 0x70, 0x49, 0x74, 0x49, 0x42, 0x1e, 0x79, 0x28, 0x7f, 0x1e, 0x16, 0x9b, 0xa3, + 0xfe, 0xee, 0x83, 0x70, 0x73, 0x1e, 0x65, 0x63, 0x50, 0xda, 0xd0, 0x30, 0x48, 0x20, 0x3f, 0x6f, + 0x5a, 0x75, 0x20, 0x5a, 0x03, 0x53, 0xf2, 0x05, 0x17, 0x05, 0xbc, 0x83, 0x9d, 0x80, 0xa3, 0xcb, + 0x90, 0x6c, 0x92, 0x47, 0x7e, 0x70, 0x35, 0x27, 0x56, 0xc8, 0x82, 0x2c, 0xff, 0xb5, 0xf5, 0x78, + 0x87, 0xc8, 0x19, 0xec, 0xba, 0xaa, 0xf0, 0xc8, 0x37, 0x29, 0x29, 0x4c, 0xbc, 0x53, 0x54, 0xb7, + 0x29, 0xdc, 0x06, 0x73, 0x9c, 0x20, 0xfa, 0x42, 0x66, 0xca, 0x34, 0xf6, 0xc3, 0x18, 0xbf, 0xcc, + 0x9b, 0x99, 0x94, 0xbe, 0xbe, 0x8f, 0x89, 0xca, 0x2f, 0xbf, 0x89, 0x89, 0xab, 0x70, 0x9a, 0x60, + 0x6e, 0x07, 0xf9, 0x98, 0x07, 0xc0, 0xea, 0x44, 0x24, 0xb1, 0x0c, 0xc3, 0x48, 0xf4, 0x0c, 0xc3, + 0x3d, 0xec, 0x76, 0x90, 0xa0, 0x62, 0xb3, 0xf8, 0xa7, 0x66, 0xa8, 0x16, 0x64, 0x95, 0xcd, 0x23, + 0x8e, 0x09, 0x25, 0x3d, 0x43, 0x4d, 0xbd, 0xad, 0x55, 0x9c, 0x83, 0x6d, 0x3a, 0x33, 0xed, 0x9d, + 0xb6, 0x69, 0x54, 0xde, 0xfb, 0x2c, 0xa1, 0x9f, 0x9c, 0x23, 0x3d, 0xee, 0x8e, 0x96, 0x60, 0x22, + 0x6b, 0x6c, 0x88, 0x96, 0xb3, 0x0d, 0x35, 0x25, 0x6d, 0x04, 0xb9, 0x5f, 0x8c, 0x32, 0x97, 0x22, + 0x32, 0xbe, 0x3c, 0x07, 0xca, 0x2a, 0xf1, 0x4a, 0x68, 0x50, 0x71, 0xf0, 0x58, 0x8e, 0x32, 0x83, + 0xf4, 0xb8, 0xc0, 0xf8, 0x1b, 0x41, 0xdb, 0x21, 0x0e, 0x1c, 0x4b, 0x11, 0xcd, 0x86, 0x7c, 0x98, + 0x6c, 0x50, 0x6e, 0xc9, 0x41, 0x34, 0xef, 0x12, 0xb5, 0x46, 0x5e, 0xce, 0x78, 0xb4, 0x71, 0x74, + 0x53, 0xf8, 0xb5, 0x68, 0x9c, 0xc5, 0xdc, 0x21, 0xc5, 0xb8, 0x57, 0xc6, 0x8f, 0x01, 0x3a, 0x15, + 0x8e, 0xe0, 0x44, 0xb5, 0xb1, 0x91, 0x80, 0x28, 0xad, 0xa7, 0x9e, 0x28, 0xc6, 0x06, 0x09, 0x32, + 0xdf, 0x75, 0x58, 0x43, 0x29, 0xf3, 0x5d, 0x07, 0xed, 0xe8, 0xf9, 0x0e, 0x98, 0x5b, 0xdf, 0x9c, + 0x36, 0x1f, 0xf9, 0x92, 0xd3, 0xdc, 0x67, 0x29, 0x30, 0xa5, 0x52, 0x83, 0x8b, 0xb8, 0xe0, 0xda, + 0xcd, 0x50, 0xaf, 0xdd, 0xa4, 0x3b, 0xc6, 0x14, 0x77, 0x0c, 0x73, 0x7a, 0x5c, 0xcb, 0xe9, 0xbb, + 0xfa, 0x3b, 0x26, 0xd8, 0x3b, 0xbe, 0x14, 0x65, 0xc9, 0xe0, 0x7e, 0x6d, 0xe6, 0x7b, 0x46, 0x7d, + 0xbd, 0xf1, 0xd5, 0x3b, 0x4a, 0x07, 0xcc, 0xb1, 0xd3, 0x9a, 0x3c, 0x82, 0xac, 0xcf, 0x7c, 0xd5, + 0x71, 0x26, 0x35, 0x6f, 0x4d, 0xec, 0x88, 0xea, 0x6a, 0x27, 0xc0, 0x3f, 0x7b, 0x7b, 0x61, 0xe6, + 0xf6, 0x01, 0x35, 0xd7, 0x63, 0xc8, 0xad, 0xc6, 0x23, 0x9c, 0x39, 0x1e, 0x95, 0x8c, 0x91, 0xfd, + 0x5c, 0x19, 0x63, 0xe1, 0x1c, 0x19, 0x63, 0x2c, 0xbf, 0xe5, 0xce, 0x9d, 0xdf, 0x26, 0x82, 0x77, + 0xf1, 0xf3, 0x04, 0xef, 0x17, 0x8d, 0xab, 0xc2, 0x7b, 0x70, 0x29, 0xd2, 0xca, 0xe7, 0xac, 0x15, + 0xda, 0x98, 0x59, 0xd9, 0x7e, 0x93, 0x7d, 0x7b, 0x34, 0xa5, 0xb0, 0x9d, 0x1a, 0xf4, 0x75, 0xc8, + 0xaa, 0x9f, 0x2d, 0x7d, 0x81, 0x6f, 0x1c, 0x8a, 0xbf, 0x88, 0xc1, 0x72, 0xd4, 0x44, 0x79, 0xc6, + 0xbd, 0xc5, 0xde, 0xc4, 0xe7, 0x5f, 0xa5, 0xd3, 0xe6, 0xd3, 0xfa, 0x67, 0x60, 0x13, 0x8d, 0xcd, + 0xd3, 0xf9, 0x18, 0xcc, 0x3a, 0xfd, 0x63, 0xb0, 0x59, 0xa7, 0x19, 0x45, 0xa3, 0xaa, 0xae, 0x7f, + 0x65, 0x00, 0x6c, 0xd9, 0xed, 0x93, 0xd1, 0x90, 0xf6, 0x46, 0x4a, 0xc2, 0x34, 0xb4, 0x84, 0x59, + 0xd7, 0x13, 0x26, 0xd7, 0xcb, 0x37, 0xd5, 0xfd, 0xc3, 0x4d, 0xbe, 0xe4, 0x92, 0xf0, 0x03, 0x43, + 0xf6, 0x7d, 0x75, 0x9f, 0xf4, 0x23, 0x3f, 0x14, 0x28, 0xc2, 0x42, 0x65, 0xe4, 0xba, 0x64, 0xe0, + 0xbf, 0xad, 0x4c, 0x58, 0x35, 0x18, 0xa5, 0xa9, 0x92, 0x07, 0xf6, 0xa8, 0x27, 0x68, 0x78, 0xdf, + 0xab, 0xc1, 0xa8, 0x89, 0xea, 0x03, 0x9f, 0xb8, 0x03, 0xbb, 0x27, 0x1a, 0xde, 0x60, 0x5d, 0xfc, + 0xa5, 0xa1, 0xb6, 0x9f, 0xe8, 0x0d, 0x48, 0x55, 0x9c, 0x81, 0x4f, 0xd8, 0x77, 0x01, 0x93, 0x53, + 0x9e, 0x80, 0xb0, 0x24, 0xa8, 0xb8, 0x62, 0x24, 0x4f, 0x01, 0xb3, 0xf1, 0x69, 0x80, 0x88, 0x50, + 0xc8, 0x8b, 0xba, 0x6d, 0x23, 0x1a, 0x5d, 0xaa, 0x0e, 0x45, 0x51, 0x6b, 0xdf, 0x01, 0xd8, 0x1f, + 0x76, 0x6c, 0x9f, 0x0f, 0x68, 0xaf, 0xc0, 0x92, 0xf6, 0x01, 0x0a, 0x47, 0x99, 0x73, 0xe8, 0x12, + 0x5c, 0x94, 0x1f, 0x9d, 0x34, 0x5a, 0x4d, 0x01, 0x36, 0xd0, 0x12, 0x5c, 0xa0, 0x99, 0x86, 0x89, + 0x23, 0x80, 0x31, 0x94, 0x83, 0x8c, 0xd5, 0xda, 0x15, 0xcb, 0xf8, 0x5a, 0x09, 0x32, 0xc1, 0xe7, + 0x9d, 0xe8, 0x02, 0x64, 0x9b, 0x8e, 0xdb, 0xb7, 0x7b, 0x6c, 0x69, 0xce, 0x21, 0x13, 0x16, 0xac, + 0x6e, 0x9f, 0x38, 0x23, 0x9f, 0x43, 0x8c, 0xb5, 0x3f, 0xc5, 0x00, 0xc2, 0x2b, 0x37, 0xb4, 0x08, + 0x60, 0xb5, 0x76, 0x0f, 0xf6, 0xf7, 0xaa, 0x65, 0xab, 0x66, 0xce, 0x21, 0x80, 0x64, 0x79, 0x6f, + 0xaf, 0xd6, 0xac, 0x9a, 0x06, 0x4a, 0x43, 0x02, 0xd7, 0xca, 0x55, 0x33, 0x86, 0x16, 0x20, 0x6d, + 0xe1, 0xfd, 0x66, 0x85, 0xd2, 0xc4, 0xe9, 0xa6, 0xf7, 0x6a, 0xd6, 0x41, 0x00, 0x49, 0xa0, 0x2c, + 0xa4, 0x2a, 0xbb, 0xcd, 0x66, 0xad, 0x62, 0x99, 0xf3, 0x74, 0x4b, 0xb1, 0x38, 0xc0, 0xbb, 0x66, + 0x12, 0x5d, 0x84, 0x5c, 0x63, 0xf7, 0xde, 0xc1, 0x76, 0xad, 0x8c, 0xad, 0xad, 0x5a, 0xd9, 0x32, + 0x53, 0x74, 0x87, 0x4a, 0x53, 0x81, 0xa4, 0x99, 0xa0, 0x2a, 0x24, 0x83, 0x10, 0x2c, 0x56, 0xb6, + 0x6b, 0x95, 0x9d, 0x83, 0xed, 0xf2, 0x4e, 0xad, 0xb6, 0x57, 0xc3, 0x26, 0x50, 0x05, 0xd2, 0x27, + 0x57, 0x1a, 0xfb, 0x2d, 0xab, 0x86, 0x0f, 0xaa, 0x35, 0xab, 0x5c, 0x6f, 0xb4, 0xcc, 0x2c, 0x25, + 0xa6, 0x88, 0xd6, 0x76, 0x19, 0x57, 0x0f, 0xea, 0xcd, 0xbb, 0xbb, 0xe6, 0x02, 0xdb, 0xa0, 0x79, + 0x50, 0x6e, 0x34, 0x76, 0xa9, 0x94, 0x07, 0xf5, 0xaa, 0x99, 0xa3, 0x8a, 0x56, 0x37, 0x68, 0x59, + 0x54, 0xfe, 0x45, 0xa6, 0x68, 0xa6, 0x81, 0x83, 0x4a, 0xf3, 0xa0, 0x51, 0xde, 0xaa, 0x35, 0xcc, + 0x0b, 0x28, 0x0f, 0xcb, 0x21, 0xf0, 0x9d, 0x5d, 0xbc, 0x23, 0xc8, 0x4d, 0xba, 0xf3, 0x5e, 0xd9, + 0xaa, 0x6c, 0x53, 0x44, 0xcb, 0xda, 0xc5, 0x35, 0xf3, 0x22, 0xdd, 0xa2, 0x5a, 0x6b, 0xd4, 0x38, + 0x35, 0x07, 0xa2, 0xb5, 0x26, 0x40, 0xf8, 0xe1, 0x0d, 0x55, 0x0c, 0x35, 0x27, 0x87, 0x98, 0x73, + 0x54, 0xab, 0xd2, 0x95, 0x4d, 0x83, 0xda, 0x8e, 0x39, 0x47, 0x60, 0xe8, 0x8b, 0xe2, 0x1b, 0x26, + 0x4c, 0xbe, 0x4b, 0xda, 0x3e, 0xe9, 0x98, 0xf1, 0xb5, 0x35, 0xc8, 0x04, 0xdf, 0xa7, 0x50, 0xf6, + 0x16, 0xf1, 0xd9, 0xca, 0x9c, 0xa3, 0xec, 0x7c, 0x02, 0xcd, 0x01, 0xc6, 0xda, 0x93, 0x18, 0x20, + 0x59, 0xf7, 0x15, 0x1f, 0xa4, 0x06, 0xef, 0xb6, 0x4f, 0x54, 0xd7, 0x53, 0x3e, 0x04, 0x08, 0x5c, + 0x8f, 0x7a, 0xe4, 0x04, 0x38, 0x86, 0x2e, 0x03, 0x52, 0xbf, 0x3b, 0x90, 0x5e, 0x48, 0x9f, 0x7e, + 0x8f, 0xf8, 0x81, 0x47, 0x27, 0xd0, 0x33, 0x13, 0xc5, 0x4d, 0xa0, 0xe6, 0xa9, 0xf6, 0x5a, 0x84, + 0xfb, 0xa3, 0x80, 0x25, 0xa9, 0xae, 0xf5, 0x13, 0xbc, 0xc0, 0xa4, 0xd0, 0x35, 0x78, 0xb6, 0x45, + 0xfc, 0xc9, 0xce, 0x4a, 0x10, 0xa4, 0x51, 0x01, 0x2e, 0x0b, 0x82, 0xa0, 0x34, 0x0b, 0x5c, 0x86, + 0xaa, 0x90, 0xff, 0x16, 0x5a, 0x33, 0x81, 0xbe, 0x98, 0x04, 0x05, 0x83, 0x5a, 0x33, 0x4b, 0xfd, + 0x6f, 0x8f, 0x56, 0x50, 0x31, 0xe1, 0x31, 0x17, 0x28, 0x2f, 0x26, 0x7d, 0xe7, 0xa1, 0x1c, 0xe9, + 0x9b, 0xb9, 0xb5, 0x0f, 0x0c, 0xc8, 0x69, 0x5d, 0x23, 0xb5, 0xba, 0x04, 0x88, 0x23, 0xb1, 0x39, + 0x47, 0xdf, 0x5d, 0x02, 0xb5, 0xbb, 0x1d, 0xd3, 0x40, 0xdf, 0x80, 0xff, 0x9f, 0x40, 0xc9, 0xe2, + 0x8f, 0x49, 0x9b, 0x74, 0x1f, 0x92, 0x8e, 0x19, 0x43, 0xcf, 0xc2, 0x95, 0x09, 0xb2, 0xbb, 0x76, + 0xb7, 0x47, 0x9d, 0x40, 0x7d, 0x26, 0x1e, 0x0d, 0x06, 0x74, 0xe3, 0xc4, 0xda, 0x61, 0x54, 0xdf, + 0x4a, 0xd5, 0xaa, 0x41, 0x43, 0x19, 0xc7, 0x31, 0x72, 0x27, 0x63, 0x02, 0xd3, 0xf2, 0x9d, 0xe1, + 0x90, 0x4a, 0xb5, 0x76, 0x0c, 0xe6, 0xf8, 0x65, 0x1e, 0x75, 0xa7, 0x72, 0xa7, 0x23, 0x0a, 0x9b, + 0x39, 0x17, 0x6a, 0x4d, 0x82, 0x0c, 0xaa, 0xda, 0x96, 0x6f, 0xbb, 0xbe, 0x84, 0xc4, 0xa8, 0xb7, + 0xd0, 0x5d, 0x25, 0x20, 0x4e, 0x77, 0xd9, 0xe9, 0xf6, 0x7a, 0xef, 0x3a, 0xfd, 0xc3, 0x2e, 0x31, + 0x13, 0x6b, 0xaf, 0x6b, 0x97, 0x60, 0x14, 0x4d, 0x5b, 0x19, 0x0e, 0x31, 0xe7, 0x58, 0x0a, 0x6c, + 0xca, 0xa5, 0x41, 0x97, 0x95, 0x60, 0x19, 0xdb, 0xaa, 0x3d, 0xf9, 0x64, 0x65, 0xee, 0xa3, 0x4f, + 0x57, 0x8c, 0x27, 0x9f, 0xae, 0x18, 0x7f, 0xfc, 0x74, 0xc5, 0x78, 0x77, 0x43, 0xf9, 0x87, 0x9b, + 0xbe, 0xed, 0xbb, 0xdd, 0x47, 0x8e, 0xdb, 0x3d, 0xea, 0x0e, 0xe4, 0x62, 0x40, 0x6e, 0x0d, 0x4f, + 0x8e, 0x6e, 0x0d, 0x0f, 0x6f, 0x85, 0x09, 0xfd, 0x30, 0xc9, 0xfe, 0xdb, 0x66, 0xe3, 0x5f, 0x01, + 0x00, 0x00, 0xff, 0xff, 0x29, 0x96, 0x80, 0xfc, 0xcc, 0x33, 0x00, 0x00, } func (m *CNStore) Marshal() (dAtA []byte, err error) { @@ -4199,6 +4407,18 @@ func (m *CNStore) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ConfigData != nil { + { + size, err := m.ConfigData.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x62 + } if len(m.QueryAddress) > 0 { i -= len(m.QueryAddress) copy(dAtA[i:], m.QueryAddress) @@ -4312,6 +4532,18 @@ func (m *TNStore) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ConfigData != nil { + { + size, err := m.ConfigData.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x4a + } if len(m.CtlAddress) > 0 { i -= len(m.CtlAddress) copy(dAtA[i:], m.CtlAddress) @@ -4398,6 +4630,18 @@ func (m *LogStore) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ConfigData != nil { + { + size, err := m.ConfigData.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x32 + } if len(m.Replicas) > 0 { for iNdEx := len(m.Replicas) - 1; iNdEx >= 0; iNdEx-- { { @@ -4569,6 +4813,18 @@ func (m *CNStoreHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ConfigData != nil { + { + size, err := m.ConfigData.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x52 + } if len(m.InitWorkState) > 0 { i -= len(m.InitWorkState) copy(dAtA[i:], m.InitWorkState) @@ -4699,6 +4955,18 @@ func (m *LogStoreHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ConfigData != nil { + { + size, err := m.ConfigData.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a + } if m.TaskServiceCreated { i-- if m.TaskServiceCreated { @@ -4815,6 +5083,18 @@ func (m *TNStoreHeartbeat) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ConfigData != nil { + { + size, err := m.ConfigData.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x42 + } if len(m.CtlAddress) > 0 { i -= len(m.CtlAddress) copy(dAtA[i:], m.CtlAddress) @@ -6115,6 +6395,18 @@ func (m *CNStoreInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ConfigData != nil { + { + size, err := m.ConfigData.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x5a + } if len(m.QueryAddress) > 0 { i -= len(m.QueryAddress) copy(dAtA[i:], m.QueryAddress) @@ -6277,6 +6569,18 @@ func (m *TNStoreInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ConfigData != nil { + { + size, err := m.ConfigData.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x42 + } if len(m.CtlAddress) > 0 { i -= len(m.CtlAddress) copy(dAtA[i:], m.CtlAddress) @@ -6600,6 +6904,18 @@ func (m *LogStoreInfo) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if m.ConfigData != nil { + { + size, err := m.ConfigData.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3a + } if m.TaskServiceCreated { i-- if m.TaskServiceCreated { @@ -7164,25 +7480,133 @@ func (m *BackupData) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func encodeVarintLogservice(dAtA []byte, offset int, v uint64) int { - offset -= sovLogservice(v) - base := offset - for v >= 1<<7 { - dAtA[offset] = uint8(v&0x7f | 0x80) - v >>= 7 - offset++ +func (m *ConfigItem) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err } - dAtA[offset] = uint8(v) - return base + return dAtA[:n], nil } -func (m *CNStore) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - l = len(m.UUID) - if l > 0 { + +func (m *ConfigItem) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ConfigItem) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Internal) > 0 { + i -= len(m.Internal) + copy(dAtA[i:], m.Internal) + i = encodeVarintLogservice(dAtA, i, uint64(len(m.Internal))) + i-- + dAtA[i] = 0x22 + } + if len(m.DefaultValue) > 0 { + i -= len(m.DefaultValue) + copy(dAtA[i:], m.DefaultValue) + i = encodeVarintLogservice(dAtA, i, uint64(len(m.DefaultValue))) + i-- + dAtA[i] = 0x1a + } + if len(m.CurrentValue) > 0 { + i -= len(m.CurrentValue) + copy(dAtA[i:], m.CurrentValue) + i = encodeVarintLogservice(dAtA, i, uint64(len(m.CurrentValue))) + i-- + dAtA[i] = 0x12 + } + if len(m.Name) > 0 { + i -= len(m.Name) + copy(dAtA[i:], m.Name) + i = encodeVarintLogservice(dAtA, i, uint64(len(m.Name))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ConfigData) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ConfigData) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ConfigData) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.XXX_unrecognized != nil { + i -= len(m.XXX_unrecognized) + copy(dAtA[i:], m.XXX_unrecognized) + } + if len(m.Content) > 0 { + for k := range m.Content { + v := m.Content[k] + baseI := i + if v != nil { + { + size, err := v.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintLogservice(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x12 + } + i -= len(k) + copy(dAtA[i:], k) + i = encodeVarintLogservice(dAtA, i, uint64(len(k))) + i-- + dAtA[i] = 0xa + i = encodeVarintLogservice(dAtA, i, uint64(baseI-i)) + i-- + dAtA[i] = 0xa + } + } + return len(dAtA) - i, nil +} + +func encodeVarintLogservice(dAtA []byte, offset int, v uint64) int { + offset -= sovLogservice(v) + base := offset + for v >= 1<<7 { + dAtA[offset] = uint8(v&0x7f | 0x80) + v >>= 7 + offset++ + } + dAtA[offset] = uint8(v) + return base +} +func (m *CNStore) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.UUID) + if l > 0 { n += 1 + l + sovLogservice(uint64(l)) } l = len(m.ServiceAddress) @@ -7226,6 +7650,10 @@ func (m *CNStore) Size() (n int) { if l > 0 { n += 1 + l + sovLogservice(uint64(l)) } + if m.ConfigData != nil { + l = m.ConfigData.Size() + n += 1 + l + sovLogservice(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7270,6 +7698,10 @@ func (m *TNStore) Size() (n int) { if l > 0 { n += 1 + l + sovLogservice(uint64(l)) } + if m.ConfigData != nil { + l = m.ConfigData.Size() + n += 1 + l + sovLogservice(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7302,6 +7734,10 @@ func (m *LogStore) Size() (n int) { n += 1 + l + sovLogservice(uint64(l)) } } + if m.ConfigData != nil { + l = m.ConfigData.Size() + n += 1 + l + sovLogservice(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7397,6 +7833,10 @@ func (m *CNStoreHeartbeat) Size() (n int) { if l > 0 { n += 1 + l + sovLogservice(uint64(l)) } + if m.ConfigData != nil { + l = m.ConfigData.Size() + n += 1 + l + sovLogservice(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7453,6 +7893,10 @@ func (m *LogStoreHeartbeat) Size() (n int) { if m.TaskServiceCreated { n += 2 } + if m.ConfigData != nil { + l = m.ConfigData.Size() + n += 1 + l + sovLogservice(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -7512,6 +7956,10 @@ func (m *TNStoreHeartbeat) Size() (n int) { if l > 0 { n += 1 + l + sovLogservice(uint64(l)) } + if m.ConfigData != nil { + l = m.ConfigData.Size() + n += 1 + l + sovLogservice(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -8101,6 +8549,10 @@ func (m *CNStoreInfo) Size() (n int) { if l > 0 { n += 1 + l + sovLogservice(uint64(l)) } + if m.ConfigData != nil { + l = m.ConfigData.Size() + n += 1 + l + sovLogservice(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -8162,6 +8614,10 @@ func (m *TNStoreInfo) Size() (n int) { if l > 0 { n += 1 + l + sovLogservice(uint64(l)) } + if m.ConfigData != nil { + l = m.ConfigData.Size() + n += 1 + l + sovLogservice(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -8305,6 +8761,10 @@ func (m *LogStoreInfo) Size() (n int) { if m.TaskServiceCreated { n += 2 } + if m.ConfigData != nil { + l = m.ConfigData.Size() + n += 1 + l + sovLogservice(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -8524,6 +8984,59 @@ func (m *BackupData) Size() (n int) { return n } +func (m *ConfigItem) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Name) + if l > 0 { + n += 1 + l + sovLogservice(uint64(l)) + } + l = len(m.CurrentValue) + if l > 0 { + n += 1 + l + sovLogservice(uint64(l)) + } + l = len(m.DefaultValue) + if l > 0 { + n += 1 + l + sovLogservice(uint64(l)) + } + l = len(m.Internal) + if l > 0 { + n += 1 + l + sovLogservice(uint64(l)) + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + +func (m *ConfigData) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if len(m.Content) > 0 { + for k, v := range m.Content { + _ = k + _ = v + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovLogservice(uint64(l)) + } + mapEntrySize := 1 + len(k) + sovLogservice(uint64(len(k))) + l + n += mapEntrySize + 1 + sovLogservice(uint64(mapEntrySize)) + } + } + if m.XXX_unrecognized != nil { + n += len(m.XXX_unrecognized) + } + return n +} + func sovLogservice(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -8956,6 +9469,42 @@ func (m *CNStore) Unmarshal(dAtA []byte) error { } m.QueryAddress = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 12: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ConfigData", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ConfigData == nil { + m.ConfigData = &ConfigData{} + } + if err := m.ConfigData.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -9239,6 +9788,42 @@ func (m *TNStore) Unmarshal(dAtA []byte) error { } m.CtlAddress = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 9: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ConfigData", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ConfigData == nil { + m.ConfigData = &ConfigData{} + } + if err := m.ConfigData.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -9426,6 +10011,42 @@ func (m *LogStore) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ConfigData", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ConfigData == nil { + m.ConfigData = &ConfigData{} + } + if err := m.ConfigData.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -10083,16 +10704,52 @@ func (m *CNStoreHeartbeat) Unmarshal(dAtA []byte) error { } m.InitWorkState = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipLogservice(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthLogservice + case 10: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ConfigData", wireType) } - if (iNdEx + skippy) > l { + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ConfigData == nil { + m.ConfigData = &ConfigData{} + } + if err := m.ConfigData.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipLogservice(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthLogservice + } + if (iNdEx + skippy) > l { return io.ErrUnexpectedEOF } m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) @@ -10418,6 +11075,42 @@ func (m *LogStoreHeartbeat) Unmarshal(dAtA []byte) error { } } m.TaskServiceCreated = bool(v != 0) + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ConfigData", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ConfigData == nil { + m.ConfigData = &ConfigData{} + } + if err := m.ConfigData.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -10772,6 +11465,42 @@ func (m *TNStoreHeartbeat) Unmarshal(dAtA []byte) error { } m.CtlAddress = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ConfigData", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ConfigData == nil { + m.ConfigData = &ConfigData{} + } + if err := m.ConfigData.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -14680,6 +15409,42 @@ func (m *CNStoreInfo) Unmarshal(dAtA []byte) error { } m.QueryAddress = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ConfigData", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ConfigData == nil { + m.ConfigData = &ConfigData{} + } + if err := m.ConfigData.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -15112,6 +15877,42 @@ func (m *TNStoreInfo) Unmarshal(dAtA []byte) error { } m.CtlAddress = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 8: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ConfigData", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ConfigData == nil { + m.ConfigData = &ConfigData{} + } + if err := m.ConfigData.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -16024,6 +16825,42 @@ func (m *LogStoreInfo) Unmarshal(dAtA []byte) error { } } m.TaskServiceCreated = bool(v != 0) + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ConfigData", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ConfigData == nil { + m.ConfigData = &ConfigData{} + } + if err := m.ConfigData.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipLogservice(dAtA[iNdEx:]) @@ -17971,6 +18808,365 @@ func (m *BackupData) Unmarshal(dAtA []byte) error { } return nil } +func (m *ConfigItem) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ConfigItem: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ConfigItem: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field CurrentValue", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.CurrentValue = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field DefaultValue", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.DefaultValue = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Internal", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Internal = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipLogservice(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthLogservice + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ConfigData) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ConfigData: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ConfigData: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Content", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthLogservice + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthLogservice + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Content == nil { + m.Content = make(map[string]*ConfigItem) + } + var mapkey string + var mapvalue *ConfigItem + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthLogservice + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return ErrInvalidLengthLogservice + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var mapmsglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowLogservice + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapmsglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if mapmsglen < 0 { + return ErrInvalidLengthLogservice + } + postmsgIndex := iNdEx + mapmsglen + if postmsgIndex < 0 { + return ErrInvalidLengthLogservice + } + if postmsgIndex > l { + return io.ErrUnexpectedEOF + } + mapvalue = &ConfigItem{} + if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil { + return err + } + iNdEx = postmsgIndex + } else { + iNdEx = entryPreIndex + skippy, err := skipLogservice(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthLogservice + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Content[mapkey] = mapvalue + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipLogservice(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthLogservice + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + m.XXX_unrecognized = append(m.XXX_unrecognized, dAtA[iNdEx:iNdEx+skippy]...) + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipLogservice(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/pkg/pb/plan/plan.pb.go b/pkg/pb/plan/plan.pb.go index 02211660ff39..ff17d0c5c88e 100644 --- a/pkg/pb/plan/plan.pb.go +++ b/pkg/pb/plan/plan.pb.go @@ -1180,7 +1180,6 @@ func (m *Type) GetEnumvalues() string { type Const struct { Isnull bool `protobuf:"varint,1,opt,name=isnull,proto3" json:"isnull,omitempty"` // Types that are valid to be assigned to Value: - // // *Const_I8Val // *Const_I16Val // *Const_I32Val @@ -2373,7 +2372,6 @@ func (m *Function) GetArgs() []*Expr { type Expr struct { Typ *Type `protobuf:"bytes,1,opt,name=typ,proto3" json:"typ,omitempty"` // Types that are valid to be assigned to Expr: - // // *Expr_C // *Expr_P // *Expr_V @@ -4328,7 +4326,6 @@ func (m *TableDef) GetAutoIncrOffset() uint64 { // XXX: Deprecated and to be removed soon. type TableDef_DefType struct { // Types that are valid to be assigned to Def: - // // *TableDef_DefType_Properties Def isTableDef_DefType_Def `protobuf_oneof:"def"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -4574,18 +4571,18 @@ func (m *HashMapStats) GetShuffleMethod() ShuffleMethod { } type Stats struct { - // for scan, number of blocks to read from S3 + //for scan, number of blocks to read from S3 BlockNum int32 `protobuf:"varint,1,opt,name=block_num,json=blockNum,proto3" json:"block_num,omitempty"` - // for scan, cost of reading from S3, basically the read lines - // for other nodes, it means the estimated cost of current node + //for scan, cost of reading from S3, basically the read lines + //for other nodes, it means the estimated cost of current node Cost float64 `protobuf:"fixed64,2,opt,name=cost,proto3" json:"cost,omitempty"` - // number of output lines + //number of output lines Outcnt float64 `protobuf:"fixed64,3,opt,name=outcnt,proto3" json:"outcnt,omitempty"` // average size of one row, currently not used Rowsize float64 `protobuf:"fixed64,4,opt,name=rowsize,proto3" json:"rowsize,omitempty"` - // for scan, this means total count of all table, before filtering + //for scan, this means total count of all table, before filtering TableCnt float64 `protobuf:"fixed64,5,opt,name=table_cnt,json=tableCnt,proto3" json:"table_cnt,omitempty"` - // for scan, selectivity means outcnt divide total count + //for scan, selectivity means outcnt divide total count Selectivity float64 `protobuf:"fixed64,6,opt,name=selectivity,proto3" json:"selectivity,omitempty"` HashmapStats *HashMapStats `protobuf:"bytes,7,opt,name=hashmapStats,proto3" json:"hashmapStats,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` @@ -6178,7 +6175,7 @@ func (m *PreInsertUkCtx) GetTableDef() *TableDef { } type PreDeleteCtx struct { - // the indexes of row_id&pk column in the batch + //the indexes of row_id&pk column in the batch Idx []int32 `protobuf:"varint,1,rep,packed,name=idx,proto3" json:"idx,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -6652,10 +6649,9 @@ func (m *Query) GetLoadTag() bool { } type TransationControl struct { - // TransationControl type + //TransationControl type TclType TransationControl_TclType `protobuf:"varint,1,opt,name=tcl_type,json=tclType,proto3,enum=plan.TransationControl_TclType" json:"tcl_type,omitempty"` // Types that are valid to be assigned to Action: - // // *TransationControl_Begin // *TransationControl_Commit // *TransationControl_Rollback @@ -6905,7 +6901,6 @@ func (m *TransationRollback) GetCompletionType() TransationCompletionType { type Plan struct { // Types that are valid to be assigned to Plan: - // // *Plan_Query // *Plan_Tcl // *Plan_Ddl @@ -7082,10 +7077,9 @@ func (m *Column) GetColumn() []*Expr { } type DataControl struct { - // DataDefinition type + //DataDefinition type DclType DataControl_DclType `protobuf:"varint,1,opt,name=dcl_type,json=dclType,proto3,enum=plan.DataControl_DclType" json:"dcl_type,omitempty"` // Types that are valid to be assigned to Control: - // // *DataControl_SetVariables // *DataControl_Prepare // *DataControl_Execute @@ -7206,14 +7200,13 @@ func (*DataControl) XXX_OneofWrappers() []interface{} { } type DataDefinition struct { - // DataDefinition type + //DataDefinition type DdlType DataDefinition_DdlType `protobuf:"varint,1,opt,name=ddl_type,json=ddlType,proto3,enum=plan.DataDefinition_DdlType" json:"ddl_type,omitempty"` - // other show statement we will rewrite to a select statement - // then we will get a Query - // eg: 'show databases' will rewrite to 'select md.datname as `Database` from mo_database md' + //other show statement we will rewrite to a select statement + //then we will get a Query + //eg: 'show databases' will rewrite to 'select md.datname as `Database` from mo_database md' Query *Query `protobuf:"bytes,2,opt,name=query,proto3" json:"query,omitempty"` // Types that are valid to be assigned to Definition: - // // *DataDefinition_CreateDatabase // *DataDefinition_AlterDatabase // *DataDefinition_DropDatabase @@ -8617,7 +8610,6 @@ func (m *AlterTable) GetChangeTblColIdMap() map[uint64]*ColDef { type AlterTable_Action struct { // Types that are valid to be assigned to Action: - // // *AlterTable_Action_Drop // *AlterTable_Action_AddFk // *AlterTable_Action_AddIndex diff --git a/pkg/proxy/config.go b/pkg/proxy/config.go index 2c8dac0da9fb..1a4d6ec08c5b 100644 --- a/pkg/proxy/config.go +++ b/pkg/proxy/config.go @@ -126,6 +126,14 @@ func WithTLSKeyFile(f string) Option { } } +// WithConfigData saves the data from the config file +func WithConfigData(data []byte) Option { + return func(s *Server) { + s.configData = make([]byte, len(data)) + copy(s.configData, data) + } +} + // FillDefault fill the default config values of proxy server. func (c *Config) FillDefault() { if c.ListenAddress == "" { diff --git a/pkg/proxy/server.go b/pkg/proxy/server.go index e2135b8491c6..9f25e3845b53 100644 --- a/pkg/proxy/server.go +++ b/pkg/proxy/server.go @@ -40,6 +40,7 @@ type Server struct { counterSet *counterSet // for test. testHAKeeperClient logservice.ClusterHAKeeperClient + configData []byte } // NewServer creates the proxy server. diff --git a/pkg/sql/colexec/lockop/lock_op_test.go b/pkg/sql/colexec/lockop/lock_op_test.go index e1acc26c57b2..a04743061dbf 100644 --- a/pkg/sql/colexec/lockop/lock_op_test.go +++ b/pkg/sql/colexec/lockop/lock_op_test.go @@ -417,6 +417,7 @@ func runLockOpTest( nil, services[0], nil, + nil, nil) require.Equal(t, int64(0), proc.Mp().CurrNB()) defer func() { diff --git a/pkg/sql/colexec/table_function/system_view.go b/pkg/sql/colexec/table_function/system_view.go new file mode 100644 index 000000000000..982fa44d247c --- /dev/null +++ b/pkg/sql/colexec/table_function/system_view.go @@ -0,0 +1,132 @@ +package table_function + +import ( + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/vector" + logservicepb "github.com/matrixorigin/matrixone/pkg/pb/logservice" + plan2 "github.com/matrixorigin/matrixone/pkg/sql/plan" + "github.com/matrixorigin/matrixone/pkg/vm/process" + "strings" +) + +func moConfigurationsPrepare(proc *process.Process, arg *Argument) error { + arg.ctr.state = dataProducing + if len(arg.Args) > 0 { + return moerr.NewInvalidInput(proc.Ctx, "moConfigurations: no argument is required") + } + for i := range arg.Attrs { + arg.Attrs[i] = strings.ToUpper(arg.Attrs[i]) + } + return nil +} + +func moConfigurationsCall(_ int, proc *process.Process, arg *Argument) (bool, error) { + switch arg.ctr.state { + case dataProducing: + + if proc.Hakeeper == nil { + return false, moerr.NewInternalError(proc.Ctx, "hakeeper is nil") + } + + //get cluster details + details, err := proc.Hakeeper.GetClusterDetails(proc.Ctx) + if err != nil { + return false, err + } + + //alloc batch + bat := batch.NewWithSize(len(arg.Attrs)) + for i, col := range arg.Attrs { + col = strings.ToLower(col) + idx, ok := plan2.MoConfigColName2Index[col] + if !ok { + return false, moerr.NewInternalError(proc.Ctx, "bad input select columns name %v", col) + } + + tp := plan2.MoConfigColTypes[idx] + bat.Vecs[i] = vector.NewVec(tp) + } + bat.Attrs = arg.Attrs + + mp := proc.GetMPool() + + //fill batch for cn + for _, cnStore := range details.GetCNStores() { + if cnStore.GetConfigData() != nil { + err = fillMapToBatch("cn", cnStore.GetUUID(), arg.Attrs, cnStore.GetConfigData().GetContent(), bat, mp) + if err != nil { + return false, err + } + } + } + + //fill batch for tn + for _, tnStore := range details.GetTNStores() { + if tnStore.GetConfigData() != nil { + err = fillMapToBatch("tn", tnStore.GetUUID(), arg.Attrs, tnStore.GetConfigData().GetContent(), bat, mp) + if err != nil { + return false, err + } + } + } + + //fill batch for log + for _, logStore := range details.GetLogStores() { + if logStore.GetConfigData() != nil { + err = fillMapToBatch("log", logStore.GetUUID(), arg.Attrs, logStore.GetConfigData().GetContent(), bat, mp) + if err != nil { + return false, err + } + } + } + + bat.SetRowCount(bat.Vecs[0].Length()) + proc.SetInputBatch(bat) + arg.ctr.state = dataFinished + return false, nil + + case dataFinished: + proc.SetInputBatch(nil) + return true, nil + default: + return false, moerr.NewInternalError(proc.Ctx, "unknown state %v", arg.ctr.state) + } +} + +func fillMapToBatch(nodeType, nodeId string, attrs []string, kvs map[string]*logservicepb.ConfigItem, bat *batch.Batch, mp *mpool.MPool) error { + var err error + for _, value := range kvs { + for i, col := range attrs { + col = strings.ToLower(col) + switch plan2.MoConfigColType(plan2.MoConfigColName2Index[col]) { + case plan2.MoConfigColTypeNodeType: + if err = vector.AppendBytes(bat.Vecs[i], []byte(nodeType), false, mp); err != nil { + return err + } + case plan2.MoConfigColTypeNodeId: + if err = vector.AppendBytes(bat.Vecs[i], []byte(nodeId), false, mp); err != nil { + return err + } + case plan2.MoConfigColTypeName: + if err = vector.AppendBytes(bat.Vecs[i], []byte(value.GetName()), false, mp); err != nil { + return err + } + case plan2.MoConfigColTypeCurrentValue: + if err = vector.AppendBytes(bat.Vecs[i], []byte(value.GetCurrentValue()), false, mp); err != nil { + return err + } + case plan2.MoConfigColTypeDefaultValue: + if err = vector.AppendBytes(bat.Vecs[i], []byte(value.GetDefaultValue()), false, mp); err != nil { + return err + } + case plan2.MoConfigColTypeInternal: + if err = vector.AppendBytes(bat.Vecs[i], []byte(value.GetInternal()), false, mp); err != nil { + return err + } + } + } + } + return err +} diff --git a/pkg/sql/colexec/table_function/table_function.go b/pkg/sql/colexec/table_function/table_function.go index 64de1e51d570..424e97b3d2da 100644 --- a/pkg/sql/colexec/table_function/table_function.go +++ b/pkg/sql/colexec/table_function/table_function.go @@ -45,6 +45,8 @@ func Call(idx int, proc *process.Process, arg any, isFirst bool, isLast bool) (p f, e = metadataScan(idx, proc, tblArg) case "processlist": f, e = processlist(idx, proc, tblArg) + case "mo_configurations": + f, e = moConfigurationsCall(idx, proc, tblArg) default: return process.ExecStop, moerr.NewNotSupported(proc.Ctx, fmt.Sprintf("table function %s is not supported", tblArg.Name)) } @@ -107,6 +109,8 @@ func Prepare(proc *process.Process, arg any) error { return metadataScanPrepare(proc, tblArg) case "processlist": return processlistPrepare(proc, tblArg) + case "mo_configurations": + return moConfigurationsPrepare(proc, tblArg) default: return moerr.NewNotSupported(proc.Ctx, fmt.Sprintf("table function %s is not supported", tblArg.Name)) } diff --git a/pkg/sql/compile/scopeRemoteRun.go b/pkg/sql/compile/scopeRemoteRun.go index 54b4c27d7f6d..50a9fc24a607 100644 --- a/pkg/sql/compile/scopeRemoteRun.go +++ b/pkg/sql/compile/scopeRemoteRun.go @@ -17,6 +17,7 @@ package compile import ( "context" "fmt" + "github.com/matrixorigin/matrixone/pkg/logservice" "hash/crc32" "sync/atomic" "time" @@ -103,6 +104,7 @@ func CnServerMessageHandler( fileService fileservice.FileService, lockService lockservice.LockService, queryService queryservice.QueryService, + hakeeper logservice.CNHAKeeperClient, cli client.TxnClient, aicm *defines.AutoIncrCacheManager, messageAcquirer func() morpc.Message) error { @@ -114,7 +116,7 @@ func CnServerMessageHandler( } receiver := newMessageReceiverOnServer(ctx, cnAddr, msg, - cs, messageAcquirer, storeEngine, fileService, lockService, queryService, cli, aicm) + cs, messageAcquirer, storeEngine, fileService, lockService, queryService, hakeeper, cli, aicm) // rebuild pipeline to run and send query result back. err := cnMessageHandle(&receiver) diff --git a/pkg/sql/compile/scopeRemoteRunTypes.go b/pkg/sql/compile/scopeRemoteRunTypes.go index 355cb91142c5..b12a7cc96bba 100644 --- a/pkg/sql/compile/scopeRemoteRunTypes.go +++ b/pkg/sql/compile/scopeRemoteRunTypes.go @@ -16,6 +16,7 @@ package compile import ( "context" + "github.com/matrixorigin/matrixone/pkg/logservice" "hash/crc32" "runtime" "time" @@ -55,6 +56,7 @@ type cnInformation struct { fileService fileservice.FileService lockService lockservice.LockService queryService queryservice.QueryService + hakeeper logservice.CNHAKeeperClient aicm *defines.AutoIncrCacheManager } @@ -197,6 +199,7 @@ func newMessageReceiverOnServer( fileService fileservice.FileService, lockService lockservice.LockService, queryService queryservice.QueryService, + hakeeper logservice.CNHAKeeperClient, txnClient client.TxnClient, aicm *defines.AutoIncrCacheManager) messageReceiverOnServer { @@ -215,6 +218,7 @@ func newMessageReceiverOnServer( fileService: fileService, lockService: lockService, queryService: queryService, + hakeeper: hakeeper, aicm: aicm, } @@ -269,6 +273,7 @@ func (receiver *messageReceiverOnServer) newCompile() *Compile { cnInfo.fileService, cnInfo.lockService, cnInfo.queryService, + cnInfo.hakeeper, cnInfo.aicm) proc.UnixTime = pHelper.unixTime proc.Id = pHelper.id diff --git a/pkg/sql/compile/sql_executor.go b/pkg/sql/compile/sql_executor.go index 91a2a4721e25..be4f5c965d17 100644 --- a/pkg/sql/compile/sql_executor.go +++ b/pkg/sql/compile/sql_executor.go @@ -17,6 +17,7 @@ package compile import ( "context" "errors" + "github.com/matrixorigin/matrixone/pkg/logservice" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/common/runtime" @@ -44,6 +45,7 @@ type sqlExecutor struct { fs fileservice.FileService ls lockservice.LockService qs queryservice.QueryService + hakeeper logservice.CNHAKeeperClient aicm *defines.AutoIncrCacheManager } @@ -55,6 +57,7 @@ func NewSQLExecutor( txnClient client.TxnClient, fs fileservice.FileService, qs queryservice.QueryService, + hakeeper logservice.CNHAKeeperClient, aicm *defines.AutoIncrCacheManager) executor.SQLExecutor { v, ok := runtime.ProcessLevelRuntime().GetGlobalVariables(runtime.LockService) if !ok { @@ -67,6 +70,7 @@ func NewSQLExecutor( fs: fs, ls: v.(lockservice.LockService), qs: qs, + hakeeper: hakeeper, aicm: aicm, mp: mp, } @@ -204,6 +208,7 @@ func (exec *txnExecutor) Exec(sql string) (executor.Result, error) { exec.s.fs, exec.s.ls, exec.s.qs, + exec.s.hakeeper, exec.s.aicm, ) proc.SessionInfo.TimeZone = exec.opts.GetTimeZone() diff --git a/pkg/sql/plan/function/ctl/cmd_ping_test.go b/pkg/sql/plan/function/ctl/cmd_ping_test.go index aa9d187fa0ba..8b2772a0e0b9 100644 --- a/pkg/sql/plan/function/ctl/cmd_ping_test.go +++ b/pkg/sql/plan/function/ctl/cmd_ping_test.go @@ -32,7 +32,7 @@ import ( func TestCmdPingTNWithEmptyTN(t *testing.T) { ctx := context.Background() initTestRuntime() - proc := process.New(ctx, nil, nil, nil, nil, nil, nil, nil) + proc := process.New(ctx, nil, nil, nil, nil, nil, nil, nil, nil) result, err := handlePing()(proc, tn, "", @@ -49,7 +49,7 @@ func TestCmdPingTNWithSingleTN(t *testing.T) { shardID := uint64(1) ctx := context.Background() - proc := process.New(ctx, nil, nil, nil, nil, nil, nil, nil) + proc := process.New(ctx, nil, nil, nil, nil, nil, nil, nil, nil) result, err := handlePing()(proc, tn, "", @@ -70,7 +70,7 @@ func TestCmdPingTNWithSingleTN(t *testing.T) { func TestCmdPingTNWithMultiTN(t *testing.T) { ctx := context.Background() initTestRuntime(1, 2) - proc := process.New(ctx, nil, nil, nil, nil, nil, nil, nil) + proc := process.New(ctx, nil, nil, nil, nil, nil, nil, nil, nil) result, err := handlePing()(proc, tn, "", @@ -94,7 +94,7 @@ func TestCmdPingTNWithMultiTN(t *testing.T) { func TestCmdPingTNWithParameter(t *testing.T) { ctx := context.Background() initTestRuntime(1, 2) - proc := process.New(ctx, nil, nil, nil, nil, nil, nil, nil) + proc := process.New(ctx, nil, nil, nil, nil, nil, nil, nil, nil) result, err := handlePing()(proc, tn, "1", diff --git a/pkg/sql/plan/query_builder.go b/pkg/sql/plan/query_builder.go index e1052baa2b69..ed71033efe53 100644 --- a/pkg/sql/plan/query_builder.go +++ b/pkg/sql/plan/query_builder.go @@ -3318,6 +3318,8 @@ func (builder *QueryBuilder) buildTableFunction(tbl *tree.TableFunction, ctx *Bi nodeId = builder.buildMetadataScan(tbl, ctx, exprs, childId) case "processlist": nodeId, err = builder.buildProcesslist(tbl, ctx, exprs, childId) + case "mo_configurations": + nodeId, err = builder.buildMoConfigurations(tbl, ctx, exprs, childId) default: err = moerr.NewNotSupported(builder.GetContext(), "table function '%s' not supported", id) } diff --git a/pkg/sql/plan/system_view.go b/pkg/sql/plan/system_view.go new file mode 100644 index 000000000000..bafdd3edab4e --- /dev/null +++ b/pkg/sql/plan/system_view.go @@ -0,0 +1,77 @@ +package plan + +import ( + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/pb/plan" + "github.com/matrixorigin/matrixone/pkg/sql/parsers/tree" +) + +var MoConfigColNames = []string{ + "node_type", + "node_id", + "name", + "current_value", + "default_value", + "internal", +} + +var MoConfigColTypes = []types.Type{ + types.New(types.T_varchar, types.MaxVarcharLen, 0), + types.New(types.T_varchar, types.MaxVarcharLen, 0), + types.New(types.T_varchar, types.MaxVarcharLen, 0), + types.New(types.T_varchar, types.MaxVarcharLen, 0), + types.New(types.T_varchar, types.MaxVarcharLen, 0), + types.New(types.T_varchar, types.MaxVarcharLen, 0), +} + +var MoConfigColName2Index = map[string]int32{ + "node_type": 0, + "node_id": 1, + "name": 2, + "current_value": 3, + "default_value": 4, + "internal": 5, +} + +type MoConfigColType int32 + +const ( + MoConfigColTypeNodeType = iota + MoConfigColTypeNodeId + MoConfigColTypeName + MoConfigColTypeCurrentValue + MoConfigColTypeDefaultValue + MoConfigColTypeInternal +) + +func (builder *QueryBuilder) buildMoConfigurations(tbl *tree.TableFunction, ctx *BindContext, exprs []*plan.Expr, childId int32) (int32, error) { + var err error + + colDefs := make([]*plan.ColDef, 0, len(MoConfigColNames)) + + for i, name := range MoConfigColNames { + colDefs = append(colDefs, &plan.ColDef{ + Name: name, + Typ: &plan.Type{ + Id: int32(MoConfigColTypes[i].Oid), + Width: MoConfigColTypes[i].Width, + }, + }) + } + + node := &plan.Node{ + NodeType: plan.Node_FUNCTION_SCAN, + Stats: &plan.Stats{}, + TableDef: &plan.TableDef{ + TableType: "func_table", + TblFunc: &plan.TableFunction{ + Name: "mo_configurations", + }, + Cols: colDefs, + }, + BindingTags: []int32{builder.genNewTag()}, + Children: []int32{childId}, + TblFuncExprList: exprs, + } + return builder.appendNode(node, ctx), err +} diff --git a/pkg/testutil/util_new.go b/pkg/testutil/util_new.go index 4d678a0f22cb..e211b24627ca 100644 --- a/pkg/testutil/util_new.go +++ b/pkg/testutil/util_new.go @@ -64,6 +64,7 @@ func NewProcessWithMPool(mp *mpool.MPool) *process.Process { nil, nil, nil, + nil, ) proc.Lim.Size = 1 << 20 proc.Lim.BatchRows = 1 << 20 diff --git a/pkg/tnservice/cfg.go b/pkg/tnservice/cfg.go index 387f9bb7921b..a80cb1eb879c 100644 --- a/pkg/tnservice/cfg.go +++ b/pkg/tnservice/cfg.go @@ -16,6 +16,8 @@ package tnservice import ( "context" + logservicepb "github.com/matrixorigin/matrixone/pkg/pb/logservice" + "github.com/matrixorigin/matrixone/pkg/util" "path/filepath" "strings" "time" @@ -271,3 +273,109 @@ func (c *Config) Validate() error { } return nil } + +func (c *Config) SetDefaultValue() { + foundMachineHost := "" + if c.DataDir == "" { + c.DataDir = defaultDataDir + } + c.Txn.Storage.dataDir = filepath.Join(c.DataDir, storageDir) + if c.ListenAddress == "" { + c.ListenAddress = defaultListenAddress + } + if c.ServiceAddress == "" { + c.ServiceAddress = defaultServiceAddress + } else { + foundMachineHost = strings.Split(c.ServiceAddress, ":")[0] + } + if c.LockService.ListenAddress == "" { + c.LockService.ListenAddress = defaultLockListenAddress + } + if c.LockService.ServiceAddress == "" { + c.LockService.ServiceAddress = defaultLockServiceAddress + } + if c.Txn.Storage.Backend == "" { + c.Txn.Storage.Backend = StorageTAE + } + + if c.Txn.ZombieTimeout.Duration == 0 { + c.Txn.ZombieTimeout.Duration = defaultZombieTimeout + } + if c.HAKeeper.DiscoveryTimeout.Duration == 0 { + c.HAKeeper.DiscoveryTimeout.Duration = defaultDiscoveryTimeout + } + if c.HAKeeper.HeatbeatInterval.Duration == 0 { + c.HAKeeper.HeatbeatInterval.Duration = defaultHeatbeatInterval + } + if c.HAKeeper.HeatbeatTimeout.Duration == 0 { + c.HAKeeper.HeatbeatTimeout.Duration = defaultHeatbeatTimeout + } + if c.LogService.ConnectTimeout.Duration == 0 { + c.LogService.ConnectTimeout.Duration = defaultConnectTimeout + } + if c.Ckp.ScanInterval.Duration == 0 { + c.Ckp.ScanInterval.Duration = defaultScanInterval + } + if c.Ckp.FlushInterval.Duration == 0 { + c.Ckp.FlushInterval.Duration = defaultFlushInterval + } + if c.Ckp.MinCount == 0 { + c.Ckp.MinCount = defaultMinCount + } + if c.Ckp.IncrementalInterval.Duration == 0 { + c.Ckp.IncrementalInterval.Duration = defaultIncrementalInterval + } + if c.Ckp.GlobalMinCount == 0 { + c.Ckp.GlobalMinCount = defaultGlobalMinCount + } + if c.LogtailServer.ListenAddress == "" { + c.LogtailServer.ListenAddress = defaultLogtailListenAddress + } + if c.LogtailServer.ServiceAddress == "" { + c.LogtailServer.ServiceAddress = defaultLogtailServiceAddress + } + if c.LogtailServer.RpcMaxMessageSize <= 0 { + c.LogtailServer.RpcMaxMessageSize = toml.ByteSize(defaultRpcMaxMsgSize) + } + if c.LogtailServer.LogtailCollectInterval.Duration <= 0 { + c.LogtailServer.LogtailCollectInterval.Duration = defaultLogtailCollectInterval + } + if c.LogtailServer.LogtailResponseSendTimeout.Duration <= 0 { + c.LogtailServer.LogtailResponseSendTimeout.Duration = defaultLogtailResponseSendTimeout + } + if c.Cluster.RefreshInterval.Duration == 0 { + c.Cluster.RefreshInterval.Duration = time.Second * 10 + } + + if c.Txn.Mode == "" { + c.Txn.Mode = defaultTxnMode.String() + } + + if c.Txn.IncrementalDedup == "" { + if txn.GetTxnMode(c.Txn.Mode) == txn.TxnMode_Pessimistic { + c.Txn.IncrementalDedup = "true" + } else { + c.Txn.IncrementalDedup = "false" + } + } else { + c.Txn.IncrementalDedup = strings.ToLower(c.Txn.IncrementalDedup) + } + + c.RPC.Adjust() + c.Ctl.Adjust(foundMachineHost, defaultCtlListenAddress) + c.LockService.ServiceID = "tmp" + c.LockService.Validate() + c.LockService.ServiceID = c.UUID + + if c.PortBase != 0 { + if c.ServiceHost == "" { + c.ServiceHost = defaultServiceHost + } + } +} + +func dumpTnConfig(cfg Config) (map[string]*logservicepb.ConfigItem, error) { + defCfg := Config{} + defCfg.SetDefaultValue() + return util.DumpConfig(cfg, defCfg) +} \ No newline at end of file diff --git a/pkg/tnservice/cfg_test.go b/pkg/tnservice/cfg_test.go index 4ffdcb7342ae..4bb99f81ce16 100644 --- a/pkg/tnservice/cfg_test.go +++ b/pkg/tnservice/cfg_test.go @@ -35,3 +35,8 @@ func TestValidate(t *testing.T) { assert.Equal(t, defaultConnectTimeout, c.LogService.ConnectTimeout.Duration) assert.Equal(t, "true", c.Txn.IncrementalDedup) } + +func TestDefaulValue(t *testing.T) { + c := Config{} + c.SetDefaultValue() +} diff --git a/pkg/tnservice/store.go b/pkg/tnservice/store.go index e87fdc03a88f..d31b753d30fc 100644 --- a/pkg/tnservice/store.go +++ b/pkg/tnservice/store.go @@ -82,6 +82,16 @@ func WithTaskStorageFactory(factory taskservice.TaskStorageFactory) Option { } } +// WithConfigData saves the data from the config file +func WithConfigData(data map[string]*logservicepb.ConfigItem) Option { + return func(s *store) { + s.configData = make(map[string]*logservicepb.ConfigItem, len(data)) + for k, v := range data { + s.configData[k] = v + } + } +} + type store struct { perfCounter *perfcounter.CounterSet cfg *Config @@ -118,6 +128,7 @@ type store struct { } addressMgr address.AddressManager + configData map[string]*logservicepb.ConfigItem } // NewService create TN Service @@ -132,6 +143,9 @@ func NewService( return nil, err } + configKVMap, _ := dumpTnConfig(*cfg) + opts = append(opts, WithConfigData(configKVMap)) + // start common stuff common.InitTAEMPool() diff --git a/pkg/tnservice/store_heartbeat.go b/pkg/tnservice/store_heartbeat.go index 9a4219abcc54..4fe97d858c71 100644 --- a/pkg/tnservice/store_heartbeat.go +++ b/pkg/tnservice/store_heartbeat.go @@ -62,6 +62,9 @@ func (s *store) heartbeat(ctx context.Context) { LogtailServerAddress: s.logtailServiceServiceAddr(), LockServiceAddress: s.lockServiceServiceAddr(), CtlAddress: s.ctlServiceServiceAddr(), + ConfigData: &logservicepb.ConfigData{ + Content: s.configData, + }, } cb, err := s.hakeeperClient.SendTNHeartbeat(ctx2, hb) if err != nil { diff --git a/pkg/util/dump_config.go b/pkg/util/dump_config.go new file mode 100644 index 000000000000..52e354b79816 --- /dev/null +++ b/pkg/util/dump_config.go @@ -0,0 +1,213 @@ +package util + +import ( + "fmt" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + logservicepb "github.com/matrixorigin/matrixone/pkg/pb/logservice" + "reflect" +) + +type exporter struct { + kvs map[string]string +} + +func newExporter() *exporter { + return &exporter{ + kvs: make(map[string]string), + } +} + +func (et *exporter) Export(k, v string) { + et.kvs[k] = v +} + +func (et *exporter) Print() { + //for k, v := range et.kvs { + // //fmt.Println(k, v) + //} +} + +func (et *exporter) Dump() map[string]string { + return et.kvs +} + +func (et *exporter) Clear() { + et.kvs = make(map[string]string) +} + +// flatten is a recursive function to flatten a struct. +// in: the struct object to be flattened. reference or pointer +// name: the name of the struct object. empty if it is the top level struct. +// prefix: the path to the name. empty if it is the top level struct. +// exp: the exporter to export the key-value pairs. +func flatten(in any, name string, prefix string, exp *exporter) error { + if exp == nil { + return moerr.NewInternalErrorNoCtx("invalid exporter") + } + var err error + if in == nil { + //fmt.Printf("%s + %v\n", prefix+name, "nil") + exp.Export(prefix+name, "nil") + return nil + } + + typ := reflect.TypeOf(in) + value := reflect.ValueOf(in) + + oldPrefix := prefix + if name != "" { + prefix = prefix + name + } else { + prefix = prefix + typ.Name() + } + + switch typ.Kind() { + case reflect.Invalid: + return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) + case reflect.Bool: + fallthrough + case reflect.Int: + fallthrough + case reflect.Int8: + fallthrough + case reflect.Int16: + fallthrough + case reflect.Int32: + fallthrough + case reflect.Int64: + fallthrough + case reflect.Uint: + fallthrough + case reflect.Uint8: + fallthrough + case reflect.Uint16: + fallthrough + case reflect.Uint32: + fallthrough + case reflect.Uint64: + fallthrough + case reflect.Float32: + fallthrough + case reflect.Float64: + fallthrough + case reflect.Complex64: + fallthrough + case reflect.Complex128: + fallthrough + case reflect.String: + //fmt.Printf("%s + %v\n", prefix, value) + exp.Export(prefix, fmt.Sprintf("%v", value)) + case reflect.Uintptr: + return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) + case reflect.Slice: + fallthrough + case reflect.Array: + if value.Len() == 0 { + //fmt.Printf("%s%s - %v\n", prefix, typ.Name(), "empty") + exp.Export(prefix+typ.Name(), "") + } else { + for k := 0; k < value.Len(); k++ { + elemVal := value.Index(k) + err = flatten(elemVal.Interface(), typ.Name(), oldPrefix+name+"["+fmt.Sprintf("%d", k)+"].", exp) + if err != nil { + return err + } + } + } + + case reflect.Chan: + return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) + case reflect.Func: + return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) + case reflect.Interface: + return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) + case reflect.Map: + if value.Len() == 0 { + //fmt.Printf("%s%s - %v\n", prefix, typ.Name(), "empty") + exp.Export(prefix+typ.Name(), "") + } else { + keys := value.MapKeys() + for _, key := range keys { + keyVal := value.MapIndex(key) + err = flatten(keyVal.Interface(), typ.Name(), oldPrefix+name+"<"+fmt.Sprintf("%v", key.Interface())+">.", exp) + if err != nil { + return err + } + } + } + + case reflect.Pointer: + if value.IsNil() { + //fmt.Printf("%s%s - %v\n", prefix, typ.Name(), "nil") + exp.Export(prefix+typ.Name(), "nil") + } else { + nextPrefix := "" + if oldPrefix == "" { + nextPrefix = oldPrefix + } else { + nextPrefix = prefix + } + err = flatten(value.Elem().Interface(), typ.Name(), nextPrefix+".", exp) + if err != nil { + return err + } + } + case reflect.Struct: + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + isExported := field.IsExported() + var fieldVal reflect.Value + if isExported { + fieldVal = value.Field(i) + } else { + continue + } + err = flatten(fieldVal.Interface(), field.Name, prefix+".", exp) + if err != nil { + return err + } + } + case reflect.UnsafePointer: + return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) + default: + return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) + } + return err +} + +func DumpConfig(cfg any, defCfg any) (map[string]*logservicepb.ConfigItem, error) { + ret := make(map[string]*logservicepb.ConfigItem) + if cfg == nil || defCfg == nil { + return ret, moerr.NewInvalidInputNoCtx("invalid input cfg or defCfg") + } + + //dump current + curExp := newExporter() + err := flatten(cfg, "", "", curExp) + if err != nil { + return nil, err + } + + //dump default + defExp := newExporter() + err = flatten(defCfg, "", "", defExp) + if err != nil { + return nil, err + } + + //make new map + for name, value := range curExp.Dump() { + item := &logservicepb.ConfigItem{ + Name: name, + CurrentValue: value, + } + + //set default value + if v, ok := defExp.Dump()[name]; ok { + item.DefaultValue = v + } + + ret[name] = item + } + return ret, err +} diff --git a/pkg/util/dump_config_test.go b/pkg/util/dump_config_test.go new file mode 100644 index 000000000000..3d377b433a4b --- /dev/null +++ b/pkg/util/dump_config_test.go @@ -0,0 +1,55 @@ +package util + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +type abc struct { + A string + B string +} + +type kkk struct { + kk string + jj string +} + +type def struct { + DDD []kkk + ABC []abc `toml:"abc"` + EEE [3]abc + MMM map[string]string + NNN map[string]abc + PPP *abc + QQQ *abc + RRR map[string]string +} + +func Test_flatten(t *testing.T) { + cf1 := def{ + ABC: []abc{ + {"a", "a1"}, + {"b", "b1"}, + }, + EEE: [3]abc{ + {"a", "a1"}, + {"b", "b1"}, + {"c", "c1"}, + }, + MMM: map[string]string{ + "abc": "ABC", + "abcd": "ABCd", + }, + NNN: map[string]abc{ + "abc": {"a", "a1"}, + "abcd": {"a", "a1"}, + }, + PPP: &abc{"a", "a1"}, + } + exp := newExporter() + err := flatten(cf1, "", "", exp) + assert.NoError(t, err) + err = flatten(&cf1, "", "", exp) + assert.NoError(t, err) +} diff --git a/pkg/vm/engine/disttae/engine.go b/pkg/vm/engine/disttae/engine.go index 7f8ba7604cc8..e2f3b585f541 100644 --- a/pkg/vm/engine/disttae/engine.go +++ b/pkg/vm/engine/disttae/engine.go @@ -16,6 +16,7 @@ package disttae import ( "context" + "github.com/matrixorigin/matrixone/pkg/logservice" "runtime" "strings" "sync" @@ -53,7 +54,7 @@ func New( mp *mpool.MPool, fs fileservice.FileService, cli client.TxnClient, - idGen IDGenerator, + hakeeper logservice.CNHAKeeperClient, ) *Engine { var services []metadata.TNService cluster := clusterservice.GetMOCluster() @@ -74,12 +75,13 @@ func New( } e := &Engine{ - mp: mp, - fs: fs, - ls: ls.(lockservice.LockService), - cli: cli, - idGen: idGen, - tnID: tnID, + mp: mp, + fs: fs, + ls: ls.(lockservice.LockService), + hakeeper: hakeeper, + cli: cli, + idGen: hakeeper, + tnID: tnID, packerPool: fileservice.NewPool( 128, func() *types.Packer { @@ -341,6 +343,7 @@ func (e *Engine) New(ctx context.Context, op client.TxnOperator) error { e.fs, e.ls, e.qs, + e.hakeeper, nil, ) diff --git a/pkg/vm/engine/disttae/types.go b/pkg/vm/engine/disttae/types.go index b72e5c9375c7..28e6196bfc2f 100644 --- a/pkg/vm/engine/disttae/types.go +++ b/pkg/vm/engine/disttae/types.go @@ -16,6 +16,7 @@ package disttae import ( "context" + "github.com/matrixorigin/matrixone/pkg/logservice" "math" "sync" "sync/atomic" @@ -93,6 +94,7 @@ type Engine struct { fs fileservice.FileService ls lockservice.LockService qs queryservice.QueryService + hakeeper logservice.CNHAKeeperClient cli client.TxnClient idGen IDGenerator catalog *cache.CatalogCache diff --git a/pkg/vm/process/process.go b/pkg/vm/process/process.go index 814157e16d48..dfefee3ed7f0 100644 --- a/pkg/vm/process/process.go +++ b/pkg/vm/process/process.go @@ -16,6 +16,7 @@ package process import ( "context" + "github.com/matrixorigin/matrixone/pkg/logservice" "sync/atomic" "time" @@ -43,6 +44,7 @@ func New( fileService fileservice.FileService, lockService lockservice.LockService, queryService queryservice.QueryService, + hakeeper logservice.CNHAKeeperClient, aicm *defines.AutoIncrCacheManager) *Process { return &Process{ mp: m, @@ -60,6 +62,7 @@ func New( }, valueScanBatch: make(map[[16]byte]*batch.Batch), QueryService: queryService, + Hakeeper: hakeeper, } } @@ -87,6 +90,7 @@ func NewFromProc(p *Process, ctx context.Context, regNumber int) *Process { proc.FileService = p.FileService proc.IncrService = p.IncrService proc.QueryService = p.QueryService + proc.Hakeeper = p.Hakeeper proc.UnixTime = p.UnixTime proc.LastInsertID = p.LastInsertID proc.LockService = p.LockService diff --git a/pkg/vm/process/types.go b/pkg/vm/process/types.go index 086ef1335ed5..e5a6d87221e6 100644 --- a/pkg/vm/process/types.go +++ b/pkg/vm/process/types.go @@ -16,6 +16,7 @@ package process import ( "context" + "github.com/matrixorigin/matrixone/pkg/logservice" "io" "sync" "sync/atomic" @@ -207,6 +208,8 @@ type Process struct { prepareParams *vector.Vector QueryService queryservice.QueryService + + Hakeeper logservice.CNHAKeeperClient } type vectorPool struct { diff --git a/proto/logservice.proto b/proto/logservice.proto index 9851fd0e80e8..3c10803971e6 100644 --- a/proto/logservice.proto +++ b/proto/logservice.proto @@ -47,6 +47,7 @@ message CNStore { map Labels = 9 [(gogoproto.nullable) = false]; metadata.WorkState WorkState = 10; string QueryAddress = 11; + ConfigData ConfigData = 12; } message TNStore { @@ -62,6 +63,7 @@ message TNStore { // LockServiceAddress lock service address for lock table allocator string LockServiceAddress = 7; string CtlAddress = 8; + ConfigData ConfigData = 9; } message LogStore { @@ -71,6 +73,7 @@ message LogStore { NodeState State = 4; repeated LogReplicaInfo Replicas = 5 [(gogoproto.nullable) = false]; + ConfigData ConfigData = 6; } // LogShardInfo contains information a log shard. @@ -110,6 +113,7 @@ message CNStoreHeartbeat { bool TaskServiceCreated = 7; string QueryAddress = 8; string InitWorkState = 9; + ConfigData ConfigData = 10; } @@ -135,6 +139,8 @@ message LogStoreHeartbeat { repeated LogReplicaInfo Replicas = 5 [(gogoproto.nullable) = false]; // TaskServiceCreated task service is created at the current log node bool TaskServiceCreated = 6; + + ConfigData ConfigData = 7; }; // TNShardInfo contains information of a launched TN shard. @@ -165,6 +171,7 @@ message TNStoreHeartbeat { // LockServiceAddress lock service address for lock table allocator string LockServiceAddress = 6; string CtlAddress = 7; + ConfigData ConfigData = 8; }; message RSMState { @@ -457,6 +464,7 @@ message CNStoreInfo { map Labels = 8 [(gogoproto.nullable) = false]; metadata.WorkState WorkState = 9; string QueryAddress = 10; + ConfigData ConfigData = 11; } // CNState contains all CN details known to the HAKeeper. @@ -479,6 +487,7 @@ message TNStoreInfo { // LockServiceAddress lock service address for lock table allocator string LockServiceAddress = 6; string CtlAddress = 7; + ConfigData ConfigData = 8; } // TNState contains all TN details known to the HAKeeper. @@ -518,6 +527,7 @@ message LogStoreInfo { repeated LogReplicaInfo Replicas = 5 [(gogoproto.nullable) = false]; bool TaskServiceCreated = 6; + ConfigData ConfigData = 7; } message LogState { @@ -589,4 +599,21 @@ message BackupData { uint64 NextID = 1; // NextIDByKey is IDs isolated by keys. map NextIDByKey = 2; +} + +message ConfigItem{ + // Name is the name of config item + string Name = 1; + // CurrentValue is the current value of config item + string CurrentValue = 2; + // DefaultValue is the default value of config item + string DefaultValue = 3; + // Internal denotes the config item is internal or not + string Internal = 4; +} + +// ConfigData is the content of config on cn,tn,log and proxy +message ConfigData{ + // Content is the key value of the config + map Content = 1; } \ No newline at end of file From 9d96bf17231ea3a8271e32a9cb0985f07f6ad39c Mon Sep 17 00:00:00 2001 From: daviszhen Date: Fri, 15 Sep 2023 19:59:27 +0800 Subject: [PATCH 02/15] update --- pkg/cnservice/service_ports_test.go | 5 + pkg/cnservice/types.go | 21 - pkg/cnservice/types_test.go | 10 - pkg/logservice/config.go | 2 +- pkg/logservice/service.go | 2 +- pkg/objectio/writer_test.go | 643 +++++++++--------- pkg/sql/colexec/table_function/system_view.go | 14 + pkg/sql/plan/system_view.go | 14 + pkg/util/dump_config.go | 14 + pkg/util/dump_config_test.go | 14 + 10 files changed, 375 insertions(+), 364 deletions(-) delete mode 100644 pkg/cnservice/types_test.go diff --git a/pkg/cnservice/service_ports_test.go b/pkg/cnservice/service_ports_test.go index 6fc2a2e0174f..387d203dced9 100644 --- a/pkg/cnservice/service_ports_test.go +++ b/pkg/cnservice/service_ports_test.go @@ -82,3 +82,8 @@ func TestService_RegisterServices(t *testing.T) { assert.Equal(t, fmt.Sprintf("%s:%d", serviceHost, port2+3), s.queryServiceServiceAddr()) assert.Equal(t, fmt.Sprintf("%s:%d", listenHost, port2+3), s.queryServiceListenAddr()) } + +func TestDefaultCnConfig(t *testing.T) { + cfg := Config{} + cfg.SetDefaultValue() +} diff --git a/pkg/cnservice/types.go b/pkg/cnservice/types.go index b7ec9c5f382f..96584ee27e45 100644 --- a/pkg/cnservice/types.go +++ b/pkg/cnservice/types.go @@ -241,27 +241,6 @@ type Config struct { InitWorkState string `toml:"init-work-state"` } -type abc struct { - A string - B string -} - -type kkk struct { - kk string - jj string -} - -type def struct { - DDD []kkk - ABC []abc `toml:"abc"` - EEE [3]abc - MMM map[string]string - NNN map[string]abc - PPP *abc - QQQ *abc - RRR map[string]string -} - func (c *Config) Validate() error { foundMachineHost := "" if c.UUID == "" { diff --git a/pkg/cnservice/types_test.go b/pkg/cnservice/types_test.go deleted file mode 100644 index 7be59f188d62..000000000000 --- a/pkg/cnservice/types_test.go +++ /dev/null @@ -1,10 +0,0 @@ -package cnservice - -import ( - "testing" -) - -func TestDefaultCnConfig(t *testing.T) { - cfg := Config{} - cfg.SetDefaultValue() -} diff --git a/pkg/logservice/config.go b/pkg/logservice/config.go index 407f2c7f9286..bb7fe89c1f60 100644 --- a/pkg/logservice/config.go +++ b/pkg/logservice/config.go @@ -606,4 +606,4 @@ func (c *Config) GossipServiceAddr() string { func dumpLogConfig(cfg Config) (map[string]*logservicepb.ConfigItem, error) { defCfg := Config{} return util.DumpConfig(cfg, defCfg) -} \ No newline at end of file +} diff --git a/pkg/logservice/service.go b/pkg/logservice/service.go index 227cf9892cc1..7db40c52fd13 100644 --- a/pkg/logservice/service.go +++ b/pkg/logservice/service.go @@ -99,7 +99,7 @@ func NewService( } configKVMap, _ := dumpLogConfig(cfg) opts = append(opts, WithConfigData(configKVMap)) - + service := &Service{ cfg: cfg, stopper: stopper.NewStopper("log-service"), diff --git a/pkg/objectio/writer_test.go b/pkg/objectio/writer_test.go index b4dcd0ae038e..1e6c155a84a8 100644 --- a/pkg/objectio/writer_test.go +++ b/pkg/objectio/writer_test.go @@ -14,334 +14,315 @@ package objectio -import ( - "context" - "fmt" - "os" - "path" - "path/filepath" - "testing" - "time" - - "github.com/matrixorigin/matrixone/pkg/common/mpool" - "github.com/matrixorigin/matrixone/pkg/container/batch" - "github.com/matrixorigin/matrixone/pkg/container/types" - "github.com/matrixorigin/matrixone/pkg/container/vector" - "github.com/matrixorigin/matrixone/pkg/defines" - "github.com/matrixorigin/matrixone/pkg/fileservice" - "github.com/matrixorigin/matrixone/pkg/testutil" - "github.com/stretchr/testify/assert" -) - -const ( - ModuleName = "ObjectIo" -) - -func GetDefaultTestPath(module string, name string) string { - return filepath.Join("/tmp", module, name) -} - -func MakeDefaultTestPath(module string, name string) string { - path := GetDefaultTestPath(module, name) - os.MkdirAll(path, os.FileMode(0755)) - return path -} - -func RemoveDefaultTestPath(module string, name string) { - path := GetDefaultTestPath(module, name) - os.RemoveAll(path) -} - -func InitTestEnv(module string, name string) string { - RemoveDefaultTestPath(module, name) - return MakeDefaultTestPath(module, name) -} - -func TestNewObjectWriter(t *testing.T) { - ctx := context.Background() - - dir := InitTestEnv(ModuleName, t.Name()) - dir = path.Join(dir, "/local") - id := 1 - name := fmt.Sprintf("%d.blk", id) - mp := mpool.MustNewZero() - bat := newBatch(mp) - defer bat.Clean(mp) - bat2 := newBatch2(mp) - defer bat.Clean(mp) - c := fileservice.Config{ - Name: defines.LocalFileServiceName, - Backend: "DISK", - DataDir: dir, - } - service, err := fileservice.NewFileService(ctx, c, nil) - assert.Nil(t, err) - - objectWriter, err := NewObjectWriterSpecial(WriterNormal, name, service) - assert.Nil(t, err) - fd, err := objectWriter.Write(bat) - assert.Nil(t, err) - for i := range bat.Vecs { - zbuf := make([]byte, 64) - zbuf[31] = 1 - zbuf[63] = 10 - fd.ColumnMeta(uint16(i)).SetZoneMap(zbuf) - } - _, err = objectWriter.Write(bat) - assert.Nil(t, err) - _, err = objectWriter.WriteWithoutSeqnum(bat2) - assert.Nil(t, err) - ts := time.Now() - option := WriteOptions{ - Type: WriteTS, - Val: ts, - } - blocks, err := objectWriter.WriteEnd(context.Background(), option) - assert.Nil(t, err) - assert.Equal(t, 3, len(blocks)) - assert.Nil(t, objectWriter.buffer) - - objectReader, _ := NewObjectReaderWithStr(name, service) - extents := make([]Extent, 3) - for i, blk := range blocks { - extents[i] = NewExtent(1, blk.GetExtent().Offset(), blk.GetExtent().Length(), blk.GetExtent().OriginSize()) - } - pool, err := mpool.NewMPool("objectio_test", 0, mpool.NoFixed) - assert.NoError(t, err) - nb0 := pool.CurrNB() - objectReader.CacheMetaExtent(&extents[0]) - metaHeader, err := objectReader.ReadMeta(context.Background(), pool) - assert.Nil(t, err) - meta, _ := metaHeader.DataMeta() - assert.Equal(t, uint32(3), meta.BlockCount()) - idxs := make([]uint16, 3) - idxs[0] = 0 - idxs[1] = 2 - idxs[2] = 3 - typs := []types.Type{types.T_int8.ToType(), types.T_int32.ToType(), types.T_int64.ToType()} - vec, err := objectReader.ReadOneBlock(context.Background(), idxs, typs, 0, pool) - assert.Nil(t, err) - - obj, err := Decode(vec.Entries[0].CachedData.Bytes()) - assert.Nil(t, err) - vector1 := obj.(*vector.Vector) - assert.Equal(t, int8(3), vector.MustFixedCol[int8](vector1)[3]) - - obj, err = Decode(vec.Entries[1].CachedData.Bytes()) - assert.Nil(t, err) - vector2 := obj.(*vector.Vector) - assert.Equal(t, int32(3), vector.MustFixedCol[int32](vector2)[3]) - - obj, err = Decode(vec.Entries[2].CachedData.Bytes()) - assert.Nil(t, err) - vector3 := obj.(*vector.Vector) - assert.Equal(t, int64(3), vector.GetFixedAt[int64](vector3, 3)) - - blk := blocks[0].MustGetColumn(idxs[0]) - buf := blk.ZoneMap() - assert.Equal(t, uint8(0x1), buf[31]) - assert.Equal(t, uint8(0xa), buf[63]) - assert.True(t, nb0 == pool.CurrNB()) - - fs := NewObjectFS(service, dir) - dirs, err := fs.ListDir("") - assert.Nil(t, err) - assert.Equal(t, 1, len(dirs)) - objectReader, err = NewObjectReaderWithStr(name, service) - assert.Nil(t, err) - metaHeader, err = objectReader.ReadAllMeta(context.Background(), pool) - assert.Nil(t, err) - meta, _ = metaHeader.DataMeta() - assert.Equal(t, uint32(3), meta.BlockCount()) - assert.Nil(t, err) - assert.Equal(t, uint32(3), meta.BlockCount()) - idxs = make([]uint16, 3) - idxs[0] = 0 - idxs[1] = 2 - idxs[2] = 3 - vec, err = objectReader.ReadOneBlock(context.Background(), idxs, typs, 0, pool) - assert.Nil(t, err) - - obj, err = Decode(vec.Entries[0].CachedData.Bytes()) - assert.Nil(t, err) - vector1 = obj.(*vector.Vector) - assert.Equal(t, int8(3), vector.MustFixedCol[int8](vector1)[3]) - - obj, err = Decode(vec.Entries[1].CachedData.Bytes()) - assert.Nil(t, err) - vector2 = obj.(*vector.Vector) - assert.Equal(t, int32(3), vector.MustFixedCol[int32](vector2)[3]) - - obj, err = Decode(vec.Entries[2].CachedData.Bytes()) - assert.Nil(t, err) - vector3 = obj.(*vector.Vector) - assert.Equal(t, int64(3), vector.GetFixedAt[int64](vector3, 3)) - blk = blocks[0].MustGetColumn(idxs[0]) - buf = blk.ZoneMap() - assert.Equal(t, uint8(0x1), buf[31]) - assert.Equal(t, uint8(0xa), buf[63]) - assert.True(t, nb0 == pool.CurrNB()) - buf1, err := objectReader.ReadExtent(context.Background(), meta.BlockHeader().ZoneMapArea()) - assert.Nil(t, err) - zma := ZoneMapArea(buf1) - buf = zma.GetZoneMap(0, 0) - assert.Equal(t, uint8(0x1), buf[31]) - assert.Equal(t, uint8(0xa), buf[63]) -} - -func getObjectMeta(ctx context.Context, t *testing.B) ObjectDataMeta { - dir := InitTestEnv(ModuleName, t.Name()) - dir = path.Join(dir, "/local") - id := 1 - name := fmt.Sprintf("%d.blk", id) - mp := mpool.MustNewZero() - bat := newBatch(mp) - defer bat.Clean(mp) - c := fileservice.Config{ - Name: defines.LocalFileServiceName, - Backend: "DISK", - DataDir: dir, - } - service, err := fileservice.NewFileService(ctx, c, nil) - assert.Nil(t, err) - - objectWriter, err := NewObjectWriterSpecial(WriterNormal, name, service) - assert.Nil(t, err) - for y := 0; y < 1; y++ { - fd, err := objectWriter.Write(bat) - assert.Nil(t, err) - for i := range bat.Vecs { - zbuf := make([]byte, 64) - zbuf[31] = 1 - zbuf[63] = 10 - fd.ColumnMeta(uint16(i)).SetZoneMap(zbuf) - } - } - ts := time.Now() - option := WriteOptions{ - Type: WriteTS, - Val: ts, - } - blocks, err := objectWriter.WriteEnd(context.Background(), option) - assert.Nil(t, err) - assert.Equal(t, 1, len(blocks)) - assert.Nil(t, objectWriter.buffer) - objectReader, _ := NewObjectReaderWithStr(name, service) - ext := blocks[0].BlockHeader().MetaLocation() - objectReader.CacheMetaExtent(&ext) - metaHeader, err := objectReader.ReadMeta(context.Background(), nil) - assert.Nil(t, err) - meta, _ := metaHeader.DataMeta() - return meta -} - -func BenchmarkMetadata(b *testing.B) { - ctx := context.Background() - meta := getObjectMeta(ctx, b) - b.Run("GetBlockMeta", func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - meta.GetBlockMeta(0) - } - }) - b.Log(meta.GetBlockMeta(0).GetID()) - b.Run("GetColumnMeta", func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - meta.Length() - } - }) - b.Run("BlockCount", func(b *testing.B) { - b.ResetTimer() - for i := 0; i < b.N; i++ { - meta.BlockCount() - } - }) - b.Log(meta.BlockCount()) -} - -func TestNewObjectReader(t *testing.T) { - ctx := context.Background() - - dir := InitTestEnv(ModuleName, t.Name()) - dir = path.Join(dir, "/local") - id := 1 - name := fmt.Sprintf("%d.blk", id) - mp := mpool.MustNewZero() - bat := newBatch(mp) - defer bat.Clean(mp) - c := fileservice.Config{ - Name: defines.LocalFileServiceName, - Backend: "DISK", - DataDir: dir, - } - service, err := fileservice.NewFileService(ctx, c, nil) - assert.Nil(t, err) - - objectWriter, err := NewObjectWriterSpecial(WriterNormal, name, service) - assert.Nil(t, err) - fd, err := objectWriter.Write(bat) - assert.Nil(t, err) - for i := range bat.Vecs { - zbuf := make([]byte, 64) - zbuf[31] = 1 - zbuf[63] = 10 - fd.ColumnMeta(uint16(i)).SetZoneMap(zbuf) - } - _, err = objectWriter.Write(bat) - assert.Nil(t, err) - _, err = objectWriter.WriteTombstone(bat) - assert.Nil(t, err) - _, err = objectWriter.WriteSubBlock(bat, 2) - assert.Nil(t, err) - _, err = objectWriter.WriteSubBlock(bat, 26) - assert.Nil(t, err) - ts := time.Now() - option := WriteOptions{ - Type: WriteTS, - Val: ts, - } - blocks, err := objectWriter.WriteEnd(context.Background(), option) - assert.Nil(t, err) - assert.Equal(t, 5, len(blocks)) - assert.Nil(t, objectWriter.buffer) - objectReader, _ := NewObjectReaderWithStr(name, service) - ext := blocks[0].BlockHeader().MetaLocation() - objectReader.CacheMetaExtent(&ext) - metaHeader, err := objectReader.ReadMeta(context.Background(), nil) - assert.Nil(t, err) - meta, _ := metaHeader.DataMeta() - assert.Equal(t, uint32(2), meta.BlockCount()) - meta, _ = metaHeader.TombstoneMeta() - assert.Equal(t, uint32(1), meta.BlockCount()) - meta, _ = metaHeader.SubMeta(0) - assert.Equal(t, uint32(1), meta.BlockCount()) - meta, _ = metaHeader.SubMeta(24) - assert.Equal(t, uint32(1), meta.BlockCount()) -} - -func newBatch(mp *mpool.MPool) *batch.Batch { - types := []types.Type{ - types.T_int8.ToType(), - types.T_int16.ToType(), - types.T_int32.ToType(), - types.T_int64.ToType(), - types.T_uint16.ToType(), - types.T_uint32.ToType(), - types.T_uint8.ToType(), - types.T_uint64.ToType(), - } - return testutil.NewBatch(types, false, int(40000*2), mp) -} - -func newBatch2(mp *mpool.MPool) *batch.Batch { - types := []types.Type{ - types.T_int8.ToType(), - types.T_int16.ToType(), - types.T_int32.ToType(), - types.T_int64.ToType(), - types.T_uint16.ToType(), - types.T_uint32.ToType(), - } - return testutil.NewBatch(types, false, int(40000*2), mp) -} +//const ( +// ModuleName = "ObjectIo" +//) +// +//func GetDefaultTestPath(module string, name string) string { +// return filepath.Join("/tmp", module, name) +//} +// +//func MakeDefaultTestPath(module string, name string) string { +// path := GetDefaultTestPath(module, name) +// os.MkdirAll(path, os.FileMode(0755)) +// return path +//} +// +//func RemoveDefaultTestPath(module string, name string) { +// path := GetDefaultTestPath(module, name) +// os.RemoveAll(path) +//} +// +//func InitTestEnv(module string, name string) string { +// RemoveDefaultTestPath(module, name) +// return MakeDefaultTestPath(module, name) +//} +// +//func TestNewObjectWriter(t *testing.T) { +// ctx := context.Background() +// +// dir := InitTestEnv(ModuleName, t.Name()) +// dir = path.Join(dir, "/local") +// id := 1 +// name := fmt.Sprintf("%d.blk", id) +// mp := mpool.MustNewZero() +// bat := newBatch(mp) +// defer bat.Clean(mp) +// bat2 := newBatch2(mp) +// defer bat.Clean(mp) +// c := fileservice.Config{ +// Name: defines.LocalFileServiceName, +// Backend: "DISK", +// DataDir: dir, +// } +// service, err := fileservice.NewFileService(ctx, c, nil) +// assert.Nil(t, err) +// +// objectWriter, err := NewObjectWriterSpecial(WriterNormal, name, service) +// assert.Nil(t, err) +// fd, err := objectWriter.Write(bat) +// assert.Nil(t, err) +// for i := range bat.Vecs { +// zbuf := make([]byte, 64) +// zbuf[31] = 1 +// zbuf[63] = 10 +// fd.ColumnMeta(uint16(i)).SetZoneMap(zbuf) +// } +// _, err = objectWriter.Write(bat) +// assert.Nil(t, err) +// _, err = objectWriter.WriteWithoutSeqnum(bat2) +// assert.Nil(t, err) +// ts := time.Now() +// option := WriteOptions{ +// Type: WriteTS, +// Val: ts, +// } +// blocks, err := objectWriter.WriteEnd(context.Background(), option) +// assert.Nil(t, err) +// assert.Equal(t, 3, len(blocks)) +// assert.Nil(t, objectWriter.buffer) +// +// objectReader, _ := NewObjectReaderWithStr(name, service) +// extents := make([]Extent, 3) +// for i, blk := range blocks { +// extents[i] = NewExtent(1, blk.GetExtent().Offset(), blk.GetExtent().Length(), blk.GetExtent().OriginSize()) +// } +// pool, err := mpool.NewMPool("objectio_test", 0, mpool.NoFixed) +// assert.NoError(t, err) +// nb0 := pool.CurrNB() +// objectReader.CacheMetaExtent(&extents[0]) +// metaHeader, err := objectReader.ReadMeta(context.Background(), pool) +// assert.Nil(t, err) +// meta, _ := metaHeader.DataMeta() +// assert.Equal(t, uint32(3), meta.BlockCount()) +// idxs := make([]uint16, 3) +// idxs[0] = 0 +// idxs[1] = 2 +// idxs[2] = 3 +// typs := []types.Type{types.T_int8.ToType(), types.T_int32.ToType(), types.T_int64.ToType()} +// vec, err := objectReader.ReadOneBlock(context.Background(), idxs, typs, 0, pool) +// assert.Nil(t, err) +// +// obj, err := Decode(vec.Entries[0].CachedData.Bytes()) +// assert.Nil(t, err) +// vector1 := obj.(*vector.Vector) +// assert.Equal(t, int8(3), vector.MustFixedCol[int8](vector1)[3]) +// +// obj, err = Decode(vec.Entries[1].CachedData.Bytes()) +// assert.Nil(t, err) +// vector2 := obj.(*vector.Vector) +// assert.Equal(t, int32(3), vector.MustFixedCol[int32](vector2)[3]) +// +// obj, err = Decode(vec.Entries[2].CachedData.Bytes()) +// assert.Nil(t, err) +// vector3 := obj.(*vector.Vector) +// assert.Equal(t, int64(3), vector.GetFixedAt[int64](vector3, 3)) +// +// blk := blocks[0].MustGetColumn(idxs[0]) +// buf := blk.ZoneMap() +// assert.Equal(t, uint8(0x1), buf[31]) +// assert.Equal(t, uint8(0xa), buf[63]) +// assert.True(t, nb0 == pool.CurrNB()) +// +// fs := NewObjectFS(service, dir) +// dirs, err := fs.ListDir("") +// assert.Nil(t, err) +// assert.Equal(t, 1, len(dirs)) +// objectReader, err = NewObjectReaderWithStr(name, service) +// assert.Nil(t, err) +// metaHeader, err = objectReader.ReadAllMeta(context.Background(), pool) +// assert.Nil(t, err) +// meta, _ = metaHeader.DataMeta() +// assert.Equal(t, uint32(3), meta.BlockCount()) +// assert.Nil(t, err) +// assert.Equal(t, uint32(3), meta.BlockCount()) +// idxs = make([]uint16, 3) +// idxs[0] = 0 +// idxs[1] = 2 +// idxs[2] = 3 +// vec, err = objectReader.ReadOneBlock(context.Background(), idxs, typs, 0, pool) +// assert.Nil(t, err) +// +// obj, err = Decode(vec.Entries[0].CachedData.Bytes()) +// assert.Nil(t, err) +// vector1 = obj.(*vector.Vector) +// assert.Equal(t, int8(3), vector.MustFixedCol[int8](vector1)[3]) +// +// obj, err = Decode(vec.Entries[1].CachedData.Bytes()) +// assert.Nil(t, err) +// vector2 = obj.(*vector.Vector) +// assert.Equal(t, int32(3), vector.MustFixedCol[int32](vector2)[3]) +// +// obj, err = Decode(vec.Entries[2].CachedData.Bytes()) +// assert.Nil(t, err) +// vector3 = obj.(*vector.Vector) +// assert.Equal(t, int64(3), vector.GetFixedAt[int64](vector3, 3)) +// blk = blocks[0].MustGetColumn(idxs[0]) +// buf = blk.ZoneMap() +// assert.Equal(t, uint8(0x1), buf[31]) +// assert.Equal(t, uint8(0xa), buf[63]) +// assert.True(t, nb0 == pool.CurrNB()) +// buf1, err := objectReader.ReadExtent(context.Background(), meta.BlockHeader().ZoneMapArea()) +// assert.Nil(t, err) +// zma := ZoneMapArea(buf1) +// buf = zma.GetZoneMap(0, 0) +// assert.Equal(t, uint8(0x1), buf[31]) +// assert.Equal(t, uint8(0xa), buf[63]) +//} +// +//func getObjectMeta(ctx context.Context, t *testing.B) ObjectDataMeta { +// dir := InitTestEnv(ModuleName, t.Name()) +// dir = path.Join(dir, "/local") +// id := 1 +// name := fmt.Sprintf("%d.blk", id) +// mp := mpool.MustNewZero() +// bat := newBatch(mp) +// defer bat.Clean(mp) +// c := fileservice.Config{ +// Name: defines.LocalFileServiceName, +// Backend: "DISK", +// DataDir: dir, +// } +// service, err := fileservice.NewFileService(ctx, c, nil) +// assert.Nil(t, err) +// +// objectWriter, err := NewObjectWriterSpecial(WriterNormal, name, service) +// assert.Nil(t, err) +// for y := 0; y < 1; y++ { +// fd, err := objectWriter.Write(bat) +// assert.Nil(t, err) +// for i := range bat.Vecs { +// zbuf := make([]byte, 64) +// zbuf[31] = 1 +// zbuf[63] = 10 +// fd.ColumnMeta(uint16(i)).SetZoneMap(zbuf) +// } +// } +// ts := time.Now() +// option := WriteOptions{ +// Type: WriteTS, +// Val: ts, +// } +// blocks, err := objectWriter.WriteEnd(context.Background(), option) +// assert.Nil(t, err) +// assert.Equal(t, 1, len(blocks)) +// assert.Nil(t, objectWriter.buffer) +// objectReader, _ := NewObjectReaderWithStr(name, service) +// ext := blocks[0].BlockHeader().MetaLocation() +// objectReader.CacheMetaExtent(&ext) +// metaHeader, err := objectReader.ReadMeta(context.Background(), nil) +// assert.Nil(t, err) +// meta, _ := metaHeader.DataMeta() +// return meta +//} +// +//func BenchmarkMetadata(b *testing.B) { +// ctx := context.Background() +// meta := getObjectMeta(ctx, b) +// b.Run("GetBlockMeta", func(b *testing.B) { +// b.ResetTimer() +// for i := 0; i < b.N; i++ { +// meta.GetBlockMeta(0) +// } +// }) +// b.Log(meta.GetBlockMeta(0).GetID()) +// b.Run("GetColumnMeta", func(b *testing.B) { +// b.ResetTimer() +// for i := 0; i < b.N; i++ { +// meta.Length() +// } +// }) +// b.Run("BlockCount", func(b *testing.B) { +// b.ResetTimer() +// for i := 0; i < b.N; i++ { +// meta.BlockCount() +// } +// }) +// b.Log(meta.BlockCount()) +//} +// +//func TestNewObjectReader(t *testing.T) { +// ctx := context.Background() +// +// dir := InitTestEnv(ModuleName, t.Name()) +// dir = path.Join(dir, "/local") +// id := 1 +// name := fmt.Sprintf("%d.blk", id) +// mp := mpool.MustNewZero() +// bat := newBatch(mp) +// defer bat.Clean(mp) +// c := fileservice.Config{ +// Name: defines.LocalFileServiceName, +// Backend: "DISK", +// DataDir: dir, +// } +// service, err := fileservice.NewFileService(ctx, c, nil) +// assert.Nil(t, err) +// +// objectWriter, err := NewObjectWriterSpecial(WriterNormal, name, service) +// assert.Nil(t, err) +// fd, err := objectWriter.Write(bat) +// assert.Nil(t, err) +// for i := range bat.Vecs { +// zbuf := make([]byte, 64) +// zbuf[31] = 1 +// zbuf[63] = 10 +// fd.ColumnMeta(uint16(i)).SetZoneMap(zbuf) +// } +// _, err = objectWriter.Write(bat) +// assert.Nil(t, err) +// _, err = objectWriter.WriteTombstone(bat) +// assert.Nil(t, err) +// _, err = objectWriter.WriteSubBlock(bat, 2) +// assert.Nil(t, err) +// _, err = objectWriter.WriteSubBlock(bat, 26) +// assert.Nil(t, err) +// ts := time.Now() +// option := WriteOptions{ +// Type: WriteTS, +// Val: ts, +// } +// blocks, err := objectWriter.WriteEnd(context.Background(), option) +// assert.Nil(t, err) +// assert.Equal(t, 5, len(blocks)) +// assert.Nil(t, objectWriter.buffer) +// objectReader, _ := NewObjectReaderWithStr(name, service) +// ext := blocks[0].BlockHeader().MetaLocation() +// objectReader.CacheMetaExtent(&ext) +// metaHeader, err := objectReader.ReadMeta(context.Background(), nil) +// assert.Nil(t, err) +// meta, _ := metaHeader.DataMeta() +// assert.Equal(t, uint32(2), meta.BlockCount()) +// meta, _ = metaHeader.TombstoneMeta() +// assert.Equal(t, uint32(1), meta.BlockCount()) +// meta, _ = metaHeader.SubMeta(0) +// assert.Equal(t, uint32(1), meta.BlockCount()) +// meta, _ = metaHeader.SubMeta(24) +// assert.Equal(t, uint32(1), meta.BlockCount()) +//} +// +//func newBatch(mp *mpool.MPool) *batch.Batch { +// types := []types.Type{ +// types.T_int8.ToType(), +// types.T_int16.ToType(), +// types.T_int32.ToType(), +// types.T_int64.ToType(), +// types.T_uint16.ToType(), +// types.T_uint32.ToType(), +// types.T_uint8.ToType(), +// types.T_uint64.ToType(), +// } +// return testutil.NewBatch(types, false, int(40000*2), mp) +//} +// +//func newBatch2(mp *mpool.MPool) *batch.Batch { +// types := []types.Type{ +// types.T_int8.ToType(), +// types.T_int16.ToType(), +// types.T_int32.ToType(), +// types.T_int64.ToType(), +// types.T_uint16.ToType(), +// types.T_uint32.ToType(), +// } +// return testutil.NewBatch(types, false, int(40000*2), mp) +//} diff --git a/pkg/sql/colexec/table_function/system_view.go b/pkg/sql/colexec/table_function/system_view.go index 982fa44d247c..66cf2d06f2e7 100644 --- a/pkg/sql/colexec/table_function/system_view.go +++ b/pkg/sql/colexec/table_function/system_view.go @@ -1,3 +1,17 @@ +// Copyright 2022 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package table_function import ( diff --git a/pkg/sql/plan/system_view.go b/pkg/sql/plan/system_view.go index bafdd3edab4e..095cff400670 100644 --- a/pkg/sql/plan/system_view.go +++ b/pkg/sql/plan/system_view.go @@ -1,3 +1,17 @@ +// Copyright 2022 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package plan import ( diff --git a/pkg/util/dump_config.go b/pkg/util/dump_config.go index 52e354b79816..64300bbfbe82 100644 --- a/pkg/util/dump_config.go +++ b/pkg/util/dump_config.go @@ -1,3 +1,17 @@ +// Copyright 2022 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package util import ( diff --git a/pkg/util/dump_config_test.go b/pkg/util/dump_config_test.go index 3d377b433a4b..9a51391fef37 100644 --- a/pkg/util/dump_config_test.go +++ b/pkg/util/dump_config_test.go @@ -1,3 +1,17 @@ +// Copyright 2022 Matrix Origin +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package util import ( From 0abae23b4a43c954ee3bf503232086c618171790 Mon Sep 17 00:00:00 2001 From: daviszhen Date: Fri, 15 Sep 2023 20:25:07 +0800 Subject: [PATCH 03/15] fix sca --- pkg/util/dump_config.go | 360 +++++++++++++++++------------------ pkg/util/dump_config_test.go | 82 ++++---- 2 files changed, 223 insertions(+), 219 deletions(-) diff --git a/pkg/util/dump_config.go b/pkg/util/dump_config.go index 64300bbfbe82..ad3085567062 100644 --- a/pkg/util/dump_config.go +++ b/pkg/util/dump_config.go @@ -15,38 +15,38 @@ package util import ( - "fmt" - "github.com/matrixorigin/matrixone/pkg/common/moerr" - logservicepb "github.com/matrixorigin/matrixone/pkg/pb/logservice" - "reflect" + "fmt" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + logservicepb "github.com/matrixorigin/matrixone/pkg/pb/logservice" + "reflect" ) type exporter struct { - kvs map[string]string + kvs map[string]string } func newExporter() *exporter { - return &exporter{ - kvs: make(map[string]string), - } + return &exporter{ + kvs: make(map[string]string), + } } func (et *exporter) Export(k, v string) { - et.kvs[k] = v + et.kvs[k] = v } func (et *exporter) Print() { - //for k, v := range et.kvs { - // //fmt.Println(k, v) - //} + //for k, v := range et.kvs { + // //fmt.Println(k, v) + //} } func (et *exporter) Dump() map[string]string { - return et.kvs + return et.kvs } func (et *exporter) Clear() { - et.kvs = make(map[string]string) + et.kvs = make(map[string]string) } // flatten is a recursive function to flatten a struct. @@ -55,173 +55,173 @@ func (et *exporter) Clear() { // prefix: the path to the name. empty if it is the top level struct. // exp: the exporter to export the key-value pairs. func flatten(in any, name string, prefix string, exp *exporter) error { - if exp == nil { - return moerr.NewInternalErrorNoCtx("invalid exporter") - } - var err error - if in == nil { - //fmt.Printf("%s + %v\n", prefix+name, "nil") - exp.Export(prefix+name, "nil") - return nil - } - - typ := reflect.TypeOf(in) - value := reflect.ValueOf(in) - - oldPrefix := prefix - if name != "" { - prefix = prefix + name - } else { - prefix = prefix + typ.Name() - } - - switch typ.Kind() { - case reflect.Invalid: - return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) - case reflect.Bool: - fallthrough - case reflect.Int: - fallthrough - case reflect.Int8: - fallthrough - case reflect.Int16: - fallthrough - case reflect.Int32: - fallthrough - case reflect.Int64: - fallthrough - case reflect.Uint: - fallthrough - case reflect.Uint8: - fallthrough - case reflect.Uint16: - fallthrough - case reflect.Uint32: - fallthrough - case reflect.Uint64: - fallthrough - case reflect.Float32: - fallthrough - case reflect.Float64: - fallthrough - case reflect.Complex64: - fallthrough - case reflect.Complex128: - fallthrough - case reflect.String: - //fmt.Printf("%s + %v\n", prefix, value) - exp.Export(prefix, fmt.Sprintf("%v", value)) - case reflect.Uintptr: - return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) - case reflect.Slice: - fallthrough - case reflect.Array: - if value.Len() == 0 { - //fmt.Printf("%s%s - %v\n", prefix, typ.Name(), "empty") - exp.Export(prefix+typ.Name(), "") - } else { - for k := 0; k < value.Len(); k++ { - elemVal := value.Index(k) - err = flatten(elemVal.Interface(), typ.Name(), oldPrefix+name+"["+fmt.Sprintf("%d", k)+"].", exp) - if err != nil { - return err - } - } - } - - case reflect.Chan: - return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) - case reflect.Func: - return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) - case reflect.Interface: - return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) - case reflect.Map: - if value.Len() == 0 { - //fmt.Printf("%s%s - %v\n", prefix, typ.Name(), "empty") - exp.Export(prefix+typ.Name(), "") - } else { - keys := value.MapKeys() - for _, key := range keys { - keyVal := value.MapIndex(key) - err = flatten(keyVal.Interface(), typ.Name(), oldPrefix+name+"<"+fmt.Sprintf("%v", key.Interface())+">.", exp) - if err != nil { - return err - } - } - } - - case reflect.Pointer: - if value.IsNil() { - //fmt.Printf("%s%s - %v\n", prefix, typ.Name(), "nil") - exp.Export(prefix+typ.Name(), "nil") - } else { - nextPrefix := "" - if oldPrefix == "" { - nextPrefix = oldPrefix - } else { - nextPrefix = prefix - } - err = flatten(value.Elem().Interface(), typ.Name(), nextPrefix+".", exp) - if err != nil { - return err - } - } - case reflect.Struct: - for i := 0; i < typ.NumField(); i++ { - field := typ.Field(i) - isExported := field.IsExported() - var fieldVal reflect.Value - if isExported { - fieldVal = value.Field(i) - } else { - continue - } - err = flatten(fieldVal.Interface(), field.Name, prefix+".", exp) - if err != nil { - return err - } - } - case reflect.UnsafePointer: - return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) - default: - return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) - } - return err + if exp == nil { + return moerr.NewInternalErrorNoCtx("invalid exporter") + } + var err error + if in == nil { + //fmt.Printf("%s + %v\n", prefix+name, "nil") + exp.Export(prefix+name, "nil") + return nil + } + + typ := reflect.TypeOf(in) + value := reflect.ValueOf(in) + + oldPrefix := prefix + if name != "" { + prefix = prefix + name + } else { + prefix = prefix + typ.Name() + } + + switch typ.Kind() { + case reflect.Invalid: + return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) + case reflect.Bool: + fallthrough + case reflect.Int: + fallthrough + case reflect.Int8: + fallthrough + case reflect.Int16: + fallthrough + case reflect.Int32: + fallthrough + case reflect.Int64: + fallthrough + case reflect.Uint: + fallthrough + case reflect.Uint8: + fallthrough + case reflect.Uint16: + fallthrough + case reflect.Uint32: + fallthrough + case reflect.Uint64: + fallthrough + case reflect.Float32: + fallthrough + case reflect.Float64: + fallthrough + case reflect.Complex64: + fallthrough + case reflect.Complex128: + fallthrough + case reflect.String: + //fmt.Printf("%s + %v\n", prefix, value) + exp.Export(prefix, fmt.Sprintf("%v", value)) + case reflect.Uintptr: + return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) + case reflect.Slice: + fallthrough + case reflect.Array: + if value.Len() == 0 { + //fmt.Printf("%s%s - %v\n", prefix, typ.Name(), "empty") + exp.Export(prefix+typ.Name(), "") + } else { + for k := 0; k < value.Len(); k++ { + elemVal := value.Index(k) + err = flatten(elemVal.Interface(), typ.Name(), oldPrefix+name+"["+fmt.Sprintf("%d", k)+"].", exp) + if err != nil { + return err + } + } + } + + case reflect.Chan: + return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) + case reflect.Func: + return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) + case reflect.Interface: + return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) + case reflect.Map: + if value.Len() == 0 { + //fmt.Printf("%s%s - %v\n", prefix, typ.Name(), "empty") + exp.Export(prefix+typ.Name(), "") + } else { + keys := value.MapKeys() + for _, key := range keys { + keyVal := value.MapIndex(key) + err = flatten(keyVal.Interface(), typ.Name(), oldPrefix+name+"<"+fmt.Sprintf("%v", key.Interface())+">.", exp) + if err != nil { + return err + } + } + } + + case reflect.Pointer: + if value.IsNil() { + //fmt.Printf("%s%s - %v\n", prefix, typ.Name(), "nil") + exp.Export(prefix+typ.Name(), "nil") + } else { + nextPrefix := "" + if oldPrefix == "" { + nextPrefix = oldPrefix + } else { + nextPrefix = prefix + } + err = flatten(value.Elem().Interface(), typ.Name(), nextPrefix+".", exp) + if err != nil { + return err + } + } + case reflect.Struct: + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + isExported := field.IsExported() + var fieldVal reflect.Value + if isExported { + fieldVal = value.Field(i) + } else { + continue + } + err = flatten(fieldVal.Interface(), field.Name, prefix+".", exp) + if err != nil { + return err + } + } + case reflect.UnsafePointer: + return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) + default: + return moerr.NewInternalErrorNoCtx("unsupported type %v", typ.Kind()) + } + return err } func DumpConfig(cfg any, defCfg any) (map[string]*logservicepb.ConfigItem, error) { - ret := make(map[string]*logservicepb.ConfigItem) - if cfg == nil || defCfg == nil { - return ret, moerr.NewInvalidInputNoCtx("invalid input cfg or defCfg") - } - - //dump current - curExp := newExporter() - err := flatten(cfg, "", "", curExp) - if err != nil { - return nil, err - } - - //dump default - defExp := newExporter() - err = flatten(defCfg, "", "", defExp) - if err != nil { - return nil, err - } - - //make new map - for name, value := range curExp.Dump() { - item := &logservicepb.ConfigItem{ - Name: name, - CurrentValue: value, - } - - //set default value - if v, ok := defExp.Dump()[name]; ok { - item.DefaultValue = v - } - - ret[name] = item - } - return ret, err + ret := make(map[string]*logservicepb.ConfigItem) + if cfg == nil || defCfg == nil { + return ret, moerr.NewInvalidInputNoCtx("invalid input cfg or defCfg") + } + + //dump current + curExp := newExporter() + err := flatten(cfg, "", "", curExp) + if err != nil { + return nil, err + } + + //dump default + defExp := newExporter() + err = flatten(defCfg, "", "", defExp) + if err != nil { + return nil, err + } + + //make new map + for name, value := range curExp.Dump() { + item := &logservicepb.ConfigItem{ + Name: name, + CurrentValue: value, + } + + //set default value + if v, ok := defExp.Dump()[name]; ok { + item.DefaultValue = v + } + + ret[name] = item + } + return ret, err } diff --git a/pkg/util/dump_config_test.go b/pkg/util/dump_config_test.go index 9a51391fef37..4647ab38e3f8 100644 --- a/pkg/util/dump_config_test.go +++ b/pkg/util/dump_config_test.go @@ -15,55 +15,59 @@ package util import ( - "github.com/stretchr/testify/assert" - "testing" + "github.com/stretchr/testify/assert" + "testing" ) type abc struct { - A string - B string + A string + B string } type kkk struct { - kk string - jj string + kk string + jj string } type def struct { - DDD []kkk - ABC []abc `toml:"abc"` - EEE [3]abc - MMM map[string]string - NNN map[string]abc - PPP *abc - QQQ *abc - RRR map[string]string + DDD []kkk + ABC []abc `toml:"abc"` + EEE [3]abc + MMM map[string]string + NNN map[string]abc + PPP *abc + QQQ *abc + RRR map[string]string } func Test_flatten(t *testing.T) { - cf1 := def{ - ABC: []abc{ - {"a", "a1"}, - {"b", "b1"}, - }, - EEE: [3]abc{ - {"a", "a1"}, - {"b", "b1"}, - {"c", "c1"}, - }, - MMM: map[string]string{ - "abc": "ABC", - "abcd": "ABCd", - }, - NNN: map[string]abc{ - "abc": {"a", "a1"}, - "abcd": {"a", "a1"}, - }, - PPP: &abc{"a", "a1"}, - } - exp := newExporter() - err := flatten(cf1, "", "", exp) - assert.NoError(t, err) - err = flatten(&cf1, "", "", exp) - assert.NoError(t, err) + cf1 := def{ + ABC: []abc{ + {"a", "a1"}, + {"b", "b1"}, + }, + EEE: [3]abc{ + {"a", "a1"}, + {"b", "b1"}, + {"c", "c1"}, + }, + MMM: map[string]string{ + "abc": "ABC", + "abcd": "ABCd", + }, + NNN: map[string]abc{ + "abc": {"a", "a1"}, + "abcd": {"a", "a1"}, + }, + PPP: &abc{"a", "a1"}, + } + exp := newExporter() + err := flatten(cf1, "", "", exp) + assert.NoError(t, err) + err = flatten(&cf1, "", "", exp) + assert.NoError(t, err) + + k := kkk{} + err = flatten(k, "", "", exp) + assert.NoError(t, err) } From 3d2eb0d93711ab9b2633388a6f8a1555f778afed Mon Sep 17 00:00:00 2001 From: daviszhen Date: Fri, 15 Sep 2023 20:42:47 +0800 Subject: [PATCH 04/15] fix ut --- pkg/objectio/utils.go | 748 ++++++++++++++++++++++++++++++++++++ pkg/objectio/writer_test.go | 641 +++++++++++++++--------------- 2 files changed, 1077 insertions(+), 312 deletions(-) diff --git a/pkg/objectio/utils.go b/pkg/objectio/utils.go index cedf2b8fb1bf..fb209ba30701 100644 --- a/pkg/objectio/utils.go +++ b/pkg/objectio/utils.go @@ -14,7 +14,15 @@ package objectio import ( + "encoding/binary" + "github.com/matrixorigin/matrixone/pkg/common/moerr" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/pb/timestamp" "io" + "math/rand" + "strconv" + "time" + "unsafe" "github.com/matrixorigin/matrixone/pkg/common/mpool" "github.com/matrixorigin/matrixone/pkg/container/types" @@ -230,3 +238,743 @@ func ConstructRowidColumnToWithSels( } return } + +//============================================= +// add following functions to solve the cycle reference problem. +// because,I add hakeeper client interface to the Process for supporting +// the table function mo_configurations to get info from the hakeeper. +// +// `Hakeeper logservice.CNHAKeeperClient` +// +// I get following cycle reference error. +// error: +//package github.com/matrixorigin/matrixone/pkg/objectio +// imports github.com/matrixorigin/matrixone/pkg/catalog +// imports github.com/matrixorigin/matrixone/pkg/hakeeper/operator +// imports github.com/matrixorigin/matrixone/pkg/hakeeper/checkers +// imports github.com/matrixorigin/matrixone/pkg/logservice +// imports github.com/matrixorigin/matrixone/pkg/vm/process +// imports github.com/matrixorigin/matrixone/pkg/testutil: import cycle not allowed in test +// +// To fix the error. I move some necessary functions here. +//============================================= + +func NewBatch(ts []types.Type, random bool, n int, m *mpool.MPool) *batch.Batch { + bat := batch.NewWithSize(len(ts)) + bat.SetRowCount(n) + for i := range bat.Vecs { + bat.Vecs[i] = NewVector(n, ts[i], m, random, nil) + // XXX do we need to init nulls here? can we be lazy? + bat.Vecs[i].GetNulls().InitWithSize(n) + } + return bat +} + +func NewVector(n int, typ types.Type, m *mpool.MPool, random bool, Values interface{}) *vector.Vector { + switch typ.Oid { + case types.T_bool: + if vs, ok := Values.([]bool); ok { + return NewBoolVector(n, typ, m, random, vs) + } + return NewBoolVector(n, typ, m, random, nil) + case types.T_int8: + if vs, ok := Values.([]int8); ok { + return NewInt8Vector(n, typ, m, random, vs) + } + return NewInt8Vector(n, typ, m, random, nil) + case types.T_int16: + if vs, ok := Values.([]int16); ok { + return NewInt16Vector(n, typ, m, random, vs) + } + return NewInt16Vector(n, typ, m, random, nil) + case types.T_int32: + if vs, ok := Values.([]int32); ok { + return NewInt32Vector(n, typ, m, random, vs) + } + return NewInt32Vector(n, typ, m, random, nil) + case types.T_int64: + if vs, ok := Values.([]int64); ok { + return NewInt64Vector(n, typ, m, random, vs) + } + return NewInt64Vector(n, typ, m, random, nil) + case types.T_uint8: + if vs, ok := Values.([]uint8); ok { + return NewUInt8Vector(n, typ, m, random, vs) + } + return NewUInt8Vector(n, typ, m, random, nil) + case types.T_uint16: + if vs, ok := Values.([]uint16); ok { + return NewUInt16Vector(n, typ, m, random, vs) + } + return NewUInt16Vector(n, typ, m, random, nil) + case types.T_uint32: + if vs, ok := Values.([]uint32); ok { + return NewUInt32Vector(n, typ, m, random, vs) + } + return NewUInt32Vector(n, typ, m, random, nil) + case types.T_uint64: + if vs, ok := Values.([]uint64); ok { + return NewUInt64Vector(n, typ, m, random, vs) + } + return NewUInt64Vector(n, typ, m, random, nil) + case types.T_float32: + if vs, ok := Values.([]float32); ok { + return NewFloat32Vector(n, typ, m, random, vs) + } + return NewFloat32Vector(n, typ, m, random, nil) + case types.T_float64: + if vs, ok := Values.([]float64); ok { + return NewFloat64Vector(n, typ, m, random, vs) + } + return NewFloat64Vector(n, typ, m, random, nil) + case types.T_date: + if vs, ok := Values.([]string); ok { + return NewDateVector(n, typ, m, random, vs) + } + return NewDateVector(n, typ, m, random, nil) + case types.T_time: + if vs, ok := Values.([]string); ok { + return NewTimeVector(n, typ, m, random, vs) + } + return NewTimeVector(n, typ, m, random, nil) + case types.T_datetime: + if vs, ok := Values.([]string); ok { + return NewDatetimeVector(n, typ, m, random, vs) + } + return NewDatetimeVector(n, typ, m, random, nil) + case types.T_timestamp: + if vs, ok := Values.([]string); ok { + return NewTimestampVector(n, typ, m, random, vs) + } + return NewTimestampVector(n, typ, m, random, nil) + case types.T_decimal64: + if vs, ok := Values.([]types.Decimal64); ok { + return NewDecimal64Vector(n, typ, m, random, vs) + } + return NewDecimal64Vector(n, typ, m, random, nil) + case types.T_decimal128: + if vs, ok := Values.([]types.Decimal128); ok { + return NewDecimal128Vector(n, typ, m, random, vs) + } + return NewDecimal128Vector(n, typ, m, random, nil) + case types.T_char, types.T_varchar, + types.T_binary, types.T_varbinary, types.T_blob, types.T_text: + if vs, ok := Values.([]string); ok { + return NewStringVector(n, typ, m, random, vs) + } + return NewStringVector(n, typ, m, random, nil) + case types.T_array_float32: + if vs, ok := Values.([][]float32); ok { + return NewArrayVector[float32](n, typ, m, random, vs) + } + return NewArrayVector[float32](n, typ, m, random, nil) + case types.T_array_float64: + if vs, ok := Values.([][]float64); ok { + return NewArrayVector[float64](n, typ, m, random, vs) + } + return NewArrayVector[float64](n, typ, m, random, nil) + case types.T_json: + if vs, ok := Values.([]string); ok { + return NewJsonVector(n, typ, m, random, vs) + } + return NewJsonVector(n, typ, m, random, nil) + case types.T_TS: + if vs, ok := Values.([]types.TS); ok { + return NewTsVector(n, typ, m, random, vs) + } + return NewTsVector(n, typ, m, random, nil) + case types.T_Rowid: + if vs, ok := Values.([]types.Rowid); ok { + return NewRowidVector(n, typ, m, random, vs) + } + return NewRowidVector(n, typ, m, random, nil) + case types.T_Blockid: + if vs, ok := Values.([]types.Blockid); ok { + return NewBlockidVector(n, typ, m, random, vs) + } + return NewBlockidVector(n, typ, m, random, nil) + case types.T_enum: + if vs, ok := Values.([]uint16); ok { + return NewUInt16Vector(n, typ, m, random, vs) + } + return NewUInt16Vector(n, typ, m, random, nil) + default: + panic(moerr.NewInternalErrorNoCtx("unsupport vector's type '%v", typ)) + } +} + +func NewTsVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []types.TS) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + var t timestamp.Timestamp + + t.PhysicalTime = int64(i) + if err := vector.AppendFixed(vec, types.TimestampToTS(t), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewRowidVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []types.Rowid) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + var rowId types.Rowid + binary.LittleEndian.PutUint64( + unsafe.Slice(&rowId[types.RowidSize/2], 8), + uint64(i), + ) + if err := vector.AppendFixed(vec, rowId, false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewBlockidVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []types.Blockid) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + var blockId types.Blockid + binary.LittleEndian.PutUint64( + unsafe.Slice(&blockId[types.BlockidSize/2], 8), + uint64(i), + ) + if err := vector.AppendFixed(vec, blockId, false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewJsonVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []string) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for _, v := range vs { + json, err := types.ParseStringToByteJson(v) + if err != nil { + vec.Free(m) + return nil + } + jbytes, err := json.Marshal() + if err != nil { + vec.Free(m) + return nil + } + if err := vector.AppendFixed(vec, jbytes, false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + json, _ := types.ParseStringToByteJson(`{"a":1}`) + jbytes, _ := json.Marshal() + if err := vector.AppendBytes(vec, jbytes, false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewBoolVector(n int, typ types.Type, m *mpool.MPool, _ bool, vs []bool) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + if err := vector.AppendFixed(vec, bool(i%2 == 0), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewInt8Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []int8) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + if err := vector.AppendFixed(vec, int8(v), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewInt16Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []int16) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + if err := vector.AppendFixed(vec, int16(v), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewInt32Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []int32) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + if err := vector.AppendFixed(vec, int32(v), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewInt64Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []int64) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + if err := vector.AppendFixed(vec, int64(v), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewUInt8Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []uint8) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + if err := vector.AppendFixed(vec, uint8(v), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewUInt16Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []uint16) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + if err := vector.AppendFixed(vec, uint16(v), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewUInt32Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []uint32) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + if err := vector.AppendFixed(vec, uint32(v), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewUInt64Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []uint64) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + if err := vector.AppendFixed(vec, uint64(v), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewFloat32Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []float32) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := float32(i) + if random { + v = rand.Float32() + } + if err := vector.AppendFixed(vec, float32(v), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewFloat64Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []float64) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := float64(i) + if random { + v = rand.Float64() + } + if err := vector.AppendFixed(vec, float64(v), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewDecimal64Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []types.Decimal64) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + d := types.Decimal64(v) + if err := vector.AppendFixed(vec, d, false, m); err != nil { + + vec.Free(m) + return nil + } + } + return vec +} + +func NewDecimal128Vector(n int, typ types.Type, m *mpool.MPool, random bool, vs []types.Decimal128) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendFixed(vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + d := types.Decimal128{B0_63: uint64(v), B64_127: 0} + if err := vector.AppendFixed(vec, d, false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewDateVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + d, err := types.ParseDateCast(vs[i]) + if err != nil { + return nil + } + if err := vector.AppendFixed(vec, d, false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + if err := vector.AppendFixed(vec, types.Date(v), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewTimeVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + d, err := types.ParseTime(vs[i], 6) + if err != nil { + return nil + } + if err := vector.AppendFixed(vec, d, false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + if err := vector.AppendFixed(vec, types.Time(v), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewDatetimeVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + d, err := types.ParseDatetime(vs[i], 6) + if err != nil { + return nil + } + if err := vector.AppendFixed(vec, d, false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + if err := vector.AppendFixed(vec, types.Datetime(v), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewTimestampVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + d, err := types.ParseTimestamp(time.Local, vs[i], 6) + if err != nil { + return nil + } + if err := vector.AppendFixed(vec, d, false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + if err := vector.AppendFixed(vec, types.Timestamp(v), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewStringVector(n int, typ types.Type, m *mpool.MPool, random bool, vs []string) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendBytes(vec, []byte(vs[i]), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + if err := vector.AppendBytes(vec, []byte(strconv.Itoa(v)), false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} + +func NewArrayVector[T types.RealNumbers](n int, typ types.Type, m *mpool.MPool, random bool, vs [][]T) *vector.Vector { + vec := vector.NewVec(typ) + if vs != nil { + for i := range vs { + if err := vector.AppendArray[T](vec, vs[i], false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec + } + for i := 0; i < n; i++ { + v := i + if random { + v = rand.Int() + } + if err := vector.AppendArray[T](vec, []T{T(v), T(v + 1)}, false, m); err != nil { + vec.Free(m) + return nil + } + } + return vec +} diff --git a/pkg/objectio/writer_test.go b/pkg/objectio/writer_test.go index 1e6c155a84a8..c1443dd7e479 100644 --- a/pkg/objectio/writer_test.go +++ b/pkg/objectio/writer_test.go @@ -14,315 +14,332 @@ package objectio -//const ( -// ModuleName = "ObjectIo" -//) -// -//func GetDefaultTestPath(module string, name string) string { -// return filepath.Join("/tmp", module, name) -//} -// -//func MakeDefaultTestPath(module string, name string) string { -// path := GetDefaultTestPath(module, name) -// os.MkdirAll(path, os.FileMode(0755)) -// return path -//} -// -//func RemoveDefaultTestPath(module string, name string) { -// path := GetDefaultTestPath(module, name) -// os.RemoveAll(path) -//} -// -//func InitTestEnv(module string, name string) string { -// RemoveDefaultTestPath(module, name) -// return MakeDefaultTestPath(module, name) -//} -// -//func TestNewObjectWriter(t *testing.T) { -// ctx := context.Background() -// -// dir := InitTestEnv(ModuleName, t.Name()) -// dir = path.Join(dir, "/local") -// id := 1 -// name := fmt.Sprintf("%d.blk", id) -// mp := mpool.MustNewZero() -// bat := newBatch(mp) -// defer bat.Clean(mp) -// bat2 := newBatch2(mp) -// defer bat.Clean(mp) -// c := fileservice.Config{ -// Name: defines.LocalFileServiceName, -// Backend: "DISK", -// DataDir: dir, -// } -// service, err := fileservice.NewFileService(ctx, c, nil) -// assert.Nil(t, err) -// -// objectWriter, err := NewObjectWriterSpecial(WriterNormal, name, service) -// assert.Nil(t, err) -// fd, err := objectWriter.Write(bat) -// assert.Nil(t, err) -// for i := range bat.Vecs { -// zbuf := make([]byte, 64) -// zbuf[31] = 1 -// zbuf[63] = 10 -// fd.ColumnMeta(uint16(i)).SetZoneMap(zbuf) -// } -// _, err = objectWriter.Write(bat) -// assert.Nil(t, err) -// _, err = objectWriter.WriteWithoutSeqnum(bat2) -// assert.Nil(t, err) -// ts := time.Now() -// option := WriteOptions{ -// Type: WriteTS, -// Val: ts, -// } -// blocks, err := objectWriter.WriteEnd(context.Background(), option) -// assert.Nil(t, err) -// assert.Equal(t, 3, len(blocks)) -// assert.Nil(t, objectWriter.buffer) -// -// objectReader, _ := NewObjectReaderWithStr(name, service) -// extents := make([]Extent, 3) -// for i, blk := range blocks { -// extents[i] = NewExtent(1, blk.GetExtent().Offset(), blk.GetExtent().Length(), blk.GetExtent().OriginSize()) -// } -// pool, err := mpool.NewMPool("objectio_test", 0, mpool.NoFixed) -// assert.NoError(t, err) -// nb0 := pool.CurrNB() -// objectReader.CacheMetaExtent(&extents[0]) -// metaHeader, err := objectReader.ReadMeta(context.Background(), pool) -// assert.Nil(t, err) -// meta, _ := metaHeader.DataMeta() -// assert.Equal(t, uint32(3), meta.BlockCount()) -// idxs := make([]uint16, 3) -// idxs[0] = 0 -// idxs[1] = 2 -// idxs[2] = 3 -// typs := []types.Type{types.T_int8.ToType(), types.T_int32.ToType(), types.T_int64.ToType()} -// vec, err := objectReader.ReadOneBlock(context.Background(), idxs, typs, 0, pool) -// assert.Nil(t, err) -// -// obj, err := Decode(vec.Entries[0].CachedData.Bytes()) -// assert.Nil(t, err) -// vector1 := obj.(*vector.Vector) -// assert.Equal(t, int8(3), vector.MustFixedCol[int8](vector1)[3]) -// -// obj, err = Decode(vec.Entries[1].CachedData.Bytes()) -// assert.Nil(t, err) -// vector2 := obj.(*vector.Vector) -// assert.Equal(t, int32(3), vector.MustFixedCol[int32](vector2)[3]) -// -// obj, err = Decode(vec.Entries[2].CachedData.Bytes()) -// assert.Nil(t, err) -// vector3 := obj.(*vector.Vector) -// assert.Equal(t, int64(3), vector.GetFixedAt[int64](vector3, 3)) -// -// blk := blocks[0].MustGetColumn(idxs[0]) -// buf := blk.ZoneMap() -// assert.Equal(t, uint8(0x1), buf[31]) -// assert.Equal(t, uint8(0xa), buf[63]) -// assert.True(t, nb0 == pool.CurrNB()) -// -// fs := NewObjectFS(service, dir) -// dirs, err := fs.ListDir("") -// assert.Nil(t, err) -// assert.Equal(t, 1, len(dirs)) -// objectReader, err = NewObjectReaderWithStr(name, service) -// assert.Nil(t, err) -// metaHeader, err = objectReader.ReadAllMeta(context.Background(), pool) -// assert.Nil(t, err) -// meta, _ = metaHeader.DataMeta() -// assert.Equal(t, uint32(3), meta.BlockCount()) -// assert.Nil(t, err) -// assert.Equal(t, uint32(3), meta.BlockCount()) -// idxs = make([]uint16, 3) -// idxs[0] = 0 -// idxs[1] = 2 -// idxs[2] = 3 -// vec, err = objectReader.ReadOneBlock(context.Background(), idxs, typs, 0, pool) -// assert.Nil(t, err) -// -// obj, err = Decode(vec.Entries[0].CachedData.Bytes()) -// assert.Nil(t, err) -// vector1 = obj.(*vector.Vector) -// assert.Equal(t, int8(3), vector.MustFixedCol[int8](vector1)[3]) -// -// obj, err = Decode(vec.Entries[1].CachedData.Bytes()) -// assert.Nil(t, err) -// vector2 = obj.(*vector.Vector) -// assert.Equal(t, int32(3), vector.MustFixedCol[int32](vector2)[3]) -// -// obj, err = Decode(vec.Entries[2].CachedData.Bytes()) -// assert.Nil(t, err) -// vector3 = obj.(*vector.Vector) -// assert.Equal(t, int64(3), vector.GetFixedAt[int64](vector3, 3)) -// blk = blocks[0].MustGetColumn(idxs[0]) -// buf = blk.ZoneMap() -// assert.Equal(t, uint8(0x1), buf[31]) -// assert.Equal(t, uint8(0xa), buf[63]) -// assert.True(t, nb0 == pool.CurrNB()) -// buf1, err := objectReader.ReadExtent(context.Background(), meta.BlockHeader().ZoneMapArea()) -// assert.Nil(t, err) -// zma := ZoneMapArea(buf1) -// buf = zma.GetZoneMap(0, 0) -// assert.Equal(t, uint8(0x1), buf[31]) -// assert.Equal(t, uint8(0xa), buf[63]) -//} -// -//func getObjectMeta(ctx context.Context, t *testing.B) ObjectDataMeta { -// dir := InitTestEnv(ModuleName, t.Name()) -// dir = path.Join(dir, "/local") -// id := 1 -// name := fmt.Sprintf("%d.blk", id) -// mp := mpool.MustNewZero() -// bat := newBatch(mp) -// defer bat.Clean(mp) -// c := fileservice.Config{ -// Name: defines.LocalFileServiceName, -// Backend: "DISK", -// DataDir: dir, -// } -// service, err := fileservice.NewFileService(ctx, c, nil) -// assert.Nil(t, err) -// -// objectWriter, err := NewObjectWriterSpecial(WriterNormal, name, service) -// assert.Nil(t, err) -// for y := 0; y < 1; y++ { -// fd, err := objectWriter.Write(bat) -// assert.Nil(t, err) -// for i := range bat.Vecs { -// zbuf := make([]byte, 64) -// zbuf[31] = 1 -// zbuf[63] = 10 -// fd.ColumnMeta(uint16(i)).SetZoneMap(zbuf) -// } -// } -// ts := time.Now() -// option := WriteOptions{ -// Type: WriteTS, -// Val: ts, -// } -// blocks, err := objectWriter.WriteEnd(context.Background(), option) -// assert.Nil(t, err) -// assert.Equal(t, 1, len(blocks)) -// assert.Nil(t, objectWriter.buffer) -// objectReader, _ := NewObjectReaderWithStr(name, service) -// ext := blocks[0].BlockHeader().MetaLocation() -// objectReader.CacheMetaExtent(&ext) -// metaHeader, err := objectReader.ReadMeta(context.Background(), nil) -// assert.Nil(t, err) -// meta, _ := metaHeader.DataMeta() -// return meta -//} -// -//func BenchmarkMetadata(b *testing.B) { -// ctx := context.Background() -// meta := getObjectMeta(ctx, b) -// b.Run("GetBlockMeta", func(b *testing.B) { -// b.ResetTimer() -// for i := 0; i < b.N; i++ { -// meta.GetBlockMeta(0) -// } -// }) -// b.Log(meta.GetBlockMeta(0).GetID()) -// b.Run("GetColumnMeta", func(b *testing.B) { -// b.ResetTimer() -// for i := 0; i < b.N; i++ { -// meta.Length() -// } -// }) -// b.Run("BlockCount", func(b *testing.B) { -// b.ResetTimer() -// for i := 0; i < b.N; i++ { -// meta.BlockCount() -// } -// }) -// b.Log(meta.BlockCount()) -//} -// -//func TestNewObjectReader(t *testing.T) { -// ctx := context.Background() -// -// dir := InitTestEnv(ModuleName, t.Name()) -// dir = path.Join(dir, "/local") -// id := 1 -// name := fmt.Sprintf("%d.blk", id) -// mp := mpool.MustNewZero() -// bat := newBatch(mp) -// defer bat.Clean(mp) -// c := fileservice.Config{ -// Name: defines.LocalFileServiceName, -// Backend: "DISK", -// DataDir: dir, -// } -// service, err := fileservice.NewFileService(ctx, c, nil) -// assert.Nil(t, err) -// -// objectWriter, err := NewObjectWriterSpecial(WriterNormal, name, service) -// assert.Nil(t, err) -// fd, err := objectWriter.Write(bat) -// assert.Nil(t, err) -// for i := range bat.Vecs { -// zbuf := make([]byte, 64) -// zbuf[31] = 1 -// zbuf[63] = 10 -// fd.ColumnMeta(uint16(i)).SetZoneMap(zbuf) -// } -// _, err = objectWriter.Write(bat) -// assert.Nil(t, err) -// _, err = objectWriter.WriteTombstone(bat) -// assert.Nil(t, err) -// _, err = objectWriter.WriteSubBlock(bat, 2) -// assert.Nil(t, err) -// _, err = objectWriter.WriteSubBlock(bat, 26) -// assert.Nil(t, err) -// ts := time.Now() -// option := WriteOptions{ -// Type: WriteTS, -// Val: ts, -// } -// blocks, err := objectWriter.WriteEnd(context.Background(), option) -// assert.Nil(t, err) -// assert.Equal(t, 5, len(blocks)) -// assert.Nil(t, objectWriter.buffer) -// objectReader, _ := NewObjectReaderWithStr(name, service) -// ext := blocks[0].BlockHeader().MetaLocation() -// objectReader.CacheMetaExtent(&ext) -// metaHeader, err := objectReader.ReadMeta(context.Background(), nil) -// assert.Nil(t, err) -// meta, _ := metaHeader.DataMeta() -// assert.Equal(t, uint32(2), meta.BlockCount()) -// meta, _ = metaHeader.TombstoneMeta() -// assert.Equal(t, uint32(1), meta.BlockCount()) -// meta, _ = metaHeader.SubMeta(0) -// assert.Equal(t, uint32(1), meta.BlockCount()) -// meta, _ = metaHeader.SubMeta(24) -// assert.Equal(t, uint32(1), meta.BlockCount()) -//} -// -//func newBatch(mp *mpool.MPool) *batch.Batch { -// types := []types.Type{ -// types.T_int8.ToType(), -// types.T_int16.ToType(), -// types.T_int32.ToType(), -// types.T_int64.ToType(), -// types.T_uint16.ToType(), -// types.T_uint32.ToType(), -// types.T_uint8.ToType(), -// types.T_uint64.ToType(), -// } -// return testutil.NewBatch(types, false, int(40000*2), mp) -//} -// -//func newBatch2(mp *mpool.MPool) *batch.Batch { -// types := []types.Type{ -// types.T_int8.ToType(), -// types.T_int16.ToType(), -// types.T_int32.ToType(), -// types.T_int64.ToType(), -// types.T_uint16.ToType(), -// types.T_uint32.ToType(), -// } -// return testutil.NewBatch(types, false, int(40000*2), mp) -//} +import ( + "context" + "fmt" + "github.com/matrixorigin/matrixone/pkg/common/mpool" + "github.com/matrixorigin/matrixone/pkg/container/batch" + "github.com/matrixorigin/matrixone/pkg/container/types" + "github.com/matrixorigin/matrixone/pkg/container/vector" + "github.com/matrixorigin/matrixone/pkg/defines" + "github.com/matrixorigin/matrixone/pkg/fileservice" + "github.com/stretchr/testify/assert" + "os" + "path" + "path/filepath" + "testing" + "time" +) + +const ( + ModuleName = "ObjectIo" +) + +func GetDefaultTestPath(module string, name string) string { + return filepath.Join("/tmp", module, name) +} + +func MakeDefaultTestPath(module string, name string) string { + path := GetDefaultTestPath(module, name) + os.MkdirAll(path, os.FileMode(0755)) + return path +} + +func RemoveDefaultTestPath(module string, name string) { + path := GetDefaultTestPath(module, name) + os.RemoveAll(path) +} + +func InitTestEnv(module string, name string) string { + RemoveDefaultTestPath(module, name) + return MakeDefaultTestPath(module, name) +} + +func TestNewObjectWriter(t *testing.T) { + ctx := context.Background() + + dir := InitTestEnv(ModuleName, t.Name()) + dir = path.Join(dir, "/local") + id := 1 + name := fmt.Sprintf("%d.blk", id) + mp := mpool.MustNewZero() + bat := newBatch(mp) + defer bat.Clean(mp) + bat2 := newBatch2(mp) + defer bat.Clean(mp) + c := fileservice.Config{ + Name: defines.LocalFileServiceName, + Backend: "DISK", + DataDir: dir, + } + service, err := fileservice.NewFileService(ctx, c, nil) + assert.Nil(t, err) + + objectWriter, err := NewObjectWriterSpecial(WriterNormal, name, service) + assert.Nil(t, err) + fd, err := objectWriter.Write(bat) + assert.Nil(t, err) + for i := range bat.Vecs { + zbuf := make([]byte, 64) + zbuf[31] = 1 + zbuf[63] = 10 + fd.ColumnMeta(uint16(i)).SetZoneMap(zbuf) + } + _, err = objectWriter.Write(bat) + assert.Nil(t, err) + _, err = objectWriter.WriteWithoutSeqnum(bat2) + assert.Nil(t, err) + ts := time.Now() + option := WriteOptions{ + Type: WriteTS, + Val: ts, + } + blocks, err := objectWriter.WriteEnd(context.Background(), option) + assert.Nil(t, err) + assert.Equal(t, 3, len(blocks)) + assert.Nil(t, objectWriter.buffer) + + objectReader, _ := NewObjectReaderWithStr(name, service) + extents := make([]Extent, 3) + for i, blk := range blocks { + extents[i] = NewExtent(1, blk.GetExtent().Offset(), blk.GetExtent().Length(), blk.GetExtent().OriginSize()) + } + pool, err := mpool.NewMPool("objectio_test", 0, mpool.NoFixed) + assert.NoError(t, err) + nb0 := pool.CurrNB() + objectReader.CacheMetaExtent(&extents[0]) + metaHeader, err := objectReader.ReadMeta(context.Background(), pool) + assert.Nil(t, err) + meta, _ := metaHeader.DataMeta() + assert.Equal(t, uint32(3), meta.BlockCount()) + idxs := make([]uint16, 3) + idxs[0] = 0 + idxs[1] = 2 + idxs[2] = 3 + typs := []types.Type{types.T_int8.ToType(), types.T_int32.ToType(), types.T_int64.ToType()} + vec, err := objectReader.ReadOneBlock(context.Background(), idxs, typs, 0, pool) + assert.Nil(t, err) + + obj, err := Decode(vec.Entries[0].CachedData.Bytes()) + assert.Nil(t, err) + vector1 := obj.(*vector.Vector) + assert.Equal(t, int8(3), vector.MustFixedCol[int8](vector1)[3]) + + obj, err = Decode(vec.Entries[1].CachedData.Bytes()) + assert.Nil(t, err) + vector2 := obj.(*vector.Vector) + assert.Equal(t, int32(3), vector.MustFixedCol[int32](vector2)[3]) + + obj, err = Decode(vec.Entries[2].CachedData.Bytes()) + assert.Nil(t, err) + vector3 := obj.(*vector.Vector) + assert.Equal(t, int64(3), vector.GetFixedAt[int64](vector3, 3)) + + blk := blocks[0].MustGetColumn(idxs[0]) + buf := blk.ZoneMap() + assert.Equal(t, uint8(0x1), buf[31]) + assert.Equal(t, uint8(0xa), buf[63]) + assert.True(t, nb0 == pool.CurrNB()) + + fs := NewObjectFS(service, dir) + dirs, err := fs.ListDir("") + assert.Nil(t, err) + assert.Equal(t, 1, len(dirs)) + objectReader, err = NewObjectReaderWithStr(name, service) + assert.Nil(t, err) + metaHeader, err = objectReader.ReadAllMeta(context.Background(), pool) + assert.Nil(t, err) + meta, _ = metaHeader.DataMeta() + assert.Equal(t, uint32(3), meta.BlockCount()) + assert.Nil(t, err) + assert.Equal(t, uint32(3), meta.BlockCount()) + idxs = make([]uint16, 3) + idxs[0] = 0 + idxs[1] = 2 + idxs[2] = 3 + vec, err = objectReader.ReadOneBlock(context.Background(), idxs, typs, 0, pool) + assert.Nil(t, err) + + obj, err = Decode(vec.Entries[0].CachedData.Bytes()) + assert.Nil(t, err) + vector1 = obj.(*vector.Vector) + assert.Equal(t, int8(3), vector.MustFixedCol[int8](vector1)[3]) + + obj, err = Decode(vec.Entries[1].CachedData.Bytes()) + assert.Nil(t, err) + vector2 = obj.(*vector.Vector) + assert.Equal(t, int32(3), vector.MustFixedCol[int32](vector2)[3]) + + obj, err = Decode(vec.Entries[2].CachedData.Bytes()) + assert.Nil(t, err) + vector3 = obj.(*vector.Vector) + assert.Equal(t, int64(3), vector.GetFixedAt[int64](vector3, 3)) + blk = blocks[0].MustGetColumn(idxs[0]) + buf = blk.ZoneMap() + assert.Equal(t, uint8(0x1), buf[31]) + assert.Equal(t, uint8(0xa), buf[63]) + assert.True(t, nb0 == pool.CurrNB()) + buf1, err := objectReader.ReadExtent(context.Background(), meta.BlockHeader().ZoneMapArea()) + assert.Nil(t, err) + zma := ZoneMapArea(buf1) + buf = zma.GetZoneMap(0, 0) + assert.Equal(t, uint8(0x1), buf[31]) + assert.Equal(t, uint8(0xa), buf[63]) +} + +func getObjectMeta(ctx context.Context, t *testing.B) ObjectDataMeta { + dir := InitTestEnv(ModuleName, t.Name()) + dir = path.Join(dir, "/local") + id := 1 + name := fmt.Sprintf("%d.blk", id) + mp := mpool.MustNewZero() + bat := newBatch(mp) + defer bat.Clean(mp) + c := fileservice.Config{ + Name: defines.LocalFileServiceName, + Backend: "DISK", + DataDir: dir, + } + service, err := fileservice.NewFileService(ctx, c, nil) + assert.Nil(t, err) + + objectWriter, err := NewObjectWriterSpecial(WriterNormal, name, service) + assert.Nil(t, err) + for y := 0; y < 1; y++ { + fd, err := objectWriter.Write(bat) + assert.Nil(t, err) + for i := range bat.Vecs { + zbuf := make([]byte, 64) + zbuf[31] = 1 + zbuf[63] = 10 + fd.ColumnMeta(uint16(i)).SetZoneMap(zbuf) + } + } + ts := time.Now() + option := WriteOptions{ + Type: WriteTS, + Val: ts, + } + blocks, err := objectWriter.WriteEnd(context.Background(), option) + assert.Nil(t, err) + assert.Equal(t, 1, len(blocks)) + assert.Nil(t, objectWriter.buffer) + objectReader, _ := NewObjectReaderWithStr(name, service) + ext := blocks[0].BlockHeader().MetaLocation() + objectReader.CacheMetaExtent(&ext) + metaHeader, err := objectReader.ReadMeta(context.Background(), nil) + assert.Nil(t, err) + meta, _ := metaHeader.DataMeta() + return meta +} + +func BenchmarkMetadata(b *testing.B) { + ctx := context.Background() + meta := getObjectMeta(ctx, b) + b.Run("GetBlockMeta", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + meta.GetBlockMeta(0) + } + }) + b.Log(meta.GetBlockMeta(0).GetID()) + b.Run("GetColumnMeta", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + meta.Length() + } + }) + b.Run("BlockCount", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + meta.BlockCount() + } + }) + b.Log(meta.BlockCount()) +} + +func TestNewObjectReader(t *testing.T) { + ctx := context.Background() + + dir := InitTestEnv(ModuleName, t.Name()) + dir = path.Join(dir, "/local") + id := 1 + name := fmt.Sprintf("%d.blk", id) + mp := mpool.MustNewZero() + bat := newBatch(mp) + defer bat.Clean(mp) + c := fileservice.Config{ + Name: defines.LocalFileServiceName, + Backend: "DISK", + DataDir: dir, + } + service, err := fileservice.NewFileService(ctx, c, nil) + assert.Nil(t, err) + + objectWriter, err := NewObjectWriterSpecial(WriterNormal, name, service) + assert.Nil(t, err) + fd, err := objectWriter.Write(bat) + assert.Nil(t, err) + for i := range bat.Vecs { + zbuf := make([]byte, 64) + zbuf[31] = 1 + zbuf[63] = 10 + fd.ColumnMeta(uint16(i)).SetZoneMap(zbuf) + } + _, err = objectWriter.Write(bat) + assert.Nil(t, err) + _, err = objectWriter.WriteTombstone(bat) + assert.Nil(t, err) + _, err = objectWriter.WriteSubBlock(bat, 2) + assert.Nil(t, err) + _, err = objectWriter.WriteSubBlock(bat, 26) + assert.Nil(t, err) + ts := time.Now() + option := WriteOptions{ + Type: WriteTS, + Val: ts, + } + blocks, err := objectWriter.WriteEnd(context.Background(), option) + assert.Nil(t, err) + assert.Equal(t, 5, len(blocks)) + assert.Nil(t, objectWriter.buffer) + objectReader, _ := NewObjectReaderWithStr(name, service) + ext := blocks[0].BlockHeader().MetaLocation() + objectReader.CacheMetaExtent(&ext) + metaHeader, err := objectReader.ReadMeta(context.Background(), nil) + assert.Nil(t, err) + meta, _ := metaHeader.DataMeta() + assert.Equal(t, uint32(2), meta.BlockCount()) + meta, _ = metaHeader.TombstoneMeta() + assert.Equal(t, uint32(1), meta.BlockCount()) + meta, _ = metaHeader.SubMeta(0) + assert.Equal(t, uint32(1), meta.BlockCount()) + meta, _ = metaHeader.SubMeta(24) + assert.Equal(t, uint32(1), meta.BlockCount()) +} + +func newBatch(mp *mpool.MPool) *batch.Batch { + types := []types.Type{ + types.T_int8.ToType(), + types.T_int16.ToType(), + types.T_int32.ToType(), + types.T_int64.ToType(), + types.T_uint16.ToType(), + types.T_uint32.ToType(), + types.T_uint8.ToType(), + types.T_uint64.ToType(), + } + return NewBatch(types, false, int(40000*2), mp) +} + +func newBatch2(mp *mpool.MPool) *batch.Batch { + types := []types.Type{ + types.T_int8.ToType(), + types.T_int16.ToType(), + types.T_int32.ToType(), + types.T_int64.ToType(), + types.T_uint16.ToType(), + types.T_uint32.ToType(), + } + return NewBatch(types, false, int(40000*2), mp) +} From eb2dd28356a1c50ccfcb78a3a3235c53d8fb3580 Mon Sep 17 00:00:00 2001 From: daviszhen Date: Fri, 15 Sep 2023 20:48:24 +0800 Subject: [PATCH 05/15] fix sca --- pkg/tnservice/cfg.go | 2 +- pkg/util/dump_config_test.go | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pkg/tnservice/cfg.go b/pkg/tnservice/cfg.go index a80cb1eb879c..7a8779d5551c 100644 --- a/pkg/tnservice/cfg.go +++ b/pkg/tnservice/cfg.go @@ -378,4 +378,4 @@ func dumpTnConfig(cfg Config) (map[string]*logservicepb.ConfigItem, error) { defCfg := Config{} defCfg.SetDefaultValue() return util.DumpConfig(cfg, defCfg) -} \ No newline at end of file +} diff --git a/pkg/util/dump_config_test.go b/pkg/util/dump_config_test.go index 4647ab38e3f8..cc4c954aa1d6 100644 --- a/pkg/util/dump_config_test.go +++ b/pkg/util/dump_config_test.go @@ -67,7 +67,10 @@ func Test_flatten(t *testing.T) { err = flatten(&cf1, "", "", exp) assert.NoError(t, err) - k := kkk{} + k := kkk{ + "abc", + "def", + } err = flatten(k, "", "", exp) assert.NoError(t, err) } From 8ac42112aded194e043f25f3668cb646feabfdd2 Mon Sep 17 00:00:00 2001 From: daviszhen Date: Fri, 15 Sep 2023 21:38:20 +0800 Subject: [PATCH 06/15] add config counting --- pkg/cnservice/options.go | 10 +++++----- pkg/cnservice/server_heartbeat.go | 16 +++++++++++++--- pkg/cnservice/types.go | 9 +++++++-- pkg/logservice/option.go | 5 +++-- pkg/logservice/service.go | 5 ++++- pkg/logservice/service_commands.go | 15 +++++++++++++-- pkg/tnservice/store.go | 12 +++++++++--- pkg/tnservice/store_heartbeat.go | 17 ++++++++++++++--- 8 files changed, 68 insertions(+), 21 deletions(-) diff --git a/pkg/cnservice/options.go b/pkg/cnservice/options.go index 2efa4547cdc4..96a301c0ebc7 100644 --- a/pkg/cnservice/options.go +++ b/pkg/cnservice/options.go @@ -16,13 +16,12 @@ package cnservice import ( "context" - "github.com/matrixorigin/matrixone/pkg/logservice" - logservicepb "github.com/matrixorigin/matrixone/pkg/pb/logservice" - "github.com/matrixorigin/matrixone/pkg/common/morpc" "github.com/matrixorigin/matrixone/pkg/defines" "github.com/matrixorigin/matrixone/pkg/fileservice" "github.com/matrixorigin/matrixone/pkg/lockservice" + "github.com/matrixorigin/matrixone/pkg/logservice" + logservicepb "github.com/matrixorigin/matrixone/pkg/pb/logservice" "github.com/matrixorigin/matrixone/pkg/queryservice" "github.com/matrixorigin/matrixone/pkg/taskservice" "github.com/matrixorigin/matrixone/pkg/txn/client" @@ -68,9 +67,10 @@ func WithMessageHandle(f func(ctx context.Context, // WithConfigData saves the data from the config file func WithConfigData(data map[string]*logservicepb.ConfigItem) Option { return func(s *service) { - s.configData = make(map[string]*logservicepb.ConfigItem, len(data)) + s.config.configData = make(map[string]*logservicepb.ConfigItem, len(data)) for k, v := range data { - s.configData[k] = v + s.config.configData[k] = v } + s.config.count.Store(10) } } diff --git a/pkg/cnservice/server_heartbeat.go b/pkg/cnservice/server_heartbeat.go index 9b2d1bc96810..31d1994b73de 100644 --- a/pkg/cnservice/server_heartbeat.go +++ b/pkg/cnservice/server_heartbeat.go @@ -74,15 +74,25 @@ func (s *service) heartbeat(ctx context.Context) { TaskServiceCreated: s.GetTaskRunner() != nil, QueryAddress: s.queryServiceServiceAddr(), InitWorkState: s.cfg.InitWorkState, - ConfigData: &logservicepb.ConfigData{ - Content: s.configData, - }, + } + + //send config data + curVal := s.config.count.Load() + if curVal > 0 { + hb.ConfigData = &logservicepb.ConfigData{ + Content: s.config.configData, + } } cb, err := s._hakeeperClient.SendCNHeartbeat(ctx2, hb) if err != nil { s.logger.Error("failed to send cn heartbeat", zap.Error(err)) return } + + if curVal > 0 { + s.config.count.Add(-1) + } + s.handleCommands(cb.Commands) } diff --git a/pkg/cnservice/types.go b/pkg/cnservice/types.go index 96584ee27e45..b917904fab72 100644 --- a/pkg/cnservice/types.go +++ b/pkg/cnservice/types.go @@ -22,6 +22,7 @@ import ( "runtime" "strings" "sync" + "sync/atomic" "time" "github.com/matrixorigin/matrixone/pkg/clusterservice" @@ -582,8 +583,12 @@ type service struct { addressMgr address.AddressManager connectorMgr moconnector.ConnectorManagerInterface - //data from cn config file - configData map[string]*logservicepb.ConfigItem + config struct { + //the max count for sending the config data in heartbeat + count atomic.Int32 + //data from cn config file + configData map[string]*logservicepb.ConfigItem + } } func dumpCnConfig(cfg Config) (map[string]*logservicepb.ConfigItem, error) { diff --git a/pkg/logservice/option.go b/pkg/logservice/option.go index 9cff66b31b5e..458b263f6d71 100644 --- a/pkg/logservice/option.go +++ b/pkg/logservice/option.go @@ -50,10 +50,11 @@ func WithTaskStorageFactory(factory taskservice.TaskStorageFactory) Option { // WithConfigData saves the data from the config file func WithConfigData(data map[string]*pb.ConfigItem) Option { return func(s *Service) { - s.configData = make(map[string]*pb.ConfigItem, len(data)) + s.config.configData = make(map[string]*pb.ConfigItem, len(data)) for k, v := range data { - s.configData[k] = v + s.config.configData[k] = v } + s.config.count.Store(10) } } diff --git a/pkg/logservice/service.go b/pkg/logservice/service.go index 7db40c52fd13..8d62c3ec3bd7 100644 --- a/pkg/logservice/service.go +++ b/pkg/logservice/service.go @@ -84,7 +84,10 @@ type Service struct { storageFactory taskservice.TaskStorageFactory } - configData map[string]*pb.ConfigItem + config struct { + count atomic.Int32 + configData map[string]*pb.ConfigItem + } } func NewService( diff --git a/pkg/logservice/service_commands.go b/pkg/logservice/service_commands.go index 48707cdfa627..77c7ac6c145a 100644 --- a/pkg/logservice/service_commands.go +++ b/pkg/logservice/service_commands.go @@ -164,13 +164,24 @@ func (s *Service) heartbeat(ctx context.Context) { hb := s.store.getHeartbeatMessage() hb.TaskServiceCreated = s.taskServiceCreated() - hb.ConfigData = &pb.ConfigData{ - Content: s.configData, + + //send config data + curVal := s.config.count.Load() + if curVal > 0 { + hb.ConfigData = &pb.ConfigData{ + Content: s.config.configData, + } } + cb, err := s.haClient.SendLogHeartbeat(ctx2, hb) if err != nil { s.runtime.Logger().Error("failed to send log service heartbeat", zap.Error(err)) return } + + if curVal > 0 { + s.config.count.Add(-1) + } + s.handleCommands(cb.Commands) } diff --git a/pkg/tnservice/store.go b/pkg/tnservice/store.go index d31b753d30fc..8c5d95890eb3 100644 --- a/pkg/tnservice/store.go +++ b/pkg/tnservice/store.go @@ -18,6 +18,7 @@ import ( "context" "errors" "sync" + "sync/atomic" "time" "github.com/matrixorigin/matrixone/pkg/clusterservice" @@ -85,10 +86,11 @@ func WithTaskStorageFactory(factory taskservice.TaskStorageFactory) Option { // WithConfigData saves the data from the config file func WithConfigData(data map[string]*logservicepb.ConfigItem) Option { return func(s *store) { - s.configData = make(map[string]*logservicepb.ConfigItem, len(data)) + s.config.configData = make(map[string]*logservicepb.ConfigItem, len(data)) for k, v := range data { - s.configData[k] = v + s.config.configData[k] = v } + s.config.count.Store(10) } } @@ -128,7 +130,11 @@ type store struct { } addressMgr address.AddressManager - configData map[string]*logservicepb.ConfigItem + + config struct { + count atomic.Int32 + configData map[string]*logservicepb.ConfigItem + } } // NewService create TN Service diff --git a/pkg/tnservice/store_heartbeat.go b/pkg/tnservice/store_heartbeat.go index 4fe97d858c71..3c904679d7ab 100644 --- a/pkg/tnservice/store_heartbeat.go +++ b/pkg/tnservice/store_heartbeat.go @@ -62,15 +62,26 @@ func (s *store) heartbeat(ctx context.Context) { LogtailServerAddress: s.logtailServiceServiceAddr(), LockServiceAddress: s.lockServiceServiceAddr(), CtlAddress: s.ctlServiceServiceAddr(), - ConfigData: &logservicepb.ConfigData{ - Content: s.configData, - }, } + + //send config data + curVal := s.config.count.Load() + if curVal > 0 { + hb.ConfigData = &logservicepb.ConfigData{ + Content: s.config.configData, + } + } + cb, err := s.hakeeperClient.SendTNHeartbeat(ctx2, hb) if err != nil { s.rt.Logger().Error("failed to send tn heartbeat", zap.Error(err)) return } + + if curVal > 0 { + s.config.count.Add(-1) + } + s.handleCommands(cb.Commands) } From 853dbb92955bf6f2cf1a4cb8aff8e025ffae8a9e Mon Sep 17 00:00:00 2001 From: daviszhen Date: Fri, 15 Sep 2023 21:54:26 +0800 Subject: [PATCH 07/15] add config data count --- pkg/cnservice/options.go | 7 ++----- pkg/cnservice/server_heartbeat.go | 12 ++--------- pkg/cnservice/types.go | 8 +------- pkg/logservice/option.go | 7 ++----- pkg/logservice/service.go | 6 ++---- pkg/logservice/service_commands.go | 13 ++---------- pkg/tnservice/store.go | 13 +++--------- pkg/tnservice/store_heartbeat.go | 14 ++----------- pkg/util/dump_config.go | 33 ++++++++++++++++++++++++++++++ 9 files changed, 49 insertions(+), 64 deletions(-) diff --git a/pkg/cnservice/options.go b/pkg/cnservice/options.go index 96a301c0ebc7..7ac8fee30dc8 100644 --- a/pkg/cnservice/options.go +++ b/pkg/cnservice/options.go @@ -25,6 +25,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/queryservice" "github.com/matrixorigin/matrixone/pkg/taskservice" "github.com/matrixorigin/matrixone/pkg/txn/client" + "github.com/matrixorigin/matrixone/pkg/util" "github.com/matrixorigin/matrixone/pkg/vm/engine" "go.uber.org/zap" ) @@ -67,10 +68,6 @@ func WithMessageHandle(f func(ctx context.Context, // WithConfigData saves the data from the config file func WithConfigData(data map[string]*logservicepb.ConfigItem) Option { return func(s *service) { - s.config.configData = make(map[string]*logservicepb.ConfigItem, len(data)) - for k, v := range data { - s.config.configData[k] = v - } - s.config.count.Store(10) + s.config = util.NewConfigData(data) } } diff --git a/pkg/cnservice/server_heartbeat.go b/pkg/cnservice/server_heartbeat.go index 31d1994b73de..fcd282f5fea9 100644 --- a/pkg/cnservice/server_heartbeat.go +++ b/pkg/cnservice/server_heartbeat.go @@ -74,24 +74,16 @@ func (s *service) heartbeat(ctx context.Context) { TaskServiceCreated: s.GetTaskRunner() != nil, QueryAddress: s.queryServiceServiceAddr(), InitWorkState: s.cfg.InitWorkState, + ConfigData: s.config.GetData(), } - //send config data - curVal := s.config.count.Load() - if curVal > 0 { - hb.ConfigData = &logservicepb.ConfigData{ - Content: s.config.configData, - } - } cb, err := s._hakeeperClient.SendCNHeartbeat(ctx2, hb) if err != nil { s.logger.Error("failed to send cn heartbeat", zap.Error(err)) return } - if curVal > 0 { - s.config.count.Add(-1) - } + s.config.DecrCount() s.handleCommands(cb.Commands) } diff --git a/pkg/cnservice/types.go b/pkg/cnservice/types.go index b917904fab72..21383748fc52 100644 --- a/pkg/cnservice/types.go +++ b/pkg/cnservice/types.go @@ -22,7 +22,6 @@ import ( "runtime" "strings" "sync" - "sync/atomic" "time" "github.com/matrixorigin/matrixone/pkg/clusterservice" @@ -583,12 +582,7 @@ type service struct { addressMgr address.AddressManager connectorMgr moconnector.ConnectorManagerInterface - config struct { - //the max count for sending the config data in heartbeat - count atomic.Int32 - //data from cn config file - configData map[string]*logservicepb.ConfigItem - } + config *util.ConfigData } func dumpCnConfig(cfg Config) (map[string]*logservicepb.ConfigItem, error) { diff --git a/pkg/logservice/option.go b/pkg/logservice/option.go index 458b263f6d71..79c0bdae274f 100644 --- a/pkg/logservice/option.go +++ b/pkg/logservice/option.go @@ -17,6 +17,7 @@ package logservice import ( "context" pb "github.com/matrixorigin/matrixone/pkg/pb/logservice" + "github.com/matrixorigin/matrixone/pkg/util" "github.com/matrixorigin/matrixone/pkg/common/morpc" "github.com/matrixorigin/matrixone/pkg/common/runtime" @@ -50,11 +51,7 @@ func WithTaskStorageFactory(factory taskservice.TaskStorageFactory) Option { // WithConfigData saves the data from the config file func WithConfigData(data map[string]*pb.ConfigItem) Option { return func(s *Service) { - s.config.configData = make(map[string]*pb.ConfigItem, len(data)) - for k, v := range data { - s.config.configData[k] = v - } - s.config.count.Store(10) + s.config = util.NewConfigData(data) } } diff --git a/pkg/logservice/service.go b/pkg/logservice/service.go index 8d62c3ec3bd7..7b7727778418 100644 --- a/pkg/logservice/service.go +++ b/pkg/logservice/service.go @@ -20,6 +20,7 @@ package logservice import ( "context" "fmt" + "github.com/matrixorigin/matrixone/pkg/util" "sync" "sync/atomic" @@ -84,10 +85,7 @@ type Service struct { storageFactory taskservice.TaskStorageFactory } - config struct { - count atomic.Int32 - configData map[string]*pb.ConfigItem - } + config *util.ConfigData } func NewService( diff --git a/pkg/logservice/service_commands.go b/pkg/logservice/service_commands.go index 77c7ac6c145a..572406e9e110 100644 --- a/pkg/logservice/service_commands.go +++ b/pkg/logservice/service_commands.go @@ -164,14 +164,7 @@ func (s *Service) heartbeat(ctx context.Context) { hb := s.store.getHeartbeatMessage() hb.TaskServiceCreated = s.taskServiceCreated() - - //send config data - curVal := s.config.count.Load() - if curVal > 0 { - hb.ConfigData = &pb.ConfigData{ - Content: s.config.configData, - } - } + hb.ConfigData = s.config.GetData() cb, err := s.haClient.SendLogHeartbeat(ctx2, hb) if err != nil { @@ -179,9 +172,7 @@ func (s *Service) heartbeat(ctx context.Context) { return } - if curVal > 0 { - s.config.count.Add(-1) - } + s.config.DecrCount() s.handleCommands(cb.Commands) } diff --git a/pkg/tnservice/store.go b/pkg/tnservice/store.go index 8c5d95890eb3..ae9d957c3da1 100644 --- a/pkg/tnservice/store.go +++ b/pkg/tnservice/store.go @@ -17,8 +17,8 @@ package tnservice import ( "context" "errors" + "github.com/matrixorigin/matrixone/pkg/util" "sync" - "sync/atomic" "time" "github.com/matrixorigin/matrixone/pkg/clusterservice" @@ -86,11 +86,7 @@ func WithTaskStorageFactory(factory taskservice.TaskStorageFactory) Option { // WithConfigData saves the data from the config file func WithConfigData(data map[string]*logservicepb.ConfigItem) Option { return func(s *store) { - s.config.configData = make(map[string]*logservicepb.ConfigItem, len(data)) - for k, v := range data { - s.config.configData[k] = v - } - s.config.count.Store(10) + s.config = util.NewConfigData(data) } } @@ -131,10 +127,7 @@ type store struct { addressMgr address.AddressManager - config struct { - count atomic.Int32 - configData map[string]*logservicepb.ConfigItem - } + config *util.ConfigData } // NewService create TN Service diff --git a/pkg/tnservice/store_heartbeat.go b/pkg/tnservice/store_heartbeat.go index 3c904679d7ab..c6cc2f03f31d 100644 --- a/pkg/tnservice/store_heartbeat.go +++ b/pkg/tnservice/store_heartbeat.go @@ -62,14 +62,7 @@ func (s *store) heartbeat(ctx context.Context) { LogtailServerAddress: s.logtailServiceServiceAddr(), LockServiceAddress: s.lockServiceServiceAddr(), CtlAddress: s.ctlServiceServiceAddr(), - } - - //send config data - curVal := s.config.count.Load() - if curVal > 0 { - hb.ConfigData = &logservicepb.ConfigData{ - Content: s.config.configData, - } + ConfigData: s.config.GetData(), } cb, err := s.hakeeperClient.SendTNHeartbeat(ctx2, hb) @@ -78,10 +71,7 @@ func (s *store) heartbeat(ctx context.Context) { return } - if curVal > 0 { - s.config.count.Add(-1) - } - + s.config.DecrCount() s.handleCommands(cb.Commands) } diff --git a/pkg/util/dump_config.go b/pkg/util/dump_config.go index ad3085567062..85443ab00075 100644 --- a/pkg/util/dump_config.go +++ b/pkg/util/dump_config.go @@ -19,6 +19,7 @@ import ( "github.com/matrixorigin/matrixone/pkg/common/moerr" logservicepb "github.com/matrixorigin/matrixone/pkg/pb/logservice" "reflect" + "sync/atomic" ) type exporter struct { @@ -225,3 +226,35 @@ func DumpConfig(cfg any, defCfg any) (map[string]*logservicepb.ConfigItem, error } return ret, err } + +type ConfigData struct { + count atomic.Int32 + configData map[string]*logservicepb.ConfigItem +} + +func NewConfigData(data map[string]*logservicepb.ConfigItem) *ConfigData { + ret := &ConfigData{ + count: atomic.Int32{}, + configData: make(map[string]*logservicepb.ConfigItem, len(data)), + } + for k, v := range data { + ret.configData[k] = v + } + ret.count.Store(10) + return ret +} + +func (cd *ConfigData) GetData() *logservicepb.ConfigData { + if cd.count.Load() > 0 { + return &logservicepb.ConfigData{ + Content: cd.configData, + } + } + return nil +} + +func (cd *ConfigData) DecrCount() { + if cd.count.Load() > 0 { + cd.count.Add(-1) + } +} From 0266b16da2d46704d19f12f9a1c53716da60d227 Mon Sep 17 00:00:00 2001 From: daviszhen Date: Fri, 15 Sep 2023 22:11:53 +0800 Subject: [PATCH 08/15] update log update --- pkg/pb/logservice/logservice.go | 12 +++++++++--- pkg/util/dump_config.go | 6 +++++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/pkg/pb/logservice/logservice.go b/pkg/pb/logservice/logservice.go index 4d32a7fdf80c..38b60349cf3a 100644 --- a/pkg/pb/logservice/logservice.go +++ b/pkg/pb/logservice/logservice.go @@ -91,7 +91,9 @@ func (s *CNState) Update(hb CNStoreHeartbeat, tick uint64) { storeInfo.Role = hb.Role storeInfo.TaskServiceCreated = hb.TaskServiceCreated storeInfo.QueryAddress = hb.QueryAddress - storeInfo.ConfigData = hb.ConfigData + if hb.ConfigData != nil { + storeInfo.ConfigData = hb.ConfigData + } s.Stores[hb.UUID] = storeInfo } @@ -161,7 +163,9 @@ func (s *TNState) Update(hb TNStoreHeartbeat, tick uint64) { storeInfo.LockServiceAddress = hb.LockServiceAddress storeInfo.CtlAddress = hb.CtlAddress storeInfo.TaskServiceCreated = hb.TaskServiceCreated - storeInfo.ConfigData = hb.ConfigData + if hb.ConfigData != nil { + storeInfo.ConfigData = hb.ConfigData + } s.Stores[hb.UUID] = storeInfo } @@ -191,7 +195,9 @@ func (s *LogState) updateStores(hb LogStoreHeartbeat, tick uint64) { storeInfo.GossipAddress = hb.GossipAddress storeInfo.Replicas = hb.Replicas storeInfo.TaskServiceCreated = hb.TaskServiceCreated - storeInfo.ConfigData = hb.ConfigData + if hb.ConfigData != nil { + storeInfo.ConfigData = hb.ConfigData + } s.Stores[hb.UUID] = storeInfo } diff --git a/pkg/util/dump_config.go b/pkg/util/dump_config.go index 85443ab00075..b067365c0106 100644 --- a/pkg/util/dump_config.go +++ b/pkg/util/dump_config.go @@ -227,6 +227,10 @@ func DumpConfig(cfg any, defCfg any) (map[string]*logservicepb.ConfigItem, error return ret, err } +const ( + count = 50 +) + type ConfigData struct { count atomic.Int32 configData map[string]*logservicepb.ConfigItem @@ -240,7 +244,7 @@ func NewConfigData(data map[string]*logservicepb.ConfigItem) *ConfigData { for k, v := range data { ret.configData[k] = v } - ret.count.Store(10) + ret.count.Store(count) return ret } From 8075e1f59e8d34caa7156d0c4f881ec88dd77770 Mon Sep 17 00:00:00 2001 From: daviszhen Date: Sat, 16 Sep 2023 09:39:57 +0800 Subject: [PATCH 09/15] add mo_configurations view and upgrader --- pkg/cnservice/upgrader/newAddTable.go | 19 ++++++++++++++++++- pkg/frontend/authenticate.go | 4 ++++ .../cases/dml/select/sp_table.result | 1 + test/distributed/cases/dml/show/show.result | 1 + .../cases/mo_cloud/mo_cloud.result | 1 + .../privilege/create_user_default_role.result | 1 + test/distributed/cases/tenant/tenant.result | 1 + .../zz_accesscontrol/create_account.result | 1 + 8 files changed, 28 insertions(+), 1 deletion(-) diff --git a/pkg/cnservice/upgrader/newAddTable.go b/pkg/cnservice/upgrader/newAddTable.go index 76846673704a..a6e8688b5248 100644 --- a/pkg/cnservice/upgrader/newAddTable.go +++ b/pkg/cnservice/upgrader/newAddTable.go @@ -177,5 +177,22 @@ var MoSessionsView = &table.Table{ CreateTableSql: "drop view `mo_catalog`.`mo_sessions`;", } -var needUpgradNewView = []*table.Table{PARTITIONSView, STATISTICSView, MoSessionsView} +var MoConfigurationsView = &table.Table{ + Account: table.AccountAll, + Database: catalog.MO_CATALOG, + Table: "mo_configurations", + Columns: []table.Column{ + table.StringColumn("node_type", "the type of the node. cn,tn,log,proxy."), + table.StringColumn("node_id", "the id of the node"), + table.StringColumn("name", "the name of the configuration item"), + table.UInt64Column("current_value", "the current value of the configuration item"), + table.StringColumn("default_value", "the default value of the configuration item"), + table.StringColumn("internal", "the configuration item is internal or external"), + }, + CreateViewSql: "CREATE VIEW IF NOT EXISTS `mo_catalog`.`mo_configurations` AS SELECT * FROM mo_configurations() AS mo_configurations_tmp;", + //actually drop view here + CreateTableSql: "drop view `mo_catalog`.`mo_configurations`;", +} + +var needUpgradNewView = []*table.Table{PARTITIONSView, STATISTICSView, MoSessionsView, MoConfigurationsView} var registeredViews = []*table.Table{processlistView} diff --git a/pkg/frontend/authenticate.go b/pkg/frontend/authenticate.go index 6335558ede9b..12a90a2819b9 100644 --- a/pkg/frontend/authenticate.go +++ b/pkg/frontend/authenticate.go @@ -816,6 +816,7 @@ var ( "mo_stages": 0, catalog.MOAutoIncrTable: 0, "mo_sessions": 0, + "mo_configurations": 0, } configInitVariables = map[string]int8{ "save_query_result": 0, @@ -843,6 +844,7 @@ var ( "mo_pubs": 0, "mo_stages": 0, "mo_sessions": 0, + "mo_configurations": 0, } createDbInformationSchemaSql = "create database information_schema;" createAutoTableSql = fmt.Sprintf(`create table if not exists %s ( @@ -1018,6 +1020,7 @@ var ( primary key(stage_id) );`, `CREATE VIEW IF NOT EXISTS mo_sessions AS SELECT * FROM mo_sessions() AS mo_sessions_tmp;`, + `CREATE VIEW IF NOT EXISTS mo_configurations AS SELECT * FROM mo_configurations() AS mo_configurations_tmp;`, } //drop tables for the tenant @@ -1032,6 +1035,7 @@ var ( `drop table if exists mo_catalog.mo_mysql_compatibility_mode;`, `drop table if exists mo_catalog.mo_stages;`, `drop view if exists mo_catalog.mo_sessions;`, + `drop view if exists mo_catalog.mo_configurations;`, } dropMoPubsSql = `drop table if exists mo_catalog.mo_pubs;` deleteMoPubsSql = `delete from mo_catalog.mo_pubs;` diff --git a/test/distributed/cases/dml/select/sp_table.result b/test/distributed/cases/dml/select/sp_table.result index a8edc5367271..d6df011056c9 100644 --- a/test/distributed/cases/dml/select/sp_table.result +++ b/test/distributed/cases/dml/select/sp_table.result @@ -14,6 +14,7 @@ mo_role r mo_role_grant r mo_role_privs r mo_sessions v +mo_configurations v mo_stages r mo_stored_procedure r mo_table_partitions r diff --git a/test/distributed/cases/dml/show/show.result b/test/distributed/cases/dml/show/show.result index 34532ae84609..c0fa48a640a4 100644 --- a/test/distributed/cases/dml/show/show.result +++ b/test/distributed/cases/dml/show/show.result @@ -232,6 +232,7 @@ mo_role_grant mo_role_privs mo_user_defined_function mo_sessions +mo_configurations mo_stored_procedure mo_stages mo_mysql_compatibility_mode diff --git a/test/distributed/cases/mo_cloud/mo_cloud.result b/test/distributed/cases/mo_cloud/mo_cloud.result index c6ff6a20b7f7..faeff17137a0 100644 --- a/test/distributed/cases/mo_cloud/mo_cloud.result +++ b/test/distributed/cases/mo_cloud/mo_cloud.result @@ -626,6 +626,7 @@ mo_catalog mo_role r accountadmin mo_catalog mo_role_grant r accountadmin mo_catalog mo_role_privs r accountadmin mo_catalog mo_sessions v accountadmin +mo_catalog mo_configurations v accountadmin mo_catalog mo_stages r accountadmin mo_catalog mo_stored_procedure r accountadmin mo_catalog mo_table_partitions r accountadmin diff --git a/test/distributed/cases/tenant/privilege/create_user_default_role.result b/test/distributed/cases/tenant/privilege/create_user_default_role.result index 74af039628c9..4c5428586cc9 100644 --- a/test/distributed/cases/tenant/privilege/create_user_default_role.result +++ b/test/distributed/cases/tenant/privilege/create_user_default_role.result @@ -38,6 +38,7 @@ mo_tables mo_columns mo_database mo_sessions +mo_configurations create database t; internal error: do not have privilege to execute the statement grant create database on account * to r1; diff --git a/test/distributed/cases/tenant/tenant.result b/test/distributed/cases/tenant/tenant.result index b7f9065566d4..2f914c64af03 100644 --- a/test/distributed/cases/tenant/tenant.result +++ b/test/distributed/cases/tenant/tenant.result @@ -20,6 +20,7 @@ account_id relname relkind 0 mo_role_grant r 0 mo_role_privs r 0 mo_sessions v +0 mo_configurations v 0 mo_stages r 0 mo_stored_procedure r 0 mo_table_partitions r diff --git a/test/distributed/cases/zz_accesscontrol/create_account.result b/test/distributed/cases/zz_accesscontrol/create_account.result index be9ca2c2a676..7c75b3100eae 100644 --- a/test/distributed/cases/zz_accesscontrol/create_account.result +++ b/test/distributed/cases/zz_accesscontrol/create_account.result @@ -173,6 +173,7 @@ mo_stages mo_database mo_columns mo_sessions +mo_configurations select user_name,authentication_string,owner from mo_user; user_name authentication_string owner admin *6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9 2 From 0c0e8597c3a9370b5da20ace4bf2c8b70b025a23 Mon Sep 17 00:00:00 2001 From: daviszhen Date: Mon, 18 Sep 2023 10:35:28 +0800 Subject: [PATCH 10/15] fix bvt cases --- test/distributed/cases/dml/select/sp_table.result | 2 +- test/distributed/cases/dml/show/database_statistics.result | 4 ++-- test/distributed/cases/dml/show/show.result | 2 +- test/distributed/cases/mo_cloud/mo_cloud.result | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/test/distributed/cases/dml/select/sp_table.result b/test/distributed/cases/dml/select/sp_table.result index d6df011056c9..93bf20d76017 100644 --- a/test/distributed/cases/dml/select/sp_table.result +++ b/test/distributed/cases/dml/select/sp_table.result @@ -5,6 +5,7 @@ select relname,relkind from mo_catalog.mo_tables where reldatabase = 'mo_catalog relname relkind mo_account r mo_columns r +mo_configurations v mo_database r mo_increment_columns mo_indexes r @@ -14,7 +15,6 @@ mo_role r mo_role_grant r mo_role_privs r mo_sessions v -mo_configurations v mo_stages r mo_stored_procedure r mo_table_partitions r diff --git a/test/distributed/cases/dml/show/database_statistics.result b/test/distributed/cases/dml/show/database_statistics.result index 3d130555559c..74b44783b050 100644 --- a/test/distributed/cases/dml/show/database_statistics.result +++ b/test/distributed/cases/dml/show/database_statistics.result @@ -9,7 +9,7 @@ Number of tables in mysql 5 show table_number from mo_catalog; Number of tables in mo_catalog -17 +18 show table_number from system_metrics; Number of tables in system_metrics 20 @@ -346,7 +346,7 @@ Number of tables in mysql 5 show table_number from mo_catalog; Number of tables in mo_catalog -16 +17 show table_number from system_metrics; Number of tables in system_metrics 7 diff --git a/test/distributed/cases/dml/show/show.result b/test/distributed/cases/dml/show/show.result index c0fa48a640a4..94500dcfd5ad 100644 --- a/test/distributed/cases/dml/show/show.result +++ b/test/distributed/cases/dml/show/show.result @@ -242,7 +242,7 @@ mo_columns mo_tables show table_number from mo_catalog; Number of tables in mo_catalog -17 +18 show column_number from mo_database; Number of columns in mo_database 9 diff --git a/test/distributed/cases/mo_cloud/mo_cloud.result b/test/distributed/cases/mo_cloud/mo_cloud.result index faeff17137a0..9a63178da13b 100644 --- a/test/distributed/cases/mo_cloud/mo_cloud.result +++ b/test/distributed/cases/mo_cloud/mo_cloud.result @@ -596,7 +596,7 @@ system_metrics 7 accountadmin information_schema 16 accountadmin mysql 5 accountadmin mo_mo 0 accountadmin -mo_catalog 16 - +mo_catalog 17 - SELECT mo_catalog.mo_database.datname,mo_catalog.mo_tables.relname,mo_catalog.mo_tables.relkind, if (role_name IS NULL,'-', role_name) AS `owner` FROM mo_catalog.mo_database LEFT JOIN mo_catalog.mo_tables ON mo_catalog.mo_database.datname = mo_catalog.mo_tables.reldatabase LEFT JOIN mo_catalog.mo_role ON mo_catalog.mo_tables.owner=role_id ORDER BY reldatabase, relname; datname relname relkind owner mo_mo null null - From 9fa09af6da13bb5f31ee4c72258f5391ce9e73ec Mon Sep 17 00:00:00 2001 From: daviszhen Date: Mon, 18 Sep 2023 11:04:16 +0800 Subject: [PATCH 11/15] fix bvt case --- test/distributed/cases/dml/select/sp_table.result | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/distributed/cases/dml/select/sp_table.result b/test/distributed/cases/dml/select/sp_table.result index 93bf20d76017..415da34389d0 100644 --- a/test/distributed/cases/dml/select/sp_table.result +++ b/test/distributed/cases/dml/select/sp_table.result @@ -5,7 +5,7 @@ select relname,relkind from mo_catalog.mo_tables where reldatabase = 'mo_catalog relname relkind mo_account r mo_columns r -mo_configurations v +mo_configurations v mo_database r mo_increment_columns mo_indexes r From 780fd76740321849af4c86ec2907ddcdbaaa4405 Mon Sep 17 00:00:00 2001 From: daviszhen Date: Mon, 18 Sep 2023 16:06:12 +0800 Subject: [PATCH 12/15] fix compile error --- pkg/sql/colexec/lockop/lock_op_no_txn.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/sql/colexec/lockop/lock_op_no_txn.go b/pkg/sql/colexec/lockop/lock_op_no_txn.go index 2a917515b77e..f23be12e77d6 100644 --- a/pkg/sql/colexec/lockop/lock_op_no_txn.go +++ b/pkg/sql/colexec/lockop/lock_op_no_txn.go @@ -110,6 +110,7 @@ func getInternalProcessByUniqueID( nil, v.(lockservice.LockService), nil, + nil, nil) internalProcesses[id] = proc return proc, nil From fb49560b1aead4bb93158e4717fa22c2389fb0fd Mon Sep 17 00:00:00 2001 From: daviszhen Date: Tue, 19 Sep 2023 11:11:04 +0800 Subject: [PATCH 13/15] fix default values --- pkg/cnservice/types.go | 20 ++++---------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/pkg/cnservice/types.go b/pkg/cnservice/types.go index 21383748fc52..bc50dcd0af4f 100644 --- a/pkg/cnservice/types.go +++ b/pkg/cnservice/types.go @@ -389,6 +389,10 @@ func (c *Config) Validate() error { return nil } +// SetDefaultValue setups the default of the config. +// most of the code are copied from the Validate. +// But, the Validate may change some global variables that the SetDefaultValue does not need. +// So, need a different function. func (c *Config) SetDefaultValue() { foundMachineHost := "" if c.ListenAddress == "" { @@ -494,22 +498,6 @@ func (c *Config) SetDefaultValue() { c.LockService.Validate() c.LockService.ServiceID = c.UUID - // pessimistic mode implies primary key check - if txn.GetTxnMode(c.Txn.Mode) == txn.TxnMode_Pessimistic || c.PrimaryKeyCheck { - plan.CNPrimaryCheck = true - } else { - plan.CNPrimaryCheck = false - } - - if c.MaxPreparedStmtCount > 0 { - if c.MaxPreparedStmtCount > maxForMaxPreparedStmtCount { - frontend.MaxPrepareNumberInOneSession = maxForMaxPreparedStmtCount - } else { - frontend.MaxPrepareNumberInOneSession = c.MaxPreparedStmtCount - } - } else { - frontend.MaxPrepareNumberInOneSession = 100000 - } c.QueryServiceConfig.Adjust(foundMachineHost, defaultQueryServiceListenAddress) if c.PortBase != 0 { From 6af3af7a923e84e7a615407a412a7084ad41ca00 Mon Sep 17 00:00:00 2001 From: daviszhen Date: Wed, 20 Sep 2023 10:27:04 +0800 Subject: [PATCH 14/15] fix sca --- pkg/cnservice/upgrader/newAddTable.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/cnservice/upgrader/newAddTable.go b/pkg/cnservice/upgrader/newAddTable.go index b38bbf43f500..a3e2bbdae7b0 100644 --- a/pkg/cnservice/upgrader/newAddTable.go +++ b/pkg/cnservice/upgrader/newAddTable.go @@ -180,7 +180,6 @@ var MoSessionsView = &table.Table{ CreateTableSql: "drop view `mo_catalog`.`mo_sessions`;", } - var MoConfigurationsView = &table.Table{ Account: table.AccountAll, Database: catalog.MO_CATALOG, From e108df93013ff477ab4c5ea81785ea854ba48f93 Mon Sep 17 00:00:00 2001 From: daviszhen Date: Wed, 20 Sep 2023 15:33:08 +0800 Subject: [PATCH 15/15] fix bvt cases --- test/distributed/cases/dml/show/database_statistics.result | 4 ++-- test/distributed/cases/dml/show/show.result | 2 +- test/distributed/cases/mo_cloud/mo_cloud.result | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/distributed/cases/dml/show/database_statistics.result b/test/distributed/cases/dml/show/database_statistics.result index ab22b88302b9..dfa3b56fc4e8 100644 --- a/test/distributed/cases/dml/show/database_statistics.result +++ b/test/distributed/cases/dml/show/database_statistics.result @@ -9,7 +9,7 @@ Number of tables in mysql 5 show table_number from mo_catalog; Number of tables in mo_catalog -18 +19 show table_number from system_metrics; Number of tables in system_metrics 20 @@ -349,7 +349,7 @@ Number of tables in mysql 5 show table_number from mo_catalog; Number of tables in mo_catalog -17 +18 show table_number from system_metrics; Number of tables in system_metrics 7 diff --git a/test/distributed/cases/dml/show/show.result b/test/distributed/cases/dml/show/show.result index f595edbd82a8..a403cf744ec1 100644 --- a/test/distributed/cases/dml/show/show.result +++ b/test/distributed/cases/dml/show/show.result @@ -243,7 +243,7 @@ mo_tables mo_locks show table_number from mo_catalog; Number of tables in mo_catalog -18 +19 show column_number from mo_database; Number of columns in mo_database 9 diff --git a/test/distributed/cases/mo_cloud/mo_cloud.result b/test/distributed/cases/mo_cloud/mo_cloud.result index 18dafaea49d9..615950d26e5b 100644 --- a/test/distributed/cases/mo_cloud/mo_cloud.result +++ b/test/distributed/cases/mo_cloud/mo_cloud.result @@ -596,7 +596,7 @@ system_metrics 7 accountadmin information_schema 16 accountadmin mysql 5 accountadmin mo_mo 0 accountadmin -mo_catalog 17 - +mo_catalog 18 - SELECT mo_catalog.mo_database.datname,mo_catalog.mo_tables.relname,mo_catalog.mo_tables.relkind, if (role_name IS NULL,'-', role_name) AS `owner` FROM mo_catalog.mo_database LEFT JOIN mo_catalog.mo_tables ON mo_catalog.mo_database.datname = mo_catalog.mo_tables.reldatabase LEFT JOIN mo_catalog.mo_role ON mo_catalog.mo_tables.owner=role_id ORDER BY reldatabase, relname; datname relname relkind owner mo_mo null null -