Skip to content

Commit

Permalink
Merge pull request #33 from dingxin-tech/feature/unit-test
Browse files Browse the repository at this point in the history 
test: 修复三层模型相关 unit test
  • Loading branch information
@fetchadd
fetchadd authored Oct 23, 2024
2 parents ddb4a09 + 6cae5b9 commit 475b91f
Show file tree
Hide file tree
Showing 11 changed files with 135 additions and 99 deletions.
25 changes: 14 additions & 11 deletions odps/example_odps_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,24 @@ package odps_test

import (
"fmt"
"github.com/aliyun/aliyun-odps-go-sdk/odps/datatype"
"github.com/aliyun/aliyun-odps-go-sdk/odps/tableschema"
"log"
"time"
)

func ExampleOdps_RunSQl() {
func ExampleOdps_RunSQL() {

column := tableschema.Column{
Name: "id",
Type: datatype.DateType,
}
tableSchema := tableschema.NewSchemaBuilder().Columns(column).Name("has_date").Build()
err2 := odpsIns.Tables().Create(tableSchema, true, nil, nil)
if err2 != nil {
log.Fatalf("%+v", err2)
}

t := time.Now()
ts := t.Format("2006-01-02")
sql := fmt.Sprintf("insert into has_date values (date'%s');", ts)
Expand All @@ -36,15 +49,5 @@ func ExampleOdps_RunSQl() {
if err != nil {
log.Fatalf("%+v", err)
}

result, err := ins.GetResult()
if err != nil {
log.Fatalf("%+v", err)
}

for _, r := range result {
fmt.Printf("%+v", r)
}

// Output:
}
18 changes: 14 additions & 4 deletions odps/example_partition_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,13 @@ import (
)

func ExamplePartition_Load() {
partition := odps.NewPartition(odpsIns, "project_1", "sale_detail", "sale_date=201910/region=shanghai")
err := partition.Load()
err := odpsIns.Table("sale_detail").AddPartitions(true, []string{"sale_date=201910/region=shanghai"})
if err != nil {
log.Fatalf("%+v", err)
}

partition := odps.NewPartition(odpsIns, defaultProjectName, "sale_detail", "sale_date=201910/region=shanghai")
err = partition.Load()
if err != nil {
log.Fatalf("%+v", err)
}
Expand All @@ -37,8 +42,13 @@ func ExamplePartition_Load() {
}

func ExamplePartition_LoadExtended() {
partition := odps.NewPartition(odpsIns, "project_1", "sale_detail", "sale_date=201910/region=shanghai")
err := partition.LoadExtended()
err := odpsIns.Table("sale_detail").AddPartitions(true, []string{"sale_date=201910/region=shanghai"})
if err != nil {
log.Fatalf("%+v", err)
}

partition := odps.NewPartition(odpsIns, defaultProjectName, "sale_detail", "sale_date=201910/region=shanghai")
err = partition.LoadExtended()
if err != nil {
log.Fatalf("%+v", err)
}
Expand Down
2 changes: 1 addition & 1 deletion odps/example_project_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ func ExampleProjects_List() {

func ExampleProjects_Exists() {
projects := odpsIns.Projects()
projectName := "project_1"
projectName := defaultProjectName

existed, err := projects.Exists(projectName)
if err != nil {
Expand Down
2 changes: 1 addition & 1 deletion odps/example_table_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ func ExampleTable_Load() {

func ExampleTable_AddPartition() {
table := odpsIns.Table("sale_detail")
err := table.AddPartition(true, "sale_date='202111',region='hangzhou'")
err := table.AddPartition(true, "sale_date=202111/region=hangzhou")
if err != nil {
log.Fatalf("%+v", err)
}
Expand Down
7 changes: 4 additions & 3 deletions odps/example_tables_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,9 @@ func ExampleTables_List() {
func ExampleTables_BatchLoadTables() {
tablesIns := odps.NewTables(odpsIns, "", "")
tableNames := []string{
"jet_mr_input",
"jet_smode_test",
"odps_smoke_table",
"has_struct",
"sale_detail",
"testtable",
"user",
}

Expand Down Expand Up @@ -96,6 +96,7 @@ func ExampleTables_Create() {
hints["odps.sql.planner.parser.odps"] = "true"
hints["odps.sql.ddl.odps"] = "true"
hints["odps.compiler.output.format"] = "lot,pot"
hints["odps.namespace.schema"] = "false"

builder := tableschema.NewSchemaBuilder()
builder.Name("user_temp").
Expand Down
36 changes: 33 additions & 3 deletions odps/example_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ import (
var account = account2.AliyunAccountFromEnv()
var endpoint = restclient.LoadEndpointFromEnv()
var odpsIns = odps.NewOdps(account, endpoint)
var defaultProjectName = "project_1"
var defaultProjectName = "go_sdk_regression_testing"

func init() {
if account.AccessId() == "" {
Expand All @@ -43,6 +43,7 @@ func init() {
createUserTable("user_temp")
createTableWithComplexData()
createSaleDetailTable()
createTestSchema()
}

func createUserTable(tableName string) {
Expand Down Expand Up @@ -75,6 +76,7 @@ func createUserTable(tableName string) {
hints["odps.sql.planner.parser.odps"] = "true"
hints["odps.sql.ddl.odps"] = "true"
hints["odps.compiler.output.format"] = "lot,pot"
hints["odps.namespace.schema"] = "false"

builder := tableschema.NewSchemaBuilder()
builder.Name(tableName).
Expand Down Expand Up @@ -102,8 +104,10 @@ func createTableWithComplexData() {
builder.Name("has_struct").Columns(column)
schema := builder.Build()

hints := make(map[string]string)
hints["odps.namespace.schema"] = "false"
tables := odps.NewTables(odpsIns, defaultProjectName, "")
err := tables.Create(schema, true, nil, nil)
err := tables.Create(schema, true, hints, nil)
if err != nil {
log.Fatalf("%+v", err)
}
Expand Down Expand Up @@ -143,8 +147,34 @@ func createSaleDetailTable() {

schema := builder.Build()
tables := odps.NewTables(odpsIns, defaultProjectName, "")
err := tables.Create(schema, true, nil, nil)

hints := make(map[string]string)
hints["odps.namespace.schema"] = "false"
err := tables.Create(schema, true, hints, nil)
if err != nil {
log.Fatalf("%+v", err)
}
}

func createTestSchema() {
schemas := odps.NewSchemas(odpsIns, defaultProjectName)
err := schemas.Create("exist_schema", true, "create by ut")
if err != nil {
log.Fatalf("%+v", err)
return
}

// create some tables in exist_schema
tables := odps.NewTables(odpsIns, defaultProjectName, "exist_schema")
c := tableschema.Column{
Name: "name",
Type: datatype2.StringType,
}
schemaBuilder := tableschema.NewSchemaBuilder()
err = tables.Create(schemaBuilder.Name("table1").Column(c).Build(), true, nil, nil)
err = tables.Create(schemaBuilder.Name("table2").Build(), true, nil, nil)
if err != nil {
log.Fatalf("%+v", err)
return
}
}
90 changes: 24 additions & 66 deletions odps/schemas_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,77 +14,35 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package odps
package odps_test

import (
"fmt"
account2 "github.com/aliyun/aliyun-odps-go-sdk/odps/account"
"github.com/aliyun/aliyun-odps-go-sdk/odps/datatype"
"github.com/aliyun/aliyun-odps-go-sdk/odps/restclient"
"github.com/aliyun/aliyun-odps-go-sdk/odps/tableschema"
"log"
"github.com/aliyun/aliyun-odps-go-sdk/odps"
"testing"
)

var (
account account2.Account
endpoint string
odpsIns *Odps
project string
)

func setup() {
account = account2.AliyunAccountFromEnv()
endpoint = restclient.LoadEndpointFromEnv()
odpsIns = NewOdps(account, endpoint)
project = "dingxin"

schemas := NewSchemas(odpsIns, project)
err := schemas.Create("exist_schema", true, "create by ut")
if err != nil {
log.Fatalf("%+v", err)
return
}

// create some tables in exist_schema
tables := NewTables(odpsIns, project, "exist_schema")
c := tableschema.Column{
Name: "name",
Type: datatype.StringType,
}
schemaBuilder := tableschema.NewSchemaBuilder()
err = tables.Create(schemaBuilder.Name("table1").Column(c).Build(), true, nil, nil)
err = tables.Create(schemaBuilder.Name("table2").Build(), true, nil, nil)
if err != nil {
log.Fatalf("%+v", err)
return
}
}

func TestMain(m *testing.M) {
// 在运行所有测试前进行设置
setup()
m.Run()
}

func TestSchemas_List(t *testing.T) {
schemas := NewSchemas(odpsIns, project)
schemas.List(func(schema *Schema, err error) {
schemas := odps.NewSchemas(odpsIns, defaultProjectName)
schemas.List(func(schema *odps.Schema, err error) {
print(schema.Name() + "\n")
})
}

func TestSchemas_GetSchema(t *testing.T) {
schema := NewSchema(odpsIns, project, "exist_schema")
schema.Load()
schema := odps.NewSchema(odpsIns, defaultProjectName, "exist_schema")
err := schema.Load()
if err != nil {
t.Fatalf("%+v", err)
}
print(schema.ModifiedTime().String())
}

func TestSchemas_ListTableBySchema(t *testing.T) {
ts := NewTables(odpsIns, project, "exist_schema")
var f = func(t *Table, err error) {
func TestSchemas_ListTableBySchema(te *testing.T) {
ts := odps.NewTables(odpsIns, defaultProjectName, "exist_schema")
var f = func(t *odps.Table, err error) {
if err != nil {
log.Fatalf("%+v", err)
te.Fatalf("%+v", err)
}

println(fmt.Sprintf("%s, %s, %s", t.Name(), t.Owner(), t.Type()))
Expand All @@ -93,45 +51,45 @@ func TestSchemas_ListTableBySchema(t *testing.T) {
}

func TestSchemas_CheckExists(t *testing.T) {
schema := NewSchema(odpsIns, project, "not_exists")
schemas := odps.NewSchemas(odpsIns, defaultProjectName)
schema := schemas.Get("not_exists")
exists, err := schema.Exists()
if err != nil {
log.Fatalf("%+v", err)
t.Fatalf("%+v", err)
}
print(exists)

schema = NewSchema(odpsIns, project, "exist_schema")
schema = odps.NewSchema(odpsIns, defaultProjectName, "exist_schema")
exists, err = schema.Exists()
if err != nil {
log.Fatalf("%+v", err)
t.Fatalf("%+v", err)
}
print(exists)
}

func TestSchemas_CreateSchema(t *testing.T) {
schemas := NewSchemas(odpsIns, project)
schemas := odps.NewSchemas(odpsIns, defaultProjectName)
err := schemas.Create("new_schema", true, "new comment")
if err != nil {
log.Fatalf("%+v", err)
t.Fatalf("%+v", err)
}

schema := NewSchema(odpsIns, project, "new_schema")
schema := odps.NewSchema(odpsIns, defaultProjectName, "new_schema")
schema.Load()
println(schema.Comment())
}

func TestSchemas_DeleteSchema(t *testing.T) {
schemas := NewSchemas(odpsIns, project)
schemas := odps.NewSchemas(odpsIns, defaultProjectName)

err := schemas.Create("to_delete_schema", true, "to delete")

schema := NewSchema(odpsIns, project, "to_delete_schema")
schema := odps.NewSchema(odpsIns, defaultProjectName, "to_delete_schema")
exists, err := schema.Exists()
print(exists)

err = schemas.Delete("to_delete_schema")
if err != nil {
log.Fatalf("%+v", err)
t.Fatalf("%+v", err)
}
exists, err = schema.Exists()
print(exists)
Expand Down
15 changes: 12 additions & 3 deletions odps/table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -405,10 +405,13 @@ func partitionValueToSpec(value string) string {
func (t *Table) AddPartitions(ifNotExists bool, partitionValues []string) error {
var sb strings.Builder
sb.WriteString("alter table ")
hints := make(map[string]string)
if t.SchemaName() == "" {
sb.WriteString(fmt.Sprintf("%s.%s", t.ProjectName(), t.Name()))
hints["odps.namespace.schema"] = "false"
} else {
sb.WriteString(fmt.Sprintf("%s.%s.%s", t.ProjectName(), t.SchemaName(), t.Name()))
hints["odps.namespace.schema"] = "true"
}
sb.WriteString(" add")
if ifNotExists {
Expand All @@ -424,7 +427,7 @@ func (t *Table) AddPartitions(ifNotExists bool, partitionValues []string) error
sb.WriteString(";")
println(sb.String())

i, err := t.ExecSql("SQLAddPartitionTask", sb.String())
i, err := t.ExecSqlWithHints("SQLAddPartitionTask", sb.String(), hints)
if err != nil {
return errors.WithStack(err)
}
Expand All @@ -441,9 +444,12 @@ func (t *Table) AddPartition(ifNotExists bool, partitionValue string) error {
func (t *Table) DeletePartitions(ifExists bool, partitionValues []string) error {
var sb strings.Builder
sb.WriteString("alter table ")
hints := make(map[string]string)
if t.SchemaName() == "" {
hints["odps.namespace.schema"] = "false"
sb.WriteString(fmt.Sprintf("%s.%s", t.ProjectName(), t.Name()))
} else {
hints["odps.namespace.schema"] = "true"
sb.WriteString(fmt.Sprintf("%s.%s.%s", t.ProjectName(), t.SchemaName(), t.Name()))
}
sb.WriteString(" drop")
Expand All @@ -464,7 +470,7 @@ func (t *Table) DeletePartitions(ifExists bool, partitionValues []string) error

sb.WriteString(";")

ins, err := t.ExecSql("SQLDropPartitionTask", sb.String())
ins, err := t.ExecSqlWithHints("SQLDropPartitionTask", sb.String(), hints)
if err != nil {
return errors.WithStack(err)
}
Expand Down Expand Up @@ -675,13 +681,16 @@ func (t *Table) getPartitions(partitionSpec string) ([]Partition, error) {
func (t *Table) CreateShards(shardCount int) error {
var sb strings.Builder
sb.WriteString("alter table ")
hints := make(map[string]string)
if t.SchemaName() == "" {
sb.WriteString(fmt.Sprintf("%s.%s", t.ProjectName(), t.Name()))
hints["odps.namespace.schema"] = "false"
} else {
sb.WriteString(fmt.Sprintf("%s.%s.%s", t.ProjectName(), t.SchemaName(), t.Name()))
hints["odps.namespace.schema"] = "true"
}
sb.WriteString(fmt.Sprintf("\ninto %d shards;", shardCount))
ins, err := t.ExecSql("SQLCreateShardsTask", sb.String())
ins, err := t.ExecSqlWithHints("SQLCreateShardsTask", sb.String(), hints)
if err != nil {
return errors.WithStack(err)
}
Expand Down
Loading

0 comments on commit 475b91f

Please sign in to comment.