feat: 增加 CI 测试，修复若干回归问题 (#36)

* Create ci.yml (#1) * Create ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * Update ci.yml * ut: fix all unit test * add endpoint to ci.yml * fix ExampleManager_RunQuery * Set CI timezone=Asia/Shanghai * try to fix sqldriver_test * bugfix * chore: fix accord by review * fix: fix Example_tunnel_upload_arrow * fix: fix Example_tunnel_upload_arrow * fix: fix Example_tunnel_upload_arrow
aliyun · Oct 29, 2024 · c7b6985 · c7b6985
1 parent 475b91f
commit c7b6985
Show file tree

Hide file tree

Showing 12 changed files with 214 additions and 53 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -0,0 +1,47 @@
+# This workflow will build a golang project
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
+
+name: CI
+
+on:
+  workflow_dispatch: # 支持手动触发
+  push:
+    branches: [ "master" ]
+  pull_request:
+    branches: [ "master" ]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-latest
+    permissions:
+      id-token: write     # 获取 oidc token 的权限
+    steps:
+    - name: get credentials
+      id: get-credentials
+      uses: aliyun/[email protected]
+      with:
+        role-to-assume: '${{ secrets.ROLE_ARN }}'
+        oidc-provider-arn: '${{ secrets.OIDC_ARN }}'
+        audience: 'actions.github.com' # default value
+
+    - name: Set up environment
+      run: |
+        echo "ALIBABA_CLOUD_ACCESS_KEY_ID=${{ steps.get-credentials.outputs.aliyun-access-key-id }}" >> $GITHUB_ENV
+        echo "ALIBABA_CLOUD_ACCESS_KEY_SECRET=${{ steps.get-credentials.outputs.aliyun-access-key-secret }}" >> $GITHUB_ENV
+        echo "ALIBABA_CLOUD_SECURITY_TOKEN=${{ steps.get-credentials.outputs.aliyun-security-token }}" >> $GITHUB_ENV
+        echo "odps_endpoint=${{ secrets.ENDPOINT }}" >> $GITHUB_ENV
+        echo "TZ=Asia/Shanghai" >> $GITHUB_ENV
+    
+    - uses: actions/checkout@v4
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.15'
+
+    - name: Build
+      run: go build -v ./...
+
+    - name: Test (Expect Arrow Package)
+      run: go test -v $(go list ./... | grep -v './arrow')
diff --git a/odps/account/aliyun_account.go b/odps/account/aliyun_account.go
@@ -41,17 +41,31 @@ func NewAliyunAccount(accessId string, accessKey string) *AliyunAccount {
 func AliyunAccountFromEnv() *AliyunAccount {
 	account := AliyunAccount{}
 
-	if accessId, found := os.LookupEnv("odps_accessId"); found {
+	if accessId, found := os.LookupEnv("ALIBABA_CLOUD_ACCESS_KEY_ID"); found {
 		account.accessId = accessId
 	}
 
-	if accessKey, found := os.LookupEnv("odps_accessKey"); found {
+	if accessKey, found := os.LookupEnv("ALIBABA_CLOUD_ACCESS_KEY_SECRET"); found {
 		account.accessKey = accessKey
 	}
 
 	return &account
 }
 
+func AccountFromEnv() Account {
+	accessId, found := os.LookupEnv("ALIBABA_CLOUD_ACCESS_KEY_ID")
+	accessKey, found := os.LookupEnv("ALIBABA_CLOUD_ACCESS_KEY_SECRET")
+	if !found {
+		return nil
+	}
+	securityToken, found := os.LookupEnv("ALIBABA_CLOUD_SECURITY_TOKEN")
+	if found {
+		return NewStsAccount(accessId, accessKey, securityToken)
+	} else {
+		return NewAliyunAccount(accessId, accessKey)
+	}
+}
+
 func (account *AliyunAccount) AccessId() string {
 	return account.accessId
 }

diff --git a/odps/account/sts_account.go b/odps/account/sts_account.go
@@ -33,6 +33,7 @@ type CredentialProvider interface {
 
 type stsAccountProvider interface {
 	_signRequest(req *http.Request, endpoint string) error
+	Credential() (*credentials.CredentialModel, error)
 }
 
 type stsStringProvider struct {
@@ -50,6 +51,14 @@ func (sp *stsStringProvider) _signRequest(req *http.Request, endpoint string) er
 	return nil
 }
 
+func (sp *stsStringProvider) Credential() (*credentials.CredentialModel, error) {
+	return &credentials.CredentialModel{
+		AccessKeyId:     &sp.accessId,
+		AccessKeySecret: &sp.accessKey,
+		SecurityToken:   &sp.stsToken,
+	}, nil
+}
+
 type stsAliyunCredentialProvider struct {
 	aliyunCredential credentials.Credential
 }
@@ -71,6 +80,10 @@ func (sp *stsAliyunCredentialProvider) _signRequest(req *http.Request, endpoint
 	return nil
 }
 
+func (sp *stsAliyunCredentialProvider) Credential() (*credentials.CredentialModel, error) {
+	return sp.aliyunCredential.GetCredential()
+}
+
 type stsCustomCredentialProvider struct {
 	provider CredentialProvider
 }
@@ -92,6 +105,10 @@ func (sp *stsCustomCredentialProvider) _signRequest(req *http.Request, endpoint
 	return nil
 }
 
+func (sp *stsCustomCredentialProvider) Credential() (*credentials.CredentialModel, error) {
+	return sp.provider.GetCredential()
+}
+
 func NewStsAccount(accessId, accessKey, securityToken string) *StsAccount {
 	sp := &stsStringProvider{
 		stsToken: securityToken,
@@ -133,3 +150,8 @@ func (account *StsAccount) GetType() Provider {
 func (account *StsAccount) SignRequest(req *http.Request, endpoint string) error {
 	return account.sp._signRequest(req, endpoint)
 }
+
+func (account *StsAccount) Credential() (*credentials.CredentialModel, error) {
+
+	return account.sp.Credential()
+}
diff --git a/odps/example_table_test.go b/odps/example_table_test.go
@@ -68,13 +68,13 @@ func ExampleTableSchema_ToSQLString() {
 
 	schema := builder.Build()
 
-	sql, _ := schema.ToSQLString("project_1", "schema", true)
+	sql, _ := schema.ToSQLString("go_sdk_regression_testing", "schema", true)
 	println("sql of create table:")
 	println(sql)
 	println()
 
 	externalSql, err := schema.ToExternalSQLString(
-		"project_1",
+		"go_sdk_regression_testing",
 		"",
 		true,
 		serdeProperties,
@@ -132,7 +132,7 @@ func ExampleTable_GetPartitions() {
 }
 
 func ExampleTable_ExecSql() {
-	//table := odps.NewTable(odpsIns, "project_1", "sale_detail")
+	//table := odps.NewTable(odpsIns, "go_sdk_regression_testing", "sale_detail")
 	table := odpsIns.Table("has_struct")
 	//instance, err := table.ExecSql("SelectSale_detail", "select * from sale_detail;")
 	instance, err := table.ExecSql("Select_has_struct", "select * from has_struct;")

diff --git a/odps/example_test.go b/odps/example_test.go
@@ -26,13 +26,13 @@ import (
 	"github.com/aliyun/aliyun-odps-go-sdk/odps/tableschema"
 )
 
-var account = account2.AliyunAccountFromEnv()
+var account = account2.AccountFromEnv()
 var endpoint = restclient.LoadEndpointFromEnv()
 var odpsIns = odps.NewOdps(account, endpoint)
 var defaultProjectName = "go_sdk_regression_testing"
 
 func init() {
-	if account.AccessId() == "" {
+	if account == nil {
 		panic("account environments are not set")
 	}
 

diff --git a/odps/security/example_manager_test.go b/odps/security/example_manager_test.go
@@ -24,12 +24,13 @@ import (
 	"log"
 )
 
-var account = account2.AliyunAccountFromEnv()
+var account = account2.AccountFromEnv()
 var endpoint = restclient.LoadEndpointFromEnv()
 var restClient = restclient.NewOdpsRestClient(account, endpoint)
+var projectName = "go_sdk_regression_testing"
 
 func ExampleManager_GetSecurityConfig() {
-	sm := security.NewSecurityManager(restClient, "project_1")
+	sm := security.NewSecurityManager(restClient, projectName)
 	sc, err := sm.GetSecurityConfig(true)
 	if err != nil {
 		log.Fatalf("%+v", err)
@@ -41,9 +42,9 @@ func ExampleManager_GetSecurityConfig() {
 }
 
 func ExampleManager_CheckPermissionV1() {
-	sm := security.NewSecurityManager(restClient, "project_1")
+	sm := security.NewSecurityManager(restClient, projectName)
 	p := security.NewPermission(
-		"project_1",
+		projectName,
 		security.ObjectTypeTable,
 		"sale_detail",
 		security.ActionTypeAll,
@@ -61,7 +62,7 @@ func ExampleManager_CheckPermissionV1() {
 }
 
 func ExampleManager_CheckPermissionV0() {
-	sm := security.NewSecurityManager(restClient, "project_1")
+	sm := security.NewSecurityManager(restClient, projectName)
 
 	r, err := sm.CheckPermissionV0(
 		security.ObjectTypeTable,
@@ -78,7 +79,7 @@ func ExampleManager_CheckPermissionV0() {
 }
 
 func ExampleManager_GetPolicy() {
-	sm := security.NewSecurityManager(restClient, "project_1")
+	sm := security.NewSecurityManager(restClient, projectName)
 	policy, err := sm.GetPolicy()
 	if err != nil {
 		log.Fatalf("%+v", err)
@@ -89,7 +90,7 @@ func ExampleManager_GetPolicy() {
 }
 
 func ExampleManager_ListUsers() {
-	sm := security.NewSecurityManager(restClient, "project_1")
+	sm := security.NewSecurityManager(restClient, projectName)
 	users, err := sm.ListUsers()
 	if err != nil {
 		log.Fatalf("%+v", err)
@@ -103,8 +104,8 @@ func ExampleManager_ListUsers() {
 }
 
 func ExampleManager_RunQuery() {
-	sm := security.NewSecurityManager(restClient, "project_1")
-	result, err := sm.RunQuery("show grants for [email protected];", true, "")
+	sm := security.NewSecurityManager(restClient, projectName)
+	result, err := sm.RunQuery("whoami;", true, "")
 	if err != nil {
 		log.Fatalf("%+v", err)
 	}

diff --git a/odps/tunnel/example_instance_result_download_test.go b/odps/tunnel/example_instance_result_download_test.go
@@ -27,11 +27,11 @@ import (
 )
 
 func Example_tunnel_download_instance_result() {
-	var account = account2.AliyunAccountFromEnv()
+	var account = account2.AccountFromEnv()
 	var endpoint = restclient.LoadEndpointFromEnv()
 	var odpsIns = odps.NewOdps(account, endpoint)
 
-	projectName := "project_1"
+	projectName := "go_sdk_regression_testing"
 	odpsIns.SetDefaultProjectName(projectName)
 	project := odpsIns.DefaultProject()
 	tunnelIns, err := tunnel.NewTunnelFromProject(project)

diff --git a/odps/tunnel/example_test.go b/odps/tunnel/example_test.go
@@ -20,24 +20,24 @@ import (
 	"github.com/aliyun/aliyun-odps-go-sdk/odps"
 	account2 "github.com/aliyun/aliyun-odps-go-sdk/odps/account"
 	"github.com/aliyun/aliyun-odps-go-sdk/odps/datatype"
+	"github.com/aliyun/aliyun-odps-go-sdk/odps/restclient"
 	"github.com/aliyun/aliyun-odps-go-sdk/odps/tableschema"
 	"github.com/aliyun/aliyun-odps-go-sdk/odps/tunnel"
 	"log"
 	"os"
 )
 
-var tunnelIns tunnel.Tunnel
+var tunnelIns *tunnel.Tunnel
 var odpsIns *odps.Odps
-var ProjectName = "test_new_console_gcc"
+var ProjectName = "go_sdk_regression_testing"
 
 func init() {
-	accessId := os.Getenv("tunnel_odps_accessId")
-	accessKey := os.Getenv("tunnel_odps_accessKey")
-	odpsEndpoint := os.Getenv("odps_endpoint")
+	account := account2.AccountFromEnv()
+	odpsEndpoint := restclient.LoadEndpointFromEnv()
 	tunnelEndpoint := os.Getenv("tunnel_odps_endpoint")
 
-	account := account2.NewAliyunAccount(accessId, accessKey)
 	odpsIns = odps.NewOdps(account, odpsEndpoint)
+	odpsIns.SetDefaultProjectName(ProjectName)
 	tunnelIns = tunnel.NewTunnel(odpsIns, tunnelEndpoint)
 
 	//createTableWithComplexData()
@@ -104,7 +104,7 @@ func createSaleDetailTable() {
 }
 
 func createUploadSampleArrowTable() {
-	ins, err := odpsIns.ExecSQl("CREATE TABLE IF NOT EXISTS project_1.upload_sample_arrow(payload STRING);")
+	ins, err := odpsIns.ExecSQl("CREATE TABLE IF NOT EXISTS go_sdk_regression_testing.upload_sample_arrow(payload STRING);")
 	if err != nil {
 		log.Fatalf("%+v", err)
 	}

diff --git a/odps/tunnel/example_tunnel_test.go b/odps/tunnel/example_tunnel_test.go
@@ -28,6 +28,12 @@ import (
 )
 
 func Example_tunnel_upload_arrow() {
+
+	err := odpsIns.Tables().Get("sale_detail").AddPartition(true, "sale_date=202111/region=hangzhou")
+	if err != nil {
+		log.Fatalf("%+v", err)
+	}
+
 	tunnelIns.SetHttpTimeout(10 * time.Second)
 
 	session, err := tunnelIns.CreateUploadSession(
@@ -86,12 +92,14 @@ func Example_tunnel_upload_arrow() {
 			case "shop_name":
 				builder := fieldBuilder.(*array.StringBuilder)
 				builder.AppendValues(data.ShopNames, nil)
-			case "customer_id":
+			case "custom_id":
 				builder := fieldBuilder.(*array.StringBuilder)
 				builder.AppendValues(data.CustomIDs, nil)
 			case "total_price":
 				builder := fieldBuilder.(*array.Float64Builder)
 				builder.AppendValues(data.totalPrice, nil)
+			default:
+				log.Fatalf("unknown field: %s", field.Name)
 			}
 		}
 
@@ -137,7 +145,7 @@ func Example_tunnel_upload_arrow() {
 
 func Example_tunnel_download_arrow_simple() {
 	session, err := tunnelIns.CreateDownloadSession(
-		"test_new_console_gcc",
+		ProjectName,
 		// "upload_sample_arrow",
 		"has_struct",
 	)
@@ -177,7 +185,7 @@ func Example_tunnel_download_arrow_simple() {
 
 func Example_tunnel_download_arrow_with_partition() {
 	session, err := tunnelIns.CreateDownloadSession(
-		"test_new_console_gcc",
+		"go_sdk_regression_testing",
 		"sale_detail",
 		tunnel2.SessionCfg.WithPartitionKey("sale_date='202111',region='hangzhou'"),
 	)

diff --git a/odps/tunnel/record_protoc_writer.go b/odps/tunnel/record_protoc_writer.go
@@ -189,6 +189,10 @@ func (r *RecordProtocWriter) writeField(val data.Data) error {
 	case data.TinyInt:
 		r.recordCrc.Update(int64(val))
 		return errors.WithStack(r.protocWriter.WriteSInt64(int64(val)))
+	case *data.String:
+		b := []byte(string(*val))
+		r.recordCrc.Update(b)
+		return errors.WithStack(r.protocWriter.WriteBytes(b))
 	case data.String:
 		b := []byte(string(val))
 		r.recordCrc.Update(b)

diff --git a/odps/tunnel/tunnel.go b/odps/tunnel/tunnel.go
@@ -50,29 +50,29 @@ type Tunnel struct {
 }
 
 // Once the tunnel endpoint is set, it cannot be modified anymore.
-func NewTunnel(odpsIns *odps.Odps, endpoint ...string) Tunnel {
+func NewTunnel(odpsIns *odps.Odps, endpoint ...string) *Tunnel {
 	tunnel := Tunnel{
 		odpsIns: odpsIns,
 	}
 	if len(endpoint) > 0 {
 		tunnel.endpoint = endpoint[0]
 	}
 
-	return tunnel
+	return &tunnel
 }
 
-func NewTunnelFromProject(project odps.Project) (Tunnel, error) {
+func NewTunnelFromProject(project *odps.Project) (*Tunnel, error) {
 	endpoint, err := project.GetTunnelEndpoint()
 	if err != nil {
-		return Tunnel{}, errors.WithStack(err)
+		return &Tunnel{}, errors.WithStack(err)
 	}
 
 	tunnel := Tunnel{
 		odpsIns:  project.OdpsIns(),
 		endpoint: endpoint,
 	}
 
-	return tunnel, nil
+	return &tunnel, nil
 }
 
 func (t *Tunnel) HttpTimeout() time.Duration {