diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..e3c93d4 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,47 @@ +# This workflow will build a golang project +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go + +name: CI + +on: + workflow_dispatch: # 支持手动触发 + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + +jobs: + + build: + runs-on: ubuntu-latest + permissions: + id-token: write # 获取 oidc token 的权限 + steps: + - name: get credentials + id: get-credentials + uses: aliyun/configure-aliyun-credentials-action@v1.0.3 + with: + role-to-assume: '${{ secrets.ROLE_ARN }}' + oidc-provider-arn: '${{ secrets.OIDC_ARN }}' + audience: 'actions.github.com' # default value + + - name: Set up environment + run: | + echo "ALIBABA_CLOUD_ACCESS_KEY_ID=${{ steps.get-credentials.outputs.aliyun-access-key-id }}" >> $GITHUB_ENV + echo "ALIBABA_CLOUD_ACCESS_KEY_SECRET=${{ steps.get-credentials.outputs.aliyun-access-key-secret }}" >> $GITHUB_ENV + echo "ALIBABA_CLOUD_SECURITY_TOKEN=${{ steps.get-credentials.outputs.aliyun-security-token }}" >> $GITHUB_ENV + echo "odps_endpoint=${{ secrets.ENDPOINT }}" >> $GITHUB_ENV + echo "TZ=Asia/Shanghai" >> $GITHUB_ENV + + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.15' + + - name: Build + run: go build -v ./... + + - name: Test (Expect Arrow Package) + run: go test -v $(go list ./... | grep -v './arrow') diff --git a/odps/account/aliyun_account.go b/odps/account/aliyun_account.go index 3d352e3..ee132a6 100644 --- a/odps/account/aliyun_account.go +++ b/odps/account/aliyun_account.go @@ -41,17 +41,31 @@ func NewAliyunAccount(accessId string, accessKey string) *AliyunAccount { func AliyunAccountFromEnv() *AliyunAccount { account := AliyunAccount{} - if accessId, found := os.LookupEnv("odps_accessId"); found { + if accessId, found := os.LookupEnv("ALIBABA_CLOUD_ACCESS_KEY_ID"); found { account.accessId = accessId } - if accessKey, found := os.LookupEnv("odps_accessKey"); found { + if accessKey, found := os.LookupEnv("ALIBABA_CLOUD_ACCESS_KEY_SECRET"); found { account.accessKey = accessKey } return &account } +func AccountFromEnv() Account { + accessId, found := os.LookupEnv("ALIBABA_CLOUD_ACCESS_KEY_ID") + accessKey, found := os.LookupEnv("ALIBABA_CLOUD_ACCESS_KEY_SECRET") + if !found { + return nil + } + securityToken, found := os.LookupEnv("ALIBABA_CLOUD_SECURITY_TOKEN") + if found { + return NewStsAccount(accessId, accessKey, securityToken) + } else { + return NewAliyunAccount(accessId, accessKey) + } +} + func (account *AliyunAccount) AccessId() string { return account.accessId } diff --git a/odps/account/sts_account.go b/odps/account/sts_account.go index fbfc31b..34fe8b4 100644 --- a/odps/account/sts_account.go +++ b/odps/account/sts_account.go @@ -33,6 +33,7 @@ type CredentialProvider interface { type stsAccountProvider interface { _signRequest(req *http.Request, endpoint string) error + Credential() (*credentials.CredentialModel, error) } type stsStringProvider struct { @@ -50,6 +51,14 @@ func (sp *stsStringProvider) _signRequest(req *http.Request, endpoint string) er return nil } +func (sp *stsStringProvider) Credential() (*credentials.CredentialModel, error) { + return &credentials.CredentialModel{ + AccessKeyId: &sp.accessId, + AccessKeySecret: &sp.accessKey, + SecurityToken: &sp.stsToken, + }, nil +} + type stsAliyunCredentialProvider struct { aliyunCredential credentials.Credential } @@ -71,6 +80,10 @@ func (sp *stsAliyunCredentialProvider) _signRequest(req *http.Request, endpoint return nil } +func (sp *stsAliyunCredentialProvider) Credential() (*credentials.CredentialModel, error) { + return sp.aliyunCredential.GetCredential() +} + type stsCustomCredentialProvider struct { provider CredentialProvider } @@ -92,6 +105,10 @@ func (sp *stsCustomCredentialProvider) _signRequest(req *http.Request, endpoint return nil } +func (sp *stsCustomCredentialProvider) Credential() (*credentials.CredentialModel, error) { + return sp.provider.GetCredential() +} + func NewStsAccount(accessId, accessKey, securityToken string) *StsAccount { sp := &stsStringProvider{ stsToken: securityToken, @@ -133,3 +150,8 @@ func (account *StsAccount) GetType() Provider { func (account *StsAccount) SignRequest(req *http.Request, endpoint string) error { return account.sp._signRequest(req, endpoint) } + +func (account *StsAccount) Credential() (*credentials.CredentialModel, error) { + + return account.sp.Credential() +} diff --git a/odps/example_table_test.go b/odps/example_table_test.go index b41dae7..b18919f 100644 --- a/odps/example_table_test.go +++ b/odps/example_table_test.go @@ -68,13 +68,13 @@ func ExampleTableSchema_ToSQLString() { schema := builder.Build() - sql, _ := schema.ToSQLString("project_1", "schema", true) + sql, _ := schema.ToSQLString("go_sdk_regression_testing", "schema", true) println("sql of create table:") println(sql) println() externalSql, err := schema.ToExternalSQLString( - "project_1", + "go_sdk_regression_testing", "", true, serdeProperties, @@ -132,7 +132,7 @@ func ExampleTable_GetPartitions() { } func ExampleTable_ExecSql() { - //table := odps.NewTable(odpsIns, "project_1", "sale_detail") + //table := odps.NewTable(odpsIns, "go_sdk_regression_testing", "sale_detail") table := odpsIns.Table("has_struct") //instance, err := table.ExecSql("SelectSale_detail", "select * from sale_detail;") instance, err := table.ExecSql("Select_has_struct", "select * from has_struct;") diff --git a/odps/example_test.go b/odps/example_test.go index f638672..09fce69 100644 --- a/odps/example_test.go +++ b/odps/example_test.go @@ -26,13 +26,13 @@ import ( "github.com/aliyun/aliyun-odps-go-sdk/odps/tableschema" ) -var account = account2.AliyunAccountFromEnv() +var account = account2.AccountFromEnv() var endpoint = restclient.LoadEndpointFromEnv() var odpsIns = odps.NewOdps(account, endpoint) var defaultProjectName = "go_sdk_regression_testing" func init() { - if account.AccessId() == "" { + if account == nil { panic("account environments are not set") } diff --git a/odps/security/example_manager_test.go b/odps/security/example_manager_test.go index 95e01ac..6a04932 100644 --- a/odps/security/example_manager_test.go +++ b/odps/security/example_manager_test.go @@ -24,12 +24,13 @@ import ( "log" ) -var account = account2.AliyunAccountFromEnv() +var account = account2.AccountFromEnv() var endpoint = restclient.LoadEndpointFromEnv() var restClient = restclient.NewOdpsRestClient(account, endpoint) +var projectName = "go_sdk_regression_testing" func ExampleManager_GetSecurityConfig() { - sm := security.NewSecurityManager(restClient, "project_1") + sm := security.NewSecurityManager(restClient, projectName) sc, err := sm.GetSecurityConfig(true) if err != nil { log.Fatalf("%+v", err) @@ -41,9 +42,9 @@ func ExampleManager_GetSecurityConfig() { } func ExampleManager_CheckPermissionV1() { - sm := security.NewSecurityManager(restClient, "project_1") + sm := security.NewSecurityManager(restClient, projectName) p := security.NewPermission( - "project_1", + projectName, security.ObjectTypeTable, "sale_detail", security.ActionTypeAll, @@ -61,7 +62,7 @@ func ExampleManager_CheckPermissionV1() { } func ExampleManager_CheckPermissionV0() { - sm := security.NewSecurityManager(restClient, "project_1") + sm := security.NewSecurityManager(restClient, projectName) r, err := sm.CheckPermissionV0( security.ObjectTypeTable, @@ -78,7 +79,7 @@ func ExampleManager_CheckPermissionV0() { } func ExampleManager_GetPolicy() { - sm := security.NewSecurityManager(restClient, "project_1") + sm := security.NewSecurityManager(restClient, projectName) policy, err := sm.GetPolicy() if err != nil { log.Fatalf("%+v", err) @@ -89,7 +90,7 @@ func ExampleManager_GetPolicy() { } func ExampleManager_ListUsers() { - sm := security.NewSecurityManager(restClient, "project_1") + sm := security.NewSecurityManager(restClient, projectName) users, err := sm.ListUsers() if err != nil { log.Fatalf("%+v", err) @@ -103,8 +104,8 @@ func ExampleManager_ListUsers() { } func ExampleManager_RunQuery() { - sm := security.NewSecurityManager(restClient, "project_1") - result, err := sm.RunQuery("show grants for aliyun$odpstest1@aliyun.com;", true, "") + sm := security.NewSecurityManager(restClient, projectName) + result, err := sm.RunQuery("whoami;", true, "") if err != nil { log.Fatalf("%+v", err) } diff --git a/odps/tunnel/example_instance_result_download_test.go b/odps/tunnel/example_instance_result_download_test.go index ac1526d..5e23583 100644 --- a/odps/tunnel/example_instance_result_download_test.go +++ b/odps/tunnel/example_instance_result_download_test.go @@ -27,11 +27,11 @@ import ( ) func Example_tunnel_download_instance_result() { - var account = account2.AliyunAccountFromEnv() + var account = account2.AccountFromEnv() var endpoint = restclient.LoadEndpointFromEnv() var odpsIns = odps.NewOdps(account, endpoint) - projectName := "project_1" + projectName := "go_sdk_regression_testing" odpsIns.SetDefaultProjectName(projectName) project := odpsIns.DefaultProject() tunnelIns, err := tunnel.NewTunnelFromProject(project) diff --git a/odps/tunnel/example_test.go b/odps/tunnel/example_test.go index c9cd63b..dd4147b 100644 --- a/odps/tunnel/example_test.go +++ b/odps/tunnel/example_test.go @@ -20,24 +20,24 @@ import ( "github.com/aliyun/aliyun-odps-go-sdk/odps" account2 "github.com/aliyun/aliyun-odps-go-sdk/odps/account" "github.com/aliyun/aliyun-odps-go-sdk/odps/datatype" + "github.com/aliyun/aliyun-odps-go-sdk/odps/restclient" "github.com/aliyun/aliyun-odps-go-sdk/odps/tableschema" "github.com/aliyun/aliyun-odps-go-sdk/odps/tunnel" "log" "os" ) -var tunnelIns tunnel.Tunnel +var tunnelIns *tunnel.Tunnel var odpsIns *odps.Odps -var ProjectName = "test_new_console_gcc" +var ProjectName = "go_sdk_regression_testing" func init() { - accessId := os.Getenv("tunnel_odps_accessId") - accessKey := os.Getenv("tunnel_odps_accessKey") - odpsEndpoint := os.Getenv("odps_endpoint") + account := account2.AccountFromEnv() + odpsEndpoint := restclient.LoadEndpointFromEnv() tunnelEndpoint := os.Getenv("tunnel_odps_endpoint") - account := account2.NewAliyunAccount(accessId, accessKey) odpsIns = odps.NewOdps(account, odpsEndpoint) + odpsIns.SetDefaultProjectName(ProjectName) tunnelIns = tunnel.NewTunnel(odpsIns, tunnelEndpoint) //createTableWithComplexData() @@ -104,7 +104,7 @@ func createSaleDetailTable() { } func createUploadSampleArrowTable() { - ins, err := odpsIns.ExecSQl("CREATE TABLE IF NOT EXISTS project_1.upload_sample_arrow(payload STRING);") + ins, err := odpsIns.ExecSQl("CREATE TABLE IF NOT EXISTS go_sdk_regression_testing.upload_sample_arrow(payload STRING);") if err != nil { log.Fatalf("%+v", err) } diff --git a/odps/tunnel/example_tunnel_test.go b/odps/tunnel/example_tunnel_test.go index e6c6632..26eb146 100644 --- a/odps/tunnel/example_tunnel_test.go +++ b/odps/tunnel/example_tunnel_test.go @@ -28,6 +28,12 @@ import ( ) func Example_tunnel_upload_arrow() { + + err := odpsIns.Tables().Get("sale_detail").AddPartition(true, "sale_date=202111/region=hangzhou") + if err != nil { + log.Fatalf("%+v", err) + } + tunnelIns.SetHttpTimeout(10 * time.Second) session, err := tunnelIns.CreateUploadSession( @@ -86,12 +92,14 @@ func Example_tunnel_upload_arrow() { case "shop_name": builder := fieldBuilder.(*array.StringBuilder) builder.AppendValues(data.ShopNames, nil) - case "customer_id": + case "custom_id": builder := fieldBuilder.(*array.StringBuilder) builder.AppendValues(data.CustomIDs, nil) case "total_price": builder := fieldBuilder.(*array.Float64Builder) builder.AppendValues(data.totalPrice, nil) + default: + log.Fatalf("unknown field: %s", field.Name) } } @@ -137,7 +145,7 @@ func Example_tunnel_upload_arrow() { func Example_tunnel_download_arrow_simple() { session, err := tunnelIns.CreateDownloadSession( - "test_new_console_gcc", + ProjectName, // "upload_sample_arrow", "has_struct", ) @@ -177,7 +185,7 @@ func Example_tunnel_download_arrow_simple() { func Example_tunnel_download_arrow_with_partition() { session, err := tunnelIns.CreateDownloadSession( - "test_new_console_gcc", + "go_sdk_regression_testing", "sale_detail", tunnel2.SessionCfg.WithPartitionKey("sale_date='202111',region='hangzhou'"), ) diff --git a/odps/tunnel/record_protoc_writer.go b/odps/tunnel/record_protoc_writer.go index c4dc057..04c03ac 100644 --- a/odps/tunnel/record_protoc_writer.go +++ b/odps/tunnel/record_protoc_writer.go @@ -189,6 +189,10 @@ func (r *RecordProtocWriter) writeField(val data.Data) error { case data.TinyInt: r.recordCrc.Update(int64(val)) return errors.WithStack(r.protocWriter.WriteSInt64(int64(val))) + case *data.String: + b := []byte(string(*val)) + r.recordCrc.Update(b) + return errors.WithStack(r.protocWriter.WriteBytes(b)) case data.String: b := []byte(string(val)) r.recordCrc.Update(b) diff --git a/odps/tunnel/tunnel.go b/odps/tunnel/tunnel.go index 257a9b1..a7492c8 100644 --- a/odps/tunnel/tunnel.go +++ b/odps/tunnel/tunnel.go @@ -50,7 +50,7 @@ type Tunnel struct { } // Once the tunnel endpoint is set, it cannot be modified anymore. -func NewTunnel(odpsIns *odps.Odps, endpoint ...string) Tunnel { +func NewTunnel(odpsIns *odps.Odps, endpoint ...string) *Tunnel { tunnel := Tunnel{ odpsIns: odpsIns, } @@ -58,13 +58,13 @@ func NewTunnel(odpsIns *odps.Odps, endpoint ...string) Tunnel { tunnel.endpoint = endpoint[0] } - return tunnel + return &tunnel } -func NewTunnelFromProject(project odps.Project) (Tunnel, error) { +func NewTunnelFromProject(project *odps.Project) (*Tunnel, error) { endpoint, err := project.GetTunnelEndpoint() if err != nil { - return Tunnel{}, errors.WithStack(err) + return &Tunnel{}, errors.WithStack(err) } tunnel := Tunnel{ @@ -72,7 +72,7 @@ func NewTunnelFromProject(project odps.Project) (Tunnel, error) { endpoint: endpoint, } - return tunnel, nil + return &tunnel, nil } func (t *Tunnel) HttpTimeout() time.Duration { diff --git a/sqldriver/example_test.go b/sqldriver/example_test.go index aa32520..38ac429 100644 --- a/sqldriver/example_test.go +++ b/sqldriver/example_test.go @@ -18,7 +18,6 @@ package sqldriver_test import ( "database/sql" - "fmt" "log" "time" @@ -30,20 +29,41 @@ import ( ) func Example() { - var account = account2.AliyunAccountFromEnv() + var account = account2.AccountFromEnv() var endpoint = restclient.LoadEndpointFromEnv() config := sqldriver.NewConfig() config.Endpoint = endpoint - config.AccessId = account.AccessId() - config.AccessKey = account.AccessKey() - config.ProjectName = "project_1" + + if account.GetType() == account2.STS { + stsAccount, _ := account.(*account2.StsAccount) + credential, err := stsAccount.Credential() + if err != nil { + log.Fatalf("%+v", err) + } + config.AccessId = *credential.AccessKeyId + config.AccessKey = *credential.AccessKeySecret + config.StsToken = *credential.SecurityToken + + } else if account.GetType() == account2.Aliyun { + akAccount, _ := account.(*account2.AliyunAccount) + config.AccessId = akAccount.AccessId() + config.AccessKey = akAccount.AccessKey() + } else { + errMsg := "unknown account type: %s" + account.GetType().String() + panic(errMsg) + } + config.ProjectName = "go_sdk_regression_testing" dsn := config.FormatDsn() db, err := sql.Open("odps", dsn) if err != nil { log.Fatalf("%+v", err) } + _, err = db.Exec("create table if not exists data_type_demo(ti tinyint, si smallint, i int, bi bigint, b binary, f float, d double);", nil) + if err != nil { + log.Fatalf("%+v", err) + } rows, err := db.Query("select ti, si, i, bi, b, f, d from data_type_demo;", nil) if err != nil { @@ -64,21 +84,35 @@ func Example() { log.Fatalf("%+v", err) } - println(ti, si, i, bi, b, f, d) + log.Println(ti, si, i, bi, b, f, d) } // Output: } func ExampleStructField() { - var account = account2.AliyunAccountFromEnv() + var account = account2.AccountFromEnv() var endpoint = restclient.LoadEndpointFromEnv() config := sqldriver.NewConfig() config.Endpoint = endpoint - config.AccessId = account.AccessId() - config.AccessKey = account.AccessKey() - config.ProjectName = "project_1" + if account.GetType() == account2.STS { + stsAccount, _ := account.(*account2.StsAccount) + credential, err := stsAccount.Credential() + if err != nil { + log.Fatalf("%+v", err) + } + config.AccessId = *credential.AccessKeyId + config.AccessKey = *credential.AccessKeySecret + config.StsToken = *credential.SecurityToken + } else if account.GetType() == account2.Aliyun { + akAccount, _ := account.(*account2.AliyunAccount) + config.AccessId = akAccount.AccessId() + config.AccessKey = akAccount.AccessKey() + } else { + log.Fatalf("unknown account type: %s", account.GetType()) + } + config.ProjectName = "go_sdk_regression_testing" dsn := config.FormatDsn() db, err := sql.Open("odps", dsn) @@ -104,14 +138,28 @@ func ExampleStructField() { } func ExampleInsert() { - var account = account2.AliyunAccountFromEnv() + var account = account2.AccountFromEnv() var endpoint = restclient.LoadEndpointFromEnv() config := sqldriver.NewConfig() config.Endpoint = endpoint - config.AccessId = account.AccessId() - config.AccessKey = account.AccessKey() - config.ProjectName = "project_1" + if account.GetType() == account2.STS { + stsAccount, _ := account.(*account2.StsAccount) + credential, err := stsAccount.Credential() + if err != nil { + log.Fatalf("%+v", err) + } + config.AccessId = *credential.AccessKeyId + config.AccessKey = *credential.AccessKeySecret + config.StsToken = *credential.SecurityToken + } else if account.GetType() == account2.Aliyun { + akAccount, _ := account.(*account2.AliyunAccount) + config.AccessId = akAccount.AccessId() + config.AccessKey = akAccount.AccessKey() + } else { + log.Fatalf("unknown account type: %s", account.GetType()) + } + config.ProjectName = "go_sdk_regression_testing" dsn := config.FormatDsn() db, err := sql.Open("odps", dsn) @@ -119,6 +167,9 @@ func ExampleInsert() { log.Fatalf("%+v", err) } + db.Exec("drop table if exists simple_struct;") + db.Exec("create table if not exists simple_struct (col struct>);") + type SimpleStruct struct { A int32 B struct { @@ -132,7 +183,7 @@ func ExampleInsert() { } odpsStruct, err := data.StructFromGoStruct(simpleStruct) - fmt.Println(odpsStruct) + log.Println(odpsStruct) if err != nil { log.Fatalf("%+v", err) @@ -147,14 +198,28 @@ func ExampleInsert() { } func ExampleCreateTable() { - var account = account2.AliyunAccountFromEnv() + var account = account2.AccountFromEnv() var endpoint = restclient.LoadEndpointFromEnv() config := sqldriver.NewConfig() config.Endpoint = endpoint - config.AccessId = account.AccessId() - config.AccessKey = account.AccessKey() - config.ProjectName = "project_1" + if account.GetType() == account2.STS { + stsAccount, _ := account.(*account2.StsAccount) + credential, err := stsAccount.Credential() + if err != nil { + log.Fatalf("%+v", err) + } + config.AccessId = *credential.AccessKeyId + config.AccessKey = *credential.AccessKeySecret + config.StsToken = *credential.SecurityToken + } else if account.GetType() == account2.Aliyun { + akAccount, _ := account.(*account2.AliyunAccount) + config.AccessId = akAccount.AccessId() + config.AccessKey = akAccount.AccessKey() + } else { + log.Fatalf("unknown account type: %s", account.GetType()) + } + config.ProjectName = "go_sdk_regression_testing" dsn := config.FormatDsn() db, err := sql.Open("odps", dsn) @@ -169,7 +234,7 @@ func ExampleCreateTable() { )), ) - _, err = db.Exec("create table simple_struct (struct_field @f);", sql.Named("f", structType.String())) + _, err = db.Exec("create table if not exists simple_struct (struct_field @f);", sql.Named("f", structType.String())) if err != nil { log.Fatalf("%+v", err) }