forked from raystack/transformers
-
Notifications
You must be signed in to change notification settings - Fork 1
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* feat: initial max2max poc * chore: use amd64 alpine for max2max * feat: use service account format for odps client * restructure: max2max project * chore: setup github workflow for max2max
- Loading branch information
1 parent
64a2bb8
commit 2bb3887
Showing
19 changed files
with
687 additions
and
13 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -7,20 +7,21 @@ on: | |
workflow_dispatch: | ||
|
||
jobs: | ||
build: | ||
build-max2max: | ||
runs-on: ubuntu-latest | ||
steps: | ||
- name: Checkout | ||
uses: actions/checkout@v2 | ||
with: | ||
fetch-depth: 0 | ||
- name: Set up Go | ||
uses: actions/setup-go@v2 | ||
- name: Setup Go | ||
uses: actions/setup-go@v5 | ||
with: | ||
go-version: '1.20' | ||
- name: Run GoReleaser for branch [Main] | ||
uses: goreleaser/[email protected] | ||
with: | ||
distribution: goreleaser | ||
version: latest | ||
args: -f .goreleaser.yml --snapshot --clean | ||
go-version: 1.22 | ||
# Build the Go binary | ||
- name: Build | ||
run: | | ||
cd max2max | ||
mkdir build | ||
go get . | ||
env GOOS=linux GOARCH=amd64 go build -o ./build/max2max . |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
build/ | ||
dist/ | ||
.env |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
FROM amd64/alpine:3 | ||
|
||
RUN apk --no-cache add tzdata | ||
COPY ./build/max2max /usr/local/bin/max2max | ||
|
||
ENTRYPOINT ["/usr/local/bin/max2max"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
module github.com/goto/maxcompute-transformation | ||
|
||
go 1.22.3 | ||
|
||
require github.com/aliyun/aliyun-odps-go-sdk v0.3.4 | ||
|
||
require ( | ||
github.com/golang/snappy v0.0.4 // indirect | ||
github.com/google/flatbuffers v23.5.26+incompatible // indirect | ||
github.com/google/uuid v1.3.0 // indirect | ||
github.com/klauspost/compress v1.15.9 // indirect | ||
github.com/pierrec/lz4/v4 v4.1.18 // indirect | ||
github.com/pkg/errors v0.9.1 // indirect | ||
github.com/stretchr/testify v1.9.0 // indirect | ||
golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect | ||
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect | ||
google.golang.org/protobuf v1.31.0 // indirect | ||
gopkg.in/ini.v1 v1.67.0 // indirect | ||
) |
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,49 @@ | ||
package client | ||
|
||
import ( | ||
"fmt" | ||
"log/slog" | ||
"os" | ||
|
||
"github.com/aliyun/aliyun-odps-go-sdk/odps" | ||
) | ||
|
||
type Loader interface { | ||
GetQuery(tableID, query string) string | ||
} | ||
|
||
type client struct { | ||
logger *slog.Logger | ||
odpsIns *odps.Odps | ||
} | ||
|
||
func NewClient(logger *slog.Logger, odpsIns *odps.Odps) *client { | ||
return &client{ | ||
logger: logger, | ||
odpsIns: odpsIns, | ||
} | ||
} | ||
|
||
func (c *client) Execute(loader Loader, tableID, queryFilePath string) error { | ||
// read query from filepath | ||
c.logger.Info(fmt.Sprintf("executing query from %s", queryFilePath)) | ||
queryRaw, err := os.ReadFile(queryFilePath) | ||
if err != nil { | ||
return err | ||
} | ||
|
||
// execute query with odps client | ||
c.logger.Info(fmt.Sprintf("execute: %s", string(queryRaw))) | ||
ins, err := c.odpsIns.ExecSQl(loader.GetQuery(tableID, string(queryRaw))) | ||
if err != nil { | ||
return err | ||
} | ||
c.logger.Info(fmt.Sprintf("taskId: %s", ins.Id())) | ||
|
||
// wait execution success | ||
if err := ins.WaitForSuccess(); err != nil { | ||
return err | ||
} | ||
c.logger.Info("execution done") | ||
return nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
package config | ||
|
||
import ( | ||
"encoding/json" | ||
|
||
"github.com/aliyun/aliyun-odps-go-sdk/odps" | ||
) | ||
|
||
type Config struct { | ||
*odps.Config | ||
LogLevel string | ||
LoadMethod string | ||
QueryFilePath string | ||
DestinationTableID string | ||
} | ||
|
||
type maxComputeCredentials struct { | ||
AccessId string `json:"access_id"` | ||
AccessKey string `json:"access_key"` | ||
Endpoint string `json:"endpoint"` | ||
ProjectName string `json:"project_name"` | ||
} | ||
|
||
func NewConfig() (*Config, error) { | ||
cfg := &Config{ | ||
Config: odps.NewConfig(), | ||
// max2max related config | ||
LogLevel: getEnv("LOG_LEVEL", "INFO"), | ||
LoadMethod: getEnv("LOAD_METHOD", "APPEND"), | ||
QueryFilePath: getEnv("QUERY_FILE_PATH", ""), | ||
DestinationTableID: getEnv("DESTINATION_TABLE_ID", ""), | ||
} | ||
// ali-odps-go-sdk related config | ||
scvAcc := getEnv("SERVICE_ACCOUNT", "") | ||
cred, err := collectMaxComputeCredential([]byte(scvAcc)) | ||
if err != nil { | ||
return nil, err | ||
} | ||
cfg.Config.AccessId = cred.AccessId | ||
cfg.Config.AccessKey = cred.AccessKey | ||
cfg.Config.Endpoint = cred.Endpoint | ||
cfg.Config.ProjectName = cred.ProjectName | ||
cfg.Config.HttpTimeout = getEnvDuration("MAXCOMPUTE_HTTP_TIMEOUT", "10s") | ||
cfg.Config.TcpConnectionTimeout = getEnvDuration("MAXCOMPUTE_TCP_TIMEOUT", "30s") | ||
|
||
return cfg, nil | ||
} | ||
|
||
func collectMaxComputeCredential(scvAcc []byte) (*maxComputeCredentials, error) { | ||
var creds maxComputeCredentials | ||
if err := json.Unmarshal(scvAcc, &creds); err != nil { | ||
return nil, err | ||
} | ||
|
||
return &creds, nil | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
package config | ||
|
||
import ( | ||
"os" | ||
"time" | ||
) | ||
|
||
func getEnv(key, fallback string) string { | ||
if value, ok := os.LookupEnv(key); ok { | ||
return value | ||
} | ||
return fallback | ||
} | ||
|
||
func getEnvDuration(key, fallback string) time.Duration { | ||
result, _ := time.ParseDuration(getEnv(key, fallback)) | ||
return result | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,20 @@ | ||
package loader | ||
|
||
import ( | ||
"fmt" | ||
"log/slog" | ||
) | ||
|
||
type appendLoader struct { | ||
logger *slog.Logger | ||
} | ||
|
||
func NewAppendLoader(logger *slog.Logger) *appendLoader { | ||
return &appendLoader{ | ||
logger: logger, | ||
} | ||
} | ||
|
||
func (l *appendLoader) GetQuery(tableID, query string) string { | ||
return fmt.Sprintf("INSERT INTO TABLE %s %s", tableID, query) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
package loader | ||
|
||
const ( | ||
APPEND = "APPEND" | ||
REPLACE = "REPLACE" | ||
REPLACE_ALL = "REPLACE_ALL" | ||
MERGE = "MERGE" | ||
MERGE_REPLACE = "MERGE_REPLACE" | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
package loader | ||
|
||
import ( | ||
"errors" | ||
"log/slog" | ||
) | ||
|
||
type Loader interface { | ||
GetQuery(tableID, query string) string | ||
} | ||
|
||
func GetLoader(name string, logger *slog.Logger) (Loader, error) { | ||
switch name { | ||
case APPEND: | ||
return NewAppendLoader(logger), nil | ||
case REPLACE: | ||
return NewReplaceLoader(logger), nil | ||
case REPLACE_ALL: | ||
return NewReplaceAllLoader(logger), nil | ||
case MERGE: | ||
return NewMergeLoader(logger), nil | ||
case MERGE_REPLACE: | ||
return NewMergeReplaceLoader(logger), nil | ||
default: | ||
return nil, errors.New("loader not found") | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
package loader | ||
|
||
import ( | ||
"log/slog" | ||
) | ||
|
||
type mergeLoader struct { | ||
logger *slog.Logger | ||
} | ||
|
||
func NewMergeLoader(logger *slog.Logger) *mergeLoader { | ||
return &mergeLoader{ | ||
logger: logger, | ||
} | ||
} | ||
|
||
func (l *mergeLoader) GetQuery(tableID, query string) string { | ||
return "-- TODO merge loader" | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
package loader | ||
|
||
import ( | ||
"log/slog" | ||
) | ||
|
||
type mergeReplaceLoader struct { | ||
logger *slog.Logger | ||
} | ||
|
||
func NewMergeReplaceLoader(logger *slog.Logger) *mergeReplaceLoader { | ||
return &mergeReplaceLoader{ | ||
logger: logger, | ||
} | ||
} | ||
|
||
func (l *mergeReplaceLoader) GetQuery(tableID, query string) string { | ||
return "-- TODO merge replace loader" | ||
} |
Oops, something went wrong.