From e3942823523e006e546199eacb01200871bb5c31 Mon Sep 17 00:00:00 2001 From: Manjunath Tapali Date: Tue, 2 Jul 2024 22:31:40 +0530 Subject: [PATCH 1/6] interface changes revision-2 --- common.proto | 56 +++++++++++++++++++++++++------------------ connector_sdk.proto | 28 ++++------------------ destination_sdk.proto | 27 +++++++++++++++++---- 3 files changed, 61 insertions(+), 50 deletions(-) diff --git a/common.proto b/common.proto index 2f09d03..fd854fd 100644 --- a/common.proto +++ b/common.proto @@ -1,8 +1,8 @@ syntax = "proto3"; option optimize_for = SPEED; option java_multiple_files = true; -option go_package = "fivetran.com/fivetran_sdk"; -package fivetran_sdk; +option go_package = "fivetran.com/fivetran_sdk_r2"; +package fivetran_sdk.r2; import "google/protobuf/timestamp.proto"; @@ -20,10 +20,12 @@ message FormField { string label = 2; bool required = 3; optional string description = 4; + optional string default_value = 5; + optional string placeholder = 6; oneof type { - TextField text_field = 5; - DropdownField dropdown_field = 6; - ToggleField toggle_field = 7; + TextField text_field = 7; + DropdownField dropdown_field = 8; + ToggleField toggle_field = 9; } } @@ -53,7 +55,6 @@ message TestResponse { oneof response { bool success = 1; string failure = 2; - // potential future warning } } @@ -79,13 +80,21 @@ enum DataType { DECIMAL = 5; FLOAT = 6; DOUBLE = 7; - NAIVE_DATE = 8; - NAIVE_DATETIME = 9; - UTC_DATETIME = 10; - BINARY = 11; - XML = 12; - STRING = 13; - JSON = 14; + NAIVE_TIME = 8; + NAIVE_DATE = 9; + NAIVE_DATETIME = 10; + UTC_DATETIME = 11; + BINARY = 12; + XML = 13; + STRING = 14; + JSON = 15; +} + +message DataTypeParams { + oneof params { + DecimalParams decimal = 1; + int32 string_byte_length = 2; + } } message DecimalParams { @@ -93,7 +102,7 @@ message DecimalParams { uint32 scale = 2; } -enum OpType { +enum RecordType { UPSERT = 0; UPDATE = 1; DELETE = 2; @@ -109,14 +118,15 @@ message ValueType { int64 long = 5; float float = 6; double double = 7; - google.protobuf.Timestamp naive_date = 8; - google.protobuf.Timestamp naive_datetime = 9; - google.protobuf.Timestamp utc_datetime = 10; - string decimal = 11; - bytes binary = 12; - string string = 13; - string json = 14; - string xml = 15; + google.protobuf.Timestamp naive_time = 8; + google.protobuf.Timestamp naive_date = 9; + google.protobuf.Timestamp naive_datetime = 10; + google.protobuf.Timestamp utc_datetime = 11; + string decimal = 12; + bytes binary = 13; + string string = 14; + string json = 15; + string xml = 16; } } @@ -129,5 +139,5 @@ message Column { string name = 1; DataType type = 2; bool primary_key = 3; - optional DecimalParams decimal = 4; + optional DataTypeParams params = 4; } diff --git a/connector_sdk.proto b/connector_sdk.proto index 4d906ee..75da39d 100644 --- a/connector_sdk.proto +++ b/connector_sdk.proto @@ -1,13 +1,13 @@ syntax = "proto3"; option optimize_for = SPEED; option java_multiple_files = true; -option go_package = "fivetran.com/fivetran_sdk"; -package fivetran_sdk; +option go_package = "fivetran.com/fivetran_sdk_r2"; +package fivetran_sdk.r2; import "common.proto"; // Fivetran (grpc client) <> Connector (grpc server) -service Connector { +service SourceConnector { rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {} rpc Test (TestRequest) returns (TestResponse) {} rpc Schema (SchemaRequest) returns (SchemaResponse) {} @@ -65,25 +65,7 @@ message TableSelection { } message UpdateResponse { - oneof response { - LogEntry log_entry = 1; - Operation operation = 2; - } -} - -enum LogLevel { - INFO = 0; - WARNING = 1; - SEVERE = 2; -} - -message LogEntry { - LogLevel level = 1; - string message = 2; -} - -message Operation { - oneof op { + oneof Operation { Record record = 1; SchemaChange schema_change = 2; Checkpoint checkpoint = 3; @@ -100,7 +82,7 @@ message SchemaChange { message Record { optional string schema_name = 1; string table_name = 2; - OpType type = 3; + RecordType type = 3; map data = 4; } diff --git a/destination_sdk.proto b/destination_sdk.proto index 8c5635b..0a5e0a9 100644 --- a/destination_sdk.proto +++ b/destination_sdk.proto @@ -1,8 +1,8 @@ syntax = "proto3"; option optimize_for = SPEED; option java_multiple_files = true; -option go_package = "fivetran.com/fivetran_sdk"; -package fivetran_sdk; +option go_package = "fivetran.com/fivetran_sdk_r2"; +package fivetran_sdk.r2; import "google/protobuf/timestamp.proto"; import "common.proto"; @@ -55,7 +55,26 @@ message CreateTableResponse { message AlterTableRequest { map configuration = 1; string schema_name = 2; - Table table = 3; + string table_name = 3; + repeated SchemaDiff changes = 4; +} + +message SchemaDiff { + oneof change { + Column add_column = 1; + ChangeType change_column_type = 2; + PrimaryKey update_primary_keys = 3; + } +} + +message PrimaryKey { + repeated string column_name = 1; +} + +message ChangeType { + string column_name = 1; + DataType new_type = 2; + optional DataTypeParams params = 3; } message AlterTableResponse { @@ -75,7 +94,7 @@ message TruncateRequest { } message SoftTruncate { - string deleted_column = 3; + string deleted_column = 1; } message TruncateResponse { From 1bd70f9b10dac3a2e8deff4bfcb17ad633b7dd09 Mon Sep 17 00:00:00 2001 From: Manjunath Tapali Date: Tue, 2 Jul 2024 22:44:08 +0530 Subject: [PATCH 2/6] updated the name --- destination_sdk.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/destination_sdk.proto b/destination_sdk.proto index 0a5e0a9..c49ab0a 100644 --- a/destination_sdk.proto +++ b/destination_sdk.proto @@ -8,7 +8,7 @@ import "google/protobuf/timestamp.proto"; import "common.proto"; // Fivetran (grpc client) <> Destination (grpc server) -service Destination { +service DestinationConnector { rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {} rpc Capabilities (CapabilitiesRequest) returns (CapabilitiesResponse) {} rpc Test (TestRequest) returns (TestResponse) {} From 92ef8030d8dd3bc5ea2164be7ae63fb502a9cf6c Mon Sep 17 00:00:00 2001 From: Manjunath Tapali Date: Wed, 3 Jul 2024 00:05:54 +0530 Subject: [PATCH 3/6] updated the code comments --- connector_sdk.proto | 2 +- destination_sdk.proto | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/connector_sdk.proto b/connector_sdk.proto index 75da39d..77a9c3d 100644 --- a/connector_sdk.proto +++ b/connector_sdk.proto @@ -6,7 +6,7 @@ package fivetran_sdk.r2; import "common.proto"; -// Fivetran (grpc client) <> Connector (grpc server) +// Fivetran (grpc client) <> SourceConnector (grpc server) service SourceConnector { rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {} rpc Test (TestRequest) returns (TestResponse) {} diff --git a/destination_sdk.proto b/destination_sdk.proto index c49ab0a..e26678f 100644 --- a/destination_sdk.proto +++ b/destination_sdk.proto @@ -7,7 +7,7 @@ package fivetran_sdk.r2; import "google/protobuf/timestamp.proto"; import "common.proto"; -// Fivetran (grpc client) <> Destination (grpc server) +// Fivetran (grpc client) <> DestinationConnector (grpc server) service DestinationConnector { rpc ConfigurationForm (ConfigurationFormRequest) returns (ConfigurationFormResponse) {} rpc Capabilities (CapabilitiesRequest) returns (CapabilitiesResponse) {} From 759ca84a834a4e603837a1d38d86b410ae4daf95 Mon Sep 17 00:00:00 2001 From: Manjunath Tapali Date: Fri, 5 Jul 2024 12:35:22 +0530 Subject: [PATCH 4/6] renamed the operation --- connector_sdk.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/connector_sdk.proto b/connector_sdk.proto index 77a9c3d..301f32e 100644 --- a/connector_sdk.proto +++ b/connector_sdk.proto @@ -65,7 +65,7 @@ message TableSelection { } message UpdateResponse { - oneof Operation { + oneof op { Record record = 1; SchemaChange schema_change = 2; Checkpoint checkpoint = 3; From 5aeb6d3e4b7514ecc7224938ec13db8492684355 Mon Sep 17 00:00:00 2001 From: Manjunath Tapali Date: Tue, 9 Jul 2024 10:31:33 +0530 Subject: [PATCH 5/6] reverted the field renumbering --- common.proto | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/common.proto b/common.proto index fd854fd..cc39336 100644 --- a/common.proto +++ b/common.proto @@ -20,13 +20,13 @@ message FormField { string label = 2; bool required = 3; optional string description = 4; - optional string default_value = 5; - optional string placeholder = 6; oneof type { - TextField text_field = 7; - DropdownField dropdown_field = 8; - ToggleField toggle_field = 9; + TextField text_field = 5; + DropdownField dropdown_field = 6; + ToggleField toggle_field = 7; } + optional string default_value = 8; + optional string placeholder = 9; } message DropdownField { @@ -80,14 +80,14 @@ enum DataType { DECIMAL = 5; FLOAT = 6; DOUBLE = 7; - NAIVE_TIME = 8; - NAIVE_DATE = 9; - NAIVE_DATETIME = 10; - UTC_DATETIME = 11; - BINARY = 12; - XML = 13; - STRING = 14; - JSON = 15; + NAIVE_DATE = 8; + NAIVE_DATETIME = 9; + UTC_DATETIME = 10; + BINARY = 11; + XML = 12; + STRING = 13; + JSON = 14; + NAIVE_TIME = 15; } message DataTypeParams { @@ -118,15 +118,15 @@ message ValueType { int64 long = 5; float float = 6; double double = 7; - google.protobuf.Timestamp naive_time = 8; - google.protobuf.Timestamp naive_date = 9; - google.protobuf.Timestamp naive_datetime = 10; - google.protobuf.Timestamp utc_datetime = 11; - string decimal = 12; - bytes binary = 13; - string string = 14; - string json = 15; - string xml = 16; + google.protobuf.Timestamp naive_date = 8; + google.protobuf.Timestamp naive_datetime = 9; + google.protobuf.Timestamp utc_datetime = 10; + string decimal = 11; + bytes binary = 12; + string string = 13; + string json = 14; + string xml = 15; + google.protobuf.Timestamp naive_time = 16; } } From c7c4500966df1fdb72a3ba8e7bf7f9a4c0258a0b Mon Sep 17 00:00:00 2001 From: Manjunath Tapali Date: Tue, 30 Jul 2024 19:07:56 +0530 Subject: [PATCH 6/6] revert schema diff changes --- destination_sdk.proto | 21 +-------------------- 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/destination_sdk.proto b/destination_sdk.proto index e26678f..66c4c76 100644 --- a/destination_sdk.proto +++ b/destination_sdk.proto @@ -55,26 +55,7 @@ message CreateTableResponse { message AlterTableRequest { map configuration = 1; string schema_name = 2; - string table_name = 3; - repeated SchemaDiff changes = 4; -} - -message SchemaDiff { - oneof change { - Column add_column = 1; - ChangeType change_column_type = 2; - PrimaryKey update_primary_keys = 3; - } -} - -message PrimaryKey { - repeated string column_name = 1; -} - -message ChangeType { - string column_name = 1; - DataType new_type = 2; - optional DataTypeParams params = 3; + Table table = 3; } message AlterTableResponse {