Skip to content

Commit

Permalink
Added support for search indexes (#1768)
Browse files Browse the repository at this point in the history 
* Added support for search indexes

* Added HIDDEN column support to AvroSchemaToDddlConverter

* Added test for SpannerSchema parsing of TOKENLIST

* Addressed review feedback

* Ran mvn spotless:apply to fix formatting
  • Loading branch information
@atask-g
atask-g authored Aug 8, 2024
1 parent 0865f8a commit df97113
Show file tree
Hide file tree
Showing 16 changed files with 256 additions and 13 deletions.
5 changes: 5 additions & 0 deletions v1/src/main/java/com/google/cloud/teleport/spanner/AvroSchemaToDdlConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

import static com.google.cloud.teleport.spanner.AvroUtil.DEFAULT_EXPRESSION;
import static com.google.cloud.teleport.spanner.AvroUtil.GENERATION_EXPRESSION;
import static com.google.cloud.teleport.spanner.AvroUtil.HIDDEN;
import static com.google.cloud.teleport.spanner.AvroUtil.INPUT;
import static com.google.cloud.teleport.spanner.AvroUtil.NOT_NULL;
import static com.google.cloud.teleport.spanner.AvroUtil.OUTPUT;
Expand Down Expand Up @@ -258,6 +259,10 @@ public Table toTable(String tableName, Schema schema) {
if (Boolean.parseBoolean(stored)) {
column.stored();
}
String hidden = f.getProp(HIDDEN);
if (Boolean.parseBoolean(hidden)) {
column.isHidden(true);
}
} else {
boolean nullable = false;
Schema avroType = f.schema();
Expand Down
1 change: 1 addition & 0 deletions v1/src/main/java/com/google/cloud/teleport/spanner/AvroUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ private AvroUtil() {}
public static final String SPANNER_NAMED_SCHEMA = "spannerNamedSchema";
public static final String SPANNER_NAME = "spannerName";
public static final String STORED = "stored";
public static final String HIDDEN = "hidden";

public static Schema unpackNullable(Schema schema) {
if (schema.getType() != Schema.Type.UNION) {
Expand Down
3 changes: 3 additions & 0 deletions v1/src/main/java/com/google/cloud/teleport/spanner/DdlToAvroSchemaConverter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import static com.google.cloud.teleport.spanner.AvroUtil.GENERATION_EXPRESSION;
import static com.google.cloud.teleport.spanner.AvroUtil.GOOGLE_FORMAT_VERSION;
import static com.google.cloud.teleport.spanner.AvroUtil.GOOGLE_STORAGE;
import static com.google.cloud.teleport.spanner.AvroUtil.HIDDEN;
import static com.google.cloud.teleport.spanner.AvroUtil.INPUT;
import static com.google.cloud.teleport.spanner.AvroUtil.NOT_NULL;
import static com.google.cloud.teleport.spanner.AvroUtil.OUTPUT;
Expand Down Expand Up @@ -155,6 +156,7 @@ public Collection<Schema> convert(Ddl ddl) {
fieldBuilder.prop(NOT_NULL, Boolean.toString(cm.notNull()));
fieldBuilder.prop(GENERATION_EXPRESSION, cm.generationExpression());
fieldBuilder.prop(STORED, Boolean.toString(cm.isStored()));
fieldBuilder.prop(HIDDEN, Boolean.toString(cm.isHidden()));
// Make the type null to allow us not export the generated column values,
// which are semantically logical entities.
fieldBuilder.type(SchemaBuilder.builder().nullType()).withDefault(null);
Expand Down Expand Up @@ -334,6 +336,7 @@ private Schema avroType(
case BYTES:
case PG_BYTEA:
case PROTO:
case TOKENLIST:
return SchemaBuilder.builder().bytesType();
case TIMESTAMP:
case PG_TIMESTAMPTZ:
Expand Down
5 changes: 5 additions & 0 deletions v1/src/main/java/com/google/cloud/teleport/spanner/common/SizedType.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,8 @@ public static String typeString(Type type, Integer size, boolean outputAsDdlRepr
return "text";
case BYTES:
return "BYTES(" + (size == -1 ? "MAX" : Integer.toString(size)) + ")";
case TOKENLIST:
return "TOKENLIST";
case PG_BYTEA:
return "bytea";
case DATE:
Expand Down Expand Up @@ -213,6 +215,9 @@ public static SizedType parseSpannerType(String spannerType, Dialect dialect) {
if (spannerType.equals("JSON")) {
return t(Type.json(), null);
}
if (spannerType.equals("TOKENLIST")) {
return t(Type.tokenlist(), null);
}
if (spannerType.startsWith("ARRAY<")) {
// Substring "ARRAY<xxx> or ARRAY<xxx>(vector_length)"

Expand Down
9 changes: 9 additions & 0 deletions v1/src/main/java/com/google/cloud/teleport/spanner/common/Type.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ public final class Type implements Serializable {
private static final Type TYPE_STRING = new Type(Code.STRING, null, null);
private static final Type TYPE_JSON = new Type(Code.JSON, null, null);
private static final Type TYPE_BYTES = new Type(Code.BYTES, null, null);
private static final Type TYPE_TOKENLIST = new Type(Code.TOKENLIST, null, null);
private static final Type TYPE_TIMESTAMP = new Type(Code.TIMESTAMP, null, null);
private static final Type TYPE_DATE = new Type(Code.DATE, null, null);
private static final Type TYPE_ARRAY_BOOL = new Type(Code.ARRAY, TYPE_BOOL, null);
Expand Down Expand Up @@ -137,6 +138,13 @@ public static Type bytes() {
return TYPE_BYTES;
}

/**
* Returns the descriptor for the {@code TOKENLIST} type: a collection of unique token strings.
*/
public static Type tokenlist() {
return TYPE_TOKENLIST;
}

/**
* Returns the descriptor for the {@code TIMESTAMP} type: a nano precision timestamp in the range
* [0000-01-01 00:00:00, 9999-12-31 23:59:59.999999999 UTC].
Expand Down Expand Up @@ -326,6 +334,7 @@ public enum Code {
STRING("STRING", Dialect.GOOGLE_STANDARD_SQL),
JSON("JSON", Dialect.GOOGLE_STANDARD_SQL),
BYTES("BYTES", Dialect.GOOGLE_STANDARD_SQL),
TOKENLIST("TOKENLIST", Dialect.GOOGLE_STANDARD_SQL),
TIMESTAMP("TIMESTAMP", Dialect.GOOGLE_STANDARD_SQL),
DATE("DATE", Dialect.GOOGLE_STANDARD_SQL),
ARRAY("ARRAY", Dialect.GOOGLE_STANDARD_SQL),
Expand Down
14 changes: 14 additions & 0 deletions v1/src/main/java/com/google/cloud/teleport/spanner/ddl/Column.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,8 @@ public abstract class Column implements Serializable {

public abstract Dialect dialect();

public abstract boolean isHidden();

@Nullable
public abstract String defaultExpression();

Expand All @@ -64,6 +66,7 @@ public static Builder builder(Dialect dialect) {
.columnOptions(ImmutableList.of())
.notNull(false)
.isGenerated(false)
.isHidden(false)
.generationExpression("")
.isStored(false);
}
Expand Down Expand Up @@ -99,6 +102,11 @@ public void prettyPrint(Appendable appendable) throws IOException {
appendable.append(" STORED");
}
}
if (isHidden()) {
if (dialect() == Dialect.GOOGLE_STANDARD_SQL) {
appendable.append(" HIDDEN");
}
}
if (columnOptions() == null) {
return;
}
Expand Down Expand Up @@ -169,6 +177,8 @@ public Builder notNull() {
return notNull(true);
}

public abstract Builder isHidden(boolean hidden);

public abstract Builder isGenerated(boolean generated);

public abstract Builder generationExpression(String expression);
Expand Down Expand Up @@ -239,6 +249,10 @@ public Builder pgBytea() {
return type(Type.pgBytea()).max();
}

public Builder tokenlist() {
return type(Type.tokenlist());
}

public Builder timestamp() {
return type(Type.timestamp());
}
Expand Down
20 changes: 19 additions & 1 deletion v1/src/main/java/com/google/cloud/teleport/spanner/ddl/Index.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,9 @@ public abstract class Index implements Serializable {

abstract ImmutableList<IndexColumn> indexColumns();

@Nullable
abstract ImmutableList<String> options();

abstract boolean unique();

// restricted for gsql
Expand All @@ -51,6 +54,9 @@ public abstract class Index implements Serializable {
@Nullable
abstract String interleaveIn();

@Nullable
abstract String type();

public static Builder builder(Dialect dialect) {
return new AutoValue_Index.Builder().dialect(dialect).nullFiltered(false).unique(false);
}
Expand Down Expand Up @@ -111,7 +117,9 @@ private void prettyPrintPg(Appendable appendable) throws IOException {

private void prettyPrintGsql(Appendable appendable) throws IOException {
appendable.append("CREATE");
if (unique()) {
if (type() != null && type().equals("SEARCH")) {
appendable.append(" SEARCH");
} else if (unique()) {
appendable.append(" UNIQUE");
}
if (nullFiltered()) {
Expand Down Expand Up @@ -143,6 +151,12 @@ private void prettyPrintGsql(Appendable appendable) throws IOException {
if (interleaveIn() != null) {
appendable.append(", INTERLEAVE IN ").append(quoteIdentifier(interleaveIn(), dialect()));
}
if (options() != null) {
String optionsString = String.join(",", options());
if (!optionsString.isEmpty()) {
appendable.append(" OPTIONS (").append(optionsString).append(")");
}
}
}

abstract Builder autoToBuilder();
Expand Down Expand Up @@ -190,6 +204,8 @@ public IndexColumn.IndexColumnsBuilder<Builder> columns() {
return columnsBuilder();
}

abstract Builder options(ImmutableList<String> options);

public abstract Builder unique(boolean unique);

public Builder unique() {
Expand All @@ -206,6 +222,8 @@ public Builder nullFiltered() {

public abstract Builder interleaveIn(String interleaveIn);

public abstract Builder type(String type);

abstract Index autoBuild();

public Index build() {
Expand Down
Loading

0 comments on commit df97113

Please sign in to comment.