Skip to content

feat: add support for UUID and PG_UUID types across Spanner migration utilities. #3905

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
pratickchokhani wants to merge 1 commit into
base: main
Choose a base branch
from
Draft

feat: add support for UUID and PG_UUID types across Spanner migration utilities. #3905

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ...astream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/pg-dialect-spanner-schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT8, col VARCHAR, PRIMARY KEY (
CREATE TABLE IF NOT EXISTS t_tstzrange (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_tsvector (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col uuid, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT8, col BYTEA, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_varbit (id INT8, col BYTEA, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT8, col BOOL[], PRIMARY KEY (id));
Expand Down
2 changes: 1 addition & 1 deletion v2/datastream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/spanner-schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT64, col STRING(MAX)) PRIMARY
CREATE TABLE IF NOT EXISTS t_tstzrange (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_tsvector (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col UUID) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT64, col BYTES(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_varbit (id INT64, col BYTES(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT64, col ARRAY<BOOL>) PRIMARY KEY (id);
Expand Down
2 changes: 1 addition & 1 deletion ...urcedb-to-spanner/src/test/resources/DataTypesIT/postgresql-pg-dialect-spanner-schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT8, col VARCHAR, PRIMARY KEY (
CREATE TABLE IF NOT EXISTS t_tstzrange (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_tsvector (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col VARCHAR, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col uuid, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT8, col BYTEA, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_varbit (id INT8, col BYTEA, PRIMARY KEY (id));
CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT8, col BOOL[], PRIMARY KEY (id));
Expand Down
2 changes: 1 addition & 1 deletion v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/postgresql-spanner-schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT64, col STRING(MAX)) PRIMARY
CREATE TABLE IF NOT EXISTS t_tstzrange (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_tsvector (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col STRING(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col UUID) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT64, col BYTES(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_varbit (id INT64, col BYTES(MAX)) PRIMARY KEY (id);
CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT64, col ARRAY<BOOL>) PRIMARY KEY (id);
Expand Down
18 changes: 18 additions & 0 deletions v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/ddl/Column.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -164,6 +164,10 @@ private static String typeString(Type type, Integer size) {
return Type.Code.JSON.getName();
case PG_JSONB:
return Type.Code.PG_JSONB.getName();
case UUID:
return Type.Code.UUID.getName();
case PG_UUID:
return Type.Code.PG_UUID.getName();
case TOKENLIST:
return Type.Code.TOKENLIST.getName();
case ARRAY:
Expand Down Expand Up @@ -250,6 +254,10 @@ public Builder bytes() {
return type(Type.bytes());
}

public Builder uuid() {
return type(Type.uuid());
}

public Builder timestamp() {
return type(Type.timestamp());
}
Expand Down Expand Up @@ -306,6 +314,10 @@ public Builder pgJsonb() {
return type(Type.pgJsonb());
}

public Builder pgUuid() {
return type(Type.pgUuid());
}

public Builder max() {
return size(-1);
}
Expand Down Expand Up @@ -377,6 +389,9 @@ private static SizedType parseSpannerType(String spannerType, Dialect dialect) {
if (spannerType.equals(Type.Code.NUMERIC.getName())) {
return t(Type.numeric(), null);
}
if (spannerType.equals(Type.Code.UUID.getName())) {
return t(Type.uuid(), null);
}
if (spannerType.equals(Type.Code.JSON.getName())) {
return t(Type.json(), null);
}
Expand Down Expand Up @@ -432,6 +447,9 @@ private static SizedType parseSpannerType(String spannerType, Dialect dialect) {
if (spannerType.equals(Type.Code.PG_NUMERIC.getName())) {
return t(Type.pgNumeric(), null);
}
if (spannerType.equals(Type.Code.PG_UUID.getName())) {
return t(Type.pgUuid(), null);
}
if (spannerType.equals(Type.Code.PG_JSONB.getName())) {
return t(Type.pgJsonb(), null);
}
Expand Down
2 changes: 2 additions & 0 deletions ...m/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,8 @@ public static com.google.cloud.spanner.Key changeEventToPrimaryKey(
case STRING:
case PG_VARCHAR:
case PG_TEXT:
case UUID:
case PG_UUID:
pk.append(
ChangeEventTypeConvertor.toString(
changeEvent, keyColName, /* requiredField= */ true));
Expand Down
2 changes: 2 additions & 0 deletions .../com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,8 @@ public static Value toValue(
case STRING:
case PG_VARCHAR:
case PG_TEXT:
case UUID:
case PG_UUID:
return Value.string(toString(changeEvent, key, requiredField));
case NUMERIC:
case PG_NUMERIC:
Expand Down
2 changes: 2 additions & 0 deletions ...c/main/java/com/google/cloud/teleport/v2/spanner/migrations/spanner/SpannerReadUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ private static void bindGoogleSqlValue(
break;
case STRING:
case JSON:
case UUID:
stmtBuilder.bind(bindName).to((String) value);
break;
case NUMERIC:
Expand Down Expand Up @@ -153,6 +154,7 @@ private static void bindPgValue(
case PG_VARCHAR:
case PG_TEXT:
case PG_JSONB:
case PG_UUID:
stmtBuilder.bind(bindName).to((String) value);
break;
case PG_NUMERIC:
Expand Down
13 changes: 13 additions & 0 deletions v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/type/Type.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ public final class Type implements Serializable {
private static final Type TYPE_BYTES = new Type(Type.Code.BYTES, null, null);
private static final Type TYPE_TIMESTAMP = new Type(Type.Code.TIMESTAMP, null, null);
private static final Type TYPE_DATE = new Type(Type.Code.DATE, null, null);
private static final Type TYPE_UUID = new Type(Type.Code.UUID, null, null);
private static final Type TYPE_ARRAY_BOOL = new Type(Type.Code.ARRAY, TYPE_BOOL, null);
private static final Type TYPE_ARRAY_INT64 = new Type(Type.Code.ARRAY, TYPE_INT64, null);
private static final Type TYPE_ARRAY_FLOAT32 = new Type(Type.Code.ARRAY, TYPE_FLOAT32, null);
Expand All @@ -66,6 +67,7 @@ public final class Type implements Serializable {
private static final Type TYPE_PG_BYTEA = new Type(Type.Code.PG_BYTEA, null, null);
private static final Type TYPE_PG_TIMESTAMPTZ = new Type(Type.Code.PG_TIMESTAMPTZ, null, null);
private static final Type TYPE_PG_DATE = new Type(Type.Code.PG_DATE, null, null);
private static final Type TYPE_PG_UUID = new Type(Type.Code.PG_UUID, null, null);
private static final Type TYPE_PG_ARRAY_BOOL = new Type(Type.Code.PG_ARRAY, TYPE_PG_BOOL, null);
private static final Type TYPE_PG_ARRAY_INT8 = new Type(Type.Code.PG_ARRAY, TYPE_PG_INT8, null);
private static final Type TYPE_PG_ARRAY_FLOAT4 =
Expand Down Expand Up @@ -160,6 +162,11 @@ public static Type date() {
return TYPE_DATE;
}

/** Returns the descriptor for the {@code UUID} type. */
public static Type uuid() {
return TYPE_UUID;
}

public static Type pgBool() {
return TYPE_PG_BOOL;
}
Expand Down Expand Up @@ -204,6 +211,10 @@ public static Type pgDate() {
return TYPE_PG_DATE;
}

public static Type pgUuid() {
return TYPE_PG_UUID;
}

public static Type pgCommitTimestamp() {
return TYPE_PG_COMMIT_TIMESTAMP;
}
Expand Down Expand Up @@ -317,6 +328,7 @@ public enum Code {
BYTES("BYTES", Dialect.GOOGLE_STANDARD_SQL),
TIMESTAMP("TIMESTAMP", Dialect.GOOGLE_STANDARD_SQL),
DATE("DATE", Dialect.GOOGLE_STANDARD_SQL),
UUID("UUID", Dialect.GOOGLE_STANDARD_SQL),
ARRAY("ARRAY", Dialect.GOOGLE_STANDARD_SQL),
STRUCT("STRUCT", Dialect.GOOGLE_STANDARD_SQL),
PG_BOOL("boolean", Dialect.POSTGRESQL),
Expand All @@ -330,6 +342,7 @@ public enum Code {
PG_BYTEA("bytea", Dialect.POSTGRESQL),
PG_TIMESTAMPTZ("timestamp with time zone", Dialect.POSTGRESQL),
PG_DATE("date", Dialect.POSTGRESQL),
PG_UUID("uuid", Dialect.POSTGRESQL),
PG_ARRAY("array", Dialect.POSTGRESQL),
PG_COMMIT_TIMESTAMP("spanner.commit_timestamp", Dialect.POSTGRESQL);

Expand Down
12 changes: 12 additions & 0 deletions v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/ddl/ColumnTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ public void testTypeString_GoogleStandardSQL() {
assertEquals("JSON", Column.builder().name("col").type(Type.json()).autoBuild().typeString());
assertEquals(
"TOKENLIST", Column.builder().name("col").type(Type.tokenlist()).autoBuild().typeString());
assertEquals("UUID", Column.builder().name("col").type(Type.uuid()).autoBuild().typeString());
assertEquals(
"ARRAY<INT64>",
Column.builder().name("col").type(Type.array(Type.int64())).autoBuild().typeString());
Expand Down Expand Up @@ -151,6 +152,13 @@ public void testTypeString_PostgreSQL() {
.type(Type.pgCommitTimestamp())
.autoBuild()
.typeString());
assertEquals(
"uuid",
Column.builder(Dialect.POSTGRESQL)
.name("col")
.type(Type.pgUuid())
.autoBuild()
.typeString());
assertEquals(
"bigint[]",
Column.builder(Dialect.POSTGRESQL)
Expand Down Expand Up @@ -180,6 +188,7 @@ public void testParseSpannerType_GoogleStandardSQL() {
assertEquals(Type.json(), Column.builder().name("col").parseType("JSON").autoBuild().type());
assertEquals(
Type.tokenlist(), Column.builder().name("col").parseType("TOKENLIST").autoBuild().type());
assertEquals(Type.uuid(), Column.builder().name("col").parseType("UUID").autoBuild().type());
assertEquals(
Type.array(Type.int64()),
Column.builder().name("col").parseType("ARRAY<INT64>").autoBuild().type());
Expand Down Expand Up @@ -239,6 +248,9 @@ public void testParseSpannerType_PostgreSQL() {
.parseType("spanner.commit_timestamp")
.autoBuild()
.type());
assertEquals(
Type.pgUuid(),
Column.builder(Dialect.POSTGRESQL).name("col").parseType("uuid").autoBuild().type());
assertEquals(
Type.pgArray(Type.pgInt8()),
Column.builder(Dialect.POSTGRESQL).name("col").parseType("bigint[]").autoBuild().type());
Expand Down
73 changes: 73 additions & 0 deletions ...ogle/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -432,4 +432,77 @@ public void mutationFromEventWithGeneratedColumn() throws ChangeEventConvertorEx
Truth.assertThat(actual.get("last_name").getAsString()).isEqualTo("B");
Truth.assertThat(actual.containsKey("full_name")).isFalse();
}

@Test
public void canConvertChangeEventWithUuidToPrimaryKey() throws Exception {
Ddl ddl =
Ddl.builder()
.createTable("UsersUuid")
.column("uuid_field")
.type(Type.uuid())
.endColumn()
.column("pg_uuid_field")
.type(Type.pgUuid())
.endColumn()
.primaryKey()
.asc("uuid_field")
.asc("pg_uuid_field")
.end()
.endTable()
.build();

JSONObject changeEvent = new JSONObject();
changeEvent.put("uuid_field", "550e8400-e29b-41d4-a716-446655440000");
changeEvent.put("pg_uuid_field", "123e4567-e89b-12d3-a456-426614174000");
changeEvent.put(Constants.EVENT_TABLE_NAME_KEY, "UsersUuid");
JsonNode ce = parseChangeEvent(changeEvent.toString());

Key key =
ChangeEventSpannerConvertor.changeEventToPrimaryKey(
"UsersUuid", ddl, ce, /* convertNameToLowerCase= */ true);

Iterable<Object> keyParts = key.getParts();
ArrayList<Object> expectedKeyParts = new ArrayList<>();
expectedKeyParts.add("550e8400-e29b-41d4-a716-446655440000");
expectedKeyParts.add("123e4567-e89b-12d3-a456-426614174000");

assertThat(keyParts, is(expectedKeyParts));
}

@Test
public void mutationFromEventWithUuid() throws Exception {
Ddl ddl =
Ddl.builder()
.createTable("UsersUuid")
.column("uuid_field")
.type(Type.uuid())
.endColumn()
.column("pg_uuid_field")
.type(Type.pgUuid())
.endColumn()
.primaryKey()
.asc("uuid_field")
.end()
.endTable()
.build();

JSONObject changeEvent = new JSONObject();
changeEvent.put("uuid_field", "550e8400-e29b-41d4-a716-446655440000");
changeEvent.put("pg_uuid_field", "123e4567-e89b-12d3-a456-426614174000");
changeEvent.put(Constants.EVENT_TABLE_NAME_KEY, "UsersUuid");
JsonNode ce = parseChangeEvent(changeEvent.toString());

Mutation mutation =
ChangeEventSpannerConvertor.mutationFromEvent(
ddl.table("UsersUuid"),
ce,
List.of("uuid_field", "pg_uuid_field"),
Set.of("uuid_field"));

Map<String, Value> actual = mutation.asMap();
Truth.assertThat(actual.get("uuid_field").getAsString())
.isEqualTo("550e8400-e29b-41d4-a716-446655440000");
Truth.assertThat(actual.get("pg_uuid_field").getAsString())
.isEqualTo("123e4567-e89b-12d3-a456-426614174000");
}
}
18 changes: 18 additions & 0 deletions .../google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -772,4 +772,22 @@ public void canConvertNullStrings() throws Exception {
assertNull(ChangeEventTypeConvertor.toDate(ce, "date_field", true));
assertEquals("NULL", ChangeEventTypeConvertor.toString(ce, "string_field", true));
}

@Test
public void canConvertToValue() throws Exception {
JSONObject changeEvent = new JSONObject();
changeEvent.put("uuid_field", "550e8400-e29b-41d4-a716-446655440000");
changeEvent.put("pg_uuid_field", "123e4567-e89b-12d3-a456-426614174000");
JsonNode ce = getJsonNode(changeEvent.toString());

assertEquals(
com.google.cloud.spanner.Value.string("550e8400-e29b-41d4-a716-446655440000"),
ChangeEventTypeConvertor.toValue(
ce, com.google.cloud.teleport.v2.spanner.type.Type.uuid(), "uuid_field", true));

assertEquals(
com.google.cloud.spanner.Value.string("123e4567-e89b-12d3-a456-426614174000"),
ChangeEventTypeConvertor.toValue(
ce, com.google.cloud.teleport.v2.spanner.type.Type.pgUuid(), "pg_uuid_field", true));
}
}
11 changes: 9 additions & 2 deletions ...st/java/com/google/cloud/teleport/v2/spanner/migrations/spanner/SpannerReadUtilsTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,9 @@ public void testGenerateReadSQLAllTypesPostgres() {
.column("date_field")
.pgDate()
.endColumn()
.column("pg_uuid_field")
.pgUuid()
.endColumn()
.column("version")
.pgInt8()
.endColumn()
Expand All @@ -133,6 +136,7 @@ public void testGenerateReadSQLAllTypesPostgres() {
.asc("bytea_field")
.asc("timestamptz_field")
.asc("date_field")
.asc("pg_uuid_field")
.end()
.endTable()
.build();
Expand All @@ -146,6 +150,7 @@ public void testGenerateReadSQLAllTypesPostgres() {
ByteArray bytesValue = ByteArray.copyFrom("test_bytes");
Timestamp timestampValue = Timestamp.ofTimeMicroseconds(1234567);
Date dateValue = Date.fromYearMonthDay(2024, 1, 1);
String uuidValue = "123e4567-e89b-12d3-a456-426614174000";

Key primaryKey =
Key.of(
Expand All @@ -156,7 +161,8 @@ public void testGenerateReadSQLAllTypesPostgres() {
textValue,
bytesValue,
timestampValue,
dateValue);
dateValue,
uuidValue);

// 3. Generate the Statement
Statement stmt =
Expand All @@ -171,7 +177,7 @@ public void testGenerateReadSQLAllTypesPostgres() {
+ "\" WHERE "
+ "\"bool_field\"=$1 AND \"int_field\"=$2 AND \"float_field\"=$3 AND "
+ "\"numeric_field\"=$4 AND \"text_field\"=$5 AND \"bytea_field\"=$6 AND "
+ "\"timestamptz_field\"=$7 AND \"date_field\"=$8";
+ "\"timestamptz_field\"=$7 AND \"date_field\"=$8 AND \"pg_uuid_field\"=$9";

assertEquals(expectedSql, stmt.getSql());

Expand All @@ -186,6 +192,7 @@ public void testGenerateReadSQLAllTypesPostgres() {
assertEquals(Value.bytes(bytesValue), params.get("p6"));
assertEquals(Value.timestamp(timestampValue), params.get("p7"));
assertEquals(Value.date(dateValue), params.get("p8"));
assertEquals(Value.string(uuidValue), params.get("p9"));
}

@Test
Expand Down
6 changes: 6 additions & 0 deletions ...cedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFn.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -586,6 +586,8 @@ private com.google.cloud.spanner.Key generateKey(String tableName, JsonNode keys
case STRING:
case PG_VARCHAR:
case PG_TEXT:
case UUID:
case PG_UUID:
pk.append(
DataChangeRecordTypeConvertor.toString(
keysJson, keyColName, /* requiredField= */ true));
Expand Down Expand Up @@ -645,6 +647,8 @@ private Object getColumnValueFromJson(Column column, JsonNode valuesJson) throws
case STRING:
case PG_VARCHAR:
case PG_TEXT:
case UUID:
case PG_UUID:
return DataChangeRecordTypeConvertor.toString(valuesJson, colName, false);
case NUMERIC:
case PG_NUMERIC:
Expand Down Expand Up @@ -690,6 +694,8 @@ private Object getColumnValueFromRow(Column column, Value value) throws Exceptio
case STRING:
case PG_VARCHAR:
case PG_TEXT:
case UUID:
case PG_UUID:
return value.getString();
case NUMERIC:
case PG_NUMERIC:
Expand Down
Loading
Loading