From e6e9753341714a39168f37ba0fe8271b6336b79b Mon Sep 17 00:00:00 2001 From: Pratick Chokhani Date: Wed, 10 Jun 2026 16:27:59 +0530 Subject: [PATCH] feat: add support for UUID and PG_UUID types across Spanner migration utilities --- .../pg-dialect-spanner-schema.sql | 2 +- .../PostgreSQLDataTypesIT/spanner-schema.sql | 2 +- .../postgresql-pg-dialect-spanner-schema.sql | 2 +- .../DataTypesIT/postgresql-spanner-schema.sql | 2 +- .../cloud/teleport/v2/spanner/ddl/Column.java | 18 +++++ .../avro/GenericRecordTypeConvertor.java | 23 ++++-- .../ChangeEventSpannerConvertor.java | 2 + .../convertors/ChangeEventTypeConvertor.java | 2 + .../migrations/spanner/SpannerReadUtils.java | 2 + .../cloud/teleport/v2/spanner/type/Type.java | 13 ++++ .../teleport/v2/spanner/ddl/ColumnTest.java | 12 +++ .../ChangeEventSpannerConvertorTest.java | 73 +++++++++++++++++++ .../ChangeEventTypeConvertorTest.java | 18 +++++ .../spanner/SpannerReadUtilsTest.java | 11 ++- .../templates/transforms/AssignShardIdFn.java | 7 ++ .../SpannerToPostgreSQLDataTypesIT.java | 5 ++ .../SpannerToSourceDbDatatypeIT.java | 13 ++++ .../postgresql-schema.sql | 3 +- .../spanner-schema.sql | 1 + .../SpannerToSourceDbDatatypeIT/session.json | 4 +- .../spanner-schema.sql | 4 +- 21 files changed, 203 insertions(+), 16 deletions(-) diff --git a/v2/datastream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/pg-dialect-spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/pg-dialect-spanner-schema.sql index ae237f29c4..94958c4eb4 100644 --- a/v2/datastream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/pg-dialect-spanner-schema.sql +++ b/v2/datastream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/pg-dialect-spanner-schema.sql @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT8, col VARCHAR, PRIMARY KEY ( CREATE TABLE IF NOT EXISTS t_tstzrange (id INT8, col VARCHAR, PRIMARY KEY (id)); CREATE TABLE IF NOT EXISTS t_tsvector (id INT8, col VARCHAR, PRIMARY KEY (id)); CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT8, col VARCHAR, PRIMARY KEY (id)); -CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col VARCHAR, PRIMARY KEY (id)); +CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col uuid, PRIMARY KEY (id)); CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT8, col BYTEA, PRIMARY KEY (id)); CREATE TABLE IF NOT EXISTS t_varbit (id INT8, col BYTEA, PRIMARY KEY (id)); CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT8, col BOOL[], PRIMARY KEY (id)); diff --git a/v2/datastream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/spanner-schema.sql b/v2/datastream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/spanner-schema.sql index c86609cf98..86e5562d11 100644 --- a/v2/datastream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/spanner-schema.sql +++ b/v2/datastream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/spanner-schema.sql @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT64, col STRING(MAX)) PRIMARY CREATE TABLE IF NOT EXISTS t_tstzrange (id INT64, col STRING(MAX)) PRIMARY KEY (id); CREATE TABLE IF NOT EXISTS t_tsvector (id INT64, col STRING(MAX)) PRIMARY KEY (id); CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT64, col STRING(MAX)) PRIMARY KEY (id); -CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col STRING(MAX)) PRIMARY KEY (id); +CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col UUID) PRIMARY KEY (id); CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT64, col BYTES(MAX)) PRIMARY KEY (id); CREATE TABLE IF NOT EXISTS t_varbit (id INT64, col BYTES(MAX)) PRIMARY KEY (id); CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT64, col ARRAY) PRIMARY KEY (id); diff --git a/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/postgresql-pg-dialect-spanner-schema.sql b/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/postgresql-pg-dialect-spanner-schema.sql index ae237f29c4..94958c4eb4 100644 --- a/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/postgresql-pg-dialect-spanner-schema.sql +++ b/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/postgresql-pg-dialect-spanner-schema.sql @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT8, col VARCHAR, PRIMARY KEY ( CREATE TABLE IF NOT EXISTS t_tstzrange (id INT8, col VARCHAR, PRIMARY KEY (id)); CREATE TABLE IF NOT EXISTS t_tsvector (id INT8, col VARCHAR, PRIMARY KEY (id)); CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT8, col VARCHAR, PRIMARY KEY (id)); -CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col VARCHAR, PRIMARY KEY (id)); +CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col uuid, PRIMARY KEY (id)); CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT8, col BYTEA, PRIMARY KEY (id)); CREATE TABLE IF NOT EXISTS t_varbit (id INT8, col BYTEA, PRIMARY KEY (id)); CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT8, col BOOL[], PRIMARY KEY (id)); diff --git a/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/postgresql-spanner-schema.sql b/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/postgresql-spanner-schema.sql index c86609cf98..86e5562d11 100644 --- a/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/postgresql-spanner-schema.sql +++ b/v2/sourcedb-to-spanner/src/test/resources/DataTypesIT/postgresql-spanner-schema.sql @@ -130,7 +130,7 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT64, col STRING(MAX)) PRIMARY CREATE TABLE IF NOT EXISTS t_tstzrange (id INT64, col STRING(MAX)) PRIMARY KEY (id); CREATE TABLE IF NOT EXISTS t_tsvector (id INT64, col STRING(MAX)) PRIMARY KEY (id); CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT64, col STRING(MAX)) PRIMARY KEY (id); -CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col STRING(MAX)) PRIMARY KEY (id); +CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col UUID) PRIMARY KEY (id); CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT64, col BYTES(MAX)) PRIMARY KEY (id); CREATE TABLE IF NOT EXISTS t_varbit (id INT64, col BYTES(MAX)) PRIMARY KEY (id); CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT64, col ARRAY) PRIMARY KEY (id); diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/ddl/Column.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/ddl/Column.java index 676d8af331..00977eb9df 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/ddl/Column.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/ddl/Column.java @@ -164,6 +164,10 @@ private static String typeString(Type type, Integer size) { return Type.Code.JSON.getName(); case PG_JSONB: return Type.Code.PG_JSONB.getName(); + case UUID: + return Type.Code.UUID.getName(); + case PG_UUID: + return Type.Code.PG_UUID.getName(); case TOKENLIST: return Type.Code.TOKENLIST.getName(); case ARRAY: @@ -250,6 +254,10 @@ public Builder bytes() { return type(Type.bytes()); } + public Builder uuid() { + return type(Type.uuid()); + } + public Builder timestamp() { return type(Type.timestamp()); } @@ -306,6 +314,10 @@ public Builder pgJsonb() { return type(Type.pgJsonb()); } + public Builder pgUuid() { + return type(Type.pgUuid()); + } + public Builder max() { return size(-1); } @@ -377,6 +389,9 @@ private static SizedType parseSpannerType(String spannerType, Dialect dialect) { if (spannerType.equals(Type.Code.NUMERIC.getName())) { return t(Type.numeric(), null); } + if (spannerType.equals(Type.Code.UUID.getName())) { + return t(Type.uuid(), null); + } if (spannerType.equals(Type.Code.JSON.getName())) { return t(Type.json(), null); } @@ -432,6 +447,9 @@ private static SizedType parseSpannerType(String spannerType, Dialect dialect) { if (spannerType.equals(Type.Code.PG_NUMERIC.getName())) { return t(Type.pgNumeric(), null); } + if (spannerType.equals(Type.Code.PG_UUID.getName())) { + return t(Type.pgUuid(), null); + } if (spannerType.equals(Type.Code.PG_JSONB.getName())) { return t(Type.pgJsonb(), null); } diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertor.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertor.java index 35472a767d..fd89f2890a 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertor.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertor.java @@ -215,7 +215,9 @@ public Map transformChangeEvent(GenericRecord record, String srcT throw e; } catch (Exception e) { throw new RuntimeException( - String.format("Unable to convert spanner value for spanner col: %s", spannerColName), + String.format( + "Unable to convert spanner value for spanner col: %s. table: %s", + spannerColName, srcTableName), e); } } @@ -570,10 +572,21 @@ private Value getSpannerValueFromObject( throw new NullPointerException("schemaMapper returned null spanner dialect."); } if (AvroToValueMapper.convertorMap().get(dialect).containsKey(spannerType)) { - return AvroToValueMapper.convertorMap() - .get(dialect) - .get(spannerType) - .apply(value, fieldSchema); + try { + return AvroToValueMapper.convertorMap() + .get(dialect) + .get(spannerType) + .apply(value, fieldSchema); + } catch (Exception e) { + LOG.error( + "Exception while converting record value {} to Field Schema: {}, colName: {}, spannerType: {}", + value, + fieldSchema, + recordColName, + spannerType, + e); + throw e; + } } else { throw new IllegalArgumentException( "Found unsupported Spanner column type(" diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertor.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertor.java index 8637763ba5..f960414e38 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertor.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertor.java @@ -111,6 +111,8 @@ public static com.google.cloud.spanner.Key changeEventToPrimaryKey( case STRING: case PG_VARCHAR: case PG_TEXT: + case UUID: + case PG_UUID: pk.append( ChangeEventTypeConvertor.toString( changeEvent, keyColName, /* requiredField= */ true)); diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertor.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertor.java index 38784dc599..0b483a9e96 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertor.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertor.java @@ -71,6 +71,8 @@ public static Value toValue( case STRING: case PG_VARCHAR: case PG_TEXT: + case UUID: + case PG_UUID: return Value.string(toString(changeEvent, key, requiredField)); case NUMERIC: case PG_NUMERIC: diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/spanner/SpannerReadUtils.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/spanner/SpannerReadUtils.java index 278352eb26..abf91d3762 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/spanner/SpannerReadUtils.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/spanner/SpannerReadUtils.java @@ -111,6 +111,7 @@ private static void bindGoogleSqlValue( break; case STRING: case JSON: + case UUID: stmtBuilder.bind(bindName).to((String) value); break; case NUMERIC: @@ -153,6 +154,7 @@ private static void bindPgValue( case PG_VARCHAR: case PG_TEXT: case PG_JSONB: + case PG_UUID: stmtBuilder.bind(bindName).to((String) value); break; case PG_NUMERIC: diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/type/Type.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/type/Type.java index a24b2236f8..24434e9a31 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/type/Type.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/type/Type.java @@ -44,6 +44,7 @@ public final class Type implements Serializable { private static final Type TYPE_BYTES = new Type(Type.Code.BYTES, null, null); private static final Type TYPE_TIMESTAMP = new Type(Type.Code.TIMESTAMP, null, null); private static final Type TYPE_DATE = new Type(Type.Code.DATE, null, null); + private static final Type TYPE_UUID = new Type(Type.Code.UUID, null, null); private static final Type TYPE_ARRAY_BOOL = new Type(Type.Code.ARRAY, TYPE_BOOL, null); private static final Type TYPE_ARRAY_INT64 = new Type(Type.Code.ARRAY, TYPE_INT64, null); private static final Type TYPE_ARRAY_FLOAT32 = new Type(Type.Code.ARRAY, TYPE_FLOAT32, null); @@ -66,6 +67,7 @@ public final class Type implements Serializable { private static final Type TYPE_PG_BYTEA = new Type(Type.Code.PG_BYTEA, null, null); private static final Type TYPE_PG_TIMESTAMPTZ = new Type(Type.Code.PG_TIMESTAMPTZ, null, null); private static final Type TYPE_PG_DATE = new Type(Type.Code.PG_DATE, null, null); + private static final Type TYPE_PG_UUID = new Type(Type.Code.PG_UUID, null, null); private static final Type TYPE_PG_ARRAY_BOOL = new Type(Type.Code.PG_ARRAY, TYPE_PG_BOOL, null); private static final Type TYPE_PG_ARRAY_INT8 = new Type(Type.Code.PG_ARRAY, TYPE_PG_INT8, null); private static final Type TYPE_PG_ARRAY_FLOAT4 = @@ -160,6 +162,11 @@ public static Type date() { return TYPE_DATE; } + /** Returns the descriptor for the {@code UUID} type. */ + public static Type uuid() { + return TYPE_UUID; + } + public static Type pgBool() { return TYPE_PG_BOOL; } @@ -204,6 +211,10 @@ public static Type pgDate() { return TYPE_PG_DATE; } + public static Type pgUuid() { + return TYPE_PG_UUID; + } + public static Type pgCommitTimestamp() { return TYPE_PG_COMMIT_TIMESTAMP; } @@ -317,6 +328,7 @@ public enum Code { BYTES("BYTES", Dialect.GOOGLE_STANDARD_SQL), TIMESTAMP("TIMESTAMP", Dialect.GOOGLE_STANDARD_SQL), DATE("DATE", Dialect.GOOGLE_STANDARD_SQL), + UUID("UUID", Dialect.GOOGLE_STANDARD_SQL), ARRAY("ARRAY", Dialect.GOOGLE_STANDARD_SQL), STRUCT("STRUCT", Dialect.GOOGLE_STANDARD_SQL), PG_BOOL("boolean", Dialect.POSTGRESQL), @@ -330,6 +342,7 @@ public enum Code { PG_BYTEA("bytea", Dialect.POSTGRESQL), PG_TIMESTAMPTZ("timestamp with time zone", Dialect.POSTGRESQL), PG_DATE("date", Dialect.POSTGRESQL), + PG_UUID("uuid", Dialect.POSTGRESQL), PG_ARRAY("array", Dialect.POSTGRESQL), PG_COMMIT_TIMESTAMP("spanner.commit_timestamp", Dialect.POSTGRESQL); diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/ddl/ColumnTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/ddl/ColumnTest.java index 2ad0ddf62c..f819b37a2f 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/ddl/ColumnTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/ddl/ColumnTest.java @@ -51,6 +51,7 @@ public void testTypeString_GoogleStandardSQL() { assertEquals("JSON", Column.builder().name("col").type(Type.json()).autoBuild().typeString()); assertEquals( "TOKENLIST", Column.builder().name("col").type(Type.tokenlist()).autoBuild().typeString()); + assertEquals("UUID", Column.builder().name("col").type(Type.uuid()).autoBuild().typeString()); assertEquals( "ARRAY", Column.builder().name("col").type(Type.array(Type.int64())).autoBuild().typeString()); @@ -151,6 +152,13 @@ public void testTypeString_PostgreSQL() { .type(Type.pgCommitTimestamp()) .autoBuild() .typeString()); + assertEquals( + "uuid", + Column.builder(Dialect.POSTGRESQL) + .name("col") + .type(Type.pgUuid()) + .autoBuild() + .typeString()); assertEquals( "bigint[]", Column.builder(Dialect.POSTGRESQL) @@ -180,6 +188,7 @@ public void testParseSpannerType_GoogleStandardSQL() { assertEquals(Type.json(), Column.builder().name("col").parseType("JSON").autoBuild().type()); assertEquals( Type.tokenlist(), Column.builder().name("col").parseType("TOKENLIST").autoBuild().type()); + assertEquals(Type.uuid(), Column.builder().name("col").parseType("UUID").autoBuild().type()); assertEquals( Type.array(Type.int64()), Column.builder().name("col").parseType("ARRAY").autoBuild().type()); @@ -239,6 +248,9 @@ public void testParseSpannerType_PostgreSQL() { .parseType("spanner.commit_timestamp") .autoBuild() .type()); + assertEquals( + Type.pgUuid(), + Column.builder(Dialect.POSTGRESQL).name("col").parseType("uuid").autoBuild().type()); assertEquals( Type.pgArray(Type.pgInt8()), Column.builder(Dialect.POSTGRESQL).name("col").parseType("bigint[]").autoBuild().type()); diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertorTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertorTest.java index 850dbd2206..43c576b258 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertorTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertorTest.java @@ -432,4 +432,77 @@ public void mutationFromEventWithGeneratedColumn() throws ChangeEventConvertorEx Truth.assertThat(actual.get("last_name").getAsString()).isEqualTo("B"); Truth.assertThat(actual.containsKey("full_name")).isFalse(); } + + @Test + public void canConvertChangeEventWithUuidToPrimaryKey() throws Exception { + Ddl ddl = + Ddl.builder() + .createTable("UsersUuid") + .column("uuid_field") + .type(Type.uuid()) + .endColumn() + .column("pg_uuid_field") + .type(Type.pgUuid()) + .endColumn() + .primaryKey() + .asc("uuid_field") + .asc("pg_uuid_field") + .end() + .endTable() + .build(); + + JSONObject changeEvent = new JSONObject(); + changeEvent.put("uuid_field", "550e8400-e29b-41d4-a716-446655440000"); + changeEvent.put("pg_uuid_field", "123e4567-e89b-12d3-a456-426614174000"); + changeEvent.put(Constants.EVENT_TABLE_NAME_KEY, "UsersUuid"); + JsonNode ce = parseChangeEvent(changeEvent.toString()); + + Key key = + ChangeEventSpannerConvertor.changeEventToPrimaryKey( + "UsersUuid", ddl, ce, /* convertNameToLowerCase= */ true); + + Iterable keyParts = key.getParts(); + ArrayList expectedKeyParts = new ArrayList<>(); + expectedKeyParts.add("550e8400-e29b-41d4-a716-446655440000"); + expectedKeyParts.add("123e4567-e89b-12d3-a456-426614174000"); + + assertThat(keyParts, is(expectedKeyParts)); + } + + @Test + public void mutationFromEventWithUuid() throws Exception { + Ddl ddl = + Ddl.builder() + .createTable("UsersUuid") + .column("uuid_field") + .type(Type.uuid()) + .endColumn() + .column("pg_uuid_field") + .type(Type.pgUuid()) + .endColumn() + .primaryKey() + .asc("uuid_field") + .end() + .endTable() + .build(); + + JSONObject changeEvent = new JSONObject(); + changeEvent.put("uuid_field", "550e8400-e29b-41d4-a716-446655440000"); + changeEvent.put("pg_uuid_field", "123e4567-e89b-12d3-a456-426614174000"); + changeEvent.put(Constants.EVENT_TABLE_NAME_KEY, "UsersUuid"); + JsonNode ce = parseChangeEvent(changeEvent.toString()); + + Mutation mutation = + ChangeEventSpannerConvertor.mutationFromEvent( + ddl.table("UsersUuid"), + ce, + List.of("uuid_field", "pg_uuid_field"), + Set.of("uuid_field")); + + Map actual = mutation.asMap(); + Truth.assertThat(actual.get("uuid_field").getAsString()) + .isEqualTo("550e8400-e29b-41d4-a716-446655440000"); + Truth.assertThat(actual.get("pg_uuid_field").getAsString()) + .isEqualTo("123e4567-e89b-12d3-a456-426614174000"); + } } diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertorTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertorTest.java index 91dd3b7d5a..a4488aaa14 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertorTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertorTest.java @@ -772,4 +772,22 @@ public void canConvertNullStrings() throws Exception { assertNull(ChangeEventTypeConvertor.toDate(ce, "date_field", true)); assertEquals("NULL", ChangeEventTypeConvertor.toString(ce, "string_field", true)); } + + @Test + public void canConvertToValue() throws Exception { + JSONObject changeEvent = new JSONObject(); + changeEvent.put("uuid_field", "550e8400-e29b-41d4-a716-446655440000"); + changeEvent.put("pg_uuid_field", "123e4567-e89b-12d3-a456-426614174000"); + JsonNode ce = getJsonNode(changeEvent.toString()); + + assertEquals( + com.google.cloud.spanner.Value.string("550e8400-e29b-41d4-a716-446655440000"), + ChangeEventTypeConvertor.toValue( + ce, com.google.cloud.teleport.v2.spanner.type.Type.uuid(), "uuid_field", true)); + + assertEquals( + com.google.cloud.spanner.Value.string("123e4567-e89b-12d3-a456-426614174000"), + ChangeEventTypeConvertor.toValue( + ce, com.google.cloud.teleport.v2.spanner.type.Type.pgUuid(), "pg_uuid_field", true)); + } } diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/spanner/SpannerReadUtilsTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/spanner/SpannerReadUtilsTest.java index 59c601db74..b335af250a 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/spanner/SpannerReadUtilsTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/spanner/SpannerReadUtilsTest.java @@ -121,6 +121,9 @@ public void testGenerateReadSQLAllTypesPostgres() { .column("date_field") .pgDate() .endColumn() + .column("pg_uuid_field") + .pgUuid() + .endColumn() .column("version") .pgInt8() .endColumn() @@ -133,6 +136,7 @@ public void testGenerateReadSQLAllTypesPostgres() { .asc("bytea_field") .asc("timestamptz_field") .asc("date_field") + .asc("pg_uuid_field") .end() .endTable() .build(); @@ -146,6 +150,7 @@ public void testGenerateReadSQLAllTypesPostgres() { ByteArray bytesValue = ByteArray.copyFrom("test_bytes"); Timestamp timestampValue = Timestamp.ofTimeMicroseconds(1234567); Date dateValue = Date.fromYearMonthDay(2024, 1, 1); + String uuidValue = "123e4567-e89b-12d3-a456-426614174000"; Key primaryKey = Key.of( @@ -156,7 +161,8 @@ public void testGenerateReadSQLAllTypesPostgres() { textValue, bytesValue, timestampValue, - dateValue); + dateValue, + uuidValue); // 3. Generate the Statement Statement stmt = @@ -171,7 +177,7 @@ public void testGenerateReadSQLAllTypesPostgres() { + "\" WHERE " + "\"bool_field\"=$1 AND \"int_field\"=$2 AND \"float_field\"=$3 AND " + "\"numeric_field\"=$4 AND \"text_field\"=$5 AND \"bytea_field\"=$6 AND " - + "\"timestamptz_field\"=$7 AND \"date_field\"=$8"; + + "\"timestamptz_field\"=$7 AND \"date_field\"=$8 AND \"pg_uuid_field\"=$9"; assertEquals(expectedSql, stmt.getSql()); @@ -186,6 +192,7 @@ public void testGenerateReadSQLAllTypesPostgres() { assertEquals(Value.bytes(bytesValue), params.get("p6")); assertEquals(Value.timestamp(timestampValue), params.get("p7")); assertEquals(Value.date(dateValue), params.get("p8")); + assertEquals(Value.string(uuidValue), params.get("p9")); } @Test diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFn.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFn.java index bc9f916764..739eb5008b 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFn.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/transforms/AssignShardIdFn.java @@ -586,6 +586,8 @@ private com.google.cloud.spanner.Key generateKey(String tableName, JsonNode keys case STRING: case PG_VARCHAR: case PG_TEXT: + case UUID: + case PG_UUID: pk.append( DataChangeRecordTypeConvertor.toString( keysJson, keyColName, /* requiredField= */ true)); @@ -645,6 +647,8 @@ private Object getColumnValueFromJson(Column column, JsonNode valuesJson) throws case STRING: case PG_VARCHAR: case PG_TEXT: + case UUID: + case PG_UUID: return DataChangeRecordTypeConvertor.toString(valuesJson, colName, false); case NUMERIC: case PG_NUMERIC: @@ -691,6 +695,9 @@ private Object getColumnValueFromRow(Column column, Value value) throws Exceptio case PG_VARCHAR: case PG_TEXT: return value.getString(); + case UUID: + case PG_UUID: + return value.getUuid(); case NUMERIC: case PG_NUMERIC: return value.getNumeric(); diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToPostgreSQLDataTypesIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToPostgreSQLDataTypesIT.java index 3aa453e76f..923e029073 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToPostgreSQLDataTypesIT.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToPostgreSQLDataTypesIT.java @@ -248,6 +248,9 @@ private Map> getSpannerTableData() { Value.json("{\"key\": \"value1\"}"), Value.json("{\"key\": \"value2\"}"), Value.json(null))); + spannerRowData.put( + "uuid_table", + List.of(Value.string("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11"), Value.string(null))); return spannerRowData; } @@ -270,6 +273,8 @@ private Map>> getExpectedData() { expectedData.put( "json_table", createRows("json_table", "{\"key\": \"value1\"}", "{\"key\": \"value2\"}", null)); + expectedData.put( + "uuid_table", createRows("uuid_table", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", null)); return expectedData; } diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbDatatypeIT.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbDatatypeIT.java index 838d40ef16..13209a1aef 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbDatatypeIT.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/SpannerToSourceDbDatatypeIT.java @@ -25,6 +25,7 @@ import com.google.cloud.Timestamp; import com.google.cloud.spanner.Key; import com.google.cloud.spanner.Mutation; +import com.google.cloud.spanner.Struct; import com.google.cloud.spanner.Value; import com.google.cloud.teleport.metadata.SkipDirectRunnerTest; import com.google.cloud.teleport.metadata.TemplateIntegrationTest; @@ -355,6 +356,13 @@ private void writeRowsInSpanner() { } private void deleteRowsInSpanner() { + List t2 = spannerResourceManager.readTableRecords(TABLE2, "uuid_column"); + List t3 = spannerResourceManager.readTableRecords(TABLE3, "uuid_column"); + + LOG.error("T2 uuid: {}", t2); + LOG.error("T3 uuid: {}", t3); + LOG.error("T2 uuid: {}", t2.get(0).getValue("uuid_column").getUuid().toString()); + LOG.error("T3 uuid: {}", t3.get(0).getValue("uuid_column").getUuid().toString()); // Write a single record to Spanner Mutation m = Mutation.delete(TABLE1, Key.newBuilder().append("value1").build()); spannerResourceManager.write(m); @@ -366,6 +374,11 @@ private void deleteRowsInSpanner() { Mutation.delete( TABLE3, Key.newBuilder().append("a5f27b71-6ffa-444e-abdb-9ce4af318865").build()); spannerResourceManager.write(m3); + t2 = spannerResourceManager.readTableRecords(TABLE2, "uuid_column"); + t3 = spannerResourceManager.readTableRecords(TABLE3, "uuid_column"); + + LOG.error("T2 after delete uuid: {}", t2); + LOG.error("T3 after delete uuid: {}", t3); } private List assertionErrors = new ArrayList<>(); diff --git a/v2/spanner-to-sourcedb/src/test/resources/SpannerToPostgreSQLDataTypesIT/postgresql-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/SpannerToPostgreSQLDataTypesIT/postgresql-schema.sql index 668adb7cc0..c7b6aa1ff1 100644 --- a/v2/spanner-to-sourcedb/src/test/resources/SpannerToPostgreSQLDataTypesIT/postgresql-schema.sql +++ b/v2/spanner-to-sourcedb/src/test/resources/SpannerToPostgreSQLDataTypesIT/postgresql-schema.sql @@ -6,4 +6,5 @@ CREATE TABLE bytes_table (id BIGINT PRIMARY KEY, bytes_col bytea); CREATE TABLE date_table (id BIGINT PRIMARY KEY, date_col date); CREATE TABLE numeric_table (id BIGINT PRIMARY KEY, numeric_col numeric); CREATE TABLE timestamp_table (id BIGINT PRIMARY KEY, timestamp_col timestamp with time zone); -CREATE TABLE json_table (id BIGINT PRIMARY KEY, json_col jsonb); \ No newline at end of file +CREATE TABLE json_table (id BIGINT PRIMARY KEY, json_col jsonb); +CREATE TABLE uuid_table (id BIGINT PRIMARY KEY, uuid_col uuid); \ No newline at end of file diff --git a/v2/spanner-to-sourcedb/src/test/resources/SpannerToPostgreSQLDataTypesIT/spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/SpannerToPostgreSQLDataTypesIT/spanner-schema.sql index 09ece7f68a..3479dbb95c 100644 --- a/v2/spanner-to-sourcedb/src/test/resources/SpannerToPostgreSQLDataTypesIT/spanner-schema.sql +++ b/v2/spanner-to-sourcedb/src/test/resources/SpannerToPostgreSQLDataTypesIT/spanner-schema.sql @@ -7,6 +7,7 @@ CREATE TABLE date_table (id INT64, date_col DATE) PRIMARY KEY(id); CREATE TABLE numeric_table (id INT64, numeric_col NUMERIC) PRIMARY KEY(id); CREATE TABLE timestamp_table (id INT64, timestamp_col TIMESTAMP) PRIMARY KEY(id); CREATE TABLE json_table (id INT64, json_col JSON) PRIMARY KEY(id); +CREATE TABLE uuid_table (id INT64, uuid_col UUID) PRIMARY KEY(id); CREATE CHANGE STREAM allstream FOR ALL OPTIONS ( diff --git a/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbDatatypeIT/session.json b/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbDatatypeIT/session.json index e813a2dc2d..024cc9f88d 100644 --- a/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbDatatypeIT/session.json +++ b/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbDatatypeIT/session.json @@ -507,7 +507,7 @@ "c1": { "Name": "uuid_column", "T": { - "Name": "STRING", + "Name": "UUID", "Len": 128, "IsArray": false }, @@ -718,7 +718,7 @@ "c1": { "Name": "uuid_column", "T": { - "Name": "STRING", + "Name": "UUID", "Len": 128, "IsArray": false }, diff --git a/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbDatatypeIT/spanner-schema.sql b/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbDatatypeIT/spanner-schema.sql index 97408d5e23..4a051d2829 100644 --- a/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbDatatypeIT/spanner-schema.sql +++ b/v2/spanner-to-sourcedb/src/test/resources/SpannerToSourceDbDatatypeIT/spanner-schema.sql @@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS AllDatatypeColumns ( CREATE TABLE IF NOT EXISTS AllDatatypePkColumns1 ( - uuid_column STRING(128) NOT NULL, + uuid_column UUID NOT NULL, varchar_column STRING(20), tinyint_column INT64, text_column STRING(MAX), @@ -61,7 +61,7 @@ CREATE TABLE IF NOT EXISTS AllDatatypePkColumns1 ( ) PRIMARY KEY(uuid_column); CREATE TABLE IF NOT EXISTS AllDatatypePkColumns2 ( - uuid_column STRING(128) NOT NULL, + uuid_column UUID NOT NULL, char_column STRING(10), tinyblob_column BYTES(MAX), tinytext_column STRING(MAX),