diff --git a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/writer/DeadLetterQueue.java b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/writer/DeadLetterQueue.java index a30532c229..d8e04c629c 100644 --- a/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/writer/DeadLetterQueue.java +++ b/v2/sourcedb-to-spanner/src/main/java/com/google/cloud/teleport/v2/writer/DeadLetterQueue.java @@ -283,6 +283,9 @@ protected FailsafeElement mutationToDlqElement(Mutation m) { case BYTES: val = Hex.encodeHexString(value.getBytes().toByteArray()); break; + case UUID: + val = value.getUuid().toString(); + break; case INT64: val = value.getInt64(); break; @@ -301,6 +304,11 @@ protected FailsafeElement mutationToDlqElement(Mutation m) { value.getBytesArray().stream() .map(v -> v == null ? null : Hex.encodeHexString(v.toByteArray())) .collect(Collectors.toList()); + } else if (value.getType().getArrayElementType().getCode() == Type.Code.UUID) { + val = + value.getUuidArray().stream() + .map(v -> v == null ? null : v.toString()) + .collect(Collectors.toList()); } else { val = value.toString(); } diff --git a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/writer/DeadLetterQueueTest.java b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/writer/DeadLetterQueueTest.java index 68631b188b..fc038f57b6 100644 --- a/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/writer/DeadLetterQueueTest.java +++ b/v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/writer/DeadLetterQueueTest.java @@ -799,6 +799,102 @@ public void testMutationToDlqElement_WithExplicitShardIdInMutationMap() { .contains("\"_metadata_shard_id_column_name\":\"migration_shard_id\""); } + @Test + public void testMutationToDlqElement_UuidType() { + Ddl ddlWithUuid = + Ddl.builder() + .createTable("uuid_table") + .column("id") + .type(com.google.cloud.teleport.v2.spanner.type.Type.uuid()) + .endColumn() + .column("uuid_array") + .type( + com.google.cloud.teleport.v2.spanner.type.Type.array( + com.google.cloud.teleport.v2.spanner.type.Type.uuid())) + .endColumn() + .primaryKey() + .asc("id") + .end() + .endTable() + .build(); + + DeadLetterQueue dlq = + DeadLetterQueue.create( + "LOG", ddlWithUuid, SQLDialect.MYSQL, getIdentityMapper(ddlWithUuid)); + + java.util.UUID idUuid = java.util.UUID.randomUUID(); + java.util.UUID elementUuid1 = java.util.UUID.randomUUID(); + java.util.UUID elementUuid2 = java.util.UUID.randomUUID(); + + Mutation m = + Mutation.newInsertBuilder("uuid_table") + .set("id") + .to(Value.uuid(idUuid)) + .set("uuid_array") + .to(Value.uuidArray(java.util.Arrays.asList(elementUuid1, elementUuid2))) + .build(); + + FailsafeElement dlqElement = dlq.mutationToDlqElement(m); + String payload = dlqElement.getPayload(); + + assertThat(payload).contains("\"id\":\"" + idUuid.toString() + "\""); + assertThat(payload) + .contains( + "\"uuid_array\":[\"" + + elementUuid1.toString() + + "\",\"" + + elementUuid2.toString() + + "\"]"); + } + + @Test + public void testMutationToDlqElement_PgUuidType() { + Ddl ddlWithPgUuid = + Ddl.builder(com.google.cloud.spanner.Dialect.POSTGRESQL) + .createTable("new_cart") + .column("new_user_id") + .type(com.google.cloud.teleport.v2.spanner.type.Type.pgUuid()) + .endColumn() + .column("uuid_array") + .type( + com.google.cloud.teleport.v2.spanner.type.Type.pgArray( + com.google.cloud.teleport.v2.spanner.type.Type.pgUuid())) + .endColumn() + .primaryKey() + .asc("new_user_id") + .end() + .endTable() + .build(); + + DeadLetterQueue dlq = + DeadLetterQueue.create( + "LOG", ddlWithPgUuid, SQLDialect.MYSQL, getIdentityMapper(ddlWithPgUuid)); + + java.util.UUID idUuid = java.util.UUID.randomUUID(); + java.util.UUID elementUuid1 = java.util.UUID.randomUUID(); + java.util.UUID elementUuid2 = java.util.UUID.randomUUID(); + + Mutation m = + Mutation.newInsertBuilder("new_cart") + .set("new_user_id") + .to(Value.uuid(idUuid)) + .set("uuid_array") + .to(Value.uuidArray(java.util.Arrays.asList(elementUuid1, elementUuid2))) + .build(); + + FailsafeElement dlqElement = dlq.mutationToDlqElement(m); + String payload = dlqElement.getPayload(); + + assertThat(payload).contains("\"new_user_id\":\"" + idUuid.toString() + "\""); + assertThat(payload) + .contains( + "\"uuid_array\":[\"" + + elementUuid1.toString() + + "\",\"" + + elementUuid2.toString() + + "\"]"); + } + private static ISchemaMapper getIdentityMapper(Ddl spannerDdl) { return new IdentityMapper(spannerDdl); } diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/ddl/Column.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/ddl/Column.java index 676d8af331..376448880a 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/ddl/Column.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/ddl/Column.java @@ -164,6 +164,10 @@ private static String typeString(Type type, Integer size) { return Type.Code.JSON.getName(); case PG_JSONB: return Type.Code.PG_JSONB.getName(); + case UUID: + return Type.Code.UUID.getName(); + case PG_UUID: + return Type.Code.PG_UUID.getName(); case TOKENLIST: return Type.Code.TOKENLIST.getName(); case ARRAY: @@ -258,6 +262,10 @@ public Builder date() { return type(Type.date()); } + public Builder uuid() { + return type(Type.uuid()); + } + public Builder numeric() { return type(Type.numeric()); } @@ -298,6 +306,10 @@ public Builder pgDate() { return type(Type.pgDate()); } + public Builder pgUuid() { + return type(Type.pgUuid()); + } + public Builder pgNumeric() { return type(Type.pgNumeric()); } @@ -374,6 +386,9 @@ private static SizedType parseSpannerType(String spannerType, Dialect dialect) { if (spannerType.equals(Type.Code.DATE.getName())) { return t(Type.date(), null); } + if (spannerType.equals(Type.Code.UUID.getName())) { + return t(Type.uuid(), null); + } if (spannerType.equals(Type.Code.NUMERIC.getName())) { return t(Type.numeric(), null); } @@ -438,6 +453,9 @@ private static SizedType parseSpannerType(String spannerType, Dialect dialect) { if (spannerType.equals(Type.Code.PG_DATE.getName())) { return t(Type.pgDate(), null); } + if (spannerType.equals(Type.Code.PG_UUID.getName())) { + return t(Type.pgUuid(), null); + } if (spannerType.equals(Type.Code.PG_COMMIT_TIMESTAMP.getName())) { return t(Type.pgCommitTimestamp(), null); } diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java index 1ac8bb41cc..849d97404f 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapper.java @@ -200,6 +200,16 @@ static Map getGsqlMap() { avroArrayFieldToSpannerArray( recordValue, fieldSchema, AvroToValueMapper::avroFieldToDate))); + gsqlFunctions.put( + Type.uuid(), + (recordValue, fieldSchema) -> Value.uuid(avroFieldToUuid(recordValue, fieldSchema))); + gsqlFunctions.put( + Type.array(Type.uuid()), + (recordValue, fieldSchema) -> + Value.uuidArray( + avroArrayFieldToSpannerArray( + recordValue, fieldSchema, AvroToValueMapper::avroFieldToUuid))); + return gsqlFunctions; } @@ -242,6 +252,16 @@ static Map getPgMap() { pgFunctions.put( Type.pgDate(), (recordValue, fieldSchema) -> Value.date(avroFieldToDate(recordValue, fieldSchema))); + pgFunctions.put( + Type.pgUuid(), + (recordValue, fieldSchema) -> Value.uuid(avroFieldToUuid(recordValue, fieldSchema))); + pgFunctions.put( + Type.pgArray(Type.pgUuid()), + (recordValue, fieldSchema) -> + Value.uuidArray( + avroArrayFieldToSpannerArray( + recordValue, fieldSchema, AvroToValueMapper::avroFieldToUuid))); + return pgFunctions; } @@ -249,6 +269,46 @@ static Map getPgMap() { * This method tries to map different kinds of source types to a boolean. This could be longs, * string as well as booleans. */ + static java.util.UUID avroFieldToUuid(Object recordValue, Schema fieldSchema) { + try { + if (recordValue == null) { + return null; + } + if (recordValue instanceof java.util.UUID) { + return (java.util.UUID) recordValue; + } + if (recordValue instanceof ByteBuffer) { + byte[] bytes = ((ByteBuffer) recordValue).array(); + if (bytes.length == 16) { + ByteBuffer bb = ByteBuffer.wrap(bytes); + return new java.util.UUID(bb.getLong(), bb.getLong()); + } else { + throw new IllegalArgumentException( + "ByteBuffer array length must be exactly 16, but was: " + bytes.length); + } + } + if (recordValue instanceof byte[]) { + byte[] bytes = (byte[]) recordValue; + if (bytes.length == 16) { + ByteBuffer bb = ByteBuffer.wrap(bytes); + return new java.util.UUID(bb.getLong(), bb.getLong()); + } else { + throw new IllegalArgumentException( + "Byte array length must be exactly 16, but was: " + bytes.length); + } + } + return java.util.UUID.fromString(recordValue.toString()); + } catch (Exception e) { + throw new AvroTypeConvertorException( + "Unable to convert " + + fieldSchema.getType() + + " to UUID, with value: " + + recordValue + + ", Exception: " + + e.getMessage()); + } + } + static Boolean avroFieldToBoolean(Object recordValue, Schema fieldSchema) { if (recordValue == null) { return null; diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertor.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertor.java index 35472a767d..2f9d009863 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertor.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertor.java @@ -154,7 +154,7 @@ public Map transformChangeEvent(GenericRecord record, String srcT } // If current column is migration shard id, populate value. if (spannerColName.equals(shardIdCol)) { - result = populateShardId(result, shardIdCol); + result = populateShardId(result, shardIdCol, spannerTableName); continue; } @@ -167,7 +167,14 @@ public Map transformChangeEvent(GenericRecord record, String srcT // the schemaMapper returns null. if (spannerColName.equals( schemaMapper.getSyntheticPrimaryKeyColName(namespace, spannerTableName))) { - result.put(spannerColName, Value.string(getUUID())); + Type spannerColumnType = + schemaMapper.getSpannerColumnType(namespace, spannerTableName, spannerColName); + if (spannerColumnType.getCode() == Type.Code.UUID + || spannerColumnType.getCode() == Type.Code.PG_UUID) { + result.put(spannerColName, Value.uuid(java.util.UUID.randomUUID())); + } else { + result.put(spannerColName, Value.string(getUUID())); + } continue; } @@ -432,11 +439,17 @@ private MigrationTransformationResponse getCustomTransformationResponse( return migrationTransformationResponse; } - private Map populateShardId(Map result, String shardIdCol) { + private Map populateShardId( + Map result, String shardIdCol, String tableName) { if (shardId == null || shardId.isBlank()) { return result; } - result.put(shardIdCol, Value.string(shardId)); + Type spannerType = schemaMapper.getSpannerColumnType(namespace, tableName, shardIdCol); + if (spannerType.getCode() == Type.Code.UUID || spannerType.getCode() == Type.Code.PG_UUID) { + result.put(shardIdCol, Value.uuid(java.util.UUID.fromString(shardId))); + } else { + result.put(shardIdCol, Value.string(shardId)); + } return result; } diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertor.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertor.java index 8637763ba5..a4bcbd3ba7 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertor.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertor.java @@ -115,6 +115,12 @@ public static com.google.cloud.spanner.Key changeEventToPrimaryKey( ChangeEventTypeConvertor.toString( changeEvent, keyColName, /* requiredField= */ true)); break; + case UUID: + case PG_UUID: + pk.appendObject( + ChangeEventTypeConvertor.toUuid( + changeEvent, keyColName, /* requiredField= */ true)); + break; case NUMERIC: case PG_NUMERIC: pk.append( diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertor.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertor.java index 38784dc599..a53fa93e83 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertor.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertor.java @@ -91,6 +91,9 @@ public static Value toValue( case ARRAY: // TODO(b/422928714): Add support for Array types. return null; + case UUID: + case PG_UUID: + return Value.uuid(toUuid(changeEvent, key, requiredField)); // TODO(b/179070999) - Add support for other data types. default: throw new IllegalArgumentException( @@ -98,6 +101,22 @@ public static Value toValue( } } + public static java.util.UUID toUuid(JsonNode changeEvent, String key, boolean requiredField) + throws ChangeEventConvertorException { + if (!containsValue(changeEvent, key, requiredField)) { + return null; + } + try { + String uuidStr = changeEvent.get(key).asText(); + if (uuidStr.equalsIgnoreCase("NULL") || uuidStr.isEmpty()) { + return null; + } + return java.util.UUID.fromString(uuidStr); + } catch (Exception e) { + throw new ChangeEventConvertorException("Unable to convert field " + key + " to UUID ", e); + } + } + public static Boolean toBoolean(JsonNode changeEvent, String key, boolean requiredField) throws ChangeEventConvertorException { diff --git a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/type/Type.java b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/type/Type.java index a24b2236f8..625900af5d 100644 --- a/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/type/Type.java +++ b/v2/spanner-common/src/main/java/com/google/cloud/teleport/v2/spanner/type/Type.java @@ -44,6 +44,7 @@ public final class Type implements Serializable { private static final Type TYPE_BYTES = new Type(Type.Code.BYTES, null, null); private static final Type TYPE_TIMESTAMP = new Type(Type.Code.TIMESTAMP, null, null); private static final Type TYPE_DATE = new Type(Type.Code.DATE, null, null); + private static final Type TYPE_UUID = new Type(Type.Code.UUID, null, null); private static final Type TYPE_ARRAY_BOOL = new Type(Type.Code.ARRAY, TYPE_BOOL, null); private static final Type TYPE_ARRAY_INT64 = new Type(Type.Code.ARRAY, TYPE_INT64, null); private static final Type TYPE_ARRAY_FLOAT32 = new Type(Type.Code.ARRAY, TYPE_FLOAT32, null); @@ -66,6 +67,8 @@ public final class Type implements Serializable { private static final Type TYPE_PG_BYTEA = new Type(Type.Code.PG_BYTEA, null, null); private static final Type TYPE_PG_TIMESTAMPTZ = new Type(Type.Code.PG_TIMESTAMPTZ, null, null); private static final Type TYPE_PG_DATE = new Type(Type.Code.PG_DATE, null, null); + private static final Type TYPE_PG_UUID = new Type(Type.Code.PG_UUID, null, null); + private static final Type TYPE_PG_ARRAY_UUID = new Type(Type.Code.PG_ARRAY, TYPE_PG_UUID, null); private static final Type TYPE_PG_ARRAY_BOOL = new Type(Type.Code.PG_ARRAY, TYPE_PG_BOOL, null); private static final Type TYPE_PG_ARRAY_INT8 = new Type(Type.Code.PG_ARRAY, TYPE_PG_INT8, null); private static final Type TYPE_PG_ARRAY_FLOAT4 = @@ -160,6 +163,11 @@ public static Type date() { return TYPE_DATE; } + /** Returns the descriptor for the {@code UUID} type. */ + public static Type uuid() { + return TYPE_UUID; + } + public static Type pgBool() { return TYPE_PG_BOOL; } @@ -204,6 +212,11 @@ public static Type pgDate() { return TYPE_PG_DATE; } + /** Returns the descriptor for the PostgreSQL {@code uuid} type. */ + public static Type pgUuid() { + return TYPE_PG_UUID; + } + public static Type pgCommitTimestamp() { return TYPE_PG_COMMIT_TIMESTAMP; } @@ -263,6 +276,8 @@ public static Type pgArray(Type elementType) { return TYPE_PG_ARRAY_TIMESTAMPTZ; case PG_DATE: return TYPE_PG_ARRAY_DATE; + case PG_UUID: + return TYPE_PG_ARRAY_UUID; default: throw new IllegalArgumentException("Unknown Array type: Array of " + elementType); } @@ -317,6 +332,7 @@ public enum Code { BYTES("BYTES", Dialect.GOOGLE_STANDARD_SQL), TIMESTAMP("TIMESTAMP", Dialect.GOOGLE_STANDARD_SQL), DATE("DATE", Dialect.GOOGLE_STANDARD_SQL), + UUID("UUID", Dialect.GOOGLE_STANDARD_SQL), ARRAY("ARRAY", Dialect.GOOGLE_STANDARD_SQL), STRUCT("STRUCT", Dialect.GOOGLE_STANDARD_SQL), PG_BOOL("boolean", Dialect.POSTGRESQL), @@ -330,6 +346,7 @@ public enum Code { PG_BYTEA("bytea", Dialect.POSTGRESQL), PG_TIMESTAMPTZ("timestamp with time zone", Dialect.POSTGRESQL), PG_DATE("date", Dialect.POSTGRESQL), + PG_UUID("uuid", Dialect.POSTGRESQL), PG_ARRAY("array", Dialect.POSTGRESQL), PG_COMMIT_TIMESTAMP("spanner.commit_timestamp", Dialect.POSTGRESQL); diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/ddl/DdlTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/ddl/DdlTest.java index 32298f6985..281b23db92 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/ddl/DdlTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/ddl/DdlTest.java @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThat; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @@ -76,6 +77,9 @@ public void testDdlGSQL() { .int64() .notNull() .endColumn() + .column("uuid_col") + .uuid() + .endColumn() .column("first_name") .string() .size(10) @@ -117,6 +121,7 @@ public void testDdlGSQL() { equalToCompressingWhiteSpace( " CREATE TABLE `Users` (" + " `id` INT64 NOT NULL," + + " `uuid_col` UUID," + " `first_name` STRING(10)," + " `last_name` STRING(MAX)," + " `full_name` STRING(MAX) AS (CONCAT(first_name, ' ', last_name)) STORED," @@ -133,6 +138,7 @@ public void testDdlGSQL() { equalToCompressingWhiteSpace( " CREATE TABLE `Users` (" + " `id` INT64 NOT NULL," + + " `uuid_col` UUID," + " `first_name` STRING(10)," + " `last_name` STRING(MAX)," + " `full_name` STRING(MAX) AS (CONCAT(first_name, ' ', last_name)) STORED," @@ -198,6 +204,9 @@ public void testDdlPG() { .pgInt8() .notNull() .endColumn() + .column("uuid_col") + .pgUuid() + .endColumn() .column("first_name") .pgVarchar() .size(10) @@ -240,6 +249,7 @@ public void testDdlPG() { equalToCompressingWhiteSpace( " CREATE TABLE \"Users\" (" + " \"id\" bigint NOT NULL," + + " \"uuid_col\" uuid," + " \"first_name\" character varying(10)," + " \"last_name\" character varying," + " \"full_name\" character varying GENERATED ALWAYS AS" @@ -1082,4 +1092,46 @@ public void testCheckConstraintBuilder() { CheckConstraint checkConstraint1 = checkConstraintBuilder.name("ck").expression("1<2").build(); assertTrue(checkConstraint.equals(checkConstraint1)); } + + @Test + public void testColumnTypeString_Uuid() { + Column gsqlUuid = Column.builder(Dialect.GOOGLE_STANDARD_SQL).name("c").uuid().autoBuild(); + assertEquals("UUID", gsqlUuid.typeString()); + + Column pgUuid = Column.builder(Dialect.POSTGRESQL).name("c").pgUuid().autoBuild(); + assertEquals("uuid", pgUuid.typeString()); + } + + @Test + public void testColumnParseType() { + // Google Standard SQL parsing tests + Column.Builder gsqlBuilder = Column.builder(Dialect.GOOGLE_STANDARD_SQL).name("col"); + + Column gsqlUuid = gsqlBuilder.parseType("UUID").autoBuild(); + assertEquals(Type.uuid(), gsqlUuid.type()); + assertNull(gsqlUuid.size()); + + Column gsqlString = gsqlBuilder.parseType("STRING(100)").autoBuild(); + assertEquals(Type.string(), gsqlString.type()); + assertEquals(Integer.valueOf(100), gsqlString.size()); + + Column gsqlInt = gsqlBuilder.parseType("INT64").autoBuild(); + assertEquals(Type.int64(), gsqlInt.type()); + assertNull(gsqlInt.size()); + + // PostgreSQL parsing tests + Column.Builder pgBuilder = Column.builder(Dialect.POSTGRESQL).name("col"); + + Column pgUuid = pgBuilder.parseType("uuid").autoBuild(); + assertEquals(Type.pgUuid(), pgUuid.type()); + assertNull(pgUuid.size()); + + Column pgVarchar = pgBuilder.parseType("character varying(100)").autoBuild(); + assertEquals(Type.pgVarchar(), pgVarchar.type()); + assertEquals(Integer.valueOf(100), pgVarchar.size()); + + Column pgInt = pgBuilder.parseType("bigint").autoBuild(); + assertEquals(Type.pgInt8(), pgInt.type()); + assertNull(pgInt.size()); + } } diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapperTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapperTest.java index 626a994266..5d13ad9ede 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapperTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/AvroToValueMapperTest.java @@ -25,6 +25,7 @@ import com.google.cloud.ByteArray; import com.google.cloud.Date; import com.google.cloud.Timestamp; +import com.google.cloud.spanner.Dialect; import com.google.cloud.spanner.Value; import com.google.cloud.teleport.v2.spanner.migrations.exceptions.AvroTypeConvertorException; import com.google.cloud.teleport.v2.spanner.type.Type; @@ -870,4 +871,146 @@ public void testAvroValueToArrayException() { avroArrayFieldToSpannerArray( genericRecord.get("arrayField"), schema, AvroToValueMapper::avroFieldToLong)); } + + @Test + public void testAvroFieldToUuid_ValidUuidString() { + java.util.UUID originalUuid = java.util.UUID.randomUUID(); + java.util.UUID result = + AvroToValueMapper.avroFieldToUuid( + originalUuid.toString(), SchemaBuilder.builder().stringType()); + assertEquals(originalUuid, result); + } + + @Test + public void testAvroFieldToUuid_ValidUuidObject() { + java.util.UUID originalUuid = java.util.UUID.randomUUID(); + java.util.UUID result = + AvroToValueMapper.avroFieldToUuid(originalUuid, SchemaBuilder.builder().stringType()); + assertEquals(originalUuid, result); + } + + @Test + public void testAvroFieldToUuid_ValidByteBuffer() { + java.util.UUID originalUuid = java.util.UUID.randomUUID(); + ByteBuffer bb = ByteBuffer.wrap(new byte[16]); + bb.putLong(originalUuid.getMostSignificantBits()) + .putLong(originalUuid.getLeastSignificantBits()); + bb.flip(); + java.util.UUID result = + AvroToValueMapper.avroFieldToUuid(bb, SchemaBuilder.builder().bytesType()); + assertEquals(originalUuid, result); + } + + @Test + public void testAvroFieldToUuid_ValidByteArray() { + java.util.UUID originalUuid = java.util.UUID.randomUUID(); + ByteBuffer bb = ByteBuffer.wrap(new byte[16]); + bb.putLong(originalUuid.getMostSignificantBits()) + .putLong(originalUuid.getLeastSignificantBits()); + java.util.UUID result = + AvroToValueMapper.avroFieldToUuid(bb.array(), SchemaBuilder.builder().bytesType()); + assertEquals(originalUuid, result); + } + + @Test + public void testAvroFieldToUuid_NullInput() { + java.util.UUID result = + AvroToValueMapper.avroFieldToUuid(null, SchemaBuilder.builder().stringType()); + assertNull(result); + } + + @Test(expected = AvroTypeConvertorException.class) + public void testAvroFieldToUuid_InvalidInput() { + AvroToValueMapper.avroFieldToUuid("invalid-uuid", SchemaBuilder.builder().stringType()); + } + + @Test + public void testGsqlUuidConverter() { + java.util.UUID originalUuid = java.util.UUID.randomUUID(); + Value value = + AvroToValueMapper.convertorMap() + .get(Dialect.GOOGLE_STANDARD_SQL) + .get(Type.uuid()) + .apply(originalUuid.toString(), SchemaBuilder.builder().stringType()); + assertEquals(Value.uuid(originalUuid), value); + } + + @Test + public void testPgUuidConverter() { + java.util.UUID originalUuid = java.util.UUID.randomUUID(); + Value value = + AvroToValueMapper.convertorMap() + .get(Dialect.POSTGRESQL) + .get(Type.pgUuid()) + .apply(originalUuid.toString(), SchemaBuilder.builder().stringType()); + assertEquals(Value.uuid(originalUuid), value); + } + + @Test + public void testAvroValueToUuidArray() { + java.util.UUID uuid1 = java.util.UUID.randomUUID(); + java.util.UUID uuid2 = java.util.UUID.randomUUID(); + String[] uuidValues = new String[] {uuid1.toString(), uuid2.toString()}; + + Schema schemaUuidArray = SchemaBuilder.array().items(SchemaBuilder.builder().stringType()); + + GenericRecord genericRecordUuidArray = + new GenericRecordBuilder( + SchemaBuilder.record("payload") + .fields() + .name("arrayField") + .type(schemaUuidArray) + .noDefault() + .endRecord()) + .set("arrayField", uuidValues) + .build(); + + assertThat( + AvroToValueMapper.getGsqlMap() + .get(Type.array(Type.uuid())) + .apply( + genericRecordUuidArray.get("arrayField"), + genericRecordUuidArray.getSchema().getField("arrayField").schema())) + .isEqualTo(Value.uuidArray(java.util.Arrays.asList(uuid1, uuid2))); + } + + @Test + public void testAvroValueToPgUuidArray() { + java.util.UUID uuid1 = java.util.UUID.randomUUID(); + java.util.UUID uuid2 = java.util.UUID.randomUUID(); + String[] uuidValues = new String[] {uuid1.toString(), uuid2.toString()}; + + Schema schemaUuidArray = SchemaBuilder.array().items(SchemaBuilder.builder().stringType()); + + GenericRecord genericRecordUuidArray = + new GenericRecordBuilder( + SchemaBuilder.record("payload") + .fields() + .name("arrayField") + .type(schemaUuidArray) + .noDefault() + .endRecord()) + .set("arrayField", uuidValues) + .build(); + + assertThat( + AvroToValueMapper.getPgMap() + .get(Type.pgArray(Type.pgUuid())) + .apply( + genericRecordUuidArray.get("arrayField"), + genericRecordUuidArray.getSchema().getField("arrayField").schema())) + .isEqualTo(Value.uuidArray(java.util.Arrays.asList(uuid1, uuid2))); + } + + @Test(expected = AvroTypeConvertorException.class) + public void testAvroFieldToUuid_InvalidByteBufferLength() { + ByteBuffer bb = ByteBuffer.wrap(new byte[10]); + AvroToValueMapper.avroFieldToUuid(bb, SchemaBuilder.builder().bytesType()); + } + + @Test(expected = AvroTypeConvertorException.class) + public void testAvroFieldToUuid_InvalidByteArrayLength() { + byte[] bytes = new byte[20]; + AvroToValueMapper.avroFieldToUuid(bytes, SchemaBuilder.builder().bytesType()); + } } diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertorTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertorTest.java index e6a226e9fb..ce37b099e8 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertorTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/avro/GenericRecordTypeConvertorTest.java @@ -17,6 +17,7 @@ import static com.google.common.truth.Truth.assertThat; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; @@ -1667,4 +1668,196 @@ public void testGeneratedColumnFiltering() throws InvalidTransformationException assertThat(result).containsEntry("val_col", Value.string("test")); assertThat(result).doesNotContainKey("gen_col"); } + + @Test + public void transformChangeEventTest_ShardIdPopulation_UuidType() + throws InvalidTransformationException { + Ddl shardedDdl = + Ddl.builder(Dialect.GOOGLE_STANDARD_SQL) + .createTable("new_people") + .column("migration_shard_id") + .uuid() + .endColumn() + .column("new_name") + .string() + .size(20) + .endColumn() + .primaryKey() + .asc("migration_shard_id") + .asc("new_name") + .end() + .endTable() + .build(); + + String shardedSessionFilePath = + Paths.get(Resources.getResource("session-file-sharded.json").getPath()).toString(); + + ISchemaMapper shardedMapper = new SessionBasedMapper(shardedSessionFilePath, shardedDdl); + GenericRecord genericRecord = + new GenericData.Record( + SchemaBuilder.record("people") + .namespace("com.test.schema") + .fields() + .name("name") + .type(unionNullType(Schema.create(Schema.Type.STRING))) + .noDefault() + .endRecord()); + genericRecord.put("name", "name1"); + + String validUuidStr = "d1a0ce61-b9dd-4169-96a8-d0d7789b61d9"; + GenericRecordTypeConvertor genericRecordTypeConvertor = + new GenericRecordTypeConvertor(shardedMapper, "", validUuidStr, null); + Map actual = + genericRecordTypeConvertor.transformChangeEvent(genericRecord, "people"); + + Map expected = + Map.of( + "new_name", Value.string("name1"), + "migration_shard_id", Value.uuid(java.util.UUID.fromString(validUuidStr))); + assertEquals(expected, actual); + } + + @Test + public void transformChangeEventTest_SynthPKPopulation_UuidType() + throws InvalidTransformationException { + String sessionFilePath = + Paths.get(Resources.getResource("session-file-with-dropped-column.json").getPath()) + .toString(); + Ddl ddl = + Ddl.builder(Dialect.GOOGLE_STANDARD_SQL) + .createTable("new_people") + .column("synth_id") + .uuid() + .notNull() + .endColumn() + .column("new_name") + .string() + .size(10) + .endColumn() + .primaryKey() + .asc("synth_id") + .end() + .endTable() + .build(); + + ISchemaMapper sessionMapper = new SessionBasedMapper(sessionFilePath, ddl); + + GenericRecord genericRecord = + new GenericData.Record( + SchemaBuilder.record("people") + .namespace("com.test.schema") + .fields() + .name("name") + .type(unionNullType(Schema.create(Schema.Type.STRING))) + .noDefault() + .endRecord()); + genericRecord.put("name", "name1"); + + GenericRecordTypeConvertor genericRecordTypeConvertor = + new GenericRecordTypeConvertor(sessionMapper, "", null, null); + Map actual = + genericRecordTypeConvertor.transformChangeEvent(genericRecord, "people"); + + assertTrue(actual.containsKey("synth_id")); + Value synthValue = actual.get("synth_id"); + assertEquals(com.google.cloud.spanner.Type.Code.UUID, synthValue.getType().getCode()); + assertNotNull(synthValue.getUuid()); + assertEquals(Value.string("name1"), actual.get("new_name")); + } + + @Test + public void transformChangeEventTest_ShardIdPopulation_PgUuidType() + throws InvalidTransformationException { + Ddl shardedDdl = + Ddl.builder(Dialect.POSTGRESQL) + .createTable("new_people") + .column("migration_shard_id") + .pgUuid() + .endColumn() + .column("new_name") + .pgVarchar() + .size(20) + .endColumn() + .primaryKey() + .asc("migration_shard_id") + .asc("new_name") + .end() + .endTable() + .build(); + + String shardedSessionFilePath = + Paths.get(Resources.getResource("session-file-sharded.json").getPath()).toString(); + + ISchemaMapper shardedMapper = new SessionBasedMapper(shardedSessionFilePath, shardedDdl); + GenericRecord genericRecord = + new GenericData.Record( + SchemaBuilder.record("people") + .namespace("com.test.schema") + .fields() + .name("name") + .type(unionNullType(Schema.create(Schema.Type.STRING))) + .noDefault() + .endRecord()); + genericRecord.put("name", "name1"); + + String validUuidStr = "d1a0ce61-b9dd-4169-96a8-d0d7789b61d9"; + GenericRecordTypeConvertor genericRecordTypeConvertor = + new GenericRecordTypeConvertor(shardedMapper, "", validUuidStr, null); + Map actual = + genericRecordTypeConvertor.transformChangeEvent(genericRecord, "people"); + + Map expected = + Map.of( + "new_name", Value.string("name1"), + "migration_shard_id", Value.uuid(java.util.UUID.fromString(validUuidStr))); + assertEquals(expected, actual); + } + + @Test + public void transformChangeEventTest_SynthPKPopulation_PgUuidType() + throws InvalidTransformationException { + String sessionFilePath = + Paths.get(Resources.getResource("session-file-with-dropped-column.json").getPath()) + .toString(); + Ddl ddl = + Ddl.builder(Dialect.POSTGRESQL) + .createTable("new_people") + .column("synth_id") + .pgUuid() + .notNull() + .endColumn() + .column("new_name") + .pgVarchar() + .size(10) + .endColumn() + .primaryKey() + .asc("synth_id") + .end() + .endTable() + .build(); + + ISchemaMapper sessionMapper = new SessionBasedMapper(sessionFilePath, ddl); + + GenericRecord genericRecord = + new GenericData.Record( + SchemaBuilder.record("people") + .namespace("com.test.schema") + .fields() + .name("name") + .type(unionNullType(Schema.create(Schema.Type.STRING))) + .noDefault() + .endRecord()); + genericRecord.put("name", "name1"); + + GenericRecordTypeConvertor genericRecordTypeConvertor = + new GenericRecordTypeConvertor(sessionMapper, "", null, null); + Map actual = + genericRecordTypeConvertor.transformChangeEvent(genericRecord, "people"); + + assertTrue(actual.containsKey("synth_id")); + Value synthValue = actual.get("synth_id"); + assertEquals(com.google.cloud.spanner.Type.Code.UUID, synthValue.getType().getCode()); + assertNotNull(synthValue.getUuid()); + assertEquals(Value.string("name1"), actual.get("new_name")); + } } diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertorTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertorTest.java index 850dbd2206..e042b39d66 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertorTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventSpannerConvertorTest.java @@ -95,6 +95,12 @@ static Ddl getTestDdl() { .column("date_field2") .date() .endColumn() + .column("uuid_col") + .uuid() + .endColumn() + .column("pg_uuid_col") + .pgUuid() + .endColumn() .primaryKey() .asc("first_name") .desc("last_name") @@ -122,6 +128,9 @@ static JSONObject getTestChangeEvent(String tableName) { "timestamp_field2", Timestamp.of(java.sql.Timestamp.valueOf("2020-12-30 4:12:12.1"))); changeEvent.put("date_field", "2020-12-30T00:00:00Z"); changeEvent.put("date_field2", "2020-12-30"); + String testUuidStr = "d1a0ce61-b9dd-4169-96a8-d0d7789b61d9"; + changeEvent.put("uuid_col", testUuidStr); + changeEvent.put("pg_uuid_col", testUuidStr); changeEvent.put(Constants.EVENT_TABLE_NAME_KEY, tableName); return changeEvent; } @@ -149,6 +158,10 @@ static Map getExpectedMapForTestChangeEvent() { Value.timestamp(Timestamp.of(java.sql.Timestamp.valueOf("2020-12-30 4:12:12.1")))); put("date_field", Value.date(Date.parseDate("2020-12-30"))); put("date_field2", Value.date(Date.parseDate("2020-12-30"))); + java.util.UUID expectedUuid = + java.util.UUID.fromString("d1a0ce61-b9dd-4169-96a8-d0d7789b61d9"); + put("uuid_col", Value.uuid(expectedUuid)); + put("pg_uuid_col", Value.uuid(expectedUuid)); } }; return expected; @@ -219,6 +232,12 @@ static Ddl getTestDdlForPrimaryKeyTest() { .column("date_field2") .date() .endColumn() + .column("uuid_col") + .uuid() + .endColumn() + .column("pg_uuid_col") + .pgUuid() + .endColumn() .primaryKey() .asc("first_name") .desc("last_name") @@ -233,6 +252,8 @@ static Ddl getTestDdlForPrimaryKeyTest() { .asc("timestamp_field2") .asc("date_field") .asc("date_field2") + .asc("uuid_col") + .asc("pg_uuid_col") .end() .endTable() .build(); @@ -261,7 +282,9 @@ public void mutationFromEventBasic() throws Exception { "timestamp_field", "timestamp_field2", "date_field", - "date_field2"); + "date_field2", + "uuid_col", + "pg_uuid_col"); Set keyNames = new HashSet<>(Arrays.asList("first_name", "last_name")); Mutation mutation = ChangeEventSpannerConvertor.mutationFromEvent(table, ce, colNames, keyNames); @@ -294,6 +317,9 @@ public void canConvertChangeEventToPrimaryKey() throws Exception { expectedKeyParts.add(Timestamp.of(java.sql.Timestamp.valueOf("2020-12-30 4:12:12.1"))); expectedKeyParts.add(Date.parseDate("2020-12-30")); expectedKeyParts.add(Date.parseDate("2020-12-30")); + java.util.UUID expectedUuid = java.util.UUID.fromString("d1a0ce61-b9dd-4169-96a8-d0d7789b61d9"); + expectedKeyParts.add(expectedUuid); + expectedKeyParts.add(expectedUuid); assertThat(keyParts, is(expectedKeyParts)); } diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertorTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertorTest.java index 91dd3b7d5a..de1652badd 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertorTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/migrations/convertors/ChangeEventTypeConvertorTest.java @@ -758,6 +758,7 @@ public void canConvertNullStrings() throws Exception { changeEvent.put("byte_field", "NULL"); changeEvent.put("timestamp_field", "NULL"); changeEvent.put("date_field", "NULL"); + changeEvent.put("uuid_field", "NULL"); changeEvent.put("string_field", "NULL"); // this should not be converted to null JsonNode ce = getJsonNode(changeEvent.toString()); @@ -770,6 +771,32 @@ public void canConvertNullStrings() throws Exception { assertNull(ChangeEventTypeConvertor.toByteArray(ce, "byte_field", true)); assertNull(ChangeEventTypeConvertor.toTimestamp(ce, "timestamp_field", true)); assertNull(ChangeEventTypeConvertor.toDate(ce, "date_field", true)); + assertNull(ChangeEventTypeConvertor.toUuid(ce, "uuid_field", true)); assertEquals("NULL", ChangeEventTypeConvertor.toString(ce, "string_field", true)); } + + @Test + public void canConvertToUuid() throws Exception { + java.util.UUID originalUuid = java.util.UUID.randomUUID(); + JSONObject changeEvent = new JSONObject(); + changeEvent.put("uuid_field1", originalUuid.toString()); + changeEvent.put("uuid_field2", JSONObject.NULL); + changeEvent.put("uuid_field3", ""); + + JsonNode ce = getJsonNode(changeEvent.toString()); + + assertEquals( + ChangeEventTypeConvertor.toUuid(ce, "uuid_field1", /* requiredField= */ true), + originalUuid); + assertNull(ChangeEventTypeConvertor.toUuid(ce, "uuid_field2", /* requiredField= */ false)); + assertNull(ChangeEventTypeConvertor.toUuid(ce, "uuid_field3", /* requiredField= */ false)); + } + + @Test(expected = ChangeEventConvertorException.class) + public void cannotConvertInvalidStringToUuid() throws Exception { + JSONObject changeEvent = new JSONObject(); + changeEvent.put("uuid_field1", "invalid-uuid"); + JsonNode ce = getJsonNode(changeEvent.toString()); + ChangeEventTypeConvertor.toUuid(ce, "uuid_field1", /* requiredField= */ true); + } } diff --git a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/type/TypeTest.java b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/type/TypeTest.java index fe49ca5088..f3a2aec642 100644 --- a/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/type/TypeTest.java +++ b/v2/spanner-common/src/test/java/com/google/cloud/teleport/v2/spanner/type/TypeTest.java @@ -34,6 +34,7 @@ public void testToString() { assertEquals("BYTES", Type.bytes().toString()); assertEquals("TIMESTAMP", Type.timestamp().toString()); assertEquals("DATE", Type.date().toString()); + assertEquals("UUID", Type.uuid().toString()); // Array types assertEquals("ARRAY", Type.array(Type.int64()).toString()); @@ -59,7 +60,9 @@ public void testToString() { assertEquals("PG_BYTEA", Type.pgBytea().toString()); assertEquals("PG_TIMESTAMPTZ", Type.pgTimestamptz().toString()); assertEquals("PG_DATE", Type.pgDate().toString()); + assertEquals("PG_UUID", Type.pgUuid().toString()); assertEquals("PG_TEXT[]", Type.pgArray(Type.pgText()).toString()); + assertEquals("PG_UUID[]", Type.pgArray(Type.pgUuid()).toString()); assertEquals("PG_COMMIT_TIMESTAMP", Type.pgCommitTimestamp().toString()); } } diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGenerator.java index d4ee9352b1..e9f49469ea 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGenerator.java @@ -196,7 +196,8 @@ private static DMLGeneratorResponse generateUpsertStatement( return getUpsertStatement(sourceTable.name(), columnNameValues); } - private static String getMappedColumnValue( + @VisibleForTesting + protected static String getMappedColumnValue( Column spannerColDef, SourceColumn sourceColDef, JSONObject valuesJson, @@ -216,10 +217,12 @@ private static String getMappedColumnValue( || colType.getCode().equals(Type.Code.PG_BOOL)) { colInputValue = (new Boolean(valuesJson.getBoolean(colName))).toString(); } else if ((colType.getCode().equals(Type.Code.ARRAY) - && colType.getArrayElementType().getCode().equals(Type.Code.STRING)) + && (colType.getArrayElementType().getCode().equals(Type.Code.STRING) + || colType.getArrayElementType().getCode().equals(Type.Code.UUID))) || (colType.getCode().equals(Type.Code.PG_ARRAY) && (colType.getArrayElementType().getCode().equals(Type.Code.PG_VARCHAR) - || colType.getArrayElementType().getCode().equals(Type.Code.PG_TEXT)))) { + || colType.getArrayElementType().getCode().equals(Type.Code.PG_TEXT) + || colType.getArrayElementType().getCode().equals(Type.Code.PG_UUID)))) { colInputValue = valuesJson.getJSONArray(colName).toList().stream() .map(String::valueOf) diff --git a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/PostgreSQLDMLGenerator.java b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/PostgreSQLDMLGenerator.java index 08b033b0a9..2ceebd2bd8 100644 --- a/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/PostgreSQLDMLGenerator.java +++ b/v2/spanner-to-sourcedb/src/main/java/com/google/cloud/teleport/v2/templates/dbutils/dml/PostgreSQLDMLGenerator.java @@ -228,10 +228,12 @@ static String getMappedColumnValue( || colType.getCode().equals(Type.Code.PG_BOOL)) { colInputValue = String.valueOf(valuesJson.getBoolean(colName)); } else if ((colType.getCode().equals(Type.Code.ARRAY) - && colType.getArrayElementType().getCode().equals(Type.Code.STRING)) + && (colType.getArrayElementType().getCode().equals(Type.Code.STRING) + || colType.getArrayElementType().getCode().equals(Type.Code.UUID))) || (colType.getCode().equals(Type.Code.PG_ARRAY) && (colType.getArrayElementType().getCode().equals(Type.Code.PG_VARCHAR) - || colType.getArrayElementType().getCode().equals(Type.Code.PG_TEXT)))) { + || colType.getArrayElementType().getCode().equals(Type.Code.PG_TEXT) + || colType.getArrayElementType().getCode().equals(Type.Code.PG_UUID)))) { colInputValue = valuesJson.getJSONArray(colName).toList().stream() diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGeneratorTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGeneratorTest.java index 117b362eae..218e626a42 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGeneratorTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/MySQLDMLGeneratorTest.java @@ -19,6 +19,8 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertThrows; import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import com.google.cloud.teleport.v2.spanner.ddl.Ddl; import com.google.cloud.teleport.v2.spanner.migrations.schema.ISchemaMapper; @@ -1557,4 +1559,127 @@ public void testGeneratedPrimaryKeyDML() { assertEquals(0, countInSQL(sql, "FirstName")); assertEquals(0, countInSQL(sql, "SingerId")); } + + @Test + public void testGetMappedColumnValue_GsqlUuidArray() { + com.google.cloud.teleport.v2.spanner.ddl.Column spannerColDef = + mock(com.google.cloud.teleport.v2.spanner.ddl.Column.class); + when(spannerColDef.name()).thenReturn("uuid_array"); + when(spannerColDef.type()) + .thenReturn( + com.google.cloud.teleport.v2.spanner.type.Type.array( + com.google.cloud.teleport.v2.spanner.type.Type.uuid())); + + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn sourceColDef = + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn.builder( + com.google.cloud.teleport.v2.spanner.sourceddl.SourceDatabaseType.MYSQL) + .name("uuid_array") + .type("varchar") + .build(); + + java.util.UUID uuid1 = java.util.UUID.randomUUID(); + java.util.UUID uuid2 = java.util.UUID.randomUUID(); + JSONObject valuesJson = new JSONObject(); + org.json.JSONArray jsonArray = new org.json.JSONArray(); + jsonArray.put(uuid1.toString()); + jsonArray.put(uuid2.toString()); + valuesJson.put("uuid_array", jsonArray); + + String mappedValue = + MySQLDMLGenerator.getMappedColumnValue(spannerColDef, sourceColDef, valuesJson, "+00:00"); + + assertEquals("'" + uuid1.toString() + "," + uuid2.toString() + "'", mappedValue); + } + + @Test + public void testGetMappedColumnValue_PgUuidArray() { + com.google.cloud.teleport.v2.spanner.ddl.Column spannerColDef = + mock(com.google.cloud.teleport.v2.spanner.ddl.Column.class); + when(spannerColDef.name()).thenReturn("uuid_array"); + when(spannerColDef.type()) + .thenReturn( + com.google.cloud.teleport.v2.spanner.type.Type.pgArray( + com.google.cloud.teleport.v2.spanner.type.Type.pgUuid())); + + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn sourceColDef = + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn.builder( + com.google.cloud.teleport.v2.spanner.sourceddl.SourceDatabaseType.MYSQL) + .name("uuid_array") + .type("varchar") + .build(); + + java.util.UUID uuid1 = java.util.UUID.randomUUID(); + java.util.UUID uuid2 = java.util.UUID.randomUUID(); + JSONObject valuesJson = new JSONObject(); + org.json.JSONArray jsonArray = new org.json.JSONArray(); + jsonArray.put(uuid1.toString()); + jsonArray.put(uuid2.toString()); + valuesJson.put("uuid_array", jsonArray); + + String mappedValue = + MySQLDMLGenerator.getMappedColumnValue(spannerColDef, sourceColDef, valuesJson, "+00:00"); + + assertEquals("'" + uuid1.toString() + "," + uuid2.toString() + "'", mappedValue); + } + + @Test + public void testGetMappedColumnValue_BasicTypes() { + // 1. Test INT64 mapping + com.google.cloud.teleport.v2.spanner.ddl.Column intCol = + mock(com.google.cloud.teleport.v2.spanner.ddl.Column.class); + when(intCol.name()).thenReturn("int_col"); + when(intCol.type()).thenReturn(com.google.cloud.teleport.v2.spanner.type.Type.int64()); + + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn sourceIntCol = + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn.builder( + com.google.cloud.teleport.v2.spanner.sourceddl.SourceDatabaseType.MYSQL) + .name("int_col") + .type("int") + .build(); + + JSONObject valuesJson = new JSONObject(); + valuesJson.put("int_col", "12345"); + + String mappedInt = + MySQLDMLGenerator.getMappedColumnValue(intCol, sourceIntCol, valuesJson, "+00:00"); + assertEquals("12345", mappedInt); + + // 2. Test BOOL mapping + com.google.cloud.teleport.v2.spanner.ddl.Column boolCol = + mock(com.google.cloud.teleport.v2.spanner.ddl.Column.class); + when(boolCol.name()).thenReturn("bool_col"); + when(boolCol.type()).thenReturn(com.google.cloud.teleport.v2.spanner.type.Type.bool()); + + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn sourceBoolCol = + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn.builder( + com.google.cloud.teleport.v2.spanner.sourceddl.SourceDatabaseType.MYSQL) + .name("bool_col") + .type("tinyint") + .build(); + + valuesJson.put("bool_col", true); + + String mappedBool = + MySQLDMLGenerator.getMappedColumnValue(boolCol, sourceBoolCol, valuesJson, "+00:00"); + assertEquals("true", mappedBool); + + // 3. Test FLOAT64 mapping + com.google.cloud.teleport.v2.spanner.ddl.Column floatCol = + mock(com.google.cloud.teleport.v2.spanner.ddl.Column.class); + when(floatCol.name()).thenReturn("float_col"); + when(floatCol.type()).thenReturn(com.google.cloud.teleport.v2.spanner.type.Type.float64()); + + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn sourceFloatCol = + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn.builder( + com.google.cloud.teleport.v2.spanner.sourceddl.SourceDatabaseType.MYSQL) + .name("float_col") + .type("double") + .build(); + + valuesJson.put("float_col", "12.345"); + + String mappedFloat = + MySQLDMLGenerator.getMappedColumnValue(floatCol, sourceFloatCol, valuesJson, "+00:00"); + assertEquals("12.345", mappedFloat); + } } diff --git a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/PostgreSQLDMLGeneratorTest.java b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/PostgreSQLDMLGeneratorTest.java index 0e9c2b95ac..2b9ca4d346 100644 --- a/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/PostgreSQLDMLGeneratorTest.java +++ b/v2/spanner-to-sourcedb/src/test/java/com/google/cloud/teleport/v2/templates/dbutils/dml/PostgreSQLDMLGeneratorTest.java @@ -445,4 +445,170 @@ public void testNonByteaDecodeBranch() { PostgreSQLDMLGenerator.getMappedColumnValue(spannerStrCol, sourceStrCol, jsonStr, "+00:00"); assertEquals("'it''s a string'", resStr); } + + @Test + public void testGetMappedColumnValue_PgUuidArray() { + com.google.cloud.teleport.v2.spanner.ddl.Column spannerColDef = + mock(com.google.cloud.teleport.v2.spanner.ddl.Column.class); + when(spannerColDef.name()).thenReturn("uuid_array"); + when(spannerColDef.type()) + .thenReturn( + com.google.cloud.teleport.v2.spanner.type.Type.pgArray( + com.google.cloud.teleport.v2.spanner.type.Type.pgUuid())); + + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn sourceColDef = + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn.builder( + com.google.cloud.teleport.v2.spanner.sourceddl.SourceDatabaseType.POSTGRESQL) + .name("uuid_array") + .type("uuid[]") + .build(); + + java.util.UUID uuid1 = java.util.UUID.randomUUID(); + java.util.UUID uuid2 = java.util.UUID.randomUUID(); + JSONObject valuesJson = new JSONObject(); + org.json.JSONArray jsonArray = new org.json.JSONArray(); + jsonArray.put(uuid1.toString()); + jsonArray.put(uuid2.toString()); + valuesJson.put("uuid_array", jsonArray); + + String mappedValue = + PostgreSQLDMLGenerator.getMappedColumnValue( + spannerColDef, sourceColDef, valuesJson, "+00:00"); + + assertEquals(uuid1.toString() + "," + uuid2.toString(), mappedValue); + } + + @Test + public void testGetMappedColumnValue_GsqlUuidArray() { + com.google.cloud.teleport.v2.spanner.ddl.Column spannerColDef = + mock(com.google.cloud.teleport.v2.spanner.ddl.Column.class); + when(spannerColDef.name()).thenReturn("uuid_array"); + when(spannerColDef.type()) + .thenReturn( + com.google.cloud.teleport.v2.spanner.type.Type.array( + com.google.cloud.teleport.v2.spanner.type.Type.uuid())); + + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn sourceColDef = + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn.builder( + com.google.cloud.teleport.v2.spanner.sourceddl.SourceDatabaseType.POSTGRESQL) + .name("uuid_array") + .type("uuid[]") + .build(); + + java.util.UUID uuid1 = java.util.UUID.randomUUID(); + java.util.UUID uuid2 = java.util.UUID.randomUUID(); + JSONObject valuesJson = new JSONObject(); + org.json.JSONArray jsonArray = new org.json.JSONArray(); + jsonArray.put(uuid1.toString()); + jsonArray.put(uuid2.toString()); + valuesJson.put("uuid_array", jsonArray); + + String mappedValue = + PostgreSQLDMLGenerator.getMappedColumnValue( + spannerColDef, sourceColDef, valuesJson, "+00:00"); + + assertEquals(uuid1.toString() + "," + uuid2.toString(), mappedValue); + } + + @Test + public void testGetMappedColumnValue_BasicTypes() { + // 1. Test INT64 mapping + com.google.cloud.teleport.v2.spanner.ddl.Column intCol = + mock(com.google.cloud.teleport.v2.spanner.ddl.Column.class); + when(intCol.name()).thenReturn("int_col"); + when(intCol.type()).thenReturn(com.google.cloud.teleport.v2.spanner.type.Type.int64()); + + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn sourceIntCol = + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn.builder( + com.google.cloud.teleport.v2.spanner.sourceddl.SourceDatabaseType.POSTGRESQL) + .name("int_col") + .type("integer") + .build(); + + JSONObject valuesJson = new JSONObject(); + valuesJson.put("int_col", "12345"); + + String mappedInt = + PostgreSQLDMLGenerator.getMappedColumnValue(intCol, sourceIntCol, valuesJson, "+00:00"); + assertEquals("12345", mappedInt); + + // 2. Test BOOL mapping + com.google.cloud.teleport.v2.spanner.ddl.Column boolCol = + mock(com.google.cloud.teleport.v2.spanner.ddl.Column.class); + when(boolCol.name()).thenReturn("bool_col"); + when(boolCol.type()).thenReturn(com.google.cloud.teleport.v2.spanner.type.Type.bool()); + + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn sourceBoolCol = + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn.builder( + com.google.cloud.teleport.v2.spanner.sourceddl.SourceDatabaseType.POSTGRESQL) + .name("bool_col") + .type("boolean") + .build(); + + valuesJson.put("bool_col", true); + + String mappedBool = + PostgreSQLDMLGenerator.getMappedColumnValue(boolCol, sourceBoolCol, valuesJson, "+00:00"); + assertEquals("true", mappedBool); + + // 3. Test FLOAT64 mapping + com.google.cloud.teleport.v2.spanner.ddl.Column floatCol = + mock(com.google.cloud.teleport.v2.spanner.ddl.Column.class); + when(floatCol.name()).thenReturn("float_col"); + when(floatCol.type()).thenReturn(com.google.cloud.teleport.v2.spanner.type.Type.float64()); + + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn sourceFloatCol = + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn.builder( + com.google.cloud.teleport.v2.spanner.sourceddl.SourceDatabaseType.POSTGRESQL) + .name("float_col") + .type("double precision") + .build(); + + valuesJson.put("float_col", "12.345"); + + String mappedFloat = + PostgreSQLDMLGenerator.getMappedColumnValue(floatCol, sourceFloatCol, valuesJson, "+00:00"); + assertEquals("12.345", mappedFloat); + + // 4. Test Standard Scalar UUID mapping + com.google.cloud.teleport.v2.spanner.ddl.Column uuidCol = + mock(com.google.cloud.teleport.v2.spanner.ddl.Column.class); + when(uuidCol.name()).thenReturn("uuid_col"); + when(uuidCol.type()).thenReturn(com.google.cloud.teleport.v2.spanner.type.Type.uuid()); + + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn sourceUuidCol = + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn.builder( + com.google.cloud.teleport.v2.spanner.sourceddl.SourceDatabaseType.POSTGRESQL) + .name("uuid_col") + .type("uuid") + .build(); + + java.util.UUID testUuid = java.util.UUID.randomUUID(); + valuesJson.put("uuid_col", testUuid.toString()); + + String mappedUuid = + PostgreSQLDMLGenerator.getMappedColumnValue(uuidCol, sourceUuidCol, valuesJson, "+00:00"); + assertEquals("'" + testUuid.toString() + "'", mappedUuid); + + // 5. Test PG Scalar UUID mapping + com.google.cloud.teleport.v2.spanner.ddl.Column pgUuidCol = + mock(com.google.cloud.teleport.v2.spanner.ddl.Column.class); + when(pgUuidCol.name()).thenReturn("pg_uuid_col"); + when(pgUuidCol.type()).thenReturn(com.google.cloud.teleport.v2.spanner.type.Type.pgUuid()); + + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn sourcePgUuidCol = + com.google.cloud.teleport.v2.spanner.sourceddl.SourceColumn.builder( + com.google.cloud.teleport.v2.spanner.sourceddl.SourceDatabaseType.POSTGRESQL) + .name("pg_uuid_col") + .type("uuid") + .build(); + + java.util.UUID testPgUuid = java.util.UUID.randomUUID(); + valuesJson.put("pg_uuid_col", testPgUuid.toString()); + + String mappedPgUuid = + PostgreSQLDMLGenerator.getMappedColumnValue( + pgUuidCol, sourcePgUuidCol, valuesJson, "+00:00"); + assertEquals("'" + testPgUuid.toString() + "'", mappedPgUuid); + } }