Skip to content

Commit 925c463

Browse files
pratickchokhanipratickchokhani
authored
feat: add support for UUID and PG_UUID types across Spanner migration utilities. (#3905)
* feat: add support for UUID and PG_UUID types across Spanner migration utilities * feat: add UUID support for GSQL and PG dialect mappings in AvroToValueMapper * UUID PK types
1 parent 707cf66 commit 925c463

40 files changed

Lines changed: 603 additions & 21 deletions

File tree

‎v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDatastreamToSpannerDataTypesAndExpressionIT.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -313,7 +313,12 @@ private List<Map<String, Object>> createRows(String colPrefix, Object... values)
313313
List<Map<String, Object>> rows = new ArrayList<>();
314314
for (int i = 0; i < vals.size(); i++) {
315315
Map<String, Object> row = new HashMap<>();
316-
row.put("id", i + 1);
316+
// We specifically want to test primary key partitioning.
317+
if (colPrefix.toLowerCase().contains("_pk")) {
318+
row.put("id", vals.get(i));
319+
} else {
320+
row.put("id", i + 1);
321+
}
317322
row.put(String.format("%s_col", colPrefix), vals.get(i));
318323
rows.add(row);
319324
}
@@ -611,6 +616,13 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
611616
expectedData.put("set", createRows("set", "v1,v2", "NULL"));
612617
expectedData.put(
613618
"integer_unsigned", createRows("integer_unsigned", "0", "42", "4294967295", "NULL"));
619+
expectedData.put("uuid", createRows("uuid", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "NULL"));
620+
expectedData.put(
621+
"uuid_pk",
622+
createRows(
623+
"uuid_pk",
624+
"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11",
625+
"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12"));
614626
return expectedData;
615627
}
616628

‎v2/datastream-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/PostgreSQLDatastreamToSpannerDataTypesIT.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -383,6 +383,18 @@ private List<Map<String, Object>> createRows(Object... values) {
383383
return rows;
384384
}
385385

386+
private List<Map<String, Object>> createUuidPkRows(Object... values) {
387+
List<Object> vals = Arrays.asList(values);
388+
List<Map<String, Object>> rows = new ArrayList<>();
389+
for (int i = 0; i < vals.size(); i++) {
390+
Map<String, Object> row = new HashMap<>();
391+
row.put("id", vals.get(i));
392+
row.put("col", vals.get(i));
393+
rows.add(row);
394+
}
395+
return rows;
396+
}
397+
386398
private List<String> getAllowedTables() {
387399
Map<String, List<Map<String, Object>>> expectedData = getExpectedData();
388400
List<String> tableNames = new ArrayList<>(expectedData.size() + UNSUPPORTED_TYPE_TABLES.size());
@@ -607,6 +619,10 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
607619
createRows("1970-02-02T18:05:06.123456Z", "1970-02-03T05:05:06.123456Z", "NULL"));
608620
result.put("timestamp_without_time_zone", createRows("1970-01-02T03:04:05.123456Z", "NULL"));
609621
result.put("uuid", createRows("a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "NULL"));
622+
result.put(
623+
"uuid_pk",
624+
createUuidPkRows(
625+
"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12"));
610626
result.put("uuid_to_bytes", createRows("oO68mZwLTvi7bWu5vTgKEQ==", "NULL"));
611627
result.put("varbit", createRows("wA==", "NULL"));
612628
result.put("varbit_to_string", createRows("wA==", "NULL"));

‎v2/datastream-to-spanner/src/test/resources/MySQLDataTypesIT/mysql-data-types.sql‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -739,4 +739,16 @@ CREATE TABLE IF NOT EXISTS spatial_geometrycollection (
739739

740740
INSERT INTO spatial_geometrycollection (geoms) VALUES (MultiPoint(Point(77.5946, 12.9716), Point(77.6100, 12.9600)));
741741

742+
CREATE TABLE IF NOT EXISTS `uuid_table` (
743+
`id` INT PRIMARY KEY,
744+
`uuid_col` VARCHAR(36) DEFAULT NULL
745+
);
746+
747+
INSERT INTO `uuid_table` (`id`, `uuid_col`) VALUES (1, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'), (2, NULL);
748+
749+
CREATE TABLE IF NOT EXISTS `uuid_pk_table` (
750+
`id` VARCHAR(36) PRIMARY KEY,
751+
`uuid_pk_col` VARCHAR(36) NOT NULL
752+
);
742753

754+
INSERT INTO `uuid_pk_table` (`id`, `uuid_pk_col`) VALUES ('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'), ('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12', 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12');

‎v2/datastream-to-spanner/src/test/resources/MySQLDataTypesIT/pg-dialect-spanner-schema.sql‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -477,3 +477,15 @@ CREATE TABLE IF NOT EXISTS spatial_geometrycollection (
477477
geom_coll VARCHAR,
478478
PRIMARY KEY (id)
479479
);
480+
481+
CREATE TABLE IF NOT EXISTS uuid_table (
482+
id INT8 NOT NULL,
483+
uuid_col UUID,
484+
PRIMARY KEY (id)
485+
);
486+
487+
CREATE TABLE IF NOT EXISTS uuid_pk_table (
488+
id VARCHAR NOT NULL,
489+
uuid_pk_col VARCHAR NOT NULL,
490+
PRIMARY KEY (id)
491+
);

‎v2/datastream-to-spanner/src/test/resources/MySQLDataTypesIT/spanner-schema.sql‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -424,3 +424,13 @@ CREATE TABLE IF NOT EXISTS `generated_to_non_generated_column_table` (
424424
`generated_column_col` STRING(100) DEFAULT(NULL),
425425
`generated_column_pk_col` STRING(100) DEFAULT(NULL),
426426
) PRIMARY KEY (`generated_column_pk_col`);
427+
428+
CREATE TABLE IF NOT EXISTS uuid_table (
429+
`id` INT64 NOT NULL,
430+
`uuid_col` UUID,
431+
) PRIMARY KEY(id);
432+
433+
CREATE TABLE IF NOT EXISTS uuid_pk_table (
434+
`id` UUID NOT NULL,
435+
`uuid_pk_col` UUID NOT NULL,
436+
) PRIMARY KEY(id);

‎v2/datastream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/pg-dialect-spanner-schema.sql‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,10 +130,11 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT8, col VARCHAR, PRIMARY KEY (
130130
CREATE TABLE IF NOT EXISTS t_tstzrange (id INT8, col VARCHAR, PRIMARY KEY (id));
131131
CREATE TABLE IF NOT EXISTS t_tsvector (id INT8, col VARCHAR, PRIMARY KEY (id));
132132
CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT8, col VARCHAR, PRIMARY KEY (id));
133-
CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col VARCHAR, PRIMARY KEY (id));
133+
CREATE TABLE IF NOT EXISTS t_uuid (id INT8, col UUID, PRIMARY KEY (id));
134134
CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT8, col BYTEA, PRIMARY KEY (id));
135135
CREATE TABLE IF NOT EXISTS t_varbit (id INT8, col BYTEA, PRIMARY KEY (id));
136136
CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT8, col BOOL[], PRIMARY KEY (id));
137137
CREATE TABLE IF NOT EXISTS t_varbit_to_string (id INT8, col VARCHAR, PRIMARY KEY (id));
138138
CREATE TABLE IF NOT EXISTS t_varchar (id INT8, col VARCHAR, PRIMARY KEY (id));
139139
CREATE TABLE IF NOT EXISTS t_xml (id INT8, col VARCHAR, PRIMARY KEY (id));
140+
CREATE TABLE IF NOT EXISTS t_uuid_pk (id UUID NOT NULL, col UUID, PRIMARY KEY (id));

‎v2/datastream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/postgresql-data-types.sql‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -279,3 +279,6 @@ INSERT INTO t_varbit_to_string (col) VALUES ('1100'::varbit(32)), (NULL);
279279
INSERT INTO t_varbit_to_bool_array (col) VALUES ('1100'::varbit(32)), (NULL);
280280
INSERT INTO t_varchar (col) VALUES ('testing varchar'), (NULL);
281281
INSERT INTO t_xml (col) VALUES ('<test>123</test>'::xml), (NULL);
282+
283+
CREATE TABLE t_uuid_pk (id uuid primary key, col uuid);
284+
INSERT INTO t_uuid_pk (id, col) VALUES ('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'::uuid, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11'::uuid), ('a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12'::uuid, 'a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12'::uuid);

‎v2/datastream-to-spanner/src/test/resources/PostgreSQLDataTypesIT/spanner-schema.sql‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,10 +130,12 @@ CREATE TABLE IF NOT EXISTS t_tstzmultirange (id INT64, col STRING(MAX)) PRIMARY
130130
CREATE TABLE IF NOT EXISTS t_tstzrange (id INT64, col STRING(MAX)) PRIMARY KEY (id);
131131
CREATE TABLE IF NOT EXISTS t_tsvector (id INT64, col STRING(MAX)) PRIMARY KEY (id);
132132
CREATE TABLE IF NOT EXISTS t_txid_snapshot (id INT64, col STRING(MAX)) PRIMARY KEY (id);
133-
CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col STRING(MAX)) PRIMARY KEY (id);
133+
CREATE TABLE IF NOT EXISTS t_uuid (id INT64, col UUID) PRIMARY KEY (id);
134134
CREATE TABLE IF NOT EXISTS t_uuid_to_bytes (id INT64, col BYTES(MAX)) PRIMARY KEY (id);
135135
CREATE TABLE IF NOT EXISTS t_varbit (id INT64, col BYTES(MAX)) PRIMARY KEY (id);
136136
CREATE TABLE IF NOT EXISTS t_varbit_to_bool_array (id INT64, col ARRAY<BOOL>) PRIMARY KEY (id);
137137
CREATE TABLE IF NOT EXISTS t_varbit_to_string (id INT64, col STRING(MAX)) PRIMARY KEY (id);
138138
CREATE TABLE IF NOT EXISTS t_varchar (id INT64, col STRING(MAX)) PRIMARY KEY (id);
139139
CREATE TABLE IF NOT EXISTS t_xml (id INT64, col STRING(MAX)) PRIMARY KEY (id);
140+
141+
CREATE TABLE IF NOT EXISTS t_uuid_pk (id UUID NOT NULL, col UUID) PRIMARY KEY (id);

‎v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesIT.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -474,6 +474,13 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
474474
"-2.2250738585072014E-308"));
475475
expectedData.put(
476476
"time_pk", createRows("time_pk", "15:50:00.200000", "838:59:58.123000", "-838:59:59"));
477+
expectedData.put("uuid", createRows("uuid", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "NULL"));
478+
expectedData.put(
479+
"uuid_pk",
480+
createRows(
481+
"uuid_pk",
482+
"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11",
483+
"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12"));
477484
return expectedData;
478485
}
479486

‎v2/sourcedb-to-spanner/src/test/java/com/google/cloud/teleport/v2/templates/MySQLDataTypesPGDialectIT.java‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -142,7 +142,12 @@ private List<Map<String, Object>> createRows(String colPrefix, Object... values)
142142
List<Map<String, Object>> rows = new ArrayList<>();
143143
for (int i = 0; i < vals.size(); i++) {
144144
Map<String, Object> row = new HashMap<>();
145-
row.put("id", i + 1);
145+
// We specifically want to test primary key partitioning.
146+
if (colPrefix.toLowerCase().contains("_pk")) {
147+
row.put("id", vals.get(i));
148+
} else {
149+
row.put("id", i + 1);
150+
}
146151
row.put(String.format("%s_col", colPrefix), vals.get(i));
147152
rows.add(row);
148153
}
@@ -400,6 +405,13 @@ private Map<String, List<Map<String, Object>>> getExpectedData() {
400405
expectedData.put("set", createRows("set", "v1,v2", "NULL"));
401406
expectedData.put(
402407
"integer_unsigned", createRows("integer_unsigned", "0", "42", "4294967295", "NULL"));
408+
expectedData.put("uuid", createRows("uuid", "a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11", "NULL"));
409+
expectedData.put(
410+
"uuid_pk",
411+
createRows(
412+
"uuid_pk",
413+
"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a11",
414+
"a0eebc99-9c0b-4ef8-bb6d-6bb9bd380a12"));
403415
return expectedData;
404416
}
405417

0 commit comments

Comments
 (0)