From df64e6c54409f61204cc2a61240c71b4838764be Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Wed, 17 Jun 2026 16:54:05 +0200 Subject: [PATCH 1/7] Introduce multiple datasources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change enables multiple datasources in Polaris, with possibility of **runtime** activation of the desired datasource. This is achieved by: 1) Introducing a new enabler property: `polaris.persistence.relational.jdbc.datasource`, which points to a named datasource to activate. 2) Declaring, **at build time**, one named datasource for each JDBC driver shipped with the application. 3) Introducing a new `DataSourceActivator` config interceptor that will, at runtime, change the `active` setting of the target datasource to `true`, and all others to `false` (`active` is a runtime setting, contrary to `db-kind`). This change also makes the server ship with the H2 driver by default, since that's the pre-condition for an H2 datasource to be selected at runtime. This change will further enable **as future improvements**: - Using to JDBC + H2 by default for server images, while still allowing users to switch to PostgreSQL *without having to rebuild Polaris*. - Enabling the persistence to "plug" into not one, but many datasources, e.g. a specific datasource for main persistence, and a separate datasource for metrics. This was outlined in https://github.com/apache/polaris/pull/3960 – but the machinery to make that happen was missing. --- helm/polaris/templates/_helpers.tpl | 6 +- helm/polaris/templates/configmap.yaml | 3 + helm/polaris/tests/configmap_test.yaml | 3 +- .../tests/cronjob_maintenance_test.yaml | 6 +- helm/polaris/tests/deployment_test.yaml | 38 +++++- helm/polaris/values.schema.json | 4 + helm/polaris/values.yaml | 6 + .../jdbc/JdbcMetaStoreManagerFactory.java | 10 -- .../jdbc/RelationalJdbcConfiguration.java | 9 +- ...lationalJdbcProductionReadinessChecks.java | 2 +- .../jdbc/JdbcGrantRecordsIdempotencyTest.java | 5 + .../jdbc/MetricsReportPersistenceTest.java | 5 + ...onalJdbcProductionReadinessChecksTest.java | 3 +- .../SimpleRelationalJdbcConfiguration.java | 6 + ...ationalJdbcIdempotencyStorePostgresIT.java | 5 + runtime/admin/build.gradle.kts | 8 +- .../src/main/resources/application.properties | 29 +++- .../polaris/admintool/AdminProfiles.java | 26 ---- runtime/common/build.gradle.kts | 5 +- .../config/jdbc/DataSourceActivator.java | 129 ++++++++++++++++++ .../common/config/jdbc/JdbcProducers.java | 48 +++++++ .../QuarkusRelationalJdbcConfiguration.java | 29 +++- ...io.smallrye.config.ConfigSourceInterceptor | 20 +++ .../config/jdbc/DataSourceActivatorTest.java | 125 +++++++++++++++++ .../main/resources/application-it.properties | 8 +- .../resources/application-test.properties | 12 +- .../src/main/resources/application.properties | 32 ++++- runtime/server/build.gradle.kts | 3 + runtime/server/distribution/LICENSE | 16 +++ runtime/service/build.gradle.kts | 10 +- .../it/relational/jdbc/JdbcApplicationIT.java | 4 +- .../jdbc/JdbcManagementServiceIT.java | 4 +- .../relational/jdbc/JdbcPolicyServiceIT.java | 4 +- .../it/relational/jdbc/JdbcRestCatalogIT.java | 4 +- .../it/relational/jdbc/JdbcViewFileIT.java | 4 +- .../org/apache/polaris/service/Profiles.java | 85 ++++++------ ...oryBufferEventListenerIntegrationTest.java | 7 +- .../InMemoryBufferEventListenerTestBase.java | 7 +- .../it/PolicyServiceIntegrationTest.java | 2 +- ...oachRelationalJdbcLifeCycleManagement.java | 12 +- .../CockroachRelationalJdbcProfile.java | 6 +- ...gresRelationalJdbcLifeCycleManagement.java | 16 +-- ...ava => PostgresRelationalJdbcProfile.java} | 2 +- .../assets/cloud_providers/deploy-aws.sh | 9 +- site/content/guides/cockroachdb/index.md | 8 +- .../jdbc/docker-compose-bootstrap-db.yml | 7 +- site/content/guides/jdbc/docker-compose.yml | 7 +- site/content/guides/jdbc/index.md | 8 +- site/content/in-dev/unreleased/admin-tool.md | 14 +- ...rye-polaris_persistence_relational_jdbc.md | 9 +- .../in-dev/unreleased/helm-chart/dev.md | 7 +- .../unreleased/helm-chart/persistence.md | 46 ++++++- .../unreleased/helm-chart/production.md | 7 +- .../in-dev/unreleased/helm-chart/reference.md | 1 + .../unreleased/metastores/relational-jdbc.md | 103 +++++++++++--- 55 files changed, 788 insertions(+), 206 deletions(-) create mode 100644 runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/DataSourceActivator.java create mode 100644 runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/JdbcProducers.java create mode 100644 runtime/common/src/main/resources/META-INF/services/io.smallrye.config.ConfigSourceInterceptor create mode 100644 runtime/common/src/test/java/org/apache/polaris/quarkus/common/config/jdbc/DataSourceActivatorTest.java rename runtime/test-common/src/main/java/org/apache/polaris/test/commons/{RelationalJdbcProfile.java => PostgresRelationalJdbcProfile.java} (94%) diff --git a/helm/polaris/templates/_helpers.tpl b/helm/polaris/templates/_helpers.tpl index 7d672d3ed61..1675a259b58 100644 --- a/helm/polaris/templates/_helpers.tpl +++ b/helm/polaris/templates/_helpers.tpl @@ -255,9 +255,9 @@ Prints an environment variable for a secret key reference. Prints database/persistence connection environment variables. */}} {{- define "polaris.persistenceEnv" -}} -{{- include "polaris.secretToEnv" (list .Values.persistence.relationalJdbc.secret "username" "quarkus.datasource.username") -}} -{{- include "polaris.secretToEnv" (list .Values.persistence.relationalJdbc.secret "password" "quarkus.datasource.password") -}} -{{- include "polaris.secretToEnv" (list .Values.persistence.relationalJdbc.secret "jdbcUrl" "quarkus.datasource.jdbc.url") -}} +{{- include "polaris.secretToEnv" (list .Values.persistence.relationalJdbc.secret "username" (printf "quarkus.datasource.%s.username" .Values.persistence.relationalJdbc.datasource)) -}} +{{- include "polaris.secretToEnv" (list .Values.persistence.relationalJdbc.secret "password" (printf "quarkus.datasource.%s.password" .Values.persistence.relationalJdbc.datasource)) -}} +{{- include "polaris.secretToEnv" (list .Values.persistence.relationalJdbc.secret "jdbcUrl" (printf "quarkus.datasource.%s.jdbc.url" .Values.persistence.relationalJdbc.datasource)) -}} {{- include "polaris.secretToEnv" (list .Values.persistence.nosql.secret "connectionString" "quarkus.mongodb.connection-string") -}} {{- end -}} diff --git a/helm/polaris/templates/configmap.yaml b/helm/polaris/templates/configmap.yaml index 75b7dcb76b8..5ef5802813e 100644 --- a/helm/polaris/templates/configmap.yaml +++ b/helm/polaris/templates/configmap.yaml @@ -51,6 +51,9 @@ data: {{- /* Persistence */ -}} {{- $_ = set $map "polaris.persistence.type" .Values.persistence.type -}} + {{- if eq .Values.persistence.type "relational-jdbc" -}} + {{- $_ = set $map "polaris.persistence.relational.jdbc.datasource" .Values.persistence.relationalJdbc.datasource -}} + {{- end -}} {{- if eq .Values.persistence.type "nosql" -}} {{- $_ = set $map "polaris.persistence.nosql.backend" .Values.persistence.nosql.backend -}} {{- $_ = set $map "quarkus.mongodb.database" .Values.persistence.nosql.database -}} diff --git a/helm/polaris/tests/configmap_test.yaml b/helm/polaris/tests/configmap_test.yaml index 5ab960bb60d..8b7f02f0a95 100644 --- a/helm/polaris/tests/configmap_test.yaml +++ b/helm/polaris/tests/configmap_test.yaml @@ -122,9 +122,10 @@ tests: - it: should configure relational-jdbc persistence set: - persistence: { type: "relational-jdbc", relationalJdbc: { secret: { name: "polaris-persistence" } } } + persistence: { type: "relational-jdbc", relationalJdbc: { datasource: "postgresql", secret: { name: "polaris-persistence" } } } asserts: - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.persistence.type=relational-jdbc" } + - matchRegex: { path: 'data["application.properties"]', pattern: "polaris.persistence.relational.jdbc.datasource=postgresql" } - it: should configure nosql persistence with default values set: diff --git a/helm/polaris/tests/cronjob_maintenance_test.yaml b/helm/polaris/tests/cronjob_maintenance_test.yaml index bf93b418f7d..c42ca655ca8 100644 --- a/helm/polaris/tests/cronjob_maintenance_test.yaml +++ b/helm/polaris/tests/cronjob_maintenance_test.yaml @@ -590,7 +590,7 @@ tests: - contains: path: spec.jobTemplate.spec.template.spec.containers[0].env content: - name: quarkus.datasource.username + name: quarkus.datasource.postgresql.username valueFrom: secretKeyRef: name: my-jdbc-secret @@ -598,7 +598,7 @@ tests: - contains: path: spec.jobTemplate.spec.template.spec.containers[0].env content: - name: quarkus.datasource.password + name: quarkus.datasource.postgresql.password valueFrom: secretKeyRef: name: my-jdbc-secret @@ -606,7 +606,7 @@ tests: - contains: path: spec.jobTemplate.spec.template.spec.containers[0].env content: - name: quarkus.datasource.jdbc.url + name: quarkus.datasource.postgresql.jdbc.url valueFrom: secretKeyRef: name: my-jdbc-secret diff --git a/helm/polaris/tests/deployment_test.yaml b/helm/polaris/tests/deployment_test.yaml index 12ef17941cf..5e5644d261f 100644 --- a/helm/polaris/tests/deployment_test.yaml +++ b/helm/polaris/tests/deployment_test.yaml @@ -1470,12 +1470,12 @@ tests: - it: should set relational-jdbc persistence environment variables template: deployment.yaml set: - persistence: { type: "relational-jdbc", relationalJdbc: { secret: { name: "polaris-persistence", username: "username", password: "password", jdbcUrl: "jdbcUrl" } } } + persistence: { type: "relational-jdbc", relationalJdbc: { datasource: "postgresql", secret: { name: "polaris-persistence", username: "username", password: "password", jdbcUrl: "jdbcUrl" } } } asserts: - contains: path: spec.template.spec.containers[0].env content: - name: quarkus.datasource.username + name: quarkus.datasource.postgresql.username valueFrom: secretKeyRef: name: polaris-persistence @@ -1483,7 +1483,7 @@ tests: - contains: path: spec.template.spec.containers[0].env content: - name: quarkus.datasource.password + name: quarkus.datasource.postgresql.password valueFrom: secretKeyRef: name: polaris-persistence @@ -1491,7 +1491,37 @@ tests: - contains: path: spec.template.spec.containers[0].env content: - name: quarkus.datasource.jdbc.url + name: quarkus.datasource.postgresql.jdbc.url + valueFrom: + secretKeyRef: + name: polaris-persistence + key: jdbcUrl + + - it: should set relational-jdbc persistence environment variables for custom datasource name + template: deployment.yaml + set: + persistence: { type: "relational-jdbc", relationalJdbc: { datasource: "mydb", secret: { name: "polaris-persistence", username: "username", password: "password", jdbcUrl: "jdbcUrl" } } } + asserts: + - contains: + path: spec.template.spec.containers[0].env + content: + name: quarkus.datasource.mydb.username + valueFrom: + secretKeyRef: + name: polaris-persistence + key: username + - contains: + path: spec.template.spec.containers[0].env + content: + name: quarkus.datasource.mydb.password + valueFrom: + secretKeyRef: + name: polaris-persistence + key: password + - contains: + path: spec.template.spec.containers[0].env + content: + name: quarkus.datasource.mydb.jdbc.url valueFrom: secretKeyRef: name: polaris-persistence diff --git a/helm/polaris/values.schema.json b/helm/polaris/values.schema.json index db9afad1e53..3f4c8b58d98 100644 --- a/helm/polaris/values.schema.json +++ b/helm/polaris/values.schema.json @@ -1157,6 +1157,10 @@ "relationalJdbc": { "type": "object", "properties": { + "datasource": { + "description": "The name of the Quarkus named datasource to use for relational persistence. Must match one of the built-in named datasources (h2, postgresql), or a custom datasource configured via extraEnv / extraConfig (in this case, you must use a custom Polaris server image including the custom datasource). Polaris will activate this datasource automatically.", + "type": "string" + }, "secret": { "type": "object", "properties": { diff --git a/helm/polaris/values.yaml b/helm/polaris/values.yaml index c539dc3be3f..3a9558644ef 100644 --- a/helm/polaris/values.yaml +++ b/helm/polaris/values.yaml @@ -922,6 +922,12 @@ persistence: type: in-memory # relational-jdbc, nosql # The configuration for the relational-jdbc persistence manager. relationalJdbc: + # -- The name of the Quarkus named datasource to use for relational persistence. + # Must match one of the built-in named datasources (h2, postgresql), or a custom datasource + # configured via extraEnv / extraConfig (in this case, you must use a custom Polaris server + # image including the custom datasource). Polaris will activate this datasource automatically. + # @section -- Persistence + datasource: postgresql # The secret name to pull the database connection properties from. secret: # -- The secret name to pull database connection properties from diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java index 1a4c7b5ba84..64e41115ffd 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/JdbcMetaStoreManagerFactory.java @@ -22,8 +22,6 @@ import io.smallrye.common.annotation.Identifier; import jakarta.enterprise.context.ApplicationScoped; -import jakarta.enterprise.inject.Instance; -import jakarta.enterprise.inject.Produces; import jakarta.inject.Inject; import java.sql.SQLException; import java.time.Clock; @@ -32,7 +30,6 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; -import javax.sql.DataSource; import org.apache.polaris.core.PolarisCallContext; import org.apache.polaris.core.PolarisDiagnostics; import org.apache.polaris.core.config.BehaviorChangeConfiguration; @@ -93,13 +90,6 @@ public class JdbcMetaStoreManagerFactory implements MetaStoreManagerFactory { protected JdbcMetaStoreManagerFactory() {} - @Produces - @ApplicationScoped - static DatasourceOperations produceDatasourceOperations( - Instance dataSource, RelationalJdbcConfiguration relationalJdbcConfiguration) { - return new DatasourceOperations(dataSource.get(), relationalJdbcConfiguration); - } - protected PrincipalSecretsGenerator secretsGenerator( String realmId, @Nullable RootCredentialsSet rootCredentialsSet) { if (rootCredentialsSet != null) { diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java index adfcae5ab1a..d0394c842ce 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcConfiguration.java @@ -21,13 +21,13 @@ import java.util.Optional; public interface RelationalJdbcConfiguration { - // max retries before giving up + /** The maximum number of retries before giving up the operation. */ Optional maxRetries(); - // max retry duration + /** The maximum retry duration in milliseconds. */ Optional maxDurationInMs(); - // initial delay + /** The initial retry delay. */ Optional initialDelayInMs(); /** @@ -35,4 +35,7 @@ public interface RelationalJdbcConfiguration { * the JDBC connection metadata. Supported values: "postgresql", "cockroachdb", "h2" */ Optional databaseType(); + + /** The datasource name to use. Required. */ + String dataSource(); } diff --git a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecks.java b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecks.java index 62c34975d93..366d79543b1 100644 --- a/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecks.java +++ b/persistence/relational-jdbc/src/main/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecks.java @@ -40,7 +40,7 @@ public ProductionReadinessCheck checkRelationalJdbc( return ProductionReadinessCheck.of( ProductionReadinessCheck.Error.of( "The current persistence (jdbc:h2) is intended for tests only.", - "quarkus.datasource.jdbc.url")); + "polaris.persistence.relational.jdbc.datasource")); } return ProductionReadinessCheck.OK; } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcGrantRecordsIdempotencyTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcGrantRecordsIdempotencyTest.java index 41e9182296c..1f1fd4108f1 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcGrantRecordsIdempotencyTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/JdbcGrantRecordsIdempotencyTest.java @@ -154,5 +154,10 @@ public Optional initialDelayInMs() { public Optional databaseType() { return Optional.of("h2"); } + + @Override + public String dataSource() { + return "h2"; + } } } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/MetricsReportPersistenceTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/MetricsReportPersistenceTest.java index ec016b8b6f4..c08babd4a8a 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/MetricsReportPersistenceTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/MetricsReportPersistenceTest.java @@ -202,5 +202,10 @@ public Optional initialDelayInMs() { public Optional databaseType() { return Optional.empty(); } + + @Override + public String dataSource() { + return "h2"; + } } } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecksTest.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecksTest.java index 3e2656b7f3f..91e0fceeb1d 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecksTest.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/RelationalJdbcProductionReadinessChecksTest.java @@ -72,7 +72,8 @@ void jdbcWithH2ReturnsWarning() { error -> { assertThat(error.message()) .isEqualTo("The current persistence (jdbc:h2) is intended for tests only."); - assertThat(error.offendingProperty()).isEqualTo("quarkus.datasource.jdbc.url"); + assertThat(error.offendingProperty()) + .isEqualTo("polaris.persistence.relational.jdbc.datasource"); assertThat(error.severe()).isFalse(); }); } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SimpleRelationalJdbcConfiguration.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SimpleRelationalJdbcConfiguration.java index bbc28c12d81..8c6a3906053 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SimpleRelationalJdbcConfiguration.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/SimpleRelationalJdbcConfiguration.java @@ -56,4 +56,10 @@ public Optional initialDelayInMs() { public Optional databaseType() { return Optional.of(databaseType); } + + @Override + public String dataSource() { + // The datasource name is irrelevant for tests that do not leverage CDI + return "datasource1"; + } } diff --git a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/idempotency/RelationalJdbcIdempotencyStorePostgresIT.java b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/idempotency/RelationalJdbcIdempotencyStorePostgresIT.java index 8c6f70b26ae..c422558992f 100644 --- a/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/idempotency/RelationalJdbcIdempotencyStorePostgresIT.java +++ b/persistence/relational-jdbc/src/test/java/org/apache/polaris/persistence/relational/jdbc/idempotency/RelationalJdbcIdempotencyStorePostgresIT.java @@ -82,6 +82,11 @@ public Optional initialDelayInMs() { public Optional databaseType() { return Optional.empty(); } + + @Override + public String dataSource() { + return "postgresql"; + } }; DatasourceOperations ops = new DatasourceOperations(dataSource, cfg); try (InputStream is = diff --git a/runtime/admin/build.gradle.kts b/runtime/admin/build.gradle.kts index c4a9985423f..5c33e2489e7 100644 --- a/runtime/admin/build.gradle.kts +++ b/runtime/admin/build.gradle.kts @@ -33,9 +33,13 @@ dependencies { compileOnly("com.fasterxml.jackson.core:jackson-annotations") + implementation(enforcedPlatform(libs.quarkus.bom)) + + // JDBC persistence + backends runtimeOnly(project(":polaris-relational-jdbc")) - runtimeOnly("org.postgresql:postgresql") + runtimeOnly("io.quarkus:quarkus-jdbc-postgresql") + // NoSQL persistence + backends implementation(project(":polaris-persistence-nosql-api")) implementation(project(":polaris-persistence-nosql-maintenance-api")) runtimeOnly(project(":polaris-persistence-nosql-metastore")) @@ -46,8 +50,6 @@ dependencies { runtimeOnly("io.quarkus:quarkus-mongodb-client") - implementation("io.quarkus:quarkus-jdbc-postgresql") - implementation(enforcedPlatform(libs.quarkus.bom)) implementation("io.quarkus:quarkus-picocli") implementation("io.quarkus:quarkus-container-image-docker") diff --git a/runtime/admin/src/main/resources/application.properties b/runtime/admin/src/main/resources/application.properties index 30791027bb7..a4ccca4e8c4 100644 --- a/runtime/admin/src/main/resources/application.properties +++ b/runtime/admin/src/main/resources/application.properties @@ -31,7 +31,12 @@ quarkus.container-image.registry=docker.io quarkus.container-image.group=apache quarkus.container-image.name=polaris-admin-tool quarkus.container-image.additional-tags=latest -quarkus.datasource.db-kind=postgresql + +# Named datasources for relational persistence backends. +# Polaris ships with many built-in datasources; by using named datasources, users can configure +# at runtime which datasources to use for relational persistence backends. +quarkus.datasource.postgresql.db-kind=postgresql + # if set to true it will try to start localstack at build and run time for the local environment # https://docs.quarkiverse.io/quarkus-amazon-services/dev/amazon-rds.html#_configuration_reference for more details quarkus.rds.devservices.enabled=false @@ -41,12 +46,13 @@ quarkus.mongodb.devservices.enabled=false # ---- Runtime Configuration ---- # Below are default values for properties that can be changed in runtime. -# Available types: +# Polaris persistence type. Available types: # - in-memory - InMemoryPolarisMetaStoreManagerFactory # - in-memory-atomic - InMemoryAtomicOperationMetaStoreManagerFactory -# - relational-jdbc - JdbcMetaStoreManagerFactory -# - nosql - NoSQL persistence backend, define the backend type via 'polaris.persistence.nosql.backend' +# - nosql (beta) - NoSQL persistence backend, define the backend type via 'polaris.persistence.nosql.backend' +# - relational-jdbc - JDBC persistence backend, define the JDBC datasource via 'polaris.persistence.relational.jdbc.datasource' and 'quarkus.datasource..*' polaris.persistence.type=relational-jdbc + # Database backend for 'nosql' persistence-type # Available backends: # - InMemory - for testing purposes @@ -55,6 +61,21 @@ polaris.persistence.type=relational-jdbc # See https://quarkus.io/guides/mongodb#configuration-reference for details about these configurations. #polaris.persistence.nosql.backend=InMemory +# Datasource to activate for the 'relational-jdbc' persistence type. +# Built-in supported datasources: +# - postgresql - activates the PostgreSQL datasource (default) +# Configure the necessary PostgreSQL properties starting with 'quarkus.datasource.postgresql.' in this file (see below). +# See https://quarkus.io/guides/datasource#configuration-reference for details about these configurations. +# Please do NOT set quarkus.datasource.active manually, Polaris will set it automatically based on +# this property. +polaris.persistence.relational.jdbc.datasource=postgresql + +# Postgres datasource configuration: +quarkus.datasource.postgresql.devservices.enabled=false +#quarkus.datasource.postgresql.jdbc.url=jdbc:postgresql://localhost:5432/my_database +#quarkus.datasource.postgresql.username= +#quarkus.datasource.postgresql.password= + ## MongoDB version store specific configuration #quarkus.mongodb.database=polaris #quarkus.mongodb.metrics.enabled=true diff --git a/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java b/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java index 3abe2b906cc..986944bad0b 100644 --- a/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java +++ b/runtime/admin/src/testFixtures/java/org/apache/polaris/admintool/AdminProfiles.java @@ -50,18 +50,6 @@ public List testResources() { } public static class RelationalJdbc implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.persistence.type", - "relational-jdbc", - // These two options are required to "trigger" the enablement of JDBC data sources in - // admin-tool tests, but do not harm other tests. - "quarkus.datasource.active", - "true", - "quarkus.datasource.db-kind", - "postgresql"); - } @Override public List testResources() { @@ -71,20 +59,6 @@ public List testResources() { } public static class CockroachJdbc implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return Map.of( - "polaris.persistence.type", - "relational-jdbc", - "polaris.persistence.relational.jdbc.database-type", - "cockroachdb", - // These two options are required to "trigger" the enablement of JDBC data sources in - // admin-tool tests, but do not harm other tests. - "quarkus.datasource.active", - "true", - "quarkus.datasource.db-kind", - "postgresql"); - } @Override public List testResources() { diff --git a/runtime/common/build.gradle.kts b/runtime/common/build.gradle.kts index e930a3e13c8..67e0e56365c 100644 --- a/runtime/common/build.gradle.kts +++ b/runtime/common/build.gradle.kts @@ -23,8 +23,11 @@ plugins { } dependencies { - compileOnly(libs.smallrye.config.core) implementation(project(":polaris-relational-jdbc")) + + implementation(platform(libs.quarkus.bom)) + implementation("io.quarkus:quarkus-agroal") + implementation(platform(libs.quarkus.amazon.services.bom)) implementation("io.quarkiverse.amazonservices:quarkus-amazon-rds") } diff --git a/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/DataSourceActivator.java b/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/DataSourceActivator.java new file mode 100644 index 00000000000..62853ab871b --- /dev/null +++ b/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/DataSourceActivator.java @@ -0,0 +1,129 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.quarkus.common.config.jdbc; + +import io.smallrye.config.ConfigSourceInterceptor; +import io.smallrye.config.ConfigSourceInterceptorContext; +import io.smallrye.config.ConfigValue; +import io.smallrye.config.ConfigValue.ConfigValueBuilder; +import io.smallrye.config.Priorities; +import jakarta.annotation.Priority; + +/** + * Activates a data source based on the current Polaris configuration under {@link + * #POLARIS_DATASOURCE_NAME_PROPERTY}, and deactivates all other data sources. + * + *

If the persistence type is not JDBC, all data sources are deactivated. + */ +@Priority(Priorities.LIBRARY + 400) +public class DataSourceActivator implements ConfigSourceInterceptor { + + /** + * The default ordinal to use for modified {@code quarkus.datasource.*.active} properties. It must + * be higher than the ordinal of the application.properties classpath config source (250) but + * lower than the ordinal of the user-supplied application.properties (260). + */ + private static final int DEFAULT_ORDINAL = 251; + + private static final String POLARIS_PERSISTENCE_TYPE_PROPERTY = "polaris.persistence.type"; + private static final String POLARIS_DATASOURCE_NAME_PROPERTY = + "polaris.persistence.relational.jdbc.datasource"; + + private static final String RELATIONAL_JDBC_PERSISTENCE_TYPE = "relational-jdbc"; + + private String activeDataSourceName; + private String persistenceType; + + @Override + public ConfigValue getValue(ConfigSourceInterceptorContext context, String name) { + ConfigValue value = context.proceed(name); + if (name.startsWith("quarkus.datasource.") && name.endsWith(".active")) { + if (name.equals("quarkus.datasource.*.active")) { + // Wildcard datasource key: return the unmodified value + return value; + } + boolean active; + if (persistenceType(context).equals(RELATIONAL_JDBC_PERSISTENCE_TYPE)) { + if (name.equals("quarkus.datasource.active")) { + // default (unnamed) datasource should never be activated + active = false; + } else { + String dataSourceName = extractDataSourceName(name); + active = dataSourceName.equals(targetDataSourceName(context)); + } + } else { + // all datasources deactivated if persistence type is not relational-jdbc + active = false; + } + if (value == null + || value.getValue() == null + || active != Boolean.parseBoolean(value.getValue())) { + value = newConfigValue(value, Boolean.toString(active)); + } + } + return value; + } + + private ConfigValue newConfigValue(ConfigValue current, String newValue) { + ConfigValueBuilder builder = current == null ? ConfigValue.builder() : current.from(); + int ordinal = current == null ? DEFAULT_ORDINAL : current.getConfigSourceOrdinal() + 1; + return builder.withConfigSourceOrdinal(ordinal).withValue(newValue).build(); + } + + private String extractDataSourceName(String property) { + String dataSourceName = property.substring("quarkus.datasource.".length()); + dataSourceName = dataSourceName.substring(0, dataSourceName.indexOf('.')); + return unquote(dataSourceName); + } + + private synchronized String persistenceType(ConfigSourceInterceptorContext context) { + if (persistenceType == null) { + ConfigValue value = context.proceed(POLARIS_PERSISTENCE_TYPE_PROPERTY); + if (value == null || value.getValue() == null) { + failOnMissingProperty(POLARIS_PERSISTENCE_TYPE_PROPERTY); + } + persistenceType = unquote(value.getValue()); + } + return persistenceType; + } + + private synchronized String targetDataSourceName(ConfigSourceInterceptorContext context) { + if (activeDataSourceName == null) { + ConfigValue value = context.proceed(POLARIS_DATASOURCE_NAME_PROPERTY); + if (value == null || value.getValue() == null) { + failOnMissingProperty(POLARIS_DATASOURCE_NAME_PROPERTY); + } + activeDataSourceName = unquote(value.getValue()); + } + return activeDataSourceName; + } + + private static void failOnMissingProperty(String polarisPersistenceTypeProperty) { + throw new IllegalStateException( + "missing required '%s' property in configuration" + .formatted(polarisPersistenceTypeProperty)); + } + + private static String unquote(String dataSourceName) { + if (dataSourceName.startsWith("\"") && dataSourceName.endsWith("\"")) { + dataSourceName = dataSourceName.substring(1, dataSourceName.length() - 1); + } + return dataSourceName; + } +} diff --git a/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/JdbcProducers.java b/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/JdbcProducers.java new file mode 100644 index 00000000000..bd74ef16c1f --- /dev/null +++ b/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/JdbcProducers.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.polaris.quarkus.common.config.jdbc; + +import io.quarkus.agroal.DataSource.DataSourceLiteral; +import jakarta.enterprise.context.ApplicationScoped; +import jakarta.enterprise.inject.Any; +import jakarta.enterprise.inject.Instance; +import jakarta.enterprise.inject.Produces; +import javax.sql.DataSource; +import org.apache.polaris.persistence.relational.jdbc.DatasourceOperations; + +public class JdbcProducers { + + @Produces + @ApplicationScoped + public DatasourceOperations produceDatasourceOperations( + @Any Instance dataSources, + QuarkusRelationalJdbcConfiguration relationalJdbcConfiguration) { + String dataSourceName = relationalJdbcConfiguration.dataSource(); + Instance dataSourceInstance = + dataSources.select(new DataSourceLiteral(dataSourceName)); + if (dataSourceInstance.isUnsatisfied()) { + throw new IllegalStateException( + "datasource %s not found; check the 'polaris.persistence.relational.jdbc.datasource' property in your configuration" + .formatted(dataSourceName)); + } + DataSource dataSource = dataSourceInstance.get(); + return new DatasourceOperations(dataSource, relationalJdbcConfiguration); + } +} diff --git a/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java b/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java index 7eba6eaad74..1a33cf12d13 100644 --- a/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java +++ b/runtime/common/src/main/java/org/apache/polaris/quarkus/common/config/jdbc/QuarkusRelationalJdbcConfiguration.java @@ -19,7 +19,34 @@ package org.apache.polaris.quarkus.common.config.jdbc; import io.smallrye.config.ConfigMapping; +import io.smallrye.config.WithName; +import java.util.Optional; import org.apache.polaris.persistence.relational.jdbc.RelationalJdbcConfiguration; @ConfigMapping(prefix = "polaris.persistence.relational.jdbc") -public interface QuarkusRelationalJdbcConfiguration extends RelationalJdbcConfiguration {} +public interface QuarkusRelationalJdbcConfiguration extends RelationalJdbcConfiguration { + + /** The maximum number of retries before giving up the operation. */ + @Override + Optional maxRetries(); + + /** The maximum retry duration in milliseconds. */ + @Override + Optional maxDurationInMs(); + + /** The initial retry delay. */ + @Override + Optional initialDelayInMs(); + + /** + * Explicitly configured database type. If not specified, the database type will be inferred from + * the JDBC connection metadata. Supported values: "postgresql", "cockroachdb", "h2" + */ + @Override + Optional databaseType(); + + /** The datasource name to use. Required. */ + @Override + @WithName("datasource") + String dataSource(); +} diff --git a/runtime/common/src/main/resources/META-INF/services/io.smallrye.config.ConfigSourceInterceptor b/runtime/common/src/main/resources/META-INF/services/io.smallrye.config.ConfigSourceInterceptor new file mode 100644 index 00000000000..d1ca142a692 --- /dev/null +++ b/runtime/common/src/main/resources/META-INF/services/io.smallrye.config.ConfigSourceInterceptor @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.quarkus.common.config.jdbc.DataSourceActivator \ No newline at end of file diff --git a/runtime/common/src/test/java/org/apache/polaris/quarkus/common/config/jdbc/DataSourceActivatorTest.java b/runtime/common/src/test/java/org/apache/polaris/quarkus/common/config/jdbc/DataSourceActivatorTest.java new file mode 100644 index 00000000000..2c42b2a70f0 --- /dev/null +++ b/runtime/common/src/test/java/org/apache/polaris/quarkus/common/config/jdbc/DataSourceActivatorTest.java @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.quarkus.common.config.jdbc; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import io.smallrye.config.PropertiesConfigSource; +import io.smallrye.config.SmallRyeConfig; +import io.smallrye.config.SmallRyeConfigBuilder; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class DataSourceActivatorTest { + + @Test + void datasourceActivation() { + SmallRyeConfig config = + buildConfig( + Map.of( + "polaris.persistence.type", "relational-jdbc", + "polaris.persistence.relational.jdbc.datasource", "mydb", + "quarkus.datasource.mydb.active", "false", + "quarkus.datasource.ds1.active", "true", + "quarkus.datasource.ds2.active", "true", + "quarkus.datasource.active", "true")); + + assertThat(isActive(config, "quarkus.datasource.mydb.active")).isTrue(); + assertThat(isActive(config, "quarkus.datasource.ds1.active")).isFalse(); + assertThat(isActive(config, "quarkus.datasource.ds2.active")).isFalse(); + assertThat(isActive(config, "quarkus.datasource.active")).isFalse(); + } + + @Test + void nonJdbcPersistenceType() { + SmallRyeConfig config = + buildConfig( + Map.of( + "polaris.persistence.type", "in-memory", + "quarkus.datasource.mydb.active", "true", + "quarkus.datasource.ds1.active", "true", + "quarkus.datasource.active", "true")); + + assertThat(isActive(config, "quarkus.datasource.mydb.active")).isFalse(); + assertThat(isActive(config, "quarkus.datasource.ds1.active")).isFalse(); + assertThat(isActive(config, "quarkus.datasource.active")).isFalse(); + } + + @Test + void missingPersistenceType() { + SmallRyeConfig config = buildConfig(Map.of("quarkus.datasource.mydb.active", "false")); + + assertThatThrownBy(() -> config.getConfigValue("quarkus.datasource.mydb.active").getValue()) + .hasMessage("missing required 'polaris.persistence.type' property in configuration"); + } + + @Test + void missingTargetDatasourceName() { + SmallRyeConfig config = + buildConfig( + Map.of( + "polaris.persistence.type", + "relational-jdbc", + "quarkus.datasource.mydb.active", + "false")); + + assertThatThrownBy(() -> config.getConfigValue("quarkus.datasource.mydb.active").getValue()) + .hasMessage( + "missing required 'polaris.persistence.relational.jdbc.datasource' property in configuration"); + } + + @Test + void wildcardActiveProperty() { + SmallRyeConfig config = + buildConfig( + Map.of( + "polaris.persistence.type", "relational-jdbc", + "polaris.persistence.relational.jdbc.datasource", "mydb", + "quarkus.datasource.*.active", "true")); + + // wildcard property should pass through unchanged + assertThat(config.getConfigValue("quarkus.datasource.*.active").getValue()).isEqualTo("true"); + } + + @Test + void quotedDatasourceNames() { + SmallRyeConfig config = + buildConfig( + Map.of( + "polaris.persistence.type", "\"relational-jdbc\"", + "polaris.persistence.relational.jdbc.datasource", "\"my-db\"", + "quarkus.datasource.\"my-db\".active", "false", + "quarkus.datasource.\"other\".active", "true")); + + assertThat(isActive(config, "quarkus.datasource.my-db.active")).isTrue(); + assertThat(isActive(config, "quarkus.datasource.other.active")).isFalse(); + } + + private static SmallRyeConfig buildConfig(Map properties) { + return new SmallRyeConfigBuilder() + .withInterceptors(new DataSourceActivator()) + .withSources(new PropertiesConfigSource(properties, "test", 100)) + .build(); + } + + private static boolean isActive(SmallRyeConfig config, String property) { + return Boolean.parseBoolean(config.getConfigValue(property).getValue()); + } +} diff --git a/runtime/defaults/src/main/resources/application-it.properties b/runtime/defaults/src/main/resources/application-it.properties index df2d9014632..592c60eb217 100644 --- a/runtime/defaults/src/main/resources/application-it.properties +++ b/runtime/defaults/src/main/resources/application-it.properties @@ -18,6 +18,7 @@ # # Configuration common to ALL integration tests (executed with "it" profile – Gradle "intTest" tasks). +# Note: build-time properties CANNOT be set here! Integration tests always run with the default build-time configuration. # Note: Quarkus integration tests cannot use QuarkusTestProfile. quarkus.http.limits.max-body-size=1000000 @@ -27,10 +28,13 @@ quarkus.http.limits.max-body-size=1000000 quarkus.http.port=0 quarkus.management.port=0 -# polaris.persistence.type=jdbc -# polaris.persistence.type=in-memory-atomic +# Use in-memory persistence backend for integration tests by default. polaris.persistence.type=in-memory +# Use PostgreSQL as the default persistence backend for integration tests +# (note: the test must be configured to use the relational-jdbc persistence type). +polaris.persistence.relational.jdbc.datasource=postgresql + polaris.secrets-manager.type=in-memory polaris.features."ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING"=false diff --git a/runtime/defaults/src/main/resources/application-test.properties b/runtime/defaults/src/main/resources/application-test.properties index c8d2d1672db..43844af2e1f 100644 --- a/runtime/defaults/src/main/resources/application-test.properties +++ b/runtime/defaults/src/main/resources/application-test.properties @@ -20,12 +20,20 @@ # Configuration common to ALL unit tests (executed with "test" profile – Gradle "test" tasks). # Per-test specific configuration should use QuarkusTestProfile +# Use H2 as the default persistence backend for unit tests +# (note: the test must be configured to use relational-jdbc persistence type). +polaris.persistence.relational.jdbc.datasource=h2 +# Use in-memory persistence backend for unit tests by default. +polaris.persistence.type=in-memory +quarkus.datasource.active=false +quarkus.datasource.devservices.enabled=false +quarkus.datasource.postgresql.devservices.enabled=false +quarkus.datasource.h2.devservices.enabled=false + # For unit tests, the default HTTP and management *test-ports* (8081 and 9001 respectively) must be # overridden to zero, so that Quarkus will automatically select an available port. quarkus.http.test-port=0 -quarkus.management.test-port=0 -quarkus.datasource.devservices.enabled=false quarkus.keycloak.devservices.enabled=false quarkus.mongodb.devservices.enabled=false diff --git a/runtime/defaults/src/main/resources/application.properties b/runtime/defaults/src/main/resources/application.properties index 63d6d48db76..ce0235717d5 100644 --- a/runtime/defaults/src/main/resources/application.properties +++ b/runtime/defaults/src/main/resources/application.properties @@ -49,7 +49,12 @@ quarkus.oidc.enabled=true quarkus.otel.enabled=true #quarkus.mongodb.metrics.enabled=true #quarkus.mongodb.connection-string=mongodb://localhost:27017 -quarkus.datasource.db-kind=postgresql + +# Named datasources for relational persistence backends. +# Polaris ships with many built-in datasources; by using named datasources, users can configure +# at runtime which datasources to use for relational persistence backends. +quarkus.datasource.h2.db-kind=h2 +quarkus.datasource.postgresql.db-kind=postgresql # ---- Runtime Configuration ---- # Below are default values for properties that can be changed in runtime. @@ -136,12 +141,13 @@ polaris.features."SUPPORTED_CATALOG_CONNECTION_TYPES"=["ICEBERG_REST"] # realm overrides # polaris.features.realm-overrides."my-realm"."SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION"=true -# Available types: +# Polaris persistence type. Available types: # - in-memory - InMemoryPolarisMetaStoreManagerFactory # - in-memory-atomic - InMemoryAtomicOperationMetaStoreManagerFactory # - nosql (beta) - NoSQL persistence backend, define the backend type via 'polaris.persistence.nosql.backend' -# - relational-jdbc +# - relational-jdbc - JDBC persistence backend, define the JDBC datasource via 'polaris.persistence.relational.jdbc.datasource' and 'quarkus.datasource..*' polaris.persistence.type=in-memory + # Database backend for 'nosql' persistence-type # Available backends: # - InMemory - for testing purposes @@ -150,6 +156,26 @@ polaris.persistence.type=in-memory # See https://quarkus.io/guides/mongodb#configuration-reference for details about these configurations. #polaris.persistence.nosql.backend=InMemory +# Datasource to activate for the 'relational-jdbc' persistence type. +# Built-in supported datasources: +# - h2 - activates the in-memory H2 datasource +# Configure the necessary H2 properties starting with 'quarkus.datasource.h2.' in this file (see below). +# Note: H2 is an in-memory database and is not suitable for production use! +# - postgresql - activates the PostgreSQL datasource (default) +# Configure the necessary PostgreSQL properties starting with 'quarkus.datasource.postgresql.' in this file (see below). +# See https://quarkus.io/guides/datasource#configuration-reference for details about these configurations. +# Please do NOT set quarkus.datasource.active manually, Polaris will set it automatically based on +# this property. +polaris.persistence.relational.jdbc.datasource=postgresql + +# H2 datasource configuration: +quarkus.datasource.h2.jdbc.url=jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE + +# Postgres datasource configuration: +#quarkus.datasource.postgresql.jdbc.url=jdbc:postgresql://localhost:5432/my_database +#quarkus.datasource.postgresql.username= +#quarkus.datasource.postgresql.password= + polaris.secrets-manager.type=in-memory # if set to true it will try to start localstack at build and run time for the local environment # https://docs.quarkiverse.io/quarkus-amazon-services/dev/amazon-rds.html#_configuration_reference for more details diff --git a/runtime/server/build.gradle.kts b/runtime/server/build.gradle.kts index 14f3524d9f2..81f7a0b1706 100644 --- a/runtime/server/build.gradle.kts +++ b/runtime/server/build.gradle.kts @@ -38,6 +38,9 @@ dependencies { runtimeOnly("org.postgresql:postgresql") runtimeOnly(project(":polaris-relational-jdbc")) runtimeOnly("io.quarkus:quarkus-jdbc-postgresql") + runtimeOnly("io.quarkus:quarkus-jdbc-h2") { + exclude(group = "org.locationtech.jts", module = "jts-core") + } runtimeOnly(project(":polaris-extensions-federation-hadoop")) runtimeOnly(project(":polaris-extensions-auth-opa")) runtimeOnly(project(":polaris-extensions-auth-ranger")) diff --git a/runtime/server/distribution/LICENSE b/runtime/server/distribution/LICENSE index b1668534103..1913b65fd68 100644 --- a/runtime/server/distribution/LICENSE +++ b/runtime/server/distribution/LICENSE @@ -309,6 +309,7 @@ This product bundles Quarkus. * Maven group:artifact IDs: io.quarkus:quarkus-grpc-common * Maven group:artifact IDs: io.quarkus:quarkus-hibernate-validator * Maven group:artifact IDs: io.quarkus:quarkus-jackson +* Maven group:artifact IDs: io.quarkus:quarkus-jdbc-h2 * Maven group:artifact IDs: io.quarkus:quarkus-jdbc-postgresql * Maven group:artifact IDs: io.quarkus:quarkus-jsonp * Maven group:artifact IDs: io.quarkus:quarkus-logging-json @@ -1783,6 +1784,21 @@ License: BSD 2-Clause -------------------------------------------------------------------------------- +This product bundles H2 JDBC Driver. + +* Maven group:artifact IDs: com.h2database:h2 + +Project URL: https://www.h2database.com/ +License: Eclipse Public License 1.0 - https://www.eclipse.org/org/documents/epl-1.0/ +| This software contains unmodified binary redistributions for +| H2 database engine (https://h2database.com/), +| which is dual licensed and available under the MPL 2.0 +| (Mozilla Public License) or under the EPL 1.0 (Eclipse Public License). +| An original copy of the license agreement can be found at: +| https://h2database.com/html/license.html + +-------------------------------------------------------------------------------- + This product bundles PostgreSQL JDBC Driver. * Maven group:artifact IDs: org.postgresql:postgresql diff --git a/runtime/service/build.gradle.kts b/runtime/service/build.gradle.kts index d19b3d8bd3a..b8007fd44d5 100644 --- a/runtime/service/build.gradle.kts +++ b/runtime/service/build.gradle.kts @@ -65,7 +65,12 @@ dependencies { implementation("io.quarkus:quarkus-security") implementation("io.quarkus:quarkus-smallrye-context-propagation") implementation("io.quarkus:quarkus-smallrye-fault-tolerance") + + // JDBC backends runtimeOnly("io.quarkus:quarkus-jdbc-postgresql") + runtimeOnly("io.quarkus:quarkus-jdbc-h2") { + exclude(group = "org.locationtech.jts", module = "jts-core") + } implementation(libs.jakarta.enterprise.cdi.api) implementation(libs.jakarta.inject.api) @@ -143,7 +148,10 @@ dependencies { testImplementation("io.quarkus:quarkus-junit-mockito") testImplementation("io.quarkus:quarkus-rest-client") testImplementation("io.quarkus:quarkus-rest-client-jackson") - testImplementation("io.quarkus:quarkus-jdbc-h2") + testImplementation("io.quarkus:quarkus-agroal") + testImplementation("io.quarkus:quarkus-jdbc-h2") { + exclude(group = "org.locationtech.jts", module = "jts-core") + } testImplementation("io.opentelemetry:opentelemetry-sdk-testing") diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcApplicationIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcApplicationIT.java index fe9b8eaf30f..2b6d088d985 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcApplicationIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcApplicationIT.java @@ -21,8 +21,8 @@ import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisApplicationIntegrationTest; -import org.apache.polaris.test.commons.RelationalJdbcProfile; +import org.apache.polaris.test.commons.PostgresRelationalJdbcProfile; -@TestProfile(RelationalJdbcProfile.class) +@TestProfile(PostgresRelationalJdbcProfile.class) @QuarkusIntegrationTest public class JdbcApplicationIT extends PolarisApplicationIntegrationTest {} diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcManagementServiceIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcManagementServiceIT.java index 045c0c20cac..ac0774a150c 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcManagementServiceIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcManagementServiceIT.java @@ -21,8 +21,8 @@ import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisManagementServiceIntegrationTest; -import org.apache.polaris.test.commons.RelationalJdbcProfile; +import org.apache.polaris.test.commons.PostgresRelationalJdbcProfile; -@TestProfile(RelationalJdbcProfile.class) +@TestProfile(PostgresRelationalJdbcProfile.class) @QuarkusIntegrationTest public class JdbcManagementServiceIT extends PolarisManagementServiceIntegrationTest {} diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcPolicyServiceIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcPolicyServiceIT.java index c2dc16568c6..cfa12defc17 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcPolicyServiceIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcPolicyServiceIT.java @@ -21,8 +21,8 @@ import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisPolicyServiceIntegrationTest; -import org.apache.polaris.test.commons.RelationalJdbcProfile; +import org.apache.polaris.test.commons.PostgresRelationalJdbcProfile; -@TestProfile(RelationalJdbcProfile.class) +@TestProfile(PostgresRelationalJdbcProfile.class) @QuarkusIntegrationTest public class JdbcPolicyServiceIT extends PolarisPolicyServiceIntegrationTest {} diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcRestCatalogIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcRestCatalogIT.java index b3ee80ae8c1..6d71efcc6fe 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcRestCatalogIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcRestCatalogIT.java @@ -21,8 +21,8 @@ import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisRestCatalogFileIntegrationTest; -import org.apache.polaris.test.commons.RelationalJdbcProfile; +import org.apache.polaris.test.commons.PostgresRelationalJdbcProfile; -@TestProfile(RelationalJdbcProfile.class) +@TestProfile(PostgresRelationalJdbcProfile.class) @QuarkusIntegrationTest public class JdbcRestCatalogIT extends PolarisRestCatalogFileIntegrationTest {} diff --git a/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcViewFileIT.java b/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcViewFileIT.java index 30b9303fd49..8311f7f5fde 100644 --- a/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcViewFileIT.java +++ b/runtime/service/src/intTest/java/org/apache/polaris/service/it/relational/jdbc/JdbcViewFileIT.java @@ -21,8 +21,8 @@ import io.quarkus.test.junit.QuarkusIntegrationTest; import io.quarkus.test.junit.TestProfile; import org.apache.polaris.service.it.test.PolarisRestCatalogViewFileIntegrationTestBase; -import org.apache.polaris.test.commons.RelationalJdbcProfile; +import org.apache.polaris.test.commons.PostgresRelationalJdbcProfile; -@TestProfile(RelationalJdbcProfile.class) +@TestProfile(PostgresRelationalJdbcProfile.class) @QuarkusIntegrationTest public class JdbcViewFileIT extends PolarisRestCatalogViewFileIntegrationTestBase {} diff --git a/runtime/service/src/test/java/org/apache/polaris/service/Profiles.java b/runtime/service/src/test/java/org/apache/polaris/service/Profiles.java index 5bda6cf6150..a38b21b3004 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/Profiles.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/Profiles.java @@ -43,6 +43,13 @@ private Profiles() {} "polaris.readiness.ignore-severe-issues", "true"); + // TODO make JDBC+H2 the default persistence type for unit tests + public static final Map RELATIONAL_JDBC_H2_PERSISTENCE = + Map.of( + "polaris.persistence.type", "relational-jdbc", + "polaris.persistence.auto-bootstrap-types", "relational-jdbc", + "polaris.persistence.relational.jdbc.datasource", "h2"); + public static class DefaultProfile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { @@ -51,10 +58,12 @@ public Map getConfigOverrides() { } public static class ApplicationIntegrationProfile extends DefaultProfile { + @Override public Map getConfigOverrides() { return ImmutableMap.builder() .putAll(super.getConfigOverrides()) + .putAll(RELATIONAL_JDBC_H2_PERSISTENCE) .put("quarkus.http.limits.max-body-size", "1000000") .put("polaris.realm-context.realms", "POLARIS,OTHER") .put("polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true") @@ -66,15 +75,15 @@ public Map getConfigOverrides() { public static class ManagementIntegrationProfile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { - return Map.of( - "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", - "true", - "polaris.features.\"ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING\"", - "true", - "polaris.storage.gcp.token", - "token", - "polaris.storage.gcp.lifespan", - "PT1H"); + return ImmutableMap.builder() + .putAll(RELATIONAL_JDBC_H2_PERSISTENCE) + .put("polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true") + .put( + "polaris.features.\"ENFORCE_PRINCIPAL_CREDENTIAL_ROTATION_REQUIRED_CHECKING\"", + "true") + .put("polaris.storage.gcp.token", "token") + .put("polaris.storage.gcp.lifespan", "PT1H") + .build(); } } @@ -83,6 +92,7 @@ public static class RestCatalogFileIntegrationProfile extends DefaultProfile { public Map getConfigOverrides() { return ImmutableMap.builder() .putAll(super.getConfigOverrides()) + .putAll(RELATIONAL_JDBC_H2_PERSISTENCE) .put("polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", "false") .put("polaris.features.\"ALLOW_NAMESPACE_CUSTOM_LOCATION\"", "true") .build(); @@ -92,13 +102,12 @@ public Map getConfigOverrides() { public static class RestCatalogViewFileIntegrationProfile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { - return Map.of( - "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", - "true", - "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", - "[\"FILE\"]", - "polaris.readiness.ignore-severe-issues", - "true"); + return ImmutableMap.builder() + .putAll(RELATIONAL_JDBC_H2_PERSISTENCE) + .put("polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true") + .put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\"]") + .put("polaris.readiness.ignore-severe-issues", "true") + .build(); } } @@ -171,13 +180,6 @@ public Map getConfigOverrides() { static final Map IN_MEMORY_BUFFER_EVENT_LISTENER_BASE_CONFIG = ImmutableMap.builder() - .put("polaris.realm-context.realms", "test1,test2") - .put("polaris.persistence.type", "relational-jdbc") - .put("polaris.persistence.auto-bootstrap-types", "relational-jdbc") - .put("quarkus.datasource.db-kind", "h2") - .put( - "quarkus.datasource.jdbc.url", - "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE") .put("polaris.event-listener.type", "persistence-in-memory-buffer") .put( "quarkus.fault-tolerance.\"org.apache.polaris.service.events.listeners.inmemory.InMemoryBufferEventListener/flush\".retry.max-retries", @@ -192,6 +194,8 @@ public static class InMemoryBufferEventListenerBufferSizeProfile implements Quar public Map getConfigOverrides() { return ImmutableMap.builder() .putAll(IN_MEMORY_BUFFER_EVENT_LISTENER_BASE_CONFIG) + .putAll(RELATIONAL_JDBC_H2_PERSISTENCE) + .put("polaris.realm-context.realms", "test1,test2") .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "60s") .put("polaris.event-listener.persistence-in-memory-buffer.max-buffer-size", "10") .build(); @@ -203,12 +207,27 @@ public static class InMemoryBufferEventListenerBufferTimeProfile implements Quar public Map getConfigOverrides() { return ImmutableMap.builder() .putAll(IN_MEMORY_BUFFER_EVENT_LISTENER_BASE_CONFIG) + .putAll(RELATIONAL_JDBC_H2_PERSISTENCE) + .put("polaris.realm-context.realms", "test1,test2") .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "100ms") .put("polaris.event-listener.persistence-in-memory-buffer.max-buffer-size", "1000") .build(); } } + public static class InMemoryBufferEventListenerIntegrationProfile extends DefaultProfile { + @Override + public Map getConfigOverrides() { + return ImmutableMap.builder() + .putAll(super.getConfigOverrides()) + .putAll(IN_MEMORY_BUFFER_EVENT_LISTENER_BASE_CONFIG) + .putAll(RELATIONAL_JDBC_H2_PERSISTENCE) + .put("quarkus.otel.sdk.disabled", "false") + .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "100ms") + .buildKeepingLast(); + } + } + public static class RealmIdTagEnabledMetricsProfile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { @@ -279,24 +298,4 @@ public Map getConfigOverrides() { .build(); } } - - public static class InMemoryBufferEventListenerIntegrationProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .put("polaris.persistence.type", "relational-jdbc") - .put("polaris.persistence.auto-bootstrap-types", "relational-jdbc") - .put("quarkus.datasource.db-kind", "h2") - .put("quarkus.otel.sdk.disabled", "false") - .put( - "quarkus.datasource.jdbc.url", - "jdbc:h2:mem:test;DB_CLOSE_DELAY=-1;MODE=PostgreSQL;DATABASE_TO_LOWER=TRUE") - .put("polaris.event-listener.type", "persistence-in-memory-buffer") - .put("polaris.event-listener.persistence-in-memory-buffer.buffer-time", "100ms") - .put("polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"", "true") - .put("polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"", "[\"FILE\",\"S3\"]") - .put("polaris.readiness.ignore-severe-issues", "true") - .build(); - } - } } diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java index 6222fdbae8d..6c8f04a08dd 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerIntegrationTest.java @@ -26,6 +26,8 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import io.agroal.api.AgroalDataSource; +import io.quarkus.agroal.DataSource; import io.quarkus.test.junit.QuarkusTest; import io.quarkus.test.junit.TestProfile; import jakarta.enterprise.inject.Instance; @@ -40,7 +42,6 @@ import java.sql.Statement; import java.time.Duration; import java.util.List; -import javax.sql.DataSource; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.SortOrder; @@ -82,7 +83,9 @@ class InMemoryBufferEventListenerIntegrationTest { private String authToken; private URI baseLocation; - @Inject Instance dataSource; + @Inject + @DataSource("h2") + Instance dataSource; @BeforeAll public void setup( diff --git a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerTestBase.java b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerTestBase.java index ba9e0626598..02db9218e41 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerTestBase.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/events/listeners/inmemory/InMemoryBufferEventListenerTestBase.java @@ -24,7 +24,9 @@ import static org.awaitility.Awaitility.await; import static org.mockito.Mockito.reset; +import io.agroal.api.AgroalDataSource; import io.netty.channel.EventLoopGroup; +import io.quarkus.agroal.DataSource; import io.quarkus.netty.MainEventLoopGroup; import io.quarkus.test.junit.mockito.InjectSpy; import io.smallrye.common.annotation.Identifier; @@ -35,7 +37,6 @@ import java.sql.Statement; import java.time.Duration; import java.util.UUID; -import javax.sql.DataSource; import org.apache.polaris.core.entity.EventEntity; import org.apache.polaris.core.persistence.MetaStoreManagerFactory; import org.junit.jupiter.api.AfterEach; @@ -59,7 +60,9 @@ abstract class InMemoryBufferEventListenerTestBase { @SuppressWarnings("CdiInjectionPointsInspection") EventLoopGroup eventLoopGroup; - @Inject Instance dataSource; + @Inject + @DataSource("h2") + Instance dataSource; @BeforeEach void resolveProducer() { diff --git a/runtime/service/src/test/java/org/apache/polaris/service/it/PolicyServiceIntegrationTest.java b/runtime/service/src/test/java/org/apache/polaris/service/it/PolicyServiceIntegrationTest.java index 46914160197..47e369c8bb0 100644 --- a/runtime/service/src/test/java/org/apache/polaris/service/it/PolicyServiceIntegrationTest.java +++ b/runtime/service/src/test/java/org/apache/polaris/service/it/PolicyServiceIntegrationTest.java @@ -24,5 +24,5 @@ import org.apache.polaris.service.it.test.PolarisPolicyServiceIntegrationTest; @QuarkusTest -@TestProfile(Profiles.DefaultProfile.class) +@TestProfile(Profiles.ApplicationIntegrationProfile.class) public class PolicyServiceIntegrationTest extends PolarisPolicyServiceIntegrationTest {} diff --git a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/CockroachRelationalJdbcLifeCycleManagement.java b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/CockroachRelationalJdbcLifeCycleManagement.java index 6a1577c7cae..001e7e50f7a 100644 --- a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/CockroachRelationalJdbcLifeCycleManagement.java +++ b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/CockroachRelationalJdbcLifeCycleManagement.java @@ -61,17 +61,17 @@ public Map start() { "relational-jdbc", "polaris.persistence.relational.jdbc.database-type", "cockroachdb", + "polaris.persistence.relational.jdbc.datasource", + "postgresql", "polaris.persistence.relational.jdbc.max-retries", "2", - "quarkus.datasource.db-kind", - "postgresql", - "quarkus.datasource.jdbc.url", + "quarkus.datasource.postgresql.jdbc.url", cockroach.getJdbcUrl(), - "quarkus.datasource.username", + "quarkus.datasource.postgresql.username", cockroach.getUsername(), - "quarkus.datasource.password", + "quarkus.datasource.postgresql.password", cockroach.getPassword(), - "quarkus.datasource.jdbc.initial-size", + "quarkus.datasource.postgresql.jdbc.initial-size", "10"); } diff --git a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/CockroachRelationalJdbcProfile.java b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/CockroachRelationalJdbcProfile.java index b86a0209b56..1648b65a822 100644 --- a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/CockroachRelationalJdbcProfile.java +++ b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/CockroachRelationalJdbcProfile.java @@ -25,11 +25,7 @@ public class CockroachRelationalJdbcProfile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { - return Map.of( - "polaris.persistence.auto-bootstrap-types", - "relational-jdbc", - "polaris.persistence.relational.jdbc.database-type", - "cockroachdb"); + return Map.of("polaris.persistence.auto-bootstrap-types", "relational-jdbc"); } @Override diff --git a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/PostgresRelationalJdbcLifeCycleManagement.java b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/PostgresRelationalJdbcLifeCycleManagement.java index 75bb868f457..a2657e406b2 100644 --- a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/PostgresRelationalJdbcLifeCycleManagement.java +++ b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/PostgresRelationalJdbcLifeCycleManagement.java @@ -61,16 +61,12 @@ public Map start() { return Map.ofEntries( Map.entry("polaris.persistence.type", "relational-jdbc"), Map.entry("polaris.persistence.relational.jdbc.max-retries", "2"), - Map.entry("quarkus.datasource.db-kind", "postgresql"), - Map.entry("quarkus.datasource.jdbc.url", postgres.getJdbcUrl()), - Map.entry("quarkus.datasource.username", postgres.getUsername()), - Map.entry("quarkus.datasource.password", postgres.getPassword()), - Map.entry("quarkus.datasource.jdbc.initial-size", "10"), - // Configure metrics named datasource to use the same PostgreSQL instance - Map.entry("quarkus.datasource.metrics.db-kind", "postgresql"), - Map.entry("quarkus.datasource.metrics.jdbc.url", postgres.getJdbcUrl()), - Map.entry("quarkus.datasource.metrics.username", postgres.getUsername()), - Map.entry("quarkus.datasource.metrics.password", postgres.getPassword())); + Map.entry("polaris.persistence.relational.jdbc.database-type", "postgresql"), + Map.entry("polaris.persistence.relational.jdbc.datasource", "postgresql"), + Map.entry("quarkus.datasource.postgresql.jdbc.url", postgres.getJdbcUrl()), + Map.entry("quarkus.datasource.postgresql.username", postgres.getUsername()), + Map.entry("quarkus.datasource.postgresql.password", postgres.getPassword()), + Map.entry("quarkus.datasource.postgresql.jdbc.initial-size", "10")); } @Override diff --git a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/RelationalJdbcProfile.java b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/PostgresRelationalJdbcProfile.java similarity index 94% rename from runtime/test-common/src/main/java/org/apache/polaris/test/commons/RelationalJdbcProfile.java rename to runtime/test-common/src/main/java/org/apache/polaris/test/commons/PostgresRelationalJdbcProfile.java index 38f7bd61a03..36412acd416 100644 --- a/runtime/test-common/src/main/java/org/apache/polaris/test/commons/RelationalJdbcProfile.java +++ b/runtime/test-common/src/main/java/org/apache/polaris/test/commons/PostgresRelationalJdbcProfile.java @@ -22,7 +22,7 @@ import java.util.List; import java.util.Map; -public class RelationalJdbcProfile implements QuarkusTestProfile { +public class PostgresRelationalJdbcProfile implements QuarkusTestProfile { @Override public Map getConfigOverrides() { return Map.of("polaris.persistence.auto-bootstrap-types", "relational-jdbc"); diff --git a/site/content/guides/assets/cloud_providers/deploy-aws.sh b/site/content/guides/assets/cloud_providers/deploy-aws.sh index 40e3f1dc642..9601eb4e527 100755 --- a/site/content/guides/assets/cloud_providers/deploy-aws.sh +++ b/site/content/guides/assets/cloud_providers/deploy-aws.sh @@ -78,10 +78,11 @@ do done POSTGRES_ADDR=$(echo $DESCRIBE_DB | jq -r '.["DBInstances"][0]["Endpoint"]' | jq -r '"\(.Address):\(.Port)"') -export QUARKUS_DATASOURCE_JDBC_URL=$(printf '%s' "jdbc:postgresql://$POSTGRES_ADDR/POLARIS") -export QUARKUS_DATASOURCE_USERNAME=postgres -export QUARKUS_DATASOURCE_PASSWORD="$POSTGRES_PASSWORD" -echo $QUARKUS_DATASOURCE_JDBC_URL +export POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATASOURCE=postgresql +export QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL=$(printf '%s' "jdbc:postgresql://$POSTGRES_ADDR/POLARIS") +export QUARKUS_DATASOURCE_POSTGRESQL_USERNAME=postgres +export QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD="$POSTGRES_PASSWORD" +echo $QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL S3_BUCKET_NAME="polaris-quickstart-s3-$RANDOM_SUFFIX" echo "S3 Bucket Name: $S3_BUCKET_NAME" diff --git a/site/content/guides/cockroachdb/index.md b/site/content/guides/cockroachdb/index.md index 13364b47e28..b4d30fe4186 100644 --- a/site/content/guides/cockroachdb/index.md +++ b/site/content/guides/cockroachdb/index.md @@ -49,10 +49,12 @@ This example requires `jq` to be installed on your machine. 2. Start the docker compose group by running the following command from the root of the repository: ```shell + export POLARIS_PERSISTENCE_TYPE=relational-jdbc + export POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATASOURCE=postgresql export POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATABASE_TYPE=cockroachdb - export QUARKUS_DATASOURCE_JDBC_URL=jdbc:postgresql://cockroachdb:26257/polaris?sslmode=disable - export QUARKUS_DATASOURCE_USERNAME=root - export QUARKUS_DATASOURCE_PASSWORD="" + export QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL=jdbc:postgresql://cockroachdb:26257/polaris?sslmode=disable + export QUARKUS_DATASOURCE_POSTGRESQL_USERNAME=root + export QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD="" export ASSETS_PATH=$(pwd)/site/content/guides/assets/ export CLIENT_ID=root export CLIENT_SECRET=s3cr3t diff --git a/site/content/guides/jdbc/docker-compose-bootstrap-db.yml b/site/content/guides/jdbc/docker-compose-bootstrap-db.yml index 924f9dc3a8b..3e1b394428d 100644 --- a/site/content/guides/jdbc/docker-compose-bootstrap-db.yml +++ b/site/content/guides/jdbc/docker-compose-bootstrap-db.yml @@ -22,10 +22,11 @@ services: image: apache/polaris-admin-tool:latest environment: - POLARIS_PERSISTENCE_TYPE=relational-jdbc + - POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATASOURCE=${POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATASOURCE:-postgresql} - POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATABASE_TYPE=${POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATABASE_TYPE:-postgresql} - - QUARKUS_DATASOURCE_JDBC_URL=${QUARKUS_DATASOURCE_JDBC_URL} - - QUARKUS_DATASOURCE_USERNAME=${QUARKUS_DATASOURCE_USERNAME} - - QUARKUS_DATASOURCE_PASSWORD=${QUARKUS_DATASOURCE_PASSWORD} + - QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL=${QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL} + - QUARKUS_DATASOURCE_POSTGRESQL_USERNAME=${QUARKUS_DATASOURCE_POSTGRESQL_USERNAME} + - QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD=${QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD} command: - "bootstrap" - "--realm=POLARIS" diff --git a/site/content/guides/jdbc/docker-compose.yml b/site/content/guides/jdbc/docker-compose.yml index 0ae04277f1e..3aa5a527c8f 100644 --- a/site/content/guides/jdbc/docker-compose.yml +++ b/site/content/guides/jdbc/docker-compose.yml @@ -32,13 +32,14 @@ services: JAVA_DEBUG: true JAVA_DEBUG_PORT: "*:5005" POLARIS_PERSISTENCE_TYPE: relational-jdbc + POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATASOURCE: ${POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATASOURCE:-postgresql} POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATABASE_TYPE: ${POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATABASE_TYPE:-postgresql} POLARIS_PERSISTENCE_RELATIONAL_JDBC_MAX_RETRIES: 5 POLARIS_PERSISTENCE_RELATIONAL_JDBC_INITIAL_DELAY_IN_MS: 100 POLARIS_PERSISTENCE_RELATIONAL_JDBC_MAX_DURATION_IN_MS: 5000 - QUARKUS_DATASOURCE_JDBC_URL: $QUARKUS_DATASOURCE_JDBC_URL - QUARKUS_DATASOURCE_USERNAME: $QUARKUS_DATASOURCE_USERNAME - QUARKUS_DATASOURCE_PASSWORD: $QUARKUS_DATASOURCE_PASSWORD + QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL: $QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL + QUARKUS_DATASOURCE_POSTGRESQL_USERNAME: $QUARKUS_DATASOURCE_POSTGRESQL_USERNAME + QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD: $QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD POLARIS_REALM_CONTEXT_REALMS: POLARIS QUARKUS_OTEL_SDK_DISABLED: true POLARIS_BOOTSTRAP_CREDENTIALS: POLARIS,$CLIENT_ID,$CLIENT_SECRET diff --git a/site/content/guides/jdbc/index.md b/site/content/guides/jdbc/index.md index cab281d3401..035aec8bf62 100644 --- a/site/content/guides/jdbc/index.md +++ b/site/content/guides/jdbc/index.md @@ -49,9 +49,11 @@ This example requires `jq` to be installed on your machine. 2. Start the docker compose group by running the following command from the root of the repository: ```shell - export QUARKUS_DATASOURCE_JDBC_URL=jdbc:postgresql://postgres:5432/POLARIS - export QUARKUS_DATASOURCE_USERNAME=postgres - export QUARKUS_DATASOURCE_PASSWORD=postgres + export POLARIS_PERSISTENCE_TYPE=relational-jdbc + export POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATASOURCE=postgresql + export QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL=jdbc:postgresql://postgres:5432/POLARIS + export QUARKUS_DATASOURCE_POSTGRESQL_USERNAME=postgres + export QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD=postgres export ASSETS_PATH=$(pwd)/site/content/guides/assets/ export CLIENT_ID=root export CLIENT_SECRET=s3cr3t diff --git a/site/content/in-dev/unreleased/admin-tool.md b/site/content/in-dev/unreleased/admin-tool.md index e9c026326d6..096043a6eda 100644 --- a/site/content/in-dev/unreleased/admin-tool.md +++ b/site/content/in-dev/unreleased/admin-tool.md @@ -133,9 +133,10 @@ Example for the PostgreSQL backend: ```bash docker run --rm -it \ --env="polaris.persistence.type=relational-jdbc" \ - --env="quarkus.datasource.username=" \ - --env="quarkus.datasource.password=" \ - --env="quarkus.datasource.jdbc.url=" \ + --env="polaris.persistence.relational.jdbc.datasource=postgresql" \ + --env="quarkus.datasource.postgresql.username=" \ + --env="quarkus.datasource.postgresql.password=" \ + --env="quarkus.datasource.postgresql.jdbc.url=" \ apache/polaris-admin-tool:latest bootstrap -r realm1 -c realm1,admin,admin ``` @@ -193,9 +194,10 @@ Example for the PostgreSQL backend: ```bash docker run --rm -it \ --env="polaris.persistence.type=relational-jdbc" \ - --env="quarkus.datasource.username=" \ - --env="quarkus.datasource.password=" \ - --env="quarkus.datasource.jdbc.url=" \ + --env="polaris.persistence.relational.jdbc.datasource=postgresql" \ + --env="quarkus.datasource.postgresql.username=" \ + --env="quarkus.datasource.postgresql.password=" \ + --env="quarkus.datasource.postgresql.jdbc.url=" \ apache/polaris-admin-tool:latest purge -r realm1 ``` diff --git a/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md b/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md index e50f6865423..5c4eee8dbe7 100644 --- a/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md +++ b/site/content/in-dev/unreleased/configuration/config-sections/smallrye-polaris_persistence_relational_jdbc.md @@ -25,7 +25,8 @@ build: | Property | Default Value | Type | Description | |----------|---------------|------|-------------| -| `polaris.persistence.relational.jdbc.max-retries` | | `int` | | -| `polaris.persistence.relational.jdbc.max-duration-in-ms` | | `long` | | -| `polaris.persistence.relational.jdbc.initial-delay-in-ms` | | `long` | | -| `polaris.persistence.relational.jdbc.database-type` | | `string` | | +| `polaris.persistence.relational.jdbc.max-retries` | | `int` | The maximum number of retries before giving up the operation. | +| `polaris.persistence.relational.jdbc.max-duration-in-ms` | | `long` | The maximum retry duration in milliseconds. | +| `polaris.persistence.relational.jdbc.initial-delay-in-ms` | | `long` | The initial retry delay. | +| `polaris.persistence.relational.jdbc.database-type` | | `string` | Explicitly configured database type. If not specified, the database type will be inferred from the JDBC connection metadata. Supported values: "postgresql", "cockroachdb", "h2" | +| `polaris.persistence.relational.jdbc.datasource` | | `string` | The datasource name to use. Required. | diff --git a/site/content/in-dev/unreleased/helm-chart/dev.md b/site/content/in-dev/unreleased/helm-chart/dev.md index 2f1f669d072..fe5d36a7ab7 100644 --- a/site/content/in-dev/unreleased/helm-chart/dev.md +++ b/site/content/in-dev/unreleased/helm-chart/dev.md @@ -147,9 +147,10 @@ kubectl run polaris-bootstrap \ --restart=Never \ --rm -it \ --env="polaris.persistence.type=relational-jdbc" \ - --env="quarkus.datasource.username=$(kubectl get secret polaris-persistence -n polaris -o jsonpath='{.data.username}' | base64 --decode)" \ - --env="quarkus.datasource.password=$(kubectl get secret polaris-persistence -n polaris -o jsonpath='{.data.password}' | base64 --decode)" \ - --env="quarkus.datasource.jdbc.url=$(kubectl get secret polaris-persistence -n polaris -o jsonpath='{.data.jdbcUrl}' | base64 --decode)" \ + --env="polaris.persistence.relational.jdbc.datasource=postgresql" \ + --env="quarkus.datasource.postgresql.username=$(kubectl get secret polaris-persistence -n polaris -o jsonpath='{.data.username}' | base64 --decode)" \ + --env="quarkus.datasource.postgresql.password=$(kubectl get secret polaris-persistence -n polaris -o jsonpath='{.data.password}' | base64 --decode)" \ + --env="quarkus.datasource.postgresql.jdbc.url=$(kubectl get secret polaris-persistence -n polaris -o jsonpath='{.data.jdbcUrl}' | base64 --decode)" \ -- \ bootstrap -r POLARIS -c POLARIS,root,s3cr3t ``` diff --git a/site/content/in-dev/unreleased/helm-chart/persistence.md b/site/content/in-dev/unreleased/helm-chart/persistence.md index cca8d97c69c..fa1cf94ae9a 100644 --- a/site/content/in-dev/unreleased/helm-chart/persistence.md +++ b/site/content/in-dev/unreleased/helm-chart/persistence.md @@ -84,6 +84,7 @@ Configure the chart to use the PostgreSQL metastore: persistence: type: relational-jdbc relationalJdbc: + datasource: "postgresql" # Named datasource to activate (default: postgresql) secret: name: "polaris-persistence" username: "username" # Key in secret containing the username @@ -91,16 +92,53 @@ persistence: jdbcUrl: "jdbcUrl" # Key in secret containing the JDBC URL ``` +The `datasource` field selects which [named Quarkus datasource](https://quarkus.io/guides/datasource#multiple-datasources) +Polaris activates at runtime. The chart maps the secret keys to +`quarkus.datasource..username`, `quarkus.datasource..password`, and +`quarkus.datasource..jdbc.url` environment variables. Polaris automatically activates the +selected datasource and deactivates all others — **do not set `quarkus.datasource.*.active` manually**. + +Two named datasources are built into the Polaris image: + +| `datasource` value | Database | Use case | +|--------------------|-----------------------------|----------------| +| `postgresql` | PostgreSQL (or CockroachDB) | Production use | +| `h2` | H2 | Testing only | + +For more details on the named datasource model, see the +[Relational JDBC metastore]({{% ref "../metastores/relational-jdbc" %}}) page. + +### CockroachDB + +CockroachDB uses the PostgreSQL wire protocol and JDBC driver, so it works with the built-in +`postgresql` named datasource. You must also set `database-type` to `cockroachdb` via +`advancedConfig` so that Polaris uses the CockroachDB-specific schema DDL: + +```yaml +persistence: + type: relational-jdbc + relationalJdbc: + datasource: postgresql + secret: + name: "polaris-persistence" + username: "username" + password: "password" + jdbcUrl: "jdbcUrl" # e.g. jdbc:postgresql://:26257/?sslmode=disable + +advancedConfig: + polaris.persistence.relational.jdbc.database-type: "cockroachdb" +``` + ### Connection Pool Settings For production deployments, you may want to tune the connection pool settings using the `advancedConfig` section: ```yaml advancedConfig: - quarkus.datasource.jdbc.min-size: "5" - quarkus.datasource.jdbc.max-size: "20" - quarkus.datasource.jdbc.acquisition-timeout: "30S" - quarkus.datasource.jdbc.idle-removal-interval: "5M" + quarkus.datasource.postgresql.jdbc.min-size: "5" + quarkus.datasource.postgresql.jdbc.max-size: "20" + quarkus.datasource.postgresql.jdbc.acquisition-timeout: "30S" + quarkus.datasource.postgresql.jdbc.idle-removal-interval: "5M" ``` See the [Quarkus Datasource configuration reference](https://quarkus.io/guides/datasource#configuration-reference) for all available options. diff --git a/site/content/in-dev/unreleased/helm-chart/production.md b/site/content/in-dev/unreleased/helm-chart/production.md index 8d0e052e561..806a0c3d93e 100644 --- a/site/content/in-dev/unreleased/helm-chart/production.md +++ b/site/content/in-dev/unreleased/helm-chart/production.md @@ -173,9 +173,10 @@ kubectl run polaris-bootstrap \ --restart=Never \ --rm -it \ --env="polaris.persistence.type=relational-jdbc" \ - --env="quarkus.datasource.username=$(kubectl get secret polaris-persistence -n polaris -o jsonpath='{.data.username}' | base64 --decode)" \ - --env="quarkus.datasource.password=$(kubectl get secret polaris-persistence -n polaris -o jsonpath='{.data.password}' | base64 --decode)" \ - --env="quarkus.datasource.jdbc.url=$(kubectl get secret polaris-persistence -n polaris -o jsonpath='{.data.jdbcUrl}' | base64 --decode)" \ + --env="polaris.persistence.relational.jdbc.datasource=postgresql" \ + --env="quarkus.datasource.postgresql.username=$(kubectl get secret polaris-persistence -n polaris -o jsonpath='{.data.username}' | base64 --decode)" \ + --env="quarkus.datasource.postgresql.password=$(kubectl get secret polaris-persistence -n polaris -o jsonpath='{.data.password}' | base64 --decode)" \ + --env="quarkus.datasource.postgresql.jdbc.url=$(kubectl get secret polaris-persistence -n polaris -o jsonpath='{.data.jdbcUrl}' | base64 --decode)" \ -- \ bootstrap -r polaris-realm1 -c polaris-realm1,root,$ROOT_PASSWORD ``` diff --git a/site/content/in-dev/unreleased/helm-chart/reference.md b/site/content/in-dev/unreleased/helm-chart/reference.md index c6366f73347..330c217602e 100644 --- a/site/content/in-dev/unreleased/helm-chart/reference.md +++ b/site/content/in-dev/unreleased/helm-chart/reference.md @@ -265,6 +265,7 @@ weight: 900 | Key | Type | Default | Description | |-----|------|---------|-------------| | persistence.type | string | `"in-memory"` | The type of persistence to use. Three built-in types are supported: in-memory, relational-jdbc, and nosql (beta). | +| persistence.relationalJdbc.datasource | string | `"postgresql"` | The name of the Quarkus named datasource to use for relational persistence. Must match one of the built-in named datasources (h2, postgresql), or a custom datasource configured via extraEnv / extraConfig (in this case, you must use a custom Polaris server image including the custom datasource). Polaris will activate this datasource automatically. | | persistence.relationalJdbc.secret.name | string | `""` | The secret name to pull database connection properties from | | persistence.relationalJdbc.secret.username | string | `"username"` | The secret key holding the database username for authentication | | persistence.relationalJdbc.secret.password | string | `"password"` | The secret key holding the database password for authentication | diff --git a/site/content/in-dev/unreleased/metastores/relational-jdbc.md b/site/content/in-dev/unreleased/metastores/relational-jdbc.md index 037cb0fafed..d51d5656564 100644 --- a/site/content/in-dev/unreleased/metastores/relational-jdbc.md +++ b/site/content/in-dev/unreleased/metastores/relational-jdbc.md @@ -25,7 +25,62 @@ weight: 100 This implementation leverages Quarkus for datasource management and supports configuration through environment variables or JVM -D flags at startup. For more information, refer to the [Quarkus configuration reference](https://quarkus.io/guides/config-reference#env-file). -We have 2 options for configuring the persistence backend: +## Named Datasources and Runtime Activation + +Polaris uses Quarkus [named datasources](https://quarkus.io/guides/datasource#multiple-datasources) to +allow selecting which database connection to use at runtime without rebuilding the application. + +Two named datasources are built into the Polaris distribution: + +| Name | Database | Suitable for | +|--------------|-----------------------------|-----------------| +| `postgresql` | PostgreSQL (or CockroachDB) | Production use | +| `h2` | H2 | Testing only | + +The `polaris.persistence.relational.jdbc.datasource` property selects which datasource Polaris +activates. Polaris automatically activates the selected datasource and deactivates all others — +**do not set `quarkus.datasource.*.active` manually**. + +### Using custom datasources + +You can also define a custom named datasource (for example, to use a different JDBC driver) by +configuring its Quarkus properties and pointing `polaris.persistence.relational.jdbc.datasource` at +it: + +```properties +polaris.persistence.type=relational-jdbc +polaris.persistence.relational.jdbc.datasource=mydb + +# Register the custom datasource with Quarkus +quarkus.datasource.mydb.db-kind=postgresql +quarkus.datasource.mydb.jdbc.url=jdbc:postgresql://myhost:5432/polaris +quarkus.datasource.mydb.username= +quarkus.datasource.mydb.password= +``` + +{{< alert warning >}} +The property `quarkus.datasource..db-kind` is a build-time property! If you intend to use +custom datasources, you must rebuild the Polaris server image. +{{< /alert >}} + +### CockroachDB + +CockroachDB uses the PostgreSQL wire protocol and JDBC driver, so it works with the built-in +`postgresql` named datasource. However, Polaris uses CockroachDB-specific schema DDL, so you must +set `polaris.persistence.relational.jdbc.database-type=cockroachdb` explicitly — auto-detection via +JDBC metadata is not reliable in all deployment configurations. + +```properties +polaris.persistence.type=relational-jdbc +polaris.persistence.relational.jdbc.datasource=postgresql +polaris.persistence.relational.jdbc.database-type=cockroachdb + +quarkus.datasource.postgresql.jdbc.url=jdbc:postgresql://:26257/?sslmode=disable +quarkus.datasource.postgresql.username= +quarkus.datasource.postgresql.password= +``` + +We have 2 options for configuring the persistence backend with the built-in datasources: ## 1. Relational JDBC metastore with username and password @@ -33,32 +88,36 @@ Using environment variables: ```properties POLARIS_PERSISTENCE_TYPE=relational-jdbc +POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATASOURCE=postgresql -QUARKUS_DATASOURCE_USERNAME= -QUARKUS_DATASOURCE_PASSWORD= -QUARKUS_DATASOURCE_JDBC_URL= +QUARKUS_DATASOURCE_POSTGRESQL_USERNAME= +QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD= +QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL= ``` Using properties file: ```properties polaris.persistence.type=relational-jdbc -quarkus.datasource.jdbc.username= -quarkus.datasource.jdbc.password= -quarkus.datasource.jdbc.jdbc-url= +polaris.persistence.relational.jdbc.datasource=postgresql + +quarkus.datasource.postgresql.jdbc.username= +quarkus.datasource.postgresql.jdbc.password= +quarkus.datasource.postgresql.jdbc.jdbc-url= ``` ## 2. AWS Aurora PostgreSQL metastore using IAM AWS authentication ```properties polaris.persistence.type=relational-jdbc -quarkus.datasource.jdbc.url=jdbc:postgresql://polaris-cluster.cluster-xyz.us-east-1.rds.amazonaws.com:6160/polaris -quarkus.datasource.jdbc.additional-jdbc-properties.wrapperPlugins=iam -quarkus.datasource.username=dbusername -quarkus.datasource.db-kind=postgresql -quarkus.datasource.jdbc.additional-jdbc-properties.ssl=true -quarkus.datasource.jdbc.additional-jdbc-properties.sslmode=require -quarkus.datasource.credentials-provider=aws +polaris.persistence.relational.jdbc.datasource=postgresql + +quarkus.datasource.postgresql.jdbc.url=jdbc:postgresql://polaris-cluster.cluster-xyz.us-east-1.rds.amazonaws.com:6160/polaris +quarkus.datasource.postgresql.jdbc.additional-jdbc-properties.wrapperPlugins=iam +quarkus.datasource.postgresql.username=dbusername +quarkus.datasource.postgresql.jdbc.additional-jdbc-properties.ssl=true +quarkus.datasource.postgresql.jdbc.additional-jdbc-properties.sslmode=require +quarkus.datasource.postgresql.credentials-provider=aws quarkus.rds.credentials-provider.aws.use-quarkus-client=true quarkus.rds.credentials-provider.aws.username=dbusername @@ -68,7 +127,7 @@ quarkus.rds.credentials-provider.aws.port=6160 This is the basic configuration. For more details, please refer to the [Quarkus plugin documentation](https://docs.quarkiverse.io/quarkus-amazon-services/dev/amazon-rds.html#_configuration_reference). -The Relational JDBC metastore currently relies on a Quarkus-managed datasource and supports only PostgresSQL and H2 databases. At this time, official documentation is provided exclusively for usage with PostgreSQL. +The Relational JDBC metastore currently relies on Quarkus-managed named datasources. The built-in datasources support PostgreSQL and H2; custom datasources can be registered for other JDBC-compatible databases. At this time, official documentation is provided exclusively for usage with PostgreSQL. Please refer to the documentation here: [Configure data sources in Quarkus](https://quarkus.io/guides/datasource). @@ -83,9 +142,10 @@ Using Docker: ```bash docker run --rm -it \ --env="polaris.persistence.type=relational-jdbc" \ - --env="quarkus.datasource.username=" \ - --env="quarkus.datasource.password=" \ - --env="quarkus.datasource.jdbc.url=" \ + --env="polaris.persistence.relational.jdbc.datasource=postgresql" \ + --env="quarkus.datasource.postgresql.username=" \ + --env="quarkus.datasource.postgresql.password=" \ + --env="quarkus.datasource.postgresql.jdbc.url=" \ apache/polaris-admin-tool:latest bootstrap -r -c ,, ``` @@ -94,9 +154,10 @@ Using the standalone JAR: ```bash java \ -Dpolaris.persistence.type=relational-jdbc \ - -Dquarkus.datasource.username= \ - -Dquarkus.datasource.password= \ - -Dquarkus.datasource.jdbc.url= \ + -Dpolaris.persistence.relational.jdbc.datasource=postgresql \ + -Dquarkus.datasource.postgresql.username= \ + -Dquarkus.datasource.postgresql.password= \ + -Dquarkus.datasource.postgresql.jdbc.url= \ -jar polaris-admin-tool.jar bootstrap -r -c ,, ``` From bb3cb700987ef30a841148392d9adaa4938df567 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Thu, 18 Jun 2026 18:38:22 +0200 Subject: [PATCH 2/7] Fix scripts --- .../guides/assets/cloud_providers/deploy-azure.sh | 9 +++++---- site/content/guides/assets/cloud_providers/deploy-gcp.sh | 9 +++++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/site/content/guides/assets/cloud_providers/deploy-azure.sh b/site/content/guides/assets/cloud_providers/deploy-azure.sh index 3b11b3550ac..3157adcdaea 100755 --- a/site/content/guides/assets/cloud_providers/deploy-azure.sh +++ b/site/content/guides/assets/cloud_providers/deploy-azure.sh @@ -43,10 +43,11 @@ CREATE_DB_RESPONSE=$(az postgres flexible-server create -l $CURRENT_REGION -g $C az postgres flexible-server db create -g $CURRENT_RESOURCE_GROUP -s $INSTANCE_NAME -d POLARIS POSTGRES_ADDR=$(echo $CREATE_DB_RESPONSE | jq -r '.host') -export QUARKUS_DATASOURCE_JDBC_URL=$(printf '%s' "jdbc:postgresql://$POSTGRES_ADDR/POLARIS") -export QUARKUS_DATASOURCE_USERNAME=postgres -export QUARKUS_DATASOURCE_PASSWORD="$POSTGRES_PASSWORD" -echo $QUARKUS_DATASOURCE_JDBC_URL +export POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATASOURCE=postgresql +export QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL=$(printf '%s' "jdbc:postgresql://$POSTGRES_ADDR/POLARIS") +export QUARKUS_DATASOURCE_POSTGRESQL_USERNAME=postgres +export QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD="$POSTGRES_PASSWORD" +echo $QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL STORAGE_ACCOUNT_NAME="polaristest$RANDOM_SUFFIX" STORAGE_CONTAINER_NAME="polaris-test-container-$RANDOM_SUFFIX" diff --git a/site/content/guides/assets/cloud_providers/deploy-gcp.sh b/site/content/guides/assets/cloud_providers/deploy-gcp.sh index 0cb686de8ec..d87e3791658 100755 --- a/site/content/guides/assets/cloud_providers/deploy-gcp.sh +++ b/site/content/guides/assets/cloud_providers/deploy-gcp.sh @@ -47,10 +47,11 @@ gcloud sql instances create $DB_INSTANCE_NAME \ gcloud sql databases create POLARIS --instance=$DB_INSTANCE_NAME -export QUARKUS_DATASOURCE_JDBC_URL=$(printf '%s' "jdbc:postgresql://$POSTGRES_ADDR/POLARIS") -export QUARKUS_DATASOURCE_USERNAME=postgres -export QUARKUS_DATASOURCE_PASSWORD="$POSTGRES_PASSWORD" -echo $QUARKUS_DATASOURCE_JDBC_URL +export POLARIS_PERSISTENCE_RELATIONAL_JDBC_DATASOURCE=postgresql +export QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL=$(printf '%s' "jdbc:postgresql://$POSTGRES_ADDR/POLARIS") +export QUARKUS_DATASOURCE_POSTGRESQL_USERNAME=postgres +export QUARKUS_DATASOURCE_POSTGRESQL_PASSWORD="$POSTGRES_PASSWORD" +echo $QUARKUS_DATASOURCE_POSTGRESQL_JDBC_URL GCS_BUCKET_NAME="polaris-test-gcs-$RANDOM_SUFFIX" echo "GCS Bucket Name: $GCS_BUCKET_NAME" From b88e82f38d663cd4efff6025d75ff003b56be5d1 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Thu, 18 Jun 2026 18:41:01 +0200 Subject: [PATCH 3/7] CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22d92db6264..c9979353ee4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti ## [Unreleased] ### Highlights +- Polaris now supports dynamic datasource activation. Out of the box, two datasources are provided: `postgresql` and `h2`. The datasource to use can be selected at runtime by setting the `polaris.persistence.relational.jdbc.datasource` configuration property (the default is `postgresql`). This allows operators to switch between supported relational databases without rebuilding Polaris. ### Upgrade notes - Event listeners are now executed on a dedicated executor. **This executor does not propagate the original request's CDI context**; listeners that were improperly relying on that should instead manage their own CDI request scope from now on. Furthermore, two new configuration options were introduced to configure the executor: From 14b5d2798e9e95dbf26a139d5b8ac7b69d577a2c Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 19 Jun 2026 18:37:01 +0200 Subject: [PATCH 4/7] rephrase changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c9979353ee4..7dcdc7306c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,12 +28,13 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti ## [Unreleased] ### Highlights -- Polaris now supports dynamic datasource activation. Out of the box, two datasources are provided: `postgresql` and `h2`. The datasource to use can be selected at runtime by setting the `polaris.persistence.relational.jdbc.datasource` configuration property (the default is `postgresql`). This allows operators to switch between supported relational databases without rebuilding Polaris. +- Polaris now supports dynamic datasource activation. This allows operators to switch between supported relational databases without rebuilding Polaris. Currently, the Polaris server ships with two datasources: `postgresql` and `h2`. The admin tool, however, includes only `postgresql` for now. Out-of-the-box support for more datasources and JDBC drivers may be added in the future. The datasource to activate can be selected at runtime by setting the `polaris.persistence.relational.jdbc.datasource` configuration property (the default is `postgresql`). ### Upgrade notes - Event listeners are now executed on a dedicated executor. **This executor does not propagate the original request's CDI context**; listeners that were improperly relying on that should instead manage their own CDI request scope from now on. Furthermore, two new configuration options were introduced to configure the executor: - `polaris.event-listener.executor.pool-size` configures the thread pool size. - `polaris.event-listener.executor.queue-size` configures the queue size for pending events when all threads are busy. +- The Helm chart now includes a new `persistence.relationalJdbc.datasource` option to select the relational database to activate. The default is `postgresql`. ### Breaking changes - The `MaintenanceService.performMaintenance()` signature now requires an explicit `OptionalLong overrideRunId` argument to supersede the latest unfinished maintenance run. From 2fab4d2d7c19acd7448aae118b54130cd02e1dfa Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 19 Jun 2026 18:46:03 +0200 Subject: [PATCH 5/7] more rephrase changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7dcdc7306c2..2a579451c68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,7 +34,7 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti - Event listeners are now executed on a dedicated executor. **This executor does not propagate the original request's CDI context**; listeners that were improperly relying on that should instead manage their own CDI request scope from now on. Furthermore, two new configuration options were introduced to configure the executor: - `polaris.event-listener.executor.pool-size` configures the thread pool size. - `polaris.event-listener.executor.queue-size` configures the queue size for pending events when all threads are busy. -- The Helm chart now includes a new `persistence.relationalJdbc.datasource` option to select the relational database to activate. The default is `postgresql`. +- The Helm chart now includes a new `persistence.relationalJdbc.datasource` option to select the relational database to activate. The default is `postgresql`, which matches pre-existing behavior. ### Breaking changes - The `MaintenanceService.performMaintenance()` signature now requires an explicit `OptionalLong overrideRunId` argument to supersede the latest unfinished maintenance run. @@ -44,6 +44,7 @@ request adding CHANGELOG notes for breaking (!) changes and possibly other secti - Names containing control (invisible) characters - Names with leading or trailing whitespace - Names containing any of these characters: /\:*?"<>|#+` +- Due to the introduction of dynamic datasource activation, the default (PostgreSQL) datasource is now unused. If you had any custom configuration for that datasource, it should be migrated from `quarkus.datasource.*` to `quarkus.datasource.postgresql.*`. The same is valid for environment variables: `QUARKUS_DATASOURCE_*` should be replaced with `QUARKUS_DATASOURCE_POSTGRESQL_*`. ### New Features - Added GCS principal attribution for vended credentials (the GCP counterpart of AWS STS session tags). Set `GCS_PRINCIPAL_ATTRIBUTION_ENABLED=true` to activate; the feature flags `GCS_PRINCIPAL_ATTRIBUTION_WIF_AUDIENCE`, `GCS_PRINCIPAL_ATTRIBUTION_TOKEN_ISSUER`, and `GCS_PRINCIPAL_ATTRIBUTION_SIGNING_KEY_FILE` are then required (a missing value is a fatal configuration error). Also requires a `gcpServiceAccount` on the catalog StorageConfiguration. When enabled, credential vending chains a catalog-signed JWT through a Workload Identity Federation token exchange and service-account impersonation, so the Polaris principal appears in GCS Data Access audit logs (`serviceAccountDelegationInfo.principalSubject`) for any client. `GCS_PRINCIPAL_ATTRIBUTION_SIGNING_KEY_ID` sets the JWT `kid` for JWKS key rotation. Attribution is keyed per-principal in the credential cache; when disabled (default), GCP vending behaviour is unchanged. From c5f6829b6a815f06d1f0a8c736f35561a0aa7ea1 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 26 Jun 2026 15:39:44 +0200 Subject: [PATCH 6/7] fix bad conflict resolution --- runtime/defaults/src/main/resources/application-test.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/runtime/defaults/src/main/resources/application-test.properties b/runtime/defaults/src/main/resources/application-test.properties index 43844af2e1f..4d91d6830ca 100644 --- a/runtime/defaults/src/main/resources/application-test.properties +++ b/runtime/defaults/src/main/resources/application-test.properties @@ -33,6 +33,7 @@ quarkus.datasource.h2.devservices.enabled=false # For unit tests, the default HTTP and management *test-ports* (8081 and 9001 respectively) must be # overridden to zero, so that Quarkus will automatically select an available port. quarkus.http.test-port=0 +quarkus.management.test-port=0 quarkus.keycloak.devservices.enabled=false quarkus.mongodb.devservices.enabled=false From b9f7d3143db9262c008af387e3a1d21e6aa07e97 Mon Sep 17 00:00:00 2001 From: Alexandre Dutra Date: Fri, 26 Jun 2026 16:01:05 +0200 Subject: [PATCH 7/7] review --- .../content/in-dev/unreleased/metastores/relational-jdbc.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/site/content/in-dev/unreleased/metastores/relational-jdbc.md b/site/content/in-dev/unreleased/metastores/relational-jdbc.md index d51d5656564..8c8579f7cfd 100644 --- a/site/content/in-dev/unreleased/metastores/relational-jdbc.md +++ b/site/content/in-dev/unreleased/metastores/relational-jdbc.md @@ -101,9 +101,9 @@ Using properties file: polaris.persistence.type=relational-jdbc polaris.persistence.relational.jdbc.datasource=postgresql -quarkus.datasource.postgresql.jdbc.username= -quarkus.datasource.postgresql.jdbc.password= -quarkus.datasource.postgresql.jdbc.jdbc-url= +quarkus.datasource.postgresql.username= +quarkus.datasource.postgresql.password= +quarkus.datasource.postgresql.jdbc.url= ``` ## 2. AWS Aurora PostgreSQL metastore using IAM AWS authentication