From cffc82a05ad0989512ec960bfabd5cbc0d662255 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 9 Jun 2026 17:19:33 +0200 Subject: [PATCH 1/2] Migrate runtime-spark-tests to server runner --- plugins/spark/v3.5/spark/build.gradle.kts | 2 +- plugins/spark/v4.0/spark/build.gradle.kts | 2 +- runtime/spark-tests/build.gradle.kts | 233 ++++++++++++++---- .../service/spark/it/CatalogFederationIT.java | 23 ++ ...olaris.service.it.ext.PolarisServerManager | 2 +- .../spark/it/HiveCatalogFederationIT.java | 32 +++ .../spark/it/PropertiesRustfsAccess.java | 128 ++++++++++ ...olaris.service.it.ext.PolarisServerManager | 20 ++ .../service/spark/it/CatalogFederationIT.java | 49 ---- .../spark/it/HiveCatalogFederationIT.java | 108 -------- .../polaris/service/spark/it/SparkIT.java | 2 - ...olaris.service.it.ext.PolarisServerManager | 20 ++ .../spark/it/SparkTestsStartupAction.java | 75 ++++++ 13 files changed, 492 insertions(+), 204 deletions(-) create mode 100644 runtime/spark-tests/src/catalogFederationIntTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java rename runtime/spark-tests/src/{intTest => catalogFederationIntTest}/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager (92%) create mode 100644 runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java create mode 100644 runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java create mode 100644 runtime/spark-tests/src/hiveFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager delete mode 100644 runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java delete mode 100644 runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java rename runtime/spark-tests/src/{intTest => sparkIntTest}/java/org/apache/polaris/service/spark/it/SparkIT.java (92%) create mode 100644 runtime/spark-tests/src/sparkIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager create mode 100644 runtime/spark-tests/src/sparkStartupAction/java/org/apache/polaris/service/spark/it/SparkTestsStartupAction.java diff --git a/plugins/spark/v3.5/spark/build.gradle.kts b/plugins/spark/v3.5/spark/build.gradle.kts index 70a7958d582..de6aad373d5 100644 --- a/plugins/spark/v3.5/spark/build.gradle.kts +++ b/plugins/spark/v3.5/spark/build.gradle.kts @@ -145,7 +145,7 @@ testing { implementation(libs.antlr4.runtime.spark35) } - targets.all { + targets.configureEach { testTask.configure { environment( "AWS_REGION", diff --git a/plugins/spark/v4.0/spark/build.gradle.kts b/plugins/spark/v4.0/spark/build.gradle.kts index 80c7ee39e30..0690bdee8f7 100644 --- a/plugins/spark/v4.0/spark/build.gradle.kts +++ b/plugins/spark/v4.0/spark/build.gradle.kts @@ -141,7 +141,7 @@ testing { implementation(libs.antlr4.runtime.spark40) } - targets.all { + targets.configureEach { testTask.configure { environment( "AWS_REGION", diff --git a/runtime/spark-tests/build.gradle.kts b/runtime/spark-tests/build.gradle.kts index 089aedb1163..cefd1ade52d 100644 --- a/runtime/spark-tests/build.gradle.kts +++ b/runtime/spark-tests/build.gradle.kts @@ -17,13 +17,33 @@ * under the License. */ +import io.quarkus.gradle.tasks.QuarkusBuild +import org.gradle.api.attributes.java.TargetJvmVersion +import org.gradle.api.plugins.jvm.JvmTestSuite +import org.gradle.language.base.plugins.LifecycleBasePlugin + plugins { alias(libs.plugins.quarkus) id("org.kordamp.gradle.jandex") id("polaris-runtime") + id("polaris-server-test-runner") +} + +val intTestJvmVersion = 21 +val sparkStartupAction = sourceSets.create("sparkStartupAction") +val sparkStartupActionCompileOnly by configurations.getting +val sparkStartupActionImplementation by configurations.getting +val quarkusBuild = tasks.named("quarkusBuild") +val localPolarisServer = files(provider { quarkusBuild.get().fastJar.resolve("quarkus-run.jar") }) + +localPolarisServer.builtBy(quarkusBuild) + +configurations.named(sparkStartupAction.runtimeOnlyConfigurationName) { + extendsFrom(configurations.named(sparkStartupAction.implementationConfigurationName)) } dependencies { + polarisServer(localPolarisServer) // must be enforced to get a consistent and validated set of dependencies implementation(enforcedPlatform(libs.quarkus.bom)) { @@ -33,6 +53,7 @@ dependencies { } implementation(project(":polaris-runtime-service")) + runtimeOnly(project(":polaris-extensions-federation-hadoop")) runtimeOnly(project(":polaris-extensions-federation-hive")) { // Brings shaded parquet 1.10 which conflicts with Iceberg's parquet 1.16 in test code. exclude("org.apache.parquet", "parquet-hadoop-bundle") @@ -54,57 +75,185 @@ dependencies { // (the bundle fat-jar provided them); add it explicitly at the BOM-managed version. runtimeOnly("software.amazon.awssdk:s3-transfer-manager") - testImplementation(project(":polaris-tests")) - testImplementation(project(":polaris-rustfs-testcontainer")) - testImplementation(testFixtures(project(":polaris-runtime-service"))) - testImplementation(project(":polaris-runtime-test-common")) - - testImplementation(platform(libs.quarkus.bom)) - testImplementation("io.quarkus:quarkus-junit") - testImplementation("io.quarkus:quarkus-rest-client") - testImplementation("io.quarkus:quarkus-rest-client-jackson") - - testImplementation(platform(libs.awssdk.bom)) - testImplementation("software.amazon.awssdk:glue") - testImplementation("software.amazon.awssdk:kms") - testImplementation("software.amazon.awssdk:dynamodb") - - testImplementation(platform(libs.testcontainers.bom)) - testImplementation("org.testcontainers:testcontainers") - testImplementation(libs.s3mock.testcontainers) - - // Required for Spark integration tests - testImplementation(enforcedPlatform(libs.scala212.lang.library)) - testImplementation(enforcedPlatform(libs.scala212.lang.reflect)) - testImplementation(libs.javax.servlet.api) - testImplementation(libs.antlr4.runtime.spark35) + sparkStartupActionCompileOnly("org.apache.polaris.server-test-runner:polaris-server-test-runner") + sparkStartupActionImplementation(project(":polaris-rustfs-testcontainer")) } -tasks.named("intTest").configure { - if (System.getenv("AWS_REGION") == null) { - environment("AWS_REGION", "us-west-2") +@Suppress("UnstableApiUsage") +fun JvmTestSuite.configureSparkIntegrationDependencies() { + dependencies { + implementation(project(":polaris-tests")) + implementation(testFixtures(project(":polaris-runtime-service"))) + implementation(project(":polaris-runtime-test-common")) + + implementation(platform(libs.awssdk.bom)) + implementation("software.amazon.awssdk:glue") + implementation("software.amazon.awssdk:kms") + implementation("software.amazon.awssdk:dynamodb") + implementation("software.amazon.awssdk:url-connection-client") + + // Required for Spark integration tests + implementation(enforcedPlatform(libs.scala212.lang.library)) + implementation(enforcedPlatform(libs.scala212.lang.reflect)) + implementation(libs.javax.servlet.api) + implementation(libs.antlr4.runtime.spark35) } - // Note: the test secrets are referenced in - // org.apache.polaris.service.it.ServerManager - environment("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") +} + +fun Test.configureSparkIntegrationTestTask( + suiteName: String, + skipCredentialSubscoping: Boolean, + storageAccessKey: String? = null, + storageSecretKey: String? = null, + withHiveRustfsStartupAction: Boolean = false, +) { + environment("AWS_REGION", providers.environmentVariable("AWS_REGION").getOrElse("us-west-2")) jvmArgs("--add-exports", "java.base/sun.nio.ch=ALL-UNNAMED") // Need to allow a java security manager after Java 21, for Subject.getSubject to work // "getSubject is supported only if a security manager is allowed". systemProperty("java.security.manager", "allow") - // Same issue as above: allow a java security manager after Java 21 - // (this setting is for the application under test, while the setting above is for test code). - systemProperty("quarkus.test.arg-line", "-Djava.security.manager=allow") - val logsDir = project.layout.buildDirectory.get().asFile.resolve("logs") - // delete files from previous runs + + val logsDir = project.layout.buildDirectory.dir("logs/$suiteName") + val hiveRustfsProperties = project.layout.buildDirectory.file("$suiteName/hive-rustfs.properties") + doFirst { - // delete log files written by Polaris - logsDir.deleteRecursively() - // delete quarkus.log file (captured Polaris stdout/stderr) - project.layout.buildDirectory.get().asFile.resolve("quarkus.log").delete() + val logsDirFile = logsDir.get().asFile + logsDirFile.deleteRecursively() + logsDirFile.mkdirs() + } + + if (withHiveRustfsStartupAction) { + systemProperty( + "polaris.spark-tests.hive-rustfs.properties", + hiveRustfsProperties.get().asFile.absolutePath, + ) + } + + withPolarisServer(configurations.polarisServer) { + if (withHiveRustfsStartupAction) { + startupActionClasspath.from(sparkStartupAction.runtimeClasspath) + startupActionClass.set("org.apache.polaris.service.spark.it.SparkTestsStartupAction") + startupActionParameters.put( + "hiveRustfsProperties", + hiveRustfsProperties.get().asFile.absolutePath, + ) + } + + environment.put("AWS_REGION", providers.environmentVariable("AWS_REGION").orElse("us-west-2")) + environment.put( + "AWS_ACCESS_KEY_ID", + providers.environmentVariable("AWS_ACCESS_KEY_ID").orElse("ap1"), + ) + environment.put( + "AWS_SECRET_ACCESS_KEY", + providers.environmentVariable("AWS_SECRET_ACCESS_KEY").orElse("s3cr3t"), + ) + environment.put("AWS_EC2_METADATA_DISABLED", "true") + environment.put("POLARIS_BOOTSTRAP_CREDENTIALS", "POLARIS,test-admin,test-secret") + + systemProperties.putAll( + mapOf( + "quarkus.profile" to "it", + "java.security.manager" to "allow", + "quarkus.log.file.enabled" to "true", + "quarkus.log.file.path" to logsDir.get().asFile.resolve("polaris.log").absolutePath, + "polaris.features.\"SUPPORTED_CATALOG_STORAGE_TYPES\"" to + "[\"FILE\",\"S3\",\"GCS\",\"AZURE\"]", + "polaris.features.\"ALLOW_INSECURE_STORAGE_TYPES\"" to "true", + "polaris.features.\"DROP_WITH_PURGE_ENABLED\"" to "true", + "polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"" to "true", + "polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"" to + skipCredentialSubscoping.toString(), + "polaris.features.\"ENABLE_CATALOG_FEDERATION\"" to "true", + "polaris.features.\"SUPPORTED_CATALOG_CONNECTION_TYPES\"" to "[\"ICEBERG_REST\",\"HIVE\"]", + "polaris.features.\"SUPPORTED_EXTERNAL_CATALOG_AUTHENTICATION_TYPES\"" to + "[\"IMPLICIT\",\"OAUTH\"]", + "polaris.features.\"ENABLE_SUB_CATALOG_RBAC_FOR_FEDERATED_CATALOGS\"" to "true", + "polaris.features.\"ALLOW_DROPPING_NON_EMPTY_PASSTHROUGH_FACADE_CATALOG\"" to "true", + "polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"" to "true", + "polaris.features.\"ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING\"" to "true", + ) + ) + storageAccessKey?.let { systemProperties.put("polaris.storage.aws.access-key", it) } + storageSecretKey?.let { systemProperties.put("polaris.storage.aws.secret-key", it) } } - // This property is not honored in a per-profile application.properties file, - // so we need to set it here. - systemProperty("quarkus.log.file.path", logsDir.resolve("polaris.log").absolutePath) // For Spark integration tests addSparkJvmOptions() } + +@Suppress("UnstableApiUsage") +testing { + suites { + register("sparkIntTest") { + useJUnitJupiter() + configureSparkIntegrationDependencies() + + targets.configureEach { + testTask.configure { + configureSparkIntegrationTestTask( + suiteName = "sparkIntTest", + skipCredentialSubscoping = true, + ) + } + } + } + + register("catalogFederationIntTest") { + useJUnitJupiter() + configureSparkIntegrationDependencies() + + targets.configureEach { + testTask.configure { + configureSparkIntegrationTestTask( + suiteName = "catalogFederationIntTest", + skipCredentialSubscoping = false, + storageAccessKey = "test-ak-123-catalog-federation", + storageSecretKey = "test-sk-123-catalog-federation", + ) + } + } + } + + register("hiveFederationIntTest") { + useJUnitJupiter() + configureSparkIntegrationDependencies() + dependencies { implementation(project(":polaris-rustfs-testcontainer")) } + + targets.configureEach { + testTask.configure { + configureSparkIntegrationTestTask( + suiteName = "hiveFederationIntTest", + skipCredentialSubscoping = false, + storageAccessKey = "test-ak-123-hive-federation", + storageSecretKey = "test-sk-123-hive-federation", + withHiveRustfsStartupAction = true, + ) + } + } + } + } +} + +listOf( + "sparkIntTestCompileClasspath", + "sparkIntTestRuntimeClasspath", + "catalogFederationIntTestCompileClasspath", + "catalogFederationIntTestRuntimeClasspath", + "hiveFederationIntTestCompileClasspath", + "hiveFederationIntTestRuntimeClasspath", + "sparkStartupActionCompileClasspath", + "sparkStartupActionRuntimeClasspath", + ) + .forEach { + configurations.named(it).configure { + attributes.attribute(TargetJvmVersion.TARGET_JVM_VERSION_ATTRIBUTE, intTestJvmVersion) + } + } + +tasks.named("intTest") { + description = "Runs all Spark integration tests." + group = LifecycleBasePlugin.VERIFICATION_GROUP + dependsOn("sparkIntTest", "catalogFederationIntTest", "hiveFederationIntTest") + testClassesDirs = files() + classpath = files() +} diff --git a/runtime/spark-tests/src/catalogFederationIntTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java b/runtime/spark-tests/src/catalogFederationIntTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java new file mode 100644 index 00000000000..f7002ce08c9 --- /dev/null +++ b/runtime/spark-tests/src/catalogFederationIntTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import org.apache.polaris.service.it.test.CatalogFederationIntegrationTest; + +public class CatalogFederationIT extends CatalogFederationIntegrationTest {} diff --git a/runtime/spark-tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/runtime/spark-tests/src/catalogFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager similarity index 92% rename from runtime/spark-tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager rename to runtime/spark-tests/src/catalogFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager index 07ecf4b1fd0..6aa2dac9e0c 100644 --- a/runtime/spark-tests/src/intTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager +++ b/runtime/spark-tests/src/catalogFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -17,4 +17,4 @@ # under the License. # -org.apache.polaris.service.it.ServerManager \ No newline at end of file +org.apache.polaris.service.it.ext.ExternalPolarisServerManager diff --git a/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java b/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java new file mode 100644 index 00000000000..9fc1f1e86a6 --- /dev/null +++ b/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import java.nio.file.Path; +import org.apache.polaris.service.it.test.HiveCatalogFederationIntegrationTest; +import org.apache.polaris.test.rustfs.RustfsAccess; + +public class HiveCatalogFederationIT extends HiveCatalogFederationIntegrationTest { + + @Override + protected RustfsAccess rustfsAccess() { + return PropertiesRustfsAccess.from( + Path.of(System.getProperty("polaris.spark-tests.hive-rustfs.properties"))); + } +} diff --git a/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java b/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java new file mode 100644 index 00000000000..d5bbada494b --- /dev/null +++ b/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import java.io.IOException; +import java.io.InputStream; +import java.net.URI; +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.Properties; +import org.apache.polaris.test.rustfs.RustfsAccess; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; + +/** Test-side view of the RustFS container owned by {@link SparkTestsStartupAction}. */ +final class PropertiesRustfsAccess implements RustfsAccess { + private final Properties properties; + + private PropertiesRustfsAccess(Properties properties) { + this.properties = properties; + } + + static PropertiesRustfsAccess from(Path path) { + Properties properties = new Properties(); + try (InputStream input = java.nio.file.Files.newInputStream(path)) { + properties.load(input); + } catch (IOException e) { + throw new IllegalStateException("Failed to load RustFS properties from " + path, e); + } + return new PropertiesRustfsAccess(properties); + } + + @Override + public String hostPort() { + return value("hostPort"); + } + + @Override + public String accessKey() { + return value("accessKey"); + } + + @Override + public String secretKey() { + return value("secretKey"); + } + + @Override + public String bucket() { + return value("bucket"); + } + + @Override + public Optional region() { + return Optional.ofNullable(properties.getProperty("region")); + } + + @Override + public String s3endpoint() { + return value("s3endpoint"); + } + + @Override + public S3Client s3Client() { + return S3Client.builder() + .httpClientBuilder(UrlConnectionHttpClient.builder()) + .endpointOverride(URI.create(s3endpoint())) + .applyMutation(builder -> region().ifPresent(region -> builder.region(Region.of(region)))) + .credentialsProvider( + StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey(), secretKey()))) + .build(); + } + + @Override + public Map icebergProperties() { + Map props = new HashMap<>(); + props.put("s3.access-key-id", accessKey()); + props.put("s3.secret-access-key", secretKey()); + props.put("s3.endpoint", s3endpoint()); + props.put("http-client.type", "urlconnection"); + region().ifPresent(region -> props.put("client.region", region)); + return props; + } + + @Override + public Map hadoopConfig() { + Map props = new HashMap<>(); + props.put("fs.s3.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); + props.put("fs.s3a.access.key", accessKey()); + props.put("fs.s3a.secret.key", secretKey()); + props.put("fs.s3a.endpoint", s3endpoint()); + return props; + } + + @Override + public URI s3BucketUri(String path) { + return URI.create(String.format("s3://%s/", bucket())).resolve(path); + } + + private String value(String name) { + String value = properties.getProperty(name); + if (value == null || value.isBlank()) { + throw new IllegalStateException("Missing RustFS property: " + name); + } + return value; + } +} diff --git a/runtime/spark-tests/src/hiveFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/runtime/spark-tests/src/hiveFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager new file mode 100644 index 00000000000..6aa2dac9e0c --- /dev/null +++ b/runtime/spark-tests/src/hiveFederationIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.it.ext.ExternalPolarisServerManager diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java deleted file mode 100644 index a05211b3016..00000000000 --- a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/CatalogFederationIT.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.spark.it; - -import com.google.common.collect.ImmutableMap; -import io.quarkus.test.junit.QuarkusIntegrationTest; -import io.quarkus.test.junit.QuarkusTestProfile; -import io.quarkus.test.junit.TestProfile; -import java.util.Map; -import org.apache.polaris.service.it.test.CatalogFederationIntegrationTest; - -@TestProfile(CatalogFederationIT.CatalogFederationProfile.class) -@QuarkusIntegrationTest -public class CatalogFederationIT extends CatalogFederationIntegrationTest { - - public static class CatalogFederationProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .put("polaris.features.\"ENABLE_CATALOG_FEDERATION\"", "true") - .put("polaris.features.\"SUPPORTED_CATALOG_CONNECTION_TYPES\"", "[\"ICEBERG_REST\"]") - .put("polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true") - .put("polaris.features.\"ENABLE_SUB_CATALOG_RBAC_FOR_FEDERATED_CATALOGS\"", "true") - .put("polaris.features.\"ALLOW_DROPPING_NON_EMPTY_PASSTHROUGH_FACADE_CATALOG\"", "true") - .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false") - .put("polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", "true") - .put("polaris.features.\"ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING\"", "true") - .put("polaris.storage.aws.access-key", CatalogFederationIntegrationTest.RUSTFS_ACCESS_KEY) - .put("polaris.storage.aws.secret-key", CatalogFederationIntegrationTest.RUSTFS_SECRET_KEY) - .build(); - } - } -} diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java b/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java deleted file mode 100644 index ccfdc4249e6..00000000000 --- a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/HiveCatalogFederationIT.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.polaris.service.spark.it; - -import com.google.common.collect.ImmutableMap; -import io.quarkus.test.common.QuarkusTestResource; -import io.quarkus.test.common.QuarkusTestResourceLifecycleManager; -import io.quarkus.test.junit.QuarkusIntegrationTest; -import io.quarkus.test.junit.QuarkusTestProfile; -import io.quarkus.test.junit.TestProfile; -import java.util.Map; -import org.apache.polaris.service.it.test.HiveCatalogFederationIntegrationTest; -import org.apache.polaris.test.rustfs.RustfsAccess; -import org.apache.polaris.test.rustfs.RustfsContainer; - -@TestProfile(HiveCatalogFederationIT.HiveFederationProfile.class) -@QuarkusTestResource(HiveCatalogFederationIT.RustfsResource.class) -@QuarkusIntegrationTest -public class HiveCatalogFederationIT extends HiveCatalogFederationIntegrationTest { - - /** Populated by {@link RustfsResource#inject} before {@code @BeforeAll} runs. */ - private RustfsAccess rustfsAccess; - - @Override - protected RustfsAccess rustfsAccess() { - return rustfsAccess; - } - - public static class HiveFederationProfile implements QuarkusTestProfile { - @Override - public Map getConfigOverrides() { - return ImmutableMap.builder() - .put("polaris.features.\"ENABLE_CATALOG_FEDERATION\"", "true") - .put("polaris.features.\"SUPPORTED_CATALOG_CONNECTION_TYPES\"", "[\"HIVE\"]") - .put( - "polaris.features.\"SUPPORTED_EXTERNAL_CATALOG_AUTHENTICATION_TYPES\"", - "[\"IMPLICIT\"]") - .put("polaris.features.\"ALLOW_OVERLAPPING_CATALOG_URLS\"", "true") - .put("polaris.features.\"ENABLE_SUB_CATALOG_RBAC_FOR_FEDERATED_CATALOGS\"", "true") - .put("polaris.features.\"ALLOW_DROPPING_NON_EMPTY_PASSTHROUGH_FACADE_CATALOG\"", "true") - .put("polaris.features.\"SKIP_CREDENTIAL_SUBSCOPING_INDIRECTION\"", "false") - .put("polaris.features.\"ALLOW_EXTERNAL_CATALOG_CREDENTIAL_VENDING\"", "true") - .put("polaris.features.\"ALLOW_FEDERATED_CATALOGS_CREDENTIAL_VENDING\"", "true") - .put( - "polaris.storage.aws.access-key", - HiveCatalogFederationIntegrationTest.RUSTFS_ACCESS_KEY) - .put( - "polaris.storage.aws.secret-key", - HiveCatalogFederationIntegrationTest.RUSTFS_SECRET_KEY) - .build(); - } - } - - /** - * Starts a RustFS container , exposes its endpoint and credentials, and injects the running - * container into {@link HiveCatalogFederationIT#rustfsAccess} on the test instance. - */ - public static class RustfsResource implements QuarkusTestResourceLifecycleManager { - - private RustfsContainer container; - - @Override - public Map start() { - container = - new RustfsContainer( - null, - HiveCatalogFederationIntegrationTest.RUSTFS_ACCESS_KEY, - HiveCatalogFederationIntegrationTest.RUSTFS_SECRET_KEY, - null, - null) - .withStartupAttempts(5); - container.start(); - return ImmutableMap.of( - "polaris.s3.endpoint", container.s3endpoint(), - "polaris.s3.access-key", container.accessKey(), - "polaris.s3.secret-key", container.secretKey()); - } - - @Override - public void inject(TestInjector testInjector) { - testInjector.injectIntoFields(container, new TestInjector.MatchesType(RustfsAccess.class)); - } - - @Override - public void stop() { - if (container != null) { - container.close(); - container = null; - } - } - } -} diff --git a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkIT.java b/runtime/spark-tests/src/sparkIntTest/java/org/apache/polaris/service/spark/it/SparkIT.java similarity index 92% rename from runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkIT.java rename to runtime/spark-tests/src/sparkIntTest/java/org/apache/polaris/service/spark/it/SparkIT.java index f82140e9c1c..4b9a21a663f 100644 --- a/runtime/spark-tests/src/intTest/java/org/apache/polaris/service/spark/it/SparkIT.java +++ b/runtime/spark-tests/src/sparkIntTest/java/org/apache/polaris/service/spark/it/SparkIT.java @@ -18,8 +18,6 @@ */ package org.apache.polaris.service.spark.it; -import io.quarkus.test.junit.QuarkusIntegrationTest; import org.apache.polaris.service.it.test.PolarisSparkIntegrationTest; -@QuarkusIntegrationTest public class SparkIT extends PolarisSparkIntegrationTest {} diff --git a/runtime/spark-tests/src/sparkIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager b/runtime/spark-tests/src/sparkIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager new file mode 100644 index 00000000000..6aa2dac9e0c --- /dev/null +++ b/runtime/spark-tests/src/sparkIntTest/resources/META-INF/services/org.apache.polaris.service.it.ext.PolarisServerManager @@ -0,0 +1,20 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +org.apache.polaris.service.it.ext.ExternalPolarisServerManager diff --git a/runtime/spark-tests/src/sparkStartupAction/java/org/apache/polaris/service/spark/it/SparkTestsStartupAction.java b/runtime/spark-tests/src/sparkStartupAction/java/org/apache/polaris/service/spark/it/SparkTestsStartupAction.java new file mode 100644 index 00000000000..df8b75fd4e6 --- /dev/null +++ b/runtime/spark-tests/src/sparkStartupAction/java/org/apache/polaris/service/spark/it/SparkTestsStartupAction.java @@ -0,0 +1,75 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.polaris.service.spark.it; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Properties; +import org.apache.polaris.server.test.runner.spi.PolarisServerStartupAction; +import org.apache.polaris.server.test.runner.spi.PolarisServerStartupContext; +import org.apache.polaris.test.rustfs.RustfsContainer; + +/** Starts services required before the custom Spark-test Polaris server process starts. */ +public class SparkTestsStartupAction implements PolarisServerStartupAction { + private static final String RUSTFS_ACCESS_KEY = "test-ak-123-hive-federation"; + private static final String RUSTFS_SECRET_KEY = "test-sk-123-hive-federation"; + + private RustfsContainer hiveRustfs; + + @Override + public void start(PolarisServerStartupContext context) throws IOException { + hiveRustfs = + new RustfsContainer(null, RUSTFS_ACCESS_KEY, RUSTFS_SECRET_KEY, null, "us-west-2") + .withStartupAttempts(5); + hiveRustfs.start(); + + context.getSystemProperties().put("polaris.s3.endpoint", hiveRustfs.s3endpoint()); + context.getSystemProperties().put("polaris.s3.access-key", hiveRustfs.accessKey()); + context.getSystemProperties().put("polaris.s3.secret-key", hiveRustfs.secretKey()); + + String propertiesFile = context.getParameters().get("hiveRustfsProperties"); + if (propertiesFile != null) { + writeHiveRustfsProperties(Path.of(propertiesFile)); + } + } + + private void writeHiveRustfsProperties(Path path) throws IOException { + Files.createDirectories(path.getParent()); + Properties properties = new Properties(); + properties.setProperty("hostPort", hiveRustfs.hostPort()); + properties.setProperty("accessKey", hiveRustfs.accessKey()); + properties.setProperty("secretKey", hiveRustfs.secretKey()); + properties.setProperty("bucket", hiveRustfs.bucket()); + properties.setProperty("s3endpoint", hiveRustfs.s3endpoint()); + hiveRustfs.region().ifPresent(region -> properties.setProperty("region", region)); + try (OutputStream output = Files.newOutputStream(path)) { + properties.store(output, "RustFS instance started by SparkTestsStartupAction"); + } + } + + @Override + public void close() { + if (hiveRustfs != null) { + hiveRustfs.close(); + hiveRustfs = null; + } + } +} From a0baa8cb8047d7cba233160afd04dbd86b7021b6 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Sat, 27 Jun 2026 09:06:54 +0200 Subject: [PATCH 2/2] review --- .../polaris/service/spark/it/PropertiesRustfsAccess.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java b/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java index d5bbada494b..ddecbe3a994 100644 --- a/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java +++ b/runtime/spark-tests/src/hiveFederationIntTest/java/org/apache/polaris/service/spark/it/PropertiesRustfsAccess.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.io.InputStream; import java.net.URI; +import java.nio.file.Files; import java.nio.file.Path; import java.util.HashMap; import java.util.Map; @@ -43,7 +44,7 @@ private PropertiesRustfsAccess(Properties properties) { static PropertiesRustfsAccess from(Path path) { Properties properties = new Properties(); - try (InputStream input = java.nio.file.Files.newInputStream(path)) { + try (InputStream input = Files.newInputStream(path)) { properties.load(input); } catch (IOException e) { throw new IllegalStateException("Failed to load RustFS properties from " + path, e);