OpenLineage : Add core lineage service scaffolding

bom/build.gradle.kts

@@ -86,6 +86,7 @@ dependencies {
     api(project(":polaris-config-docs-generator"))
 
     api(project(":polaris-core"))
+    api(project(":polaris-lineage"))
 
     api(project(":polaris-relational-jdbc"))
 

gradle/projects.main.properties

@@ -20,6 +20,7 @@
 
 polaris-bom=bom
 polaris-core=polaris-core
+polaris-lineage=lineage
 polaris-api-iceberg-service=api/iceberg-service
 polaris-api-management-model=api/management-model
 polaris-api-management-service=api/management-service

lineage/build.gradle.kts

@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+plugins {
+  id("polaris-client")
+}
+
+description = "Polaris lineage model and service contract"

lineage/src/main/java/org/apache/polaris/lineage/LineageData.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.lineage;
+
+import java.util.Objects;
+import java.util.OptionalLong;
+
+/** Dataset metadata returned in a lineage query response. */
+public record LineageData(
+    OptionalLong catalogId,
+    OptionalLong datasetId,
+    String namespace,
+    String name,
+    String subType,
+    OptionalLong createdAt,
+    OptionalLong updatedAt) {
+  public LineageData {
+    Objects.requireNonNull(catalogId, "catalogId must be non-null");
+    Objects.requireNonNull(datasetId, "datasetId must be non-null");
+    Objects.requireNonNull(namespace, "namespace must be non-null");
+    Objects.requireNonNull(name, "name must be non-null");
+    Objects.requireNonNull(createdAt, "createdAt must be non-null");
+    Objects.requireNonNull(updatedAt, "updatedAt must be non-null");
+  }
+
+  public LineageData(
+      long catalogId,
+      long datasetId,
+      String namespace,
+      String name,
+      String subType,
+      long createdAt,
+      long updatedAt) {
+    this(
+        OptionalLong.of(catalogId),
+        OptionalLong.of(datasetId),
+        namespace,
+        name,
+        subType,
+        OptionalLong.of(createdAt),
+        OptionalLong.of(updatedAt));
+  }
+}

lineage/src/main/java/org/apache/polaris/lineage/LineageDirection.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.lineage;
+
+/** Supported directions for lineage queries. */
+public enum LineageDirection {
+  UPSTREAM,
+  DOWNSTREAM,
+  BOTH
+}

lineage/src/main/java/org/apache/polaris/lineage/LineageFieldMapping.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.lineage;
+
+import java.util.Objects;
+
+/** A source-to-target field mapping returned for column-granularity queries. */
+public record LineageFieldMapping(String sourceField, String targetField) {
+  public LineageFieldMapping {
+    Objects.requireNonNull(sourceField, "sourceField must be non-null");
+    Objects.requireNonNull(targetField, "targetField must be non-null");
+  }
+}

lineage/src/main/java/org/apache/polaris/lineage/LineageGranularity.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.lineage;
+
+/** Supported query granularities for lineage lookups. */
+public enum LineageGranularity {
+  DATASET,
+  COLUMN
+}

lineage/src/main/java/org/apache/polaris/lineage/LineageGraph.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.lineage;
+
+import java.util.List;
+import java.util.Objects;
+
+/** Normalized response model for lineage queries. */
+public record LineageGraph(
+    LineageNode node, List<LineageNode> upstream, List<LineageNode> downstream) {
+  public LineageGraph {
+    Objects.requireNonNull(node, "node must be non-null");
+    upstream = List.copyOf(Objects.requireNonNull(upstream, "upstream must be non-null"));
+    downstream = List.copyOf(Objects.requireNonNull(downstream, "downstream must be non-null"));
+  }
+}

lineage/src/main/java/org/apache/polaris/lineage/LineageNode.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.lineage;
+
+import java.util.List;
+import java.util.Objects;
+
+/** A node returned in a lineage query response. */
+public record LineageNode(
+    String id,
+    LineageNodeType type,
+    LineageData data,
+    boolean opaque,
+    List<LineageFieldMapping> fieldMappings) {
+  public LineageNode {
+    Objects.requireNonNull(id, "id must be non-null");
+    Objects.requireNonNull(type, "type must be non-null");
+    fieldMappings =
+        List.copyOf(Objects.requireNonNull(fieldMappings, "fieldMappings must be non-null"));
+  }
+
+  public LineageNode(String id, LineageNodeType type, LineageData data, boolean opaque) {
+    this(id, type, data, opaque, List.of());
+  }
+}

lineage/src/main/java/org/apache/polaris/lineage/LineageNodeType.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.lineage;
+
+/** Node kinds surfaced by normalized lineage queries. */
+public enum LineageNodeType {
+  DATASET,
+  COLUMN
+}

lineage/src/main/java/org/apache/polaris/lineage/LineageQueryRequest.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.lineage;
+
+import java.util.Objects;
+
+/** Request model for normalized lineage lookups. */
+public record LineageQueryRequest(
+    String nodeId, LineageDirection direction, LineageGranularity granularity) {
+  public LineageQueryRequest {
+    Objects.requireNonNull(nodeId, "nodeId must be non-null");
+    Objects.requireNonNull(direction, "direction must be non-null");
+    Objects.requireNonNull(granularity, "granularity must be non-null");
+  }
+}

lineage/src/main/java/org/apache/polaris/lineage/LineageService.java

@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.lineage;
+
+/** Service boundary for lineage operations used by transport-layer adapters. */
+public interface LineageService {
+  LineageGraph query(LineageQueryRequest request);
+}

polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java

@@ -357,6 +357,13 @@ public static void enforceFeatureEnabledOrThrow(
           .defaultValue(true)
           .buildFeatureConfiguration();
 
+  public static final FeatureConfiguration<Boolean> ENABLE_LINEAGE =
+      PolarisConfiguration.<Boolean>builder()
+          .key("ENABLE_LINEAGE")
+          .description("If true, lineage services are enabled")
+          .defaultValue(false)
+          .buildFeatureConfiguration();
+
   public static final FeatureConfiguration<List<String>> SUPPORTED_CATALOG_CONNECTION_TYPES =
       PolarisConfiguration.<List<String>>builder()
           .key("SUPPORTED_CATALOG_CONNECTION_TYPES")

runtime/service/build.gradle.kts

@@ -26,6 +26,7 @@ plugins {
 
 dependencies {
   implementation(project(":polaris-core"))
+  implementation(project(":polaris-lineage"))
   implementation(project(":polaris-api-management-model"))
   implementation(project(":polaris-api-management-service"))
   implementation(project(":polaris-api-iceberg-service"))