OpenLineage : Add core lineage service scaffolding

polaris-core/src/main/java/org/apache/polaris/core/config/FeatureConfiguration.java

@@ -357,6 +357,13 @@ public static void enforceFeatureEnabledOrThrow(
           .defaultValue(true)
           .buildFeatureConfiguration();
 
+  public static final FeatureConfiguration<Boolean> ENABLE_LINEAGE =
+      PolarisConfiguration.<Boolean>builder()
+          .key("ENABLE_LINEAGE")
+          .description("If true, lineage services are enabled")
+          .defaultValue(false)
+          .buildFeatureConfiguration();
+
   public static final FeatureConfiguration<List<String>> SUPPORTED_CATALOG_CONNECTION_TYPES =
       PolarisConfiguration.<List<String>>builder()
           .key("SUPPORTED_CATALOG_CONNECTION_TYPES")

polaris-core/src/main/java/org/apache/polaris/core/lineage/LineageData.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.lineage;
+
+import java.util.Objects;
+import java.util.OptionalLong;
+
+/** Dataset metadata returned in a lineage query response. */
+public record LineageData(
+    OptionalLong catalogId,
+    OptionalLong datasetId,
+    String namespace,
+    String name,
+    String subType,
+    OptionalLong createdAt,
+    OptionalLong updatedAt) {
+  public LineageData {
+    Objects.requireNonNull(catalogId, "catalogId must be non-null");
+    Objects.requireNonNull(datasetId, "datasetId must be non-null");
+    Objects.requireNonNull(namespace, "namespace must be non-null");
+    Objects.requireNonNull(name, "name must be non-null");
+    Objects.requireNonNull(createdAt, "createdAt must be non-null");
+    Objects.requireNonNull(updatedAt, "updatedAt must be non-null");
+  }
+
+  public LineageData(
+      long catalogId,
+      long datasetId,
+      String namespace,
+      String name,
+      String subType,
+      long createdAt,
+      long updatedAt) {
+    this(
+        OptionalLong.of(catalogId),
+        OptionalLong.of(datasetId),
+        namespace,
+        name,
+        subType,
+        OptionalLong.of(createdAt),
+        OptionalLong.of(updatedAt));
+  }
+}

polaris-core/src/main/java/org/apache/polaris/core/lineage/LineageDirection.java

@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.lineage;
+
+/** Supported directions for lineage queries. */
+public enum LineageDirection {
+  UPSTREAM,
+  DOWNSTREAM,
+  BOTH
+}

polaris-core/src/main/java/org/apache/polaris/core/lineage/LineageFieldMapping.java

@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.lineage;
+
+import java.util.Objects;
+
+/** A source-to-target field mapping returned for column-granularity queries. */
+public record LineageFieldMapping(String sourceField, String targetField) {
+  public LineageFieldMapping {
+    Objects.requireNonNull(sourceField, "sourceField must be non-null");
+    Objects.requireNonNull(targetField, "targetField must be non-null");
+  }
+}

polaris-core/src/main/java/org/apache/polaris/core/lineage/LineageGranularity.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.lineage;
+
+/** Supported query granularities for lineage lookups. */
+public enum LineageGranularity {
+  DATASET,
+  COLUMN
+}

polaris-core/src/main/java/org/apache/polaris/core/lineage/LineageGraph.java

@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.lineage;
+
+import java.util.List;
+import java.util.Objects;
+
+/** Normalized response model for lineage queries. */
+public record LineageGraph(
+    LineageNode node, List<LineageNode> upstream, List<LineageNode> downstream) {
+  public LineageGraph {
+    Objects.requireNonNull(node, "node must be non-null");
+    upstream = List.copyOf(Objects.requireNonNull(upstream, "upstream must be non-null"));
+    downstream = List.copyOf(Objects.requireNonNull(downstream, "downstream must be non-null"));
+  }
+}

polaris-core/src/main/java/org/apache/polaris/core/lineage/LineageNode.java

@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.lineage;
+
+import java.util.List;
+import java.util.Objects;
+
+/** A node returned in a lineage query response. */
+public record LineageNode(
+    String id,
+    LineageNodeType type,
+    LineageData data,
+    boolean opaque,
+    List<LineageFieldMapping> fieldMappings) {
+  public LineageNode {
+    Objects.requireNonNull(id, "id must be non-null");
+    Objects.requireNonNull(type, "type must be non-null");
+    fieldMappings =
+        List.copyOf(Objects.requireNonNull(fieldMappings, "fieldMappings must be non-null"));
+  }
+
+  public LineageNode(String id, LineageNodeType type, LineageData data, boolean opaque) {
+    this(id, type, data, opaque, List.of());
+  }
+}

polaris-core/src/main/java/org/apache/polaris/core/lineage/LineageNodeType.java

@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.lineage;
+
+/** Node kinds surfaced by normalized lineage queries. */
+public enum LineageNodeType {
+  DATASET,
+  COLUMN
+}

polaris-core/src/main/java/org/apache/polaris/core/lineage/LineageQueryRequest.java

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.lineage;
+
+import java.util.Objects;
+
+/** Request model for normalized lineage lookups. */
+public record LineageQueryRequest(
+    String nodeId, LineageDirection direction, LineageGranularity granularity) {
+  public LineageQueryRequest {
+    Objects.requireNonNull(nodeId, "nodeId must be non-null");
+    Objects.requireNonNull(direction, "direction must be non-null");
+    Objects.requireNonNull(granularity, "granularity must be non-null");
+  }
+}

polaris-core/src/main/java/org/apache/polaris/core/lineage/LineageService.java

@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.core.lineage;
+
+/** Service boundary for lineage operations used by transport-layer adapters. */
+public interface LineageService {
+  LineageGraph query(LineageQueryRequest request);
+}

runtime/service/src/main/java/org/apache/polaris/service/lineage/DefaultLineageService.java

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.polaris.service.lineage;
+
+import jakarta.enterprise.context.RequestScoped;
+import jakarta.inject.Inject;
+import org.apache.polaris.core.config.FeatureConfiguration;
+import org.apache.polaris.core.context.CallContext;
+import org.apache.polaris.core.lineage.LineageGraph;
+import org.apache.polaris.core.lineage.LineageQueryRequest;
+import org.apache.polaris.core.lineage.LineageService;
+
+@RequestScoped
+public class DefaultLineageService implements LineageService {
+  private final CallContext callContext;
+  private final LineageConfiguration configuration;
+
+  @Inject
+  public DefaultLineageService(CallContext callContext, LineageConfiguration configuration) {
+    this.callContext = callContext;
+    this.configuration = configuration;
+  }
+
+  @Override
+  public LineageGraph query(LineageQueryRequest request) {
+    ensureEnabled();
+    throw new UnsupportedOperationException("Lineage query is not implemented yet.");
+  }
+
+  private void ensureEnabled() {
+    if (!configuration.enabled()) {
+      throw new UnsupportedOperationException(
+          "Lineage is disabled: set polaris.lineage.enabled=true to enable it.");
+    }
+
+    if (!callContext.getRealmConfig().getConfig(FeatureConfiguration.ENABLE_LINEAGE)) {
+      throw new UnsupportedOperationException(
+          "Lineage realm feature is disabled: enable "
+              + FeatureConfiguration.ENABLE_LINEAGE.key()
+              + " in the realm feature configuration.");
+    }
+  }
+}