Align native UDF builds with jar dependency pins

examples/UDF-Examples/RAPIDS-accelerated-UDFs/README.md

@@ -146,7 +146,10 @@ mvn clean package -Pudf-native-examples
 
 The build will automatically:
 - Extract `libcudf.so` from the rapids-4-spark jar
-- Clone cuDF repository for headers (shallow clone)
+- Read the embedded `spark-rapids-jni` and `cudf-java` version metadata from the jar
+- Download the matching `spark-rapids-jni` `cudf-pins` files
+- Clone the cuDF repository at the revision recorded in the jar
+- Configure rapids-cmake with the matching `rapids-cmake` sha and package override file
 - Build only your UDF native code against the prebuilt library
 
 **Native ABI compatibility note:**
@@ -160,12 +163,11 @@ the UDF may fail with undefined symbols or crash when Spark loads the native
 library.
 
 When changing the rapids-4-spark jar version, rebuild the native UDFs with
-matching cuDF/RMM/CCCL headers and libraries. For snapshot or locally built
-jars, make sure `cudf.git.branch`, the rapids-cmake branch in
-`src/main/cpp/CMakeLists.txt`, and the RMM/CCCL versions resolved by
-rapids-cmake correspond to the same source state used to build the jar. After
-changing these values, remove `target/native-deps` and `target/cudf-repo`
-before rebuilding so stale headers or libraries are not reused.
+matching cuDF/RMM/CCCL headers and libraries. In prebuilt mode, the Maven build
+derives those native dependency pins from the `spark-rapids-jni` revision
+recorded in the jar. After changing jar versions, remove `target/native-deps`,
+`target/cudf-repo`, `target/cudf-pins`, and `target/cpp-build` before
+rebuilding so stale headers, pins, or libraries are not reused.
 
 **Or manually extract first:**
 ```bash
@@ -217,7 +219,7 @@ You can customize the build by passing Maven system properties via `-D<property>
 | `BUILD_UDF_BENCHMARKS` | `OFF` | Build benchmark executables |
 | `PER_THREAD_DEFAULT_STREAM` | `ON` | Enable per-thread default CUDA streams |
 | `CUDF_ENABLE_ARROW_S3` | `OFF` | Enable Arrow S3 support in cuDF |
-| `cudf.git.branch` | `main` | cuDF git branch to clone for headers |
+| `cudf.git.branch` | `main` | cuDF git branch to clone for source builds or fallback paths |
 | `skipCudfExtraction` | `false` | Skip extracting cuDF dependencies from jar |
 
 **Example usage:**

examples/UDF-Examples/RAPIDS-accelerated-UDFs/clone-cudf-repo.sh

@@ -22,11 +22,11 @@
 # headers needed for compiling native UDF code.
 #
 # Usage:
-#   clone-cudf-repo.sh <target_directory> <branch_name>
+#   clone-cudf-repo.sh <target_directory> <git_ref>
 #
 # Arguments:
 #   target_directory - Directory where cuDF repo will be cloned
-#   branch_name      - Git branch to clone/checkout
+#   git_ref          - Git branch, tag, or commit to checkout
 #
 # Exit codes:
 #   0 - Success
@@ -38,79 +38,88 @@ set -o pipefail
 
 # Parse arguments
 if [ $# -ne 2 ]; then
-    echo "ERROR: Usage: $0 <target_directory> <branch_name>" >&2
+    echo "ERROR: Usage: $0 <target_directory> <git_ref>" >&2
     exit 1
 fi
 
 CUDF_DIR="$1"
-BRANCH="$2"
+GIT_REF="$2"
+GIT=(git -c "safe.directory=$CUDF_DIR")
 
 echo "================================================"
 echo "cuDF Repository Management"
 echo "  Target directory: $CUDF_DIR"
-echo "  Branch: $BRANCH"
+echo "  Git ref: $GIT_REF"
 echo "================================================"
 
 # Check if repository already exists
 if [ ! -d "$CUDF_DIR/.git" ]; then
     # Repository doesn't exist - clone it
-    echo "Cloning cuDF repository ($BRANCH branch)..."
+    echo "Cloning cuDF repository..."
 
-    git clone --depth 1 --branch "$BRANCH" \
-        https://github.com/rapidsai/cudf.git "$CUDF_DIR" || {
-        echo "ERROR: Failed to clone cuDF from branch $BRANCH" >&2
+    "${GIT[@]}" clone --filter=blob:none --no-checkout https://github.com/rapidsai/cudf.git "$CUDF_DIR" || {
+        echo "ERROR: Failed to clone cuDF repository" >&2
         echo "Please check:" >&2
         echo "  1. Network connectivity to GitHub" >&2
-        echo "  2. Branch '$BRANCH' exists in cuDF repository" >&2
         exit 1
     }
-
+
+    cd "$CUDF_DIR" || {
+        echo "ERROR: Cannot access directory $CUDF_DIR" >&2
+        exit 1
+    }
+
+    "${GIT[@]}" fetch --depth 1 origin "$GIT_REF" || {
+        echo "ERROR: Failed to fetch cuDF ref $GIT_REF from origin" >&2
+        echo "Please check:" >&2
+        echo "  1. Network connectivity to GitHub" >&2
+        echo "  2. Ref '$GIT_REF' exists in cuDF repository" >&2
+        exit 1
+    }
+
+    "${GIT[@]}" checkout --detach FETCH_HEAD || {
+        echo "ERROR: Failed to checkout cuDF ref $GIT_REF" >&2
+        exit 1
+    }
+
     echo "✓ Successfully cloned cuDF repository"
 else
     # Repository exists - verify and update if needed
-    echo "cuDF repository exists, verifying branch..."
+    echo "cuDF repository exists, verifying ref..."
     cd "$CUDF_DIR" || {
         echo "ERROR: Cannot access directory $CUDF_DIR" >&2
         exit 1
     }
-
-    # Get current branch
-    CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD 2>/dev/null || echo "unknown")
-
-    if [ "$CURRENT_BRANCH" != "$BRANCH" ]; then
-        # Branch mismatch - fetch and switch to correct branch
-        echo "Branch mismatch detected:"
-        echo "  Current branch: $CURRENT_BRANCH"
-        echo "  Expected branch: $BRANCH"
-        echo "Fetching and switching to $BRANCH..."
-
-        git fetch --depth 1 origin "$BRANCH" || {
-            echo "ERROR: Failed to fetch branch $BRANCH from origin" >&2
-            echo "Please check:" >&2
-            echo "  1. Network connectivity to GitHub" >&2
-            echo "  2. Branch '$BRANCH' exists in cuDF repository" >&2
-            exit 1
-        }
-
-        git checkout "$BRANCH" || {
-            echo "ERROR: Failed to checkout branch $BRANCH" >&2
-            exit 1
-        }
-
-        git reset --hard "origin/$BRANCH" || {
-            echo "ERROR: Failed to reset to origin/$BRANCH" >&2
+
+    CURRENT_COMMIT=$("${GIT[@]}" rev-parse HEAD 2>/dev/null || echo "unknown")
+
+    "${GIT[@]}" fetch --depth 1 origin "$GIT_REF" || {
+        echo "ERROR: Failed to fetch cuDF ref $GIT_REF from origin" >&2
+        echo "Please check:" >&2
+        echo "  1. Network connectivity to GitHub" >&2
+        echo "  2. Ref '$GIT_REF' exists in cuDF repository" >&2
+        exit 1
+    }
+
+    FETCHED_COMMIT=$("${GIT[@]}" rev-parse FETCH_HEAD)
+
+    if [ "$CURRENT_COMMIT" != "$FETCHED_COMMIT" ]; then
+        echo "cuDF ref mismatch detected:"
+        echo "  Current commit: $CURRENT_COMMIT"
+        echo "  Expected commit: $FETCHED_COMMIT ($GIT_REF)"
+        "${GIT[@]}" checkout --detach FETCH_HEAD || {
+            echo "ERROR: Failed to checkout cuDF ref $GIT_REF" >&2
             exit 1
         }
-
-        echo "✓ Switched to branch $BRANCH"
+        echo "✓ Switched cuDF repository to $GIT_REF"
     else
-        echo "✓ Already on correct branch ($BRANCH)"
+        echo "✓ Already on correct cuDF ref ($GIT_REF)"
     fi
 fi
 
 echo "================================================"
 echo "✓ cuDF repository ready at: $CUDF_DIR"
-echo "  Branch: $BRANCH"
+echo "  Git ref: $GIT_REF"
 echo "================================================"
 
 exit 0

examples/UDF-Examples/RAPIDS-accelerated-UDFs/extract-cudf-libs.sh

@@ -28,6 +28,7 @@
 #   ./extract-cudf-libs.sh
 #
 # Environment Variables (optional, will use pom.xml values if not set):
+#   RAPIDS_JAR_PATH      - explicit rapids-4-spark jar path
 #   RAPIDS4SPARK_VERSION - rapids-4-spark version (e.g., 26.02.0 or 26.06.0-SNAPSHOT)
 #   SCALA_VERSION        - Scala binary version (e.g., 2.12, 2.13)
 #   CUDA_VERSION         - CUDA version (e.g., cuda11, cuda12)
@@ -40,7 +41,7 @@
 set -e
 
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-TARGET_DIR="$SCRIPT_DIR/target"
+TARGET_DIR="${TARGET_DIR:-$SCRIPT_DIR/target}"
 NATIVE_DEPS_DIR="$TARGET_DIR/native-deps"
 CUDF_REPO_DIR="$TARGET_DIR/cudf-repo"
 POM_FILE="$SCRIPT_DIR/pom.xml"
@@ -50,6 +51,11 @@ POM_FILE="$SCRIPT_DIR/pom.xml"
 extract_pom_property() {
     local property_name="$1"
     local value
+
+    if [ ! -f "$POM_FILE" ]; then
+        echo ""
+        return
+    fi
 
     # Use xmllint if available (more reliable)
     if command -v xmllint >/dev/null 2>&1; then
@@ -109,10 +115,16 @@ JAR_PATH_WITH_CLASSIFIER="$MAVEN_REPO/com/nvidia/rapids-4-spark_${SCALA_VERSION}
 JAR_PATH_NO_CLASSIFIER="$MAVEN_REPO/com/nvidia/rapids-4-spark_${SCALA_VERSION}/${RAPIDS4SPARK_VERSION}/rapids-4-spark_${SCALA_VERSION}-${RAPIDS4SPARK_VERSION}.jar"
 
 echo "Looking for rapids-4-spark jar..."
+if [ -n "${RAPIDS_JAR_PATH:-}" ]; then
+    echo "  Explicit path: $RAPIDS_JAR_PATH"
+fi
 echo "  Pattern 1 (with classifier): $JAR_PATH_WITH_CLASSIFIER"
 echo "  Pattern 2 (no classifier):   $JAR_PATH_NO_CLASSIFIER"
 
-if [ -f "$JAR_PATH_WITH_CLASSIFIER" ]; then
+if [ -n "${RAPIDS_JAR_PATH:-}" ] && [ -f "$RAPIDS_JAR_PATH" ]; then
+    JAR_PATH="$RAPIDS_JAR_PATH"
+    echo "✓ Found jar (explicit path): $JAR_PATH"
+elif [ -f "$JAR_PATH_WITH_CLASSIFIER" ]; then
     JAR_PATH="$JAR_PATH_WITH_CLASSIFIER"
     echo "✓ Found jar (with classifier): $JAR_PATH"
 elif [ -f "$JAR_PATH_NO_CLASSIFIER" ]; then
@@ -122,6 +134,9 @@ else
     echo ""
     echo "ERROR: rapids-4-spark jar not found!"
     echo "Tried:"
+    if [ -n "${RAPIDS_JAR_PATH:-}" ]; then
+        echo "  $RAPIDS_JAR_PATH"
+    fi
     echo "  $JAR_PATH_WITH_CLASSIFIER"
     echo "  $JAR_PATH_NO_CLASSIFIER"
     echo ""
@@ -232,15 +247,22 @@ fi
 echo "✓ Successfully extracted libraries to: $NATIVE_DEPS_DIR"
 ls -lh "$NATIVE_DEPS_DIR"
 
+PINS_DIR="$TARGET_DIR/cudf-pins"
+PINS_PROPERTIES="$TARGET_DIR/jar-native-deps.properties"
+"$SCRIPT_DIR/resolve-jni-cudf-pins.sh" "$JAR_PATH" "$PINS_DIR" "$PINS_PROPERTIES"
+RESOLVED_CUDF_REF=$(awk -F= '$1 == "jar.cudf.revision" {print $2; exit}' "$PINS_PROPERTIES")
+if [ -z "$RESOLVED_CUDF_REF" ]; then
+    echo "ERROR: Failed to resolve cuDF revision from $PINS_PROPERTIES" >&2
+    exit 1
+fi
+
 # Clone cuDF repo for headers (shallow clone)
 if [ ! -d "$CUDF_REPO_DIR/.git" ]; then
     echo "Cloning cuDF repository for headers..."
-    git clone --depth 1 --branch "$CUDF_BRANCH" https://github.com/rapidsai/cudf.git "$CUDF_REPO_DIR"
-    echo "✓ Cloned cuDF repo to: $CUDF_REPO_DIR"
 else
     echo "✓ cuDF repo already exists at: $CUDF_REPO_DIR"
-    echo "  (Delete it to re-clone: rm -rf \"$CUDF_REPO_DIR\")"
 fi
+"$SCRIPT_DIR/clone-cudf-repo.sh" "$CUDF_REPO_DIR" "$RESOLVED_CUDF_REF"
 
 echo ""
 echo "=================================================="

examples/UDF-Examples/RAPIDS-accelerated-UDFs/pom.xml

@@ -413,6 +413,16 @@
                                                   message="Failed to extract libcudf.so from jar"/>
 
                                             <echo message="✓ libcudf.so extracted to: ${project.build.directory}/native-deps"/>
+
+                                            <!-- Resolve matching cuDF/RMM/CCCL pins from the spark-rapids-jni
+                                                 revision embedded in the rapids-4-spark jar. -->
+                                            <chmod file="${basedir}/resolve-jni-cudf-pins.sh" perm="755"/>
+                                            <exec executable="${basedir}/resolve-jni-cudf-pins.sh" failonerror="true">
+                                                <arg value="${rapids.jar.path}"/>
+                                                <arg value="${project.build.directory}/cudf-pins"/>
+                                                <arg value="${project.build.directory}/jar-native-deps.properties"/>
+                                            </exec>
+                                            <property file="${project.build.directory}/jar-native-deps.properties"/>
 
                                             <!-- Clone cuDF repo for headers only (shallow clone) -->
                                             <!-- Ensure script is executable -->
@@ -421,7 +431,7 @@
                                             <!-- Execute script to clone/update cuDF repository -->
                                             <exec executable="${basedir}/clone-cudf-repo.sh" failonerror="true">
                                                 <arg value="${project.build.directory}/cudf-repo"/>
-                                                <arg value="${cudf.git.branch}"/>
+                                                <arg value="${jar.cudf.revision}"/>
                                             </exec>
 
                                             <!-- Verify cuDF headers exist after clone/update -->
@@ -451,6 +461,7 @@
                                 <phase>compile</phase>
                                 <configuration>
                                     <target>
+                                        <property file="${project.build.directory}/jar-native-deps.properties"/>
                                         <mkdir dir="${udf.native.build.path}"/>
                                         <exec dir="${udf.native.build.path}"
                                               failonerror="true"
@@ -462,6 +473,9 @@
                                             <arg value="-DPER_THREAD_DEFAULT_STREAM=${PER_THREAD_DEFAULT_STREAM}"/>
                                             <arg value="-DCUDF_ENABLE_ARROW_S3=${CUDF_ENABLE_ARROW_S3}"/>
                                             <arg value="-DUSE_PREBUILT_CUDF=${USE_PREBUILT_CUDF}"/>
+                                            <arg value="-DRAPIDS_CMAKE_BRANCH=${jar.rapids.cmake.sha}"/>
+                                            <arg value="-DRAPIDS_CMAKE_FILE=${jar.rapids.cmake.file}"/>
+                                            <arg value="-DRAPIDS_CMAKE_CPM_OVERRIDE_VERSION_FILE=${jar.cudf.pins.file}"/>
                                         </exec>
                                         <exec failonerror="true"
                                               executable="cmake">