Adding introspection levels

benchmarks/CMakeLists.txt

@@ -148,4 +148,13 @@ if (UMPIRE_ENABLE_BENCHMARKS)
   blt_add_benchmark(
     NAME inspector_benchmarks
     COMMAND inspector_benchmarks)
+
+  blt_add_executable(
+    NAME introspection_level_benchmarks
+    SOURCES introspection_level_benchmarks.cpp
+    DEPENDS_ON ${benchmark_depends})
+
+  blt_add_benchmark(
+    NAME introspection_level_benchmarks
+    COMMAND introspection_level_benchmarks)
 endif()

benchmarks/introspection_level_benchmarks.cpp

@@ -0,0 +1,248 @@
+//////////////////////////////////////////////////////////////////////////////
+// Copyright (c) 2016-26, Lawrence Livermore National Security, LLC and Umpire
+// project contributors. See the COPYRIGHT file for details.
+//
+// SPDX-License-Identifier: (MIT)
+//////////////////////////////////////////////////////////////////////////////
+#include <vector>
+#include <iostream>
+
+#include "benchmark/benchmark.h"
+
+#include "umpire/Allocator.hpp"
+#include "umpire/ResourceManager.hpp"
+#include "umpire/Introspection.hpp"
+
+static const std::size_t NUM_ALLOCATIONS = 10000;
+static const std::size_t ALLOC_SIZE = 1024;
+
+// Get current introspection level (set via UMPIRE_INTROSPECTION_LEVEL env var)
+static umpire::IntrospectionLevel getCurrentLevel() {
+  auto& rm = umpire::ResourceManager::getInstance();
+  return rm.getIntrospectionLevel();
+}
+
+// Benchmark allocation/deallocation overhead
+static void BM_Allocate(benchmark::State& state) {
+  auto& rm = umpire::ResourceManager::getInstance();
+  auto alloc = rm.getAllocator("HOST");
+  std::vector<void*> ptrs;
+  ptrs.reserve(NUM_ALLOCATIONS);
+
+  for (auto _ : state) {
+    state.PauseTiming();
+    ptrs.clear();
+    state.ResumeTiming();
+
+    for (std::size_t i = 0; i < NUM_ALLOCATIONS; ++i) {
+      void* ptr = alloc.allocate(ALLOC_SIZE);
+      ptrs.push_back(ptr);
+    }
+
+    state.PauseTiming();
+    for (auto ptr : ptrs) {
+      alloc.deallocate(ptr);
+    }
+    state.ResumeTiming();
+  }
+
+  state.SetItemsProcessed(state.iterations() * NUM_ALLOCATIONS);
+}
+BENCHMARK(BM_Allocate);
+
+// Benchmark hasAllocator queries (Off mode: always returns false, skip)
+static void BM_HasAllocator(benchmark::State& state) {
+  auto& rm = umpire::ResourceManager::getInstance();
+
+  if (getCurrentLevel() == umpire::IntrospectionLevel::Off) {
+    state.SkipWithError("hasAllocator not available in Off mode");
+    return;
+  }
+
+  auto alloc = rm.getAllocator("HOST");
+  std::vector<void*> ptrs;
+  ptrs.reserve(NUM_ALLOCATIONS);
+
+  for (std::size_t i = 0; i < NUM_ALLOCATIONS; ++i) {
+    ptrs.push_back(alloc.allocate(ALLOC_SIZE));
+  }
+
+  for (auto _ : state) {
+    for (auto ptr : ptrs) {
+      benchmark::DoNotOptimize(rm.hasAllocator(ptr));
+    }
+  }
+
+  for (auto ptr : ptrs) {
+    alloc.deallocate(ptr);
+  }
+
+  state.SetItemsProcessed(state.iterations() * NUM_ALLOCATIONS);
+}
+BENCHMARK(BM_HasAllocator);
+
+// Benchmark getAllocator queries (Off mode: throws, skip)
+static void BM_GetAllocator(benchmark::State& state) {
+  auto& rm = umpire::ResourceManager::getInstance();
+
+  if (getCurrentLevel() == umpire::IntrospectionLevel::Off) {
+    state.SkipWithError("getAllocator not available in Off mode");
+    return;
+  }
+
+  auto alloc = rm.getAllocator("HOST");
+  std::vector<void*> ptrs;
+  ptrs.reserve(NUM_ALLOCATIONS);
+
+  for (std::size_t i = 0; i < NUM_ALLOCATIONS; ++i) {
+    ptrs.push_back(alloc.allocate(ALLOC_SIZE));
+  }
+
+  for (auto _ : state) {
+    for (auto ptr : ptrs) {
+      benchmark::DoNotOptimize(rm.getAllocator(ptr));
+    }
+  }
+
+  for (auto ptr : ptrs) {
+    alloc.deallocate(ptr);
+  }
+
+  state.SetItemsProcessed(state.iterations() * NUM_ALLOCATIONS);
+}
+BENCHMARK(BM_GetAllocator);
+
+// Benchmark getSize queries (only available in On mode)
+static void BM_GetSize(benchmark::State& state) {
+  auto& rm = umpire::ResourceManager::getInstance();
+
+  if (getCurrentLevel() != umpire::IntrospectionLevel::On) {
+    state.SkipWithError("getSize only available in On mode");
+    return;
+  }
+
+  auto alloc = rm.getAllocator("HOST");
+  std::vector<void*> ptrs;
+  ptrs.reserve(NUM_ALLOCATIONS);
+
+  for (std::size_t i = 0; i < NUM_ALLOCATIONS; ++i) {
+    ptrs.push_back(alloc.allocate(ALLOC_SIZE));
+  }
+
+  for (auto _ : state) {
+    for (auto ptr : ptrs) {
+      benchmark::DoNotOptimize(rm.getSize(ptr));
+    }
+  }
+
+  for (auto ptr : ptrs) {
+    alloc.deallocate(ptr);
+  }
+
+  state.SetItemsProcessed(state.iterations() * NUM_ALLOCATIONS);
+}
+BENCHMARK(BM_GetSize);
+
+// Benchmark copy operations (Off mode: not available, skip)
+static void BM_Copy(benchmark::State& state) {
+  auto& rm = umpire::ResourceManager::getInstance();
+
+  if (getCurrentLevel() == umpire::IntrospectionLevel::Off) {
+    state.SkipWithError("copy not available in Off mode");
+    return;
+  }
+
+  auto alloc = rm.getAllocator("HOST");
+  void* src = alloc.allocate(ALLOC_SIZE);
+  void* dst = alloc.allocate(ALLOC_SIZE);
+
+  for (auto _ : state) {
+    rm.copy(dst, src, ALLOC_SIZE);
+  }
+
+  alloc.deallocate(src);
+  alloc.deallocate(dst);
+
+  state.SetItemsProcessed(state.iterations());
+  state.SetBytesProcessed(state.iterations() * ALLOC_SIZE);
+}
+BENCHMARK(BM_Copy);
+
+// Benchmark memset operations (Off mode: not available, skip)
+static void BM_Memset(benchmark::State& state) {
+  auto& rm = umpire::ResourceManager::getInstance();
+
+  if (getCurrentLevel() == umpire::IntrospectionLevel::Off) {
+    state.SkipWithError("memset not available in Off mode");
+    return;
+  }
+
+  auto alloc = rm.getAllocator("HOST");
+  void* ptr = alloc.allocate(ALLOC_SIZE);
+
+  for (auto _ : state) {
+    rm.memset(ptr, 0, ALLOC_SIZE);
+  }
+
+  alloc.deallocate(ptr);
+
+  state.SetItemsProcessed(state.iterations());
+  state.SetBytesProcessed(state.iterations() * ALLOC_SIZE);
+}
+BENCHMARK(BM_Memset);
+
+// Benchmark memory overhead (allocation records storage)
+// Tests how performance scales with increasing allocation count
+static void BM_MemoryOverhead(benchmark::State& state) {
+  auto& rm = umpire::ResourceManager::getInstance();
+  auto alloc = rm.getAllocator("HOST");
+  std::vector<void*> ptrs;
+
+  for (auto _ : state) {
+    state.PauseTiming();
+    ptrs.clear();
+    ptrs.reserve(state.range(0));
+    state.ResumeTiming();
+
+    for (int64_t i = 0; i < state.range(0); ++i) {
+      ptrs.push_back(alloc.allocate(ALLOC_SIZE));
+    }
+
+    state.PauseTiming();
+    for (auto ptr : ptrs) {
+      alloc.deallocate(ptr);
+    }
+    state.ResumeTiming();
+  }
+
+  state.SetItemsProcessed(state.iterations() * state.range(0));
+}
+BENCHMARK(BM_MemoryOverhead)->Range(100, 100000);
+
+int main(int argc, char** argv) {
+  auto& rm = umpire::ResourceManager::getInstance();
+  auto level = rm.getIntrospectionLevel();
+
+  std::cout << "============================================\n";
+  std::cout << "Introspection Level Benchmarks\n";
+  std::cout << "============================================\n";
+  std::cout << "Current level: ";
+  switch (level) {
+    case umpire::IntrospectionLevel::Off:
+      std::cout << "Off (zero overhead, no introspection)\n";
+      break;
+    case umpire::IntrospectionLevel::Basic:
+      std::cout << "Basic (runtime API inference, zero storage)\n";
+      break;
+    case umpire::IntrospectionLevel::On:
+      std::cout << "On (full tracking with metadata)\n";
+      break;
+  }
+  std::cout << "============================================\n\n";
+
+  ::benchmark::Initialize(&argc, argv);
+  if (::benchmark::ReportUnrecognizedArguments(argc, argv)) return 1;
+  ::benchmark::RunSpecifiedBenchmarks();
+  ::benchmark::Shutdown();
+  return 0;
+}

benchmarks/run_introspection_benchmarks.sh

@@ -0,0 +1,119 @@
+#!/bin/bash
+##############################################################################
+# Copyright (c) 2016-26, Lawrence Livermore National Security, LLC and Umpire
+# project contributors. See the COPYRIGHT file for details.
+#
+# SPDX-License-Identifier: (MIT)
+##############################################################################
+
+# Script to run introspection level benchmarks and generate comparison report
+
+BENCHMARK_BIN="${1:-./bin/introspection_level_benchmarks}"
+
+if [ ! -f "$BENCHMARK_BIN" ]; then
+    echo "Error: Benchmark binary not found at $BENCHMARK_BIN"
+    echo "Usage: $0 [path_to_benchmark_binary]"
+    echo "Example: $0 ./build/bin/introspection_level_benchmarks"
+    exit 1
+fi
+
+OUTPUT_DIR="benchmark_results_$(date +%Y%m%d_%H%M%S)"
+mkdir -p "$OUTPUT_DIR"
+
+echo "============================================"
+echo "Introspection Level Benchmark Suite"
+echo "============================================"
+echo ""
+echo "Output directory: $OUTPUT_DIR"
+echo ""
+echo "Running benchmarks for all three levels..."
+echo ""
+
+# Run benchmarks for each introspection level
+for LEVEL in off basic on; do
+    echo "============================================"
+    echo "Running: Level = $LEVEL"
+    echo "============================================"
+
+    UMPIRE_INTROSPECTION_LEVEL="$LEVEL" "$BENCHMARK_BIN" \
+        --benchmark_out="$OUTPUT_DIR/results_${LEVEL}.json" \
+        --benchmark_out_format=json \
+        --benchmark_counters_tabular=true \
+        | tee "$OUTPUT_DIR/results_${LEVEL}.txt"
+
+    echo ""
+done
+
+echo ""
+echo "============================================"
+echo "Results saved to $OUTPUT_DIR/"
+echo "  - results_off.txt/json: Off mode results"
+echo "  - results_basic.txt/json: Basic mode results"
+echo "  - results_on.txt/json: On mode results"
+echo "============================================"
+echo ""
+
+# Generate comparison summary
+echo "Performance Comparison Summary" | tee "$OUTPUT_DIR/comparison.txt"
+echo "==============================" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "" | tee -a "$OUTPUT_DIR/comparison.txt"
+
+echo "ALLOCATION PERFORMANCE (lower is better):" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "-----------------------------------------" | tee -a "$OUTPUT_DIR/comparison.txt"
+for LEVEL in off basic on; do
+    echo "" | tee -a "$OUTPUT_DIR/comparison.txt"
+    echo "[$LEVEL]:" | tee -a "$OUTPUT_DIR/comparison.txt"
+    grep "^BM_Allocate " "$OUTPUT_DIR/results_${LEVEL}.txt" 2>/dev/null | head -1 | tee -a "$OUTPUT_DIR/comparison.txt"
+done
+echo "" | tee -a "$OUTPUT_DIR/comparison.txt"
+
+echo "QUERY PERFORMANCE - hasAllocator (lower is better):" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "---------------------------------------------------" | tee -a "$OUTPUT_DIR/comparison.txt"
+for LEVEL in basic on; do
+    echo "" | tee -a "$OUTPUT_DIR/comparison.txt"
+    echo "[$LEVEL]:" | tee -a "$OUTPUT_DIR/comparison.txt"
+    grep "^BM_HasAllocator " "$OUTPUT_DIR/results_${LEVEL}.txt" 2>/dev/null | head -1 | tee -a "$OUTPUT_DIR/comparison.txt"
+done
+echo "(Off mode: N/A - always returns false)" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "" | tee -a "$OUTPUT_DIR/comparison.txt"
+
+echo "QUERY PERFORMANCE - getAllocator (lower is better):" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "----------------------------------------------------" | tee -a "$OUTPUT_DIR/comparison.txt"
+for LEVEL in basic on; do
+    echo "" | tee -a "$OUTPUT_DIR/comparison.txt"
+    echo "[$LEVEL]:" | tee -a "$OUTPUT_DIR/comparison.txt"
+    grep "^BM_GetAllocator " "$OUTPUT_DIR/results_${LEVEL}.txt" 2>/dev/null | head -1 | tee -a "$OUTPUT_DIR/comparison.txt"
+done
+echo "(Off mode: N/A - throws exception)" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "" | tee -a "$OUTPUT_DIR/comparison.txt"
+
+echo "COPY PERFORMANCE (lower is better):" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "-----------------------------------" | tee -a "$OUTPUT_DIR/comparison.txt"
+for LEVEL in basic on; do
+    echo "" | tee -a "$OUTPUT_DIR/comparison.txt"
+    echo "[$LEVEL]:" | tee -a "$OUTPUT_DIR/comparison.txt"
+    grep "^BM_Copy " "$OUTPUT_DIR/results_${LEVEL}.txt" 2>/dev/null | head -1 | tee -a "$OUTPUT_DIR/comparison.txt"
+done
+echo "(Off mode: N/A - not available)" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "" | tee -a "$OUTPUT_DIR/comparison.txt"
+
+echo "MEMORY OVERHEAD SCALING (time for 100K allocations):" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "-----------------------------------------------------" | tee -a "$OUTPUT_DIR/comparison.txt"
+for LEVEL in off basic on; do
+    echo "" | tee -a "$OUTPUT_DIR/comparison.txt"
+    echo "[$LEVEL]:" | tee -a "$OUTPUT_DIR/comparison.txt"
+    grep "^BM_MemoryOverhead/100000 " "$OUTPUT_DIR/results_${LEVEL}.txt" 2>/dev/null | tee -a "$OUTPUT_DIR/comparison.txt"
+done
+echo "" | tee -a "$OUTPUT_DIR/comparison.txt"
+
+echo "" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "KEY FINDINGS:" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "------------" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "- Off/Basic should have similar allocation performance (no tracking overhead)" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "- On mode has tracking overhead (Judy array insertions)" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "- Basic mode queries are SLOWER (runtime API calls) vs On mode (map lookups)" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "- Basic mode has ZERO storage overhead, On mode has ~24-48 bytes per allocation" | tee -a "$OUTPUT_DIR/comparison.txt"
+echo "" | tee -a "$OUTPUT_DIR/comparison.txt"
+
+echo ""
+echo "See $OUTPUT_DIR/comparison.txt for side-by-side comparison"