Add end-to-end CPU vs torch bench harness + 5-scene report (Issue #21)

.gitignore

@@ -1,13 +1,15 @@
 # Bench / profile runs produce machine-dependent artifacts (wall time,
 # RSS, VRAM-derived auto chunk sizes, hostnames, /home paths) that have
 # no meaning on a different machine and previously leaked host info
-# into the public repo (audited 2026-05-02). The whole logs_*/ tree is
+# into the public repo (audited 2026-05-02). Anything matching logs_*
+# (directories like logs_baseline/ as well as the end-to-end harness's
+# top-level diff files like logs_e2e_<scene>_walls_diff.json) is
 # therefore untracked.
 #
 # What stays tracked under benchmark/: harness scripts (scripts/),
 # Python utilities (tools/), test fixtures (fixtures/), reports
 # (reports/), the top-level Makefile, and requirements.txt.
 # Numerical findings are transcribed into the corresponding report
 # by hand; raw artifacts live only on the developer's machine.
-logs_*/
+logs_*
 __pycache__/

fixtures/FernandinaSenDT128_cpu.txt

@@ -0,0 +1,44 @@
+# CPU baseline fixture for FernandinaSenDT128 — byte-for-byte identical
+# to fixtures/FernandinaSenDT128_torch.txt except for the trailing
+# `mintpy.*.solver = torch` lines, which are omitted so MintPy uses the
+# default cpu solver for both invert_network and correct_topography.
+# Used by the Issue #21 end-to-end bench harness paired with the torch
+# fixture; the harness asserts that the cpu and torch fixtures differ
+# only in the 2 solver lines so the wall comparison is fair.
+# vim: set filetype=cfg:
+##------------------------------- ISCE/topsStack ----------------------##
+#ssaraopt = --platform=SENTINEL-1A,SENTINEL-1B -r 128 -f 589,590,591,592,593  -e 2017-07-01
+#sentinelStack.boundingBox      = '-1 0.15 -91.6 -90.9'
+#sentinelStack.subswath         = 1 2     # comment
+#sentinelStack.numConnections   = 5       # comment
+#sentinelStack.azimuthLooks     = 5       # comment
+#sentinelStack.rangeLooks       = 15      # comment
+#sentinelStack.filtStrength     = 0.2     # comment
+#sentinelStack.unwMethod        = snaphu  # comment
+#sentinelStack.coregistration   = auto    # comment
+#subset.y0:y1,x0:x1 = 1150:1600,1070:1670
+
+
+##-------------------------------- MintPy -----------------------------##
+########## 1. Load Data (--load to exit after this step)
+## load_data.py -H to check more details and example inputs.
+mintpy.load.processor        = isce
+##---------for ISCE only:
+mintpy.load.metaFile         = ../reference/IW*.xml
+mintpy.load.baselineDir      = ../baselines
+##---------interferogram datasets:
+mintpy.load.unwFile          = ../merged/interferograms/*/filt_*.unw
+mintpy.load.corFile          = ../merged/interferograms/*/filt_*.cor
+mintpy.load.connCompFile     = ../merged/interferograms/*/filt_*.unw.conncomp
+##---------geometry datasets:
+mintpy.load.demFile          = ../merged/geom_reference/hgt.rdr
+mintpy.load.lookupYFile      = ../merged/geom_reference/lat.rdr
+mintpy.load.lookupXFile      = ../merged/geom_reference/lon.rdr
+mintpy.load.incAngleFile     = ../merged/geom_reference/los.rdr
+mintpy.load.azAngleFile      = ../merged/geom_reference/los.rdr
+mintpy.load.shadowMaskFile   = ../merged/geom_reference/shadowMask.rdr
+mintpy.load.waterMaskFile    = None
+
+mintpy.reference.lalo               = -0.30,-91.43
+mintpy.topographicResidual.stepDate = 20170910,20180613  #eruption dates
+mintpy.deramp                       = linear

fixtures/FernandinaSenDT128_torch.txt

@@ -1,6 +1,7 @@
 # GPU benchmark template — copy of docs/templates/FernandinaSenDT128.txt
-# with mintpy.networkInversion.solver = torch added at the end so the
-# invert_network step uses the GPU-batched solver.
+# with the two opt-in GPU solver flags added at the end so both the
+# invert_network and correct_topography steps use the GPU-batched
+# solvers. Used by the Issue #21 end-to-end bench harness.
 # vim: set filetype=cfg:
 ##------------------------------- ISCE/topsStack ----------------------##
 #ssaraopt = --platform=SENTINEL-1A,SENTINEL-1B -r 128 -f 589,590,591,592,593  -e 2017-07-01
@@ -39,5 +40,6 @@ mintpy.reference.lalo               = -0.30,-91.43
 mintpy.topographicResidual.stepDate = 20170910,20180613  #eruption dates
 mintpy.deramp                       = linear
 
-##---------GPU acceleration of invert_network (fork-only, opt-in):
+##---------GPU acceleration (fork-only, opt-in):
 mintpy.networkInversion.solver      = torch
+mintpy.topographicResidual.solver   = torch

fixtures/GalapagosSenDT128_cpu.txt

@@ -0,0 +1,56 @@
+# CPU baseline fixture for GalapagosSenDT128 — derived from the
+# dataset's embedded mintpy/inputs/GalapagosSenDT128.template (Zenodo
+# record 4743058). Differs from fixtures/GalapagosSenDT128_torch.txt
+# only by the trailing `mintpy.*.solver = torch` lines, which are
+# omitted so MintPy uses the default cpu solver for both invert_network
+# and correct_topography. Used by the Issue #21 end-to-end bench
+# harness paired with the torch fixture.
+#
+# Raw ISCE inputs referenced under $GALAPAGOS_DIR/... no longer exist
+# on disk (the Issue #6 large-scene bench cleaned them up after load).
+# The harness pre-seeds ifgramStack.h5 + geometryRadar.h5 symlinks
+# into the workdir's inputs/ from the original mintpy/inputs/ so
+# smallbaselineApp skips load_data via update-mode; the $GALAPAGOS_DIR
+# variable just needs to resolve to a valid path string for parsing.
+# vim: set filetype=cfg:
+##----------------------------- SentinelStack/ISCE ---------------------##
+cleanopt                        = 1    # [ 0 / 1 / 2 / 3 4]   0: no cleaning, 1: largest files, 2: merged/etc, PROCESS dirs, 3: SLC,RAW, 4: everything
+ssaraopt                        = --platform=SENTINEL-1A,SENTINEL-1B -r 128 -f 587,588,589,590,591,592,593  -e 2018-07-01
+processor                       = isce
+sentinelStack.demDir            = /nethome/famelung/Sentinel/GalapagosT128SenVVD/DEM
+sentinelStack.boundingBox       = '-1 0.15 -91.6 -90.9'
+sentinelStack.subswath          = 1 2  # comment
+sentinelStack.numConnections    = 5   # comment
+sentinelStack.azimuthLooks      = 5   # comment
+sentinelStack.rangeLooks        = 15  # comment
+sentinelStack.filtStrength      = 0.2  # comment
+sentinelStack.unwMethod         = snaphu  # comment
+sentinelStack.coregistration    = auto  # comment
+
+
+
+##-------------------------------- MintPy -----------------------------##
+mintpy.load.processor        = isce
+##---------for ISCE only:
+mintpy.load.metaFile         = $GALAPAGOS_DIR/GalapagosSenDT128/ISCE/master/IW*.xml
+mintpy.load.baselineDir      = $GALAPAGOS_DIR/GalapagosSenDT128/ISCE/baselines
+##---------interferogram datasets:
+mintpy.load.unwFile          = $GALAPAGOS_DIR/GalapagosSenDT128/ISCE/merged/interferograms/*/filt_*.unw
+mintpy.load.corFile          = $GALAPAGOS_DIR/GalapagosSenDT128/ISCE/merged/interferograms/*/filt_*.cor
+mintpy.load.connCompFile     = $GALAPAGOS_DIR/GalapagosSenDT128/ISCE/merged/interferograms/*/filt_*.unw.conncomp
+##---------geometry datasets:
+mintpy.load.demFile          = $GALAPAGOS_DIR/GalapagosSenDT128/ISCE/merged/geom_master/hgt.rdr
+mintpy.load.lookupYFile      = $GALAPAGOS_DIR/GalapagosSenDT128/ISCE/merged/geom_master/lat.rdr
+mintpy.load.lookupXFile      = $GALAPAGOS_DIR/GalapagosSenDT128/ISCE/merged/geom_master/lon.rdr
+mintpy.load.incAngleFile     = $GALAPAGOS_DIR/GalapagosSenDT128/ISCE/merged/geom_master/los.rdr
+mintpy.load.azAngleFile      = $GALAPAGOS_DIR/GalapagosSenDT128/ISCE/merged/geom_master/los.rdr
+mintpy.load.shadowMaskFile   = $GALAPAGOS_DIR/GalapagosSenDT128/ISCE/merged/geom_master/shadowMask.rdr
+mintpy.load.waterMaskFile    = $GALAPAGOS_DIR/GalapagosSenDT128/ISCE/merged/geom_master/waterMask.rdr
+mintpy.load.bperpFile        = $GALAPAGOS_DIR/GalapagosSenDT128/ISCE/merged/baseline_grid/*/bperp.rdr
+
+mintpy.subset.yx                         = 400:2400,0:1700
+mintpy.reference.lalo                    = -0.31,-91.22
+mintpy.network.endDate                   = 20180626
+mintpy.unwrapError.method                = bridging+phase_closure   #[bridging / phase_closure / no], auto for no
+mintpy.topographicResidual.stepFuncDate  = 20150524,20150616,20170321,20170910,20180613  #Wolf,Wolf,CerroAzul,Fernandina,Fernandina
+mintpy.deramp                            = linear

fixtures/GalapagosSenDT128_torch.txt

@@ -1,8 +1,10 @@
 # GPU benchmark template for GalapagosSenDT128 — copy of the dataset's
 # embedded inputs/GalapagosSenDT128.template (Zenodo record 4743058) with
-# mintpy.networkInversion.solver = torch added at the end so the
-# invert_network step uses the GPU-batched solver. The cpu baseline run
-# uses the embedded template directly (no solver line, defaults to cpu).
+# the two opt-in GPU solver flags added at the end so both the
+# invert_network and correct_topography steps use the GPU-batched
+# solvers. The cpu baseline run uses the embedded template directly
+# (no solver lines, defaults to cpu). Used by the Issue #21 end-to-end
+# bench harness.
 # vim: set filetype=cfg:
 ##----------------------------- SentinelStack/ISCE ---------------------##
 cleanopt                        = 1    # [ 0 / 1 / 2 / 3 4]   0: no cleaning, 1: largest files, 2: merged/etc, PROCESS dirs, 3: SLC,RAW, 4: everything
@@ -46,5 +48,6 @@ mintpy.unwrapError.method                = bridging+phase_closure   #[bridging /
 mintpy.topographicResidual.stepFuncDate  = 20150524,20150616,20170321,20170910,20180613  #Wolf,Wolf,CerroAzul,Fernandina,Fernandina
 mintpy.deramp                            = linear
 
-##---------GPU acceleration of invert_network (fork-only, opt-in):
+##---------GPU acceleration (fork-only, opt-in):
 mintpy.networkInversion.solver           = torch
+mintpy.topographicResidual.solver        = torch

fixtures/KujuAlosAT422F650_cpu.txt

@@ -0,0 +1,31 @@
+# CPU baseline fixture for KujuAlosAT422F650 — derived from
+# docs/templates/KujuAlosAT422F650.txt with the same Issue #19 bench
+# overrides as fixtures/KujuAlosAT422F650_torch.txt
+# (`troposphericDelay.method = no` to skip ERA5 download for ALOS-1
+# L-band 2007-2011 data, `save.hdfEos5 = no` to skip HE5 packaging).
+# Differs from the torch fixture only by the trailing
+# `mintpy.*.solver = torch` lines so the wall comparison is fair.
+# vim: set filetype=cfg:
+########## 1. Load Data (--load to exit after this step)
+## auto - automatic path pattern for Univ of Miami file structure
+## load_data.py -H to check more details and example inputs.
+mintpy.load.processor      = roipac  #[isce,roipac,gamma,], auto for isce
+##---------interferogram datasets:
+mintpy.load.unwFile        = ./../interferograms/*/filt_*.unw      #[path2unw_file]
+mintpy.load.corFile        = ./../interferograms/*/filt_*.cor      #[path2cor_file]
+mintpy.load.connCompFile   = None                                  #[path2conn_file]
+mintpy.load.intFile        = None                                  #[path2int_file]
+##---------geometry datasets:
+mintpy.load.demFile        = ./../geometry/radar*.hgt              #[path2hgt_file]
+mintpy.load.lookupYFile    = ./../geometry/geomap*.trans           #[path2lookup_lat/y_file]
+mintpy.load.lookupXFile    = ./../geometry/geomap*.trans           #[path2lookup_lon/x_file]
+mintpy.load.incAngleFile   = None                                  #[path2los_file]
+mintpy.load.azAngleFile    = None                                  #[path2los_file]
+mintpy.load.shadowMaskFile = None                                  #[path2shadow_file]
+mintpy.load.waterMaskFile  = None                                  #[path2water_mask_file]
+
+##————————————————————————————— PROCESSING Options ——-----————————————————————##
+mintpy.reference.lalo               = 33.0655, 131.2076
+mintpy.troposphericDelay.method     = no   # bench: run on raw timeseries.h5
+mintpy.deramp                       = no
+mintpy.save.hdfEos5                 = no   # bench: skip HE5 packaging

fixtures/KujuAlosAT422F650_torch.txt

@@ -1,8 +1,8 @@
 # GPU benchmark template — copy of docs/templates/KujuAlosAT422F650.txt
-# with mintpy.networkInversion.solver = torch added at the end so the
-# invert_network step uses the GPU-batched solver. Used by Issue #19
-# Tier 1 bench survey (extends Fernandina + Galapagos with an ALOS
-# L-band / ROI_PAC ingest path).
+# with the two opt-in GPU solver flags added at the end so both the
+# invert_network and correct_topography steps use the GPU-batched
+# solvers. Used by Issue #19 Tier 1 bench survey (ALOS L-band /
+# ROI_PAC ingest path) and the Issue #21 end-to-end bench harness.
 #
 # Differences from docs/templates beyond the solver line:
 #   - mintpy.troposphericDelay.method = no       (skip ERA5 download;
@@ -34,5 +34,6 @@ mintpy.troposphericDelay.method     = no   # bench: run on raw timeseries.h5
 mintpy.deramp                       = no
 mintpy.save.hdfEos5                 = no   # bench: skip HE5 packaging
 
-##---------GPU acceleration of invert_network (fork-only, opt-in):
+##---------GPU acceleration (fork-only, opt-in):
 mintpy.networkInversion.solver      = torch
+mintpy.topographicResidual.solver   = torch

fixtures/SanFranBaySenD42_cpu.txt

@@ -0,0 +1,34 @@
+# CPU baseline fixture for SanFranBaySenD42 — derived from
+# docs/templates/SanFranBaySenD42.txt with the Issue #19 bench
+# `troposphericDelay.method = no` override applied to match
+# fixtures/SanFranBaySenD42_torch.txt (skip ERA5 download for GMTSAR
+# bench). Differs from the torch fixture only by the trailing
+# `mintpy.*.solver = torch` lines so the wall comparison is fair.
+# vim: set filetype=cfg:
+## manually add the following metadata
+HEADING = -168
+ALOOKS  = 8
+RLOOKS  = 32
+
+########## computing resource configuration
+mintpy.compute.maxMemory    = 8
+mintpy.compute.cluster      = local
+mintpy.compute.numWorker    = 4
+
+########## 1. Load Data (--load to exit after this step)
+mintpy.load.processor       = gmtsar    #[isce, aria, hyp3, gmtsar, snap, gamma, roipac], auto for isce
+mintpy.load.metaFile        = ../supermaster.PRM
+mintpy.load.baselineDir     = ../baseline_table.dat
+##---------interferogram datasets:
+mintpy.load.unwFile         = ../interferograms/*/unwrap_ll*.grd
+mintpy.load.corFile         = ../interferograms/*/corr_ll*.grd
+##---------geometry datasets:
+mintpy.load.demFile         = ../geometry/dem.grd
+mintpy.load.incAngleFile    = ../geometry/incidence_angle.grd
+mintpy.load.azAngleFile     = ../geometry/azimuth_angle.grd
+mintpy.load.waterMaskFile   = ../geometry/water_mask.grd
+
+mintpy.reference.lalo           = 37.69, -122.07
+mintpy.troposphericDelay.method = no
+mintpy.deramp                   = no
+mintpy.topographicResidual      = yes

fixtures/SanFranBaySenD42_torch.txt

@@ -1,8 +1,8 @@
 # GPU benchmark template — copy of docs/templates/SanFranBaySenD42.txt
-# with mintpy.networkInversion.solver = torch added at the end so the
-# invert_network step uses the GPU-batched solver. Used by Issue #19
-# Tier 1 bench survey (extends Fernandina + Galapagos with a GMTSAR
-# ingest path + D = 333).
+# with the two opt-in GPU solver flags added at the end so both the
+# invert_network and correct_topography steps use the GPU-batched
+# solvers. Used by Issue #19 Tier 1 bench survey (GMTSAR ingest path
+# + D = 333) and the Issue #21 end-to-end bench harness.
 # vim: set filetype=cfg:
 ## manually add the following metadata
 HEADING = -168
@@ -32,5 +32,6 @@ mintpy.troposphericDelay.method = no
 mintpy.deramp                   = no
 mintpy.topographicResidual      = yes
 
-##---------GPU acceleration of invert_network (fork-only, opt-in):
+##---------GPU acceleration (fork-only, opt-in):
 mintpy.networkInversion.solver  = torch
+mintpy.topographicResidual.solver = torch

fixtures/SanFranSenDT42_cpu.txt

@@ -0,0 +1,25 @@
+# CPU baseline fixture for SanFranSenDT42 — derived from
+# docs/templates/SanFranSenDT42.txt with the `weightFunc = no`
+# inversion speed-up retained to match
+# fixtures/SanFranSenDT42_torch.txt. Differs from the torch fixture
+# only by the trailing `mintpy.*.solver = torch` lines so the wall
+# comparison is fair. Tropo defaults to pyaps (ERA5) on both sides
+# matching the upstream docs template.
+# vim: set filetype=cfg:
+mintpy.load.processor       = aria  #[isce, aria, snap, gamma, roipac], auto for isce
+##---------interferogram datasets:
+mintpy.load.unwFile         = ../stack/unwrapStack.vrt
+mintpy.load.corFile         = ../stack/cohStack.vrt
+mintpy.load.connCompFile    = ../stack/connCompStack.vrt
+##---------geometry datasets:
+mintpy.load.demFile         = ../DEM/*.dem
+mintpy.load.incAngleFile    = ../incidenceAngle/*.vrt
+mintpy.load.azAngleFile     = ../azimuthAngle/*.vrt
+mintpy.load.waterMaskFile   = ../mask/watermask.msk
+
+mintpy.reference.lalo       = 37.69, -122.07
+mintpy.unwrapError.method   = bridging
+mintpy.deramp               = no
+
+## options to speedup the processing (fast but not the best)
+mintpy.networkInversion.weightFunc              = no

fixtures/SanFranSenDT42_torch.txt

@@ -1,8 +1,8 @@
 # GPU benchmark template — copy of docs/templates/SanFranSenDT42.txt
-# with mintpy.networkInversion.solver = torch added at the end so the
-# invert_network step uses the GPU-batched solver. Used by Issue #19
-# Tier 1 bench survey (extends Fernandina + Galapagos with an ARIA
-# ingest path).
+# with the two opt-in GPU solver flags added at the end so both the
+# invert_network and correct_topography steps use the GPU-batched
+# solvers. Used by Issue #19 Tier 1 bench survey (ARIA ingest path)
+# and the Issue #21 end-to-end bench harness.
 # vim: set filetype=cfg:
 mintpy.load.processor       = aria  #[isce, aria, snap, gamma, roipac], auto for isce
 ##---------interferogram datasets:
@@ -21,7 +21,12 @@ mintpy.deramp               = no
 
 ## options to speedup the processing (fast but not the best)
 mintpy.networkInversion.weightFunc              = no
-mintpy.topographicResidual.pixelwiseGeometry    = no
+# `mintpy.topographicResidual.pixelwiseGeometry = no` was set in the
+# Issue #19 Tier 1 direct-call fixture to skip the pixelwise path
+# entirely. For the Issue #21 end-to-end bench we let it default to
+# `yes` so the GPU correct_topography solver is actually exercised
+# (the `= no` mean-geometry path bypasses mintpy.gpu.dem_error).
 
-##---------GPU acceleration of invert_network (fork-only, opt-in):
+##---------GPU acceleration (fork-only, opt-in):
 mintpy.networkInversion.solver                  = torch
+mintpy.topographicResidual.solver               = torch