From be75bcfe3d6ccd189572cc3527e8264a20ebf11e Mon Sep 17 00:00:00 2001 From: Saianeesh Keshav Haridas Date: Fri, 10 Apr 2026 07:56:28 -0700 Subject: [PATCH 01/15] feat: add support for ASO tubes --- sotodlib/coords/optics.py | 156 +++++++++++++++++++++++++++++++++----- 1 file changed, 136 insertions(+), 20 deletions(-) diff --git a/sotodlib/coords/optics.py b/sotodlib/coords/optics.py index 161e70919..31c19eaf3 100644 --- a/sotodlib/coords/optics.py +++ b/sotodlib/coords/optics.py @@ -4,6 +4,7 @@ LAT code adapted from code provided by Simon Dicker. """ + import logging from functools import lru_cache, partial import numpy as np @@ -47,6 +48,7 @@ SAT_R_SKY = (0.0, 0.0523597, 0.10471958, 0.15707946, 0.20943951, 0.26179764, 0.27925093, 0.30543087) # fmt: on + def _interp_func(x, y, spline): xr = np.atleast_1d(x).ravel() xa = np.argsort(xr) @@ -119,8 +121,8 @@ def get_gamma(pol_xi, pol_eta): xi = pol_xi.reshape((-1, 2)) eta = pol_eta.reshape((-1, 2)) - q0 = quat.rotation_xieta(xi[:,0], eta[:,0]) - q1 = quat.rotation_xieta(xi[:,1], eta[:,1]) + q0 = quat.rotation_xieta(xi[:, 0], eta[:, 0]) + q1 = quat.rotation_xieta(xi[:, 1], eta[:, 1]) dq = ~q0 * q1 d_xi, d_eta, _ = quat.decompose_xieta(dq) @@ -129,9 +131,10 @@ def get_gamma(pol_xi, pol_eta): @lru_cache(maxsize=None) -def load_ufm_to_fp_config(config_path): +def load_config(config_path): """ - Load and cache config file with the parameters to transform from UFM to focal_plane coordinates. + Load and cache config file with the parameters to transform from UFM to focal_plane coordinates + or focal_plane to OT coordinates. Arguments: @@ -164,7 +167,7 @@ def get_ufm_to_fp_pars(telescope_flavor, wafer_slot, config_path): transform_pars: Dict of transformation parameters that can be passed to ufm_to_fp. """ - config = load_ufm_to_fp_config(config_path) + config = load_config(config_path) return config[telescope_flavor][wafer_slot] @@ -228,6 +231,86 @@ def ufm_to_fp(aman, x=None, y=None, pol=None, theta=0, dx=0, dy=0): return x_fp, y_fp, pol_fp +@lru_cache(maxsize=None) +def get_fp_to_ot_pars(ot, config_path): + """ + Get (and cache) the parameters to transform from focal_plane to OT coordinates + for a specific slot of a given telescope's focal plane. + + Arguments: + + ot: The OT(ie. i6, st1, etc). + + config_path: Path to the yaml with the parameters. + + Returns: + + transform_pars: Dict of transformation parameters that can be passed to fp_to_ot. + """ + config = load_config(config_path) + return config[ot] + + +def fp_to_ot(aman, x=None, y=None, pol=None, phi=0, dx=0, dy=0): + """ + Transform from coords internal to focal plane to OT coordinates. + + Arguments: + + aman: AxisManager assumed to contain aman.focal_plane. + If provided outputs will be wrapped into aman.focal_plane. + + x: X position in focal_plane's internal coordinate system in mm. + If provided overrides the value from aman. + + y: Y position in focal_plane's internal coordinate system in mm. + If provided overrides the value from aman. + + pol: Polarization angle in focal_plane's internal coordinate system in deg. + If provided overrides the value from aman. + + phi: Internal rotation of the focal_plane in degrees. + + dx: X offset in mm. + + dy: Y offset in mm. + + Returns: + + x_fp: X position on focal plane. + + y_fp: Y position on focal plane. + + pol_fp: Pol angle on focal plane. + """ + if x is None: + x = aman.focal_plane.x_fp + if y is None: + y = aman.focal_plane.y_fp + if pol is None: + pol = aman.focal_plane.pol_fp + xy = np.column_stack((x, y, np.zeros_like(x))) + + rot = R.from_euler("z", phi, degrees=True) + xy = rot.apply(xy) + + x_ot = xy[:, 0] + dx + y_ot = xy[:, 1] + dy + pol_ot = pol + phi + + if aman is not None: + focal_plane = core.AxisManager(aman.dets) + focal_plane.wrap("x_ot", x_ot, [(0, focal_plane.dets)]) + focal_plane.wrap("y_ot", y_ot, [(0, focal_plane.dets)]) + focal_plane.wrap("pol_ot", pol_ot, [(0, focal_plane.dets)]) + if "focal_plane" in aman: + aman.focal_plane.merge(focal_plane) + else: + aman.wrap("focal_plane", focal_plane) + + return x_ot, y_ot, pol_ot + + def LAT_pix2sky(x, y, sec2el, sec2xel, array2secx, array2secy, rot=0, opt2cryo=30.0): """ Routine to map pixels from arrays to sky. @@ -238,7 +321,7 @@ def LAT_pix2sky(x, y, sec2el, sec2xel, array2secx, array2secy, rot=0, opt2cryo=3 y: Y position on focal plane (currently zemax coord). - sec2el: Function that maps positions on secondary to on sky elevation. + sec2el: Function that maps positions on secondary to on sky elevation. sex2xel: Function that maps positions on secondary to on sky cross-elevation. @@ -423,23 +506,27 @@ def LAT_focal_plane(aman, zemax_path, x=None, y=None, pol=None, roll=0, tube_slo If aman is provided then will be wrapped as aman.focal_plane.eta. """ if x is None: - x = aman.focal_plane.x_fp + x = aman.focal_plane.x_ot if y is None: - y = aman.focal_plane.y_fp + y = aman.focal_plane.y_ot if pol is None: - pol = aman.focal_plane.pol_fp + pol = aman.focal_plane.pol_ot sec2el, sec2xel = LAT_optics(zemax_path) array2secx, array2secy = LATR_optics(zemax_path, tube_slot) el, xel = LAT_pix2sky(x, y, sec2el, sec2xel, array2secx, array2secy, roll) - xi, eta, _ = quat.decompose_xieta(quat.euler(1, np.deg2rad(90)) * quat.rotation_lonlat(-xel, el)) + xi, eta, _ = quat.decompose_xieta( + quat.euler(1, np.deg2rad(90)) * quat.rotation_lonlat(-xel, el) + ) pol_x, pol_y = gen_pol_endpoints(x, y, pol) pol_el, pol_xel = LAT_pix2sky( pol_x, pol_y, sec2el, sec2xel, array2secx, array2secy, roll ) - pol_xi, pol_eta, _ = quat.decompose_xieta(quat.euler(1, np.deg2rad(90)) * quat.rotation_lonlat(-pol_xel, pol_el)) + pol_xi, pol_eta, _ = quat.decompose_xieta( + quat.euler(1, np.deg2rad(90)) * quat.rotation_lonlat(-pol_xel, pol_el) + ) gamma = get_gamma(pol_xi, pol_eta) if np.isscalar(xi): gamma = gamma[0] @@ -519,13 +606,13 @@ def SAT_focal_plane(aman, x=None, y=None, pol=None, roll=0, mapping_data=None): mapping_data = (tuple(val) for val in mapping_data) fp_to_sky = sat_to_sky(*mapping_data) - # Get things in polor coords - r_fp = (x**2 + y**2)**.5 + # Get things in polor coords + r_fp = (x**2 + y**2) ** 0.5 phi_fp = np.arctan2(y, x) # Now to the sky theta_iso = -fp_to_sky(r_fp) - phi_iso = np.pi/2 - phi_fp + phi_iso = np.pi / 2 - phi_fp # Flip about the origin for optics (180 deg rotation) phi_iso += np.pi @@ -539,10 +626,10 @@ def SAT_focal_plane(aman, x=None, y=None, pol=None, roll=0, mapping_data=None): # Lets do gamma as a set of endpoints pol_x, pol_y = gen_pol_endpoints(x, y, pol) - pol_r = (pol_x**2 + pol_y**2)**.5 + pol_r = (pol_x**2 + pol_y**2) ** 0.5 pol_phi = np.arctan2(pol_y, pol_x) pol_theta_iso = -fp_to_sky(pol_r) - pol_phi_iso = np.pi/2 - pol_phi + np.pi + pol_phi_iso = np.pi / 2 - pol_phi + np.pi _xi, _eta, _ = quat.decompose_xieta(quat.rotation_iso(pol_theta_iso, pol_phi_iso)) pol_xi = _xi * np.cos(np.deg2rad(roll)) - _eta * np.sin(np.deg2rad(roll)) pol_eta = _eta * np.cos(np.deg2rad(roll)) + _xi * np.sin(np.deg2rad(roll)) @@ -574,6 +661,8 @@ def get_focal_plane( wafer_slot="ws0", config_path=None, ufm_to_fp_pars=None, + ot_config_path=None, + fp_to_ot_pars=None, zemax_path=None, mapping_data=None, return_fp=False, @@ -608,9 +697,14 @@ def get_focal_plane( config_path: Path to the ufm_to_fp config file. - ufm_to_fp_pars: Loaded ufm_to_fp_parsm to focalplane params. + ufm_to_fp_pars: Loaded ufm_to_fp_pars to focalplane params. If provided config_path is is ignored. + ot_config_path: Path to the optics_tubes config file. + + fp_to_ot_pars: Loaded optics_tubes params. + If provided ot_config_path is is ignored. + zemax_path: Path to the data file from Zemax. Only used by the LAT. @@ -639,6 +733,15 @@ def get_focal_plane( pol_fp: Pol angle on focal plane. Only returned if return_fp is True. + + x_ot: X position in the OT. + Only returned if return_fp is True. + + y_ot: Y position in the OT. + Only returned if return_fp is True. + + pol_ot: Pol angle in the OT. + Only returned if return_fp is True. """ if aman is not None and "obs_info" in aman: if telescope_flavor is None: @@ -654,19 +757,32 @@ def get_focal_plane( if ufm_to_fp_pars is None: ufm_to_fp_pars = get_ufm_to_fp_pars(telescope_flavor, wafer_slot, config_path) x_fp, y_fp, pol_fp = ufm_to_fp(aman, x=x, y=y, pol=pol, **ufm_to_fp_pars) + if fp_to_ot_pars is None: + fp_to_ot_pars = get_fp_to_ot_pars(tube_slot, ot_config_path) + x_ot, y_ot, pol_ot = fp_to_ot(aman, x=x_fp, y=y_fp, pol=pol_fp, **fp_to_ot_pars) + # We dont want to use the shift of the OT when computing on sky locations + x_use, y_use, pol_use = fp_to_ot( + aman, x=x_fp, y=y_fp, pol=pol_fp, phi=fp_to_ot_pars["phi"] + ) if telescope_flavor == "LAT": if zemax_path is None: raise ValueError("Must provide zemax_path for LAT") xi, eta, gamma = LAT_focal_plane( - aman, zemax_path, x=x_fp, y=y_fp, pol=pol_fp, roll=roll, tube_slot=tube_slot + aman, + zemax_path, + x=x_use, + y=y_use, + pol=pol_use, + roll=roll, + tube_slot=tube_slot, ) else: xi, eta, gamma = SAT_focal_plane( - aman, x=x_fp, y=y_fp, pol=pol_fp, roll=roll, mapping_data=mapping_data + aman, x=x_use, y=y_use, pol=pol_use, roll=roll, mapping_data=mapping_data ) if return_fp: - return xi, eta, gamma, x_fp, y_fp, pol_fp + return xi, eta, gamma, x_fp, y_fp, pol_fp, x_ot, y_ot, pol_ot return xi, eta, gamma From 3edbc66ed055c3a8e31635a5e64bd3085dd67f43 Mon Sep 17 00:00:00 2001 From: Saianeesh Keshav Haridas Date: Fri, 10 Apr 2026 07:59:31 -0700 Subject: [PATCH 02/15] fix: supple ot path to gen_template --- sotodlib/site_pipeline/finalize_focal_plane.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sotodlib/site_pipeline/finalize_focal_plane.py b/sotodlib/site_pipeline/finalize_focal_plane.py index 02a31faf5..c45f262a9 100644 --- a/sotodlib/site_pipeline/finalize_focal_plane.py +++ b/sotodlib/site_pipeline/finalize_focal_plane.py @@ -387,6 +387,7 @@ def _load_rset(config): def _mk_pointing_config(telescope_flavor, tube_slot, wafer_slot, config): config_dir = config.get("pipeline_config_dir", os.environ["PIPELINE_CONFIG_DIR"]) config_path = os.path.join(config_dir, "shared/focalplane/ufm_to_fp.yaml") + ot_config_path = os.path.join(config_dir, "shared/focalplane/optics_tubes.yaml") zemax_path = config.get("zemax_path", None) pointing_cfg = { @@ -394,6 +395,7 @@ def _mk_pointing_config(telescope_flavor, tube_slot, wafer_slot, config): "tube_slot": tube_slot, "wafer_slot": wafer_slot, "config_path": config_path, + "ot_config_path": config_path, "zemax_path": zemax_path, "return_fp": False, } From e493b1b1937179479fe3240e8fc2c158bed1a042 Mon Sep 17 00:00:00 2001 From: Saianeesh Keshav Haridas Date: Fri, 10 Apr 2026 08:07:38 -0700 Subject: [PATCH 03/15] fix: update test --- tests/test_coords.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_coords.py b/tests/test_coords.py index e5e1ab032..d917be7fe 100644 --- a/tests/test_coords.py +++ b/tests/test_coords.py @@ -258,7 +258,7 @@ def test_sat_fp(self): y = x.copy() pol = x.copy() - xi, eta, gamma = co.get_focal_plane(None, x, y, pol, 0, "SAT", "ws1", ufm_to_fp_pars={'theta': 60.0, 'dx': 0.0, 'dy': 128.5}) + xi, eta, gamma = co.get_focal_plane(None, x, y, pol, 0, "SAT", "ws1", ufm_to_fp_pars={'theta': 60.0, 'dx': 0.0, 'dy': 128.5}, fp_to_ot_pars={'phi': 0.0, 'dx': 0.0, 'dy': 0}) self.assertTrue(np.all(np.isclose(xi, np.array([-6.4406e-02, 0, 5.58489e-02])))) self.assertTrue(np.all(np.isclose(eta, np.array([0.01425728, -0.2207397, -0.404499])))) self.assertTrue(np.all(np.isclose(gamma, np.array([5.409, 3.6846, 1.8156])))) From 410671a2f907a3f3083b9d4a92e597c35699f00e Mon Sep 17 00:00:00 2001 From: Michael McCrackan Date: Fri, 1 May 2026 13:13:02 -0700 Subject: [PATCH 04/15] add tube orientation --- sotodlib/coords/det_match.py | 21 +++++++++++++++------ sotodlib/coords/det_match_solutions.py | 5 +++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/sotodlib/coords/det_match.py b/sotodlib/coords/det_match.py index ea9bc7806..0b4e2b73d 100644 --- a/sotodlib/coords/det_match.py +++ b/sotodlib/coords/det_match.py @@ -80,6 +80,7 @@ class PointingConfig: Either the tube name as a string or the tube number as an int. """ fp_file: str + ot_file: str wafer_slot: str tel_type: str zemax_path: Optional[str] = None @@ -109,17 +110,25 @@ def __post_init__(self): self.theta = np.deg2rad(self.fp_pars['theta']) def get_pointing(self, x, y, pol=0): - xp = x * np.cos(self.theta) - y * np.sin(self.theta) + self.dx - yp = x * np.sin(self.theta) + y * np.cos(self.theta) + self.dy - if self.tel_type.upper() == 'SAT': + xp = x * np.cos(self.theta) - y * np.sin(self.theta) + self.dx + yp = x * np.sin(self.theta) + y * np.cos(self.theta) + self.dy xi, eta, gamma = optics.SAT_focal_plane( None, x=xp, y=yp, pol=pol, roll=self.roll ) elif self.tel_type.upper() == 'LAT': - xi, eta, gamma = optics.LAT_focal_plane( - None, self.zemax_path, x=xp, y=yp, pol=pol, - roll=self.roll, tube_slot=self.tube_slot + xi, eta, gamma = optics.get_focal_plane( + None, + x=x, + y=y, + pol=pol, + roll=self.roll, + telescope_flavor="LAT", + tube_slot=self.tube_slot, + wafer_slot=self.wafer_slot, + ufm_to_fp_pars=self.fp_pars, + ot_config_path=self.ot_file, + zemax_path=self.zemax_path, ) return xi, eta, gamma diff --git a/sotodlib/coords/det_match_solutions.py b/sotodlib/coords/det_match_solutions.py index 2399e04f4..f140d28a1 100644 --- a/sotodlib/coords/det_match_solutions.py +++ b/sotodlib/coords/det_match_solutions.py @@ -109,6 +109,7 @@ class SolutionsCfg: initial_pointing_offset: Tuple[float, float] = (0, 0) ufm_to_fp_path: Optional[str] = None + fp_to_ot_path: Optional[str] = None freq_correct_by_muxband: bool = True ctx: Context = field(init=False) @@ -146,6 +147,10 @@ def __post_init__(self): self.ufm_to_fp_path = os.path.join( self.site_pipeline_cfg_dir, "shared/focalplane/ufm_to_fp.yaml" ) + if self.fp_to_ot_path is None: + self.fp_to_ot_path = os.path.join( + self.site_pipeline_cfg_dir, "shared/focalplane/optics_tubes.yaml" + ) @dataclass From 8eaf7cc798a091a4d1b4c69d9572a91b15f1bb51 Mon Sep 17 00:00:00 2001 From: Michael McCrackan Date: Fri, 1 May 2026 13:33:29 -0700 Subject: [PATCH 05/15] make sat and lat consistent --- sotodlib/coords/det_match.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/sotodlib/coords/det_match.py b/sotodlib/coords/det_match.py index 0b4e2b73d..23a70f3ff 100644 --- a/sotodlib/coords/det_match.py +++ b/sotodlib/coords/det_match.py @@ -105,16 +105,23 @@ def __post_init__(self): self.fp_pars = optics.get_ufm_to_fp_pars( self.tel_type, self.wafer_slot, self.fp_file ) - self.dx = self.fp_pars['dx'] - self.dy = self.fp_pars['dy'] - self.theta = np.deg2rad(self.fp_pars['theta']) + + self.ot_pars = optics.get_fp_to_ot_pars( + self.tube_slot, self.ot_file) def get_pointing(self, x, y, pol=0): if self.tel_type.upper() == 'SAT': - xp = x * np.cos(self.theta) - y * np.sin(self.theta) + self.dx - yp = x * np.sin(self.theta) + y * np.cos(self.theta) + self.dy - xi, eta, gamma = optics.SAT_focal_plane( - None, x=xp, y=yp, pol=pol, roll=self.roll + xi, eta, gamma = optics.get_focal_plane( + None, + x=x, + y=y, + pol=pol, + roll=self.roll, + telescope_flavor="SAT", + tube_slot=self.tube_slot, + wafer_slot=self.wafer_slot, + ufm_to_fp_pars=self.fp_pars, + fp_to_ot_pars=self.ot_pars, ) elif self.tel_type.upper() == 'LAT': xi, eta, gamma = optics.get_focal_plane( @@ -127,7 +134,7 @@ def get_pointing(self, x, y, pol=0): tube_slot=self.tube_slot, wafer_slot=self.wafer_slot, ufm_to_fp_pars=self.fp_pars, - ot_config_path=self.ot_file, + fp_to_ot_pars=self.ot_pars, zemax_path=self.zemax_path, ) return xi, eta, gamma From ea72c58e1a17259afabc79a0f45cba485a43c412 Mon Sep 17 00:00:00 2001 From: Michael McCrackan Date: Sat, 2 May 2026 10:32:20 -0700 Subject: [PATCH 06/15] LF support --- sotodlib/coords/det_match.py | 51 +++++++--- sotodlib/coords/det_match_solutions.py | 110 ++++++++++++++++----- sotodlib/site_pipeline/update_det_match.py | 7 +- 3 files changed, 131 insertions(+), 37 deletions(-) diff --git a/sotodlib/coords/det_match.py b/sotodlib/coords/det_match.py index 23a70f3ff..aefa7ce2f 100644 --- a/sotodlib/coords/det_match.py +++ b/sotodlib/coords/det_match.py @@ -67,22 +67,22 @@ class PointingConfig: Path to focal-plane file that is used by the optics module. wafer_slot: str Wafer slot of the UFM. For example: "ws0" - tel_type: str - Tel type for the optics model. Either "SAT" or "LAT" + platform: str + Which platform to use for the optics model. zemax_path: str - If running for a "LAT" tel_type, the path to the zemax file must be specified. + If running for the lat platform, the path to the zemax file must be specified. roll: float Rotation about the line of sight. For the LAT this is elev - 60 - corotator. For the SAT this is -1*boresight. tube_slot: str/int - If running for a "LAT" tel_type, the tube slot must be specified. + If running for the lat platform, the tube slot must be specified. Either the tube name as a string or the tube number as an int. """ fp_file: str ot_file: str wafer_slot: str - tel_type: str + platform: str zemax_path: Optional[str] = None roll: Optional[float] = 0 tube_slot: Optional[Union[str, int]] = None @@ -93,6 +93,10 @@ class PointingConfig: fp_pars: dict = field(init=False) def __post_init__(self): + if self.platform.lower() in ["satp1", "satp2", "satp3"]: + self.tel_type = "SAT" + elif self.platform.lower() == "lat": + self.tel_type = "LAT" if self.tel_type == 'LAT': if self.zemax_path is None: raise ValueError("zemax path must be set for 'LAT' tel_type") @@ -301,7 +305,8 @@ def as_array(self): return np.array(data, dtype=dtype) @classmethod - def from_aman(cls, aman, stream_id, det_cal=None, name=None, pointing: Optional[AxisManager]=None): + def from_aman(cls, aman, stream_id, det_cal=None, name=None, pointing: Optional[AxisManager]=None, + ignore_north_south=False): """ Load a resonator set from a Context object based on an obs_id @@ -331,7 +336,10 @@ def from_aman(cls, aman, stream_id, det_cal=None, name=None, pointing: Optional[ resonators = [] for i, ri in enumerate(np.where(m)[0]): band, channel = aman.det_info.smurf.band[ri], aman.det_info.smurf.channel[ri] - is_north = north_is_highband ^ (band < 4) + if not ignore_north_south: + is_north = north_is_highband ^ (band < 4) + else: + is_north = 1 readout_id = aman.det_info.readout_id[ri] bg = det_cal.bg[ri] res_freq=aman.det_info.smurf.frequency[ri] @@ -352,7 +360,7 @@ def from_aman(cls, aman, stream_id, det_cal=None, name=None, pointing: Optional[ @classmethod def from_tunefile(cls, tunefile, name=None, north_is_highband=True, - resfit_file=None, bgmap_file=None): + resfit_file=None, bgmap_file=None, ignore_north_south=False): """ Creates an instance based on a smurf-tune file. If a resfit or bgmap file is included, that data will be added to the Resonance objects as @@ -379,7 +387,10 @@ def from_tunefile(cls, tunefile, name=None, north_is_highband=True, continue for res_idx, d in _v['resonances'].items(): - is_north = north_is_highband ^ (band < 4) + if not ignore_north_south: + is_north = north_is_highband ^ (band < 4) + else: + is_north = 1 res = Resonator( idx=idx, smurf_res_idx=res_idx, res_freq=d['freq'], smurf_band=band, is_north=is_north @@ -401,7 +412,8 @@ def from_tunefile(cls, tunefile, name=None, north_is_highband=True, @classmethod def from_wafer_info_file(cls, wafer_info_file, array_name, name=None, - pt_cfg: Optional[PointingConfig]=None): + pt_cfg: Optional[PointingConfig]=None, + ignore_north_south=False): """ Initialize a ResSet from a wafer info file. This is a file that contains detector design information. @@ -431,7 +443,11 @@ def from_wafer_info_file(cls, wafer_info_file, array_name, name=None, resonators = [] idx = 0 for r in wafer_array: - is_north = r['dets:wafer.coax'] == b'N' + # always true for LF + if not ignore_north_south: + is_north = r['dets:wafer.coax'] == b'N' + else: + is_north = 1 kwargs = dict( idx=idx, det_id=r['dets:det_id'].decode(), @@ -473,7 +489,8 @@ def from_wafer_info_file(cls, wafer_info_file, array_name, name=None, @classmethod def from_solutions(cls, sol_file, north_is_highband=True, name=None, - fp_pars=None, platform='SAT', zemax_path=None): + fp_pars=None, platform='SAT', zemax_path=None, + ignore_north_south=False): """ Creates an instance from an input-solution file. This will include both design data, along with smurf-band and smurf-channel info. Resonance frequencies used here are the VNA @@ -521,7 +538,10 @@ def _int(val, null_val=None): continue # is_north = north_is_highband ^ (_int(d['smurf_band']) < 4) # is_north = d['is_north'].lower().strip() == 'true' - is_north = _int(d['bias_line']) < 6 + if not ignore_north_south: + is_north = _int(d['bias_line']) < 6 + else: + is_north = 1 is_optical = (d['is_optical'].lower() == 'true') kwargs = dict( @@ -761,7 +781,7 @@ def _get_biadjacency_matrix(self) -> np.ndarray: mat = np.zeros((len(self.src), len(self.dst)), dtype=float) - # N/S mismatch + # N/S mismatch (all N for LF) m = src_arr['is_north'][:, None] != dst_arr['is_north'][None, :] mat[m] = np.inf @@ -954,7 +974,10 @@ def get_stats(self) -> MatchingStats: stats.unmatched_src = np.sum(~src_arr['matched'].astype(bool)) stats.unmatched_dst = np.sum(~dst_arr['matched'].astype(bool)) + # print(stats.unmatched_src, stats.unmatched_dst) + has_pt = ~np.isnan(src_arr['xi']) + # print(has_pt) stats.unmatched_src_with_pointing = np.sum( (~src_arr['matched'].astype(bool)) & has_pt ) diff --git a/sotodlib/coords/det_match_solutions.py b/sotodlib/coords/det_match_solutions.py index f140d28a1..305d54b06 100644 --- a/sotodlib/coords/det_match_solutions.py +++ b/sotodlib/coords/det_match_solutions.py @@ -32,14 +32,14 @@ class SolutionsCfg: Directory where results should be stored. wafer_info_path: str Path to the wafer_info h5 file. - tel_type: str - Tel type for the optics model. Either "SAT" or "LAT" + platform: str + Telescope platform. base_obs_id: str Obs_id to use as a base for matching when merging multiple pointing obs_ids for a wafer. Will default to the pointing obs_id with the greatest number of detectors above the min_R2 threshold. zemax_path: str - If running for a "LAT" tel_type, the path to the zemax file must be specified. + If running for the LAT, the path to the zemax file must be specified. apply_roll: bool Whether or not to apply the obs_id roll angle. Some pointing sets may already be corrected for roll angle. @@ -91,7 +91,7 @@ class SolutionsCfg: pointing_results_dir: str results_dir: str wafer_info_path: str - tel_type: str + platform: str base_obs_id: Optional[str] = None zemax_path: Optional[str] = None apply_roll: bool = True @@ -110,6 +110,7 @@ class SolutionsCfg: initial_pointing_offset: Tuple[float, float] = (0, 0) ufm_to_fp_path: Optional[str] = None fp_to_ot_path: Optional[str] = None + imprinter_path: Optional[str] = None freq_correct_by_muxband: bool = True ctx: Context = field(init=False) @@ -151,6 +152,10 @@ def __post_init__(self): self.fp_to_ot_path = os.path.join( self.site_pipeline_cfg_dir, "shared/focalplane/optics_tubes.yaml" ) + if self.imprinter_path is None: + self.imprinter_path = os.path.join( + self.site_pipeline_cfg_dir, f"{self.platform}/imprinter.yaml" + ) @dataclass @@ -235,7 +240,7 @@ def pointing_preprocess(cfg: SolutionsCfg, pinfo: PointingInfo): return meta -def merge_pointing_info(cfg: SolutionsCfg, pinfos: List[PointingInfo], base_idx=0): +def merge_pointing_info(cfg: SolutionsCfg, pinfos: List[PointingInfo], base_idx=0, ignore_north_south=False): """ Combine all pointing measurements into a single resonator set, with the median pointing info from all. This requires a base_idx to be specified, @@ -251,7 +256,13 @@ def merge_pointing_info(cfg: SolutionsCfg, pinfos: List[PointingInfo], base_idx= meta = pinfos[base_idx].meta stream_id = meta.det_info.stream_id[0] wafer_slot = meta.det_info.wafer_slot[0] - base_resset = dm.ResSet.from_aman(meta, stream_id=stream_id, pointing=meta.tod_pointing) + + base_resset = dm.ResSet.from_aman( + meta, + stream_id=stream_id, + pointing=meta.tod_pointing, + ignore_north_south=ignore_north_south + ) pointing_map = { r.idx: [(r.xi, r.eta)] for r in base_resset @@ -259,14 +270,19 @@ def merge_pointing_info(cfg: SolutionsCfg, pinfos: List[PointingInfo], base_idx= match_pars = dm.MatchParams( freq_width=cfg.match_pars["pointing"]["freq_width"], - dist_width=np.deg2rad(cfg.match_pars["pointing"]["dist_width"]) + dist_width=np.deg2rad(cfg.match_pars["pointing"]["dist_width"]), ) for i in range(len(pinfos)): if i == base_idx: continue meta = pinfos[i].meta - src = dm.ResSet.from_aman(meta, stream_id=stream_id, pointing=meta.tod_pointing) + src = dm.ResSet.from_aman( + meta, + stream_id=stream_id, + pointing=meta.tod_pointing, + ignore_north_south=ignore_north_south + ) dst = base_resset match = dm.Match(src, dst, match_pars=match_pars) for rsrc, rdst in match.get_match_iter(include_unmatched=False): @@ -395,7 +411,8 @@ def match_wafer( cfg: SolutionsCfg, am: AxisManager, stream_id: str, - meas_rset: Optional[dm.ResSet] + meas_rset: Optional[dm.ResSet], + ignore_north_south: bool = False ) -> MatchSolution: """ Create a match solution for a given wafer slot. @@ -414,19 +431,58 @@ def match_wafer( m = am.det_info.stream_id == stream_id wafer_slot = am.det_info.wafer_slot[m][0] + tube_slot = am.obs_info.tube_slot if meas_rset is None: - src = dm.ResSet.from_aman(am, stream_id, pointing=am[cfg.pointing_field]) + src = dm.ResSet.from_aman( + am, + stream_id, + pointing=am[cfg.pointing_field], + ignore_north_south=ignore_north_south + ) else: src = meas_rset - pt_cfg = dm.PointingConfig( - fp_file=cfg.ufm_to_fp_path, wafer_slot=wafer_slot, tel_type=cfg.tel_type, - zemax_path=cfg.zemax_path, - roll=np.deg2rad(am.obs_info.roll_center) if cfg.apply_roll else 0.0, - tube_slot = am.obs_info.tube_slot - ) - dst = dm.ResSet.from_wafer_info_file(cfg.wafer_info_path, stream_id, pt_cfg=pt_cfg) + if ignore_north_south: + wafer_slot = [] + wafer_dict = {} + + # get wafer and wafer_slot entry from imprinter + with open(cfg.imprinter_path, "r") as f: + imprinter = yaml.safe_load(f) + + for k, v in imprinter['tel_tubes'].items(): + if v['tube_slot'] == tube_slot: + for ws in v['wafer_slots']: + if 'wafer' in ws.keys(): + wafer_dict[ws['wafer_slot']] = (ws['wafer'].split('_')[-1].capitalize()) + wafer_slot.append(ws['wafer_slot']) + else: + wafer_slot = [wafer_slot] + wafer_dict = None + + dst = [] + for ws in wafer_slot: + pt_cfg = dm.PointingConfig( + fp_file=cfg.ufm_to_fp_path, ot_file=cfg.fp_to_ot_path, + wafer_slot=ws, platform=cfg.platform, + zemax_path=cfg.zemax_path, + roll=np.deg2rad(am.obs_info.roll_center) if cfg.apply_roll else 0.0, + tube_slot = am.obs_info.tube_slot + ) + dst_wafer = dm.ResSet.from_wafer_info_file( + cfg.wafer_info_path, + stream_id, + pt_cfg=pt_cfg, + ignore_north_south=ignore_north_south, + ).as_array() + + if wafer_dict is not None: + mask = [wafer_dict[ws] in d["det_id"].astype(str) for d in dst_wafer] + dst_wafer = dst_wafer[mask] + + dst.append(dst_wafer) + dst = dm.ResSet.from_array(np.concatenate(dst)) # first match match_pars = dm.MatchParams( @@ -434,7 +490,7 @@ def match_wafer( dist_width=np.deg2rad(cfg.match_pars["match0"]["dist_width"]), enforce_pointing_reqs=True, allow_unassigned_to_assigned=False, - unassigned_slots=cfg.unassigned_slots + unassigned_slots=cfg.unassigned_slots, ) match = dm.Match(src, dst, match_pars=match_pars, apply_dst_pointing=False) @@ -560,13 +616,23 @@ def get_wafer_solution( else: base_idx = 0 - meas_rset, pointing_map = merge_pointing_info(cfg, pointing_results, base_idx=base_idx) - # tod_pointing = get_best_tod_pointing(cfg, pointing_results) + if wafer_slot == "ws.": + ignore_north_south = True + else: + ignore_north_south = False + + meas_rset, pointing_map = merge_pointing_info( + cfg, pointing_results, base_idx=base_idx, + ignore_north_south=ignore_north_south + ) meta = pointing_results[0].meta stream_id = meta.det_info.stream_id[0] - match_solution = match_wafer(cfg, meta, stream_id, meas_rset=meas_rset) + match_solution = match_wafer( + cfg, meta, stream_id, meas_rset=meas_rset, + ignore_north_south=ignore_north_south + ) solution = FullWaferSolution( pointing_results=pointing_results, @@ -582,7 +648,7 @@ def get_wafer_solution( def solve_all(cfg) -> Dict[str, Optional[FullWaferSolution]]: - wafer_slots = ["ws0", "ws1", "ws2", "ws3", "ws4", "ws5", "ws6"] + wafer_slots = ["ws.", "ws0", "ws1", "ws2", "ws3", "ws4", "ws5", "ws6"] results = {ws: get_wafer_solution(cfg, ws, save=True) for ws in wafer_slots} return results diff --git a/sotodlib/site_pipeline/update_det_match.py b/sotodlib/site_pipeline/update_det_match.py index 3cf74ca2c..fb40bb8bb 100644 --- a/sotodlib/site_pipeline/update_det_match.py +++ b/sotodlib/site_pipeline/update_det_match.py @@ -245,7 +245,12 @@ def get_failed_detsets(cache_file): def run_match_aman(runner: Runner, aman, detset, wafer_slot=None): stream_id = aman.det_info.stream_id[aman.det_info.detset == detset][0] - rs0 = det_match.ResSet.from_aman(aman, stream_id) + if wafer_slot == "ws.": + ignore_north_south = True + else: + ignore_north_south = False + + rs0 = det_match.ResSet.from_aman(aman, stream_id, ignore_north_south=ignore_north_south) rs0.name = 'meas' rs1 = load_solution_set(runner, stream_id, wafer_slot=wafer_slot) From 6ea0c5701a6b2aaff6a8c909192ffc3e849dc97f Mon Sep 17 00:00:00 2001 From: Michael McCrackan Date: Mon, 18 May 2026 13:58:41 -0700 Subject: [PATCH 07/15] updates and fixes --- sotodlib/coords/det_match.py | 5 ++-- sotodlib/coords/det_match_solutions.py | 33 ++++++++++++---------- sotodlib/coords/optics.py | 7 ++++- sotodlib/site_pipeline/update_det_match.py | 22 ++------------- 4 files changed, 28 insertions(+), 39 deletions(-) diff --git a/sotodlib/coords/det_match.py b/sotodlib/coords/det_match.py index aefa7ce2f..44eef149c 100644 --- a/sotodlib/coords/det_match.py +++ b/sotodlib/coords/det_match.py @@ -443,7 +443,6 @@ def from_wafer_info_file(cls, wafer_info_file, array_name, name=None, resonators = [] idx = 0 for r in wafer_array: - # always true for LF if not ignore_north_south: is_north = r['dets:wafer.coax'] == b'N' else: @@ -489,7 +488,7 @@ def from_wafer_info_file(cls, wafer_info_file, array_name, name=None, @classmethod def from_solutions(cls, sol_file, north_is_highband=True, name=None, - fp_pars=None, platform='SAT', zemax_path=None, + fp_pars=None, platform='LAT', zemax_path=None, ignore_north_south=False): """ Creates an instance from an input-solution file. This will include both design data, along with smurf-band @@ -508,7 +507,7 @@ def from_solutions(cls, sol_file, north_is_highband=True, name=None, Result of the function ``sotododlib.coords.optics.get_ufm_to_fp_pars``. If this is None, detector positions will not be mapped to pointing angles. platform (str): - 'SAT' or 'LAT'. Used to determine which focal plane function to + Which platform. Used to determine which focal plane function to use for pointing zemax_path (str): zemax path, required to get pointing for LAT optics diff --git a/sotodlib/coords/det_match_solutions.py b/sotodlib/coords/det_match_solutions.py index 305d54b06..b51583fca 100644 --- a/sotodlib/coords/det_match_solutions.py +++ b/sotodlib/coords/det_match_solutions.py @@ -34,6 +34,8 @@ class SolutionsCfg: Path to the wafer_info h5 file. platform: str Telescope platform. + wafer_slots: list + Optional list of wafer slots to make solutions for. base_obs_id: str Obs_id to use as a base for matching when merging multiple pointing obs_ids for a wafer. Will default to the pointing obs_id with the @@ -92,6 +94,7 @@ class SolutionsCfg: results_dir: str wafer_info_path: str platform: str + wafer_slots: Optional[list[str]] = None base_obs_id: Optional[str] = None zemax_path: Optional[str] = None apply_roll: bool = True @@ -128,6 +131,9 @@ def from_yaml(cls, path: str) -> "SolutionsCfg": def __post_init__(self): self.ctx = Context(self.ctx_path) + if self.wafer_slots is None: + self.wafer_slots = ["ws.", "ws0", "ws1", "ws2", "ws3", "ws4", "ws5", "ws6"] + if not os.path.exists(self.results_dir): if os.path.exists(os.path.split(self.results_dir)[0]): os.makedirs(self.results_dir) @@ -444,8 +450,8 @@ def match_wafer( src = meas_rset if ignore_north_south: - wafer_slot = [] - wafer_dict = {} + wafer_slots = [] + wafers_dict = {} # get wafer and wafer_slot entry from imprinter with open(cfg.imprinter_path, "r") as f: @@ -455,14 +461,14 @@ def match_wafer( if v['tube_slot'] == tube_slot: for ws in v['wafer_slots']: if 'wafer' in ws.keys(): - wafer_dict[ws['wafer_slot']] = (ws['wafer'].split('_')[-1].capitalize()) - wafer_slot.append(ws['wafer_slot']) + wafers_dict[ws['wafer_slot']] = (ws['wafer'].split('_')[-1].capitalize()) + wafer_slots.append(ws['wafer_slot']) else: - wafer_slot = [wafer_slot] - wafer_dict = None + wafer_slots = [wafer_slot] + wafers_dict = None dst = [] - for ws in wafer_slot: + for ws in wafer_slots: pt_cfg = dm.PointingConfig( fp_file=cfg.ufm_to_fp_path, ot_file=cfg.fp_to_ot_path, wafer_slot=ws, platform=cfg.platform, @@ -477,8 +483,8 @@ def match_wafer( ignore_north_south=ignore_north_south, ).as_array() - if wafer_dict is not None: - mask = [wafer_dict[ws] in d["det_id"].astype(str) for d in dst_wafer] + if wafers_dict is not None: + mask = [wafers_dict[ws] in d["det_id"].astype(str) for d in dst_wafer] dst_wafer = dst_wafer[mask] dst.append(dst_wafer) @@ -616,10 +622,8 @@ def get_wafer_solution( else: base_idx = 0 - if wafer_slot == "ws.": - ignore_north_south = True - else: - ignore_north_south = False + # Whether or not to use North/South in match + ignore_north_south = (wafer_slot == "ws.") meas_rset, pointing_map = merge_pointing_info( cfg, pointing_results, base_idx=base_idx, @@ -648,8 +652,7 @@ def get_wafer_solution( def solve_all(cfg) -> Dict[str, Optional[FullWaferSolution]]: - wafer_slots = ["ws.", "ws0", "ws1", "ws2", "ws3", "ws4", "ws5", "ws6"] - results = {ws: get_wafer_solution(cfg, ws, save=True) for ws in wafer_slots} + results = {ws: get_wafer_solution(cfg, ws, save=True) for ws in cfg.wafer_slots} return results diff --git a/sotodlib/coords/optics.py b/sotodlib/coords/optics.py index 31c19eaf3..d11d49995 100644 --- a/sotodlib/coords/optics.py +++ b/sotodlib/coords/optics.py @@ -373,7 +373,7 @@ def load_zemax(path): zemax_dat: Dictionairy with data from zemax """ try: - zemax_dat = np.load(path, allow_pickle=True) + zemax_dat = np.load(path, allow_pickle=True, encoding='bytes').items() except Exception as e: logger.error("Can't load data from " + path) raise e @@ -398,6 +398,7 @@ def LAT_optics(zemax_path): zemax_dat = load_zemax(zemax_path) try: LAT = zemax_dat["LAT"][()] + LAT = {k.decode(): v for k, v in zemax_dat["LAT"][()].items()} except Exception as e: logger.error("LAT key missing from dictionary") raise e @@ -436,6 +437,10 @@ def LATR_optics(zemax_path, tube_slot): zemax_dat = load_zemax(zemax_path) try: LATR = zemax_dat["LATR"][()] + LATR = np.array([ + {k.decode(): v for k, v in d.items()} + for d in LATR + ], dtype=object) except Exception as e: logger.error("LATR key missing from dictionary") raise e diff --git a/sotodlib/site_pipeline/update_det_match.py b/sotodlib/site_pipeline/update_det_match.py index fb40bb8bb..e3f9f1d70 100644 --- a/sotodlib/site_pipeline/update_det_match.py +++ b/sotodlib/site_pipeline/update_det_match.py @@ -245,10 +245,7 @@ def get_failed_detsets(cache_file): def run_match_aman(runner: Runner, aman, detset, wafer_slot=None): stream_id = aman.det_info.stream_id[aman.det_info.detset == detset][0] - if wafer_slot == "ws.": - ignore_north_south = True - else: - ignore_north_south = False + ignore_north_south = True if wafer_slot == "ws." else False rs0 = det_match.ResSet.from_aman(aman, stream_id, ignore_north_south=ignore_north_south) rs0.name = 'meas' @@ -319,23 +316,8 @@ def run_match(runner: Runner, detset: str) -> bool: logger.info(f" - {ds}") for ds in new_detsets: - stream_id = aman.det_info.stream_id[aman.det_info.detset == ds][0] - # Try to get wafer slot info from book idx - if 'wafer_slots' in book_idx: - for ws in book_idx['wafer_slots']: - if ws['stream_id'] == stream_id: - wafer_slot = ws['wafer_slot'] - break - else: - logger.error( - f"Could not find wafer_slot from book index for ds={detset}, " - f"obs_id={obs_id}" - ) - raise Exception("Could not find wafer-slot") - else: - wafer_slot = None - + wafer_slot = aman.det_info.wafer_slot[aman.det_info.detset == ds][0] match = run_match_aman(runner, aman, ds, wafer_slot=wafer_slot) fpath = os.path.join(runner.match_dir, f"{ds}.h5") match.save(fpath) From 426c83d9c944bc8e387d6e01151205646080d7f6 Mon Sep 17 00:00:00 2001 From: Michael McCrackan Date: Tue, 19 May 2026 22:02:53 -0700 Subject: [PATCH 08/15] updates --- sotodlib/coords/det_match.py | 7 +++-- sotodlib/coords/det_match_solutions.py | 27 ++++++++++++++++--- sotodlib/site_pipeline/update_det_match.py | 30 +++++++++++++++++++++- 3 files changed, 56 insertions(+), 8 deletions(-) diff --git a/sotodlib/coords/det_match.py b/sotodlib/coords/det_match.py index 44eef149c..2d97d0d31 100644 --- a/sotodlib/coords/det_match.py +++ b/sotodlib/coords/det_match.py @@ -160,6 +160,8 @@ class Resonator: smurf_subband: int = -1 readout_id: str = '' + wafer_slot: str = 'ws.' + xi: float = np.nan eta: float = np.nan gamma: float = np.nan @@ -214,7 +216,7 @@ def apply_design_properties(smurf_res, design_res, in_place=False, apply_pointin 'det_pol', 'det_freq', 'det_bandpass', 'det_angle_raw_deg', 'det_angle_actual_deg', 'det_type', 'det_id', 'is_optical', 'mux_bondpad', 'mux_subband', 'mux_band', 'mux_channel', - 'mux_layout_pos' + 'mux_layout_pos', 'wafer_slot' ] # for LF @@ -973,10 +975,7 @@ def get_stats(self) -> MatchingStats: stats.unmatched_src = np.sum(~src_arr['matched'].astype(bool)) stats.unmatched_dst = np.sum(~dst_arr['matched'].astype(bool)) - # print(stats.unmatched_src, stats.unmatched_dst) - has_pt = ~np.isnan(src_arr['xi']) - # print(has_pt) stats.unmatched_src_with_pointing = np.sum( (~src_arr['matched'].astype(bool)) & has_pt ) diff --git a/sotodlib/coords/det_match_solutions.py b/sotodlib/coords/det_match_solutions.py index b51583fca..1ce14e342 100644 --- a/sotodlib/coords/det_match_solutions.py +++ b/sotodlib/coords/det_match_solutions.py @@ -467,7 +467,8 @@ def match_wafer( wafer_slots = [wafer_slot] wafers_dict = None - dst = [] + dst_merged = [] + # loop through and get pointing information for all wafers for ws in wafer_slots: pt_cfg = dm.PointingConfig( fp_file=cfg.ufm_to_fp_path, ot_file=cfg.fp_to_ot_path, @@ -483,12 +484,32 @@ def match_wafer( ignore_north_south=ignore_north_south, ).as_array() + # extract current wafer if wafers_dict is not None: mask = [wafers_dict[ws] in d["det_id"].astype(str) for d in dst_wafer] dst_wafer = dst_wafer[mask] + dst_wafer['wafer_slot'][:] = ws - dst.append(dst_wafer) - dst = dm.ResSet.from_array(np.concatenate(dst)) + dst_merged.append(dst_wafer) + dst_merged = np.concatenate(dst_merged) + + # get full wafer info and populate pointing and wafer_slot information + # from merged dst (not necesary if matching a single wafer) + if ignore_north_south: + dst = dm.ResSet.from_wafer_info_file( + cfg.wafer_info_path, + stream_id, + pt_cfg=None, + ignore_north_south=ignore_north_south, + ).as_array() + + merge_fields = ["xi", "eta", "wafer_slot"] + for i, d in enumerate(dst_all): + idx = np.where(dst_merged['det_id'] == d['det_id'])[0][0] + if idx: + for field in merge_fields: + dst[i][field] = dst_merged[idx][field] + dst = dm.ResSet.from_array(dst) # first match match_pars = dm.MatchParams( diff --git a/sotodlib/site_pipeline/update_det_match.py b/sotodlib/site_pipeline/update_det_match.py index e3f9f1d70..d33c53c04 100644 --- a/sotodlib/site_pipeline/update_det_match.py +++ b/sotodlib/site_pipeline/update_det_match.py @@ -154,6 +154,7 @@ def __init__(self, cfg: UpdateDetMatchesConfig): self.cfg = cfg self.ctx = core.Context(cfg.context_path) self.detset_db = None + self.detset_db_path = None self.detcal_db = None with open(self.cfg.wafer_map_path, 'r') as f: self.wafer_map = yaml.safe_load(f) @@ -169,7 +170,8 @@ def __init__(self, cfg: UpdateDetMatchesConfig): else: continue if entry_name == cfg.detset_meta_name: - self.detset_db = core.metadata.ManifestDb(d['db']) + self.detset_db_path = d['db'] + self.detset_db = core.metadata.ManifestDb(self.detset_db_path) elif entry_name == cfg.detcal_meta_name: self.detcal_db = core.metadata.ManifestDb(d['db']) @@ -242,6 +244,29 @@ def get_failed_detsets(cache_file): return list(x.keys()) +def update_detset_wafer_slot(runner: Runner, ds, match): + """ + Update a deset with the wafer_slot field from a match. + """ + ds_base_path = os.path.dirname(detset_db_path) + db = self.detset_db.inspect() + entry = [d for d in db if d['dataset'] == ds][0] + + h5_path = "os.path.join(ds_base_path, entry['filename'])" + with h5py.File(os.path.join(ds_base_path, entry['filename'])) as h5f: + dataset = h5f[entry['dets:detset']][:] + + # replace wafer_slot + merged = match.merged.as_array() + for i, row in enumerate(dataset): + idx = np.where(merged['readout_id'] == row['dets:readout_id'].astype(str))[0] + if idx.size != 0: + row[i]['dets:wafer_slot'] = merged[idx[0]]['wafer_slot'] + + # write entry back out to detset + write_dataset(core.metadata.ResultSet.from_friend(dataset), h5_path, ds, overwrite=True) + + def run_match_aman(runner: Runner, aman, detset, wafer_slot=None): stream_id = aman.det_info.stream_id[aman.det_info.detset == detset][0] @@ -263,6 +288,7 @@ def run_match_aman(runner: Runner, aman, detset, wafer_slot=None): match_pars.freq_offset_mhz = opt_freq match = det_match.Match(rs0, rs1, match_pars=match_pars, apply_dst_pointing=runner.cfg.apply_solution_pointing) + return match def run_match(runner: Runner, detset: str) -> bool: @@ -321,6 +347,8 @@ def run_match(runner: Runner, detset: str) -> bool: match = run_match_aman(runner, aman, ds, wafer_slot=wafer_slot) fpath = os.path.join(runner.match_dir, f"{ds}.h5") match.save(fpath) + if wafer_slot == "ws.": + update_detset_wafer_slot(runner, ds, match) logger.info(f"Saved match to file: {fpath}") return True From f768e43e83e3c7f35899023398a4019cd268232d Mon Sep 17 00:00:00 2001 From: Michael McCrackan Date: Mon, 1 Jun 2026 07:32:54 -0700 Subject: [PATCH 09/15] remove detset function --- sotodlib/site_pipeline/update_det_match.py | 25 ---------------------- 1 file changed, 25 deletions(-) diff --git a/sotodlib/site_pipeline/update_det_match.py b/sotodlib/site_pipeline/update_det_match.py index d33c53c04..ca1876049 100644 --- a/sotodlib/site_pipeline/update_det_match.py +++ b/sotodlib/site_pipeline/update_det_match.py @@ -244,29 +244,6 @@ def get_failed_detsets(cache_file): return list(x.keys()) -def update_detset_wafer_slot(runner: Runner, ds, match): - """ - Update a deset with the wafer_slot field from a match. - """ - ds_base_path = os.path.dirname(detset_db_path) - db = self.detset_db.inspect() - entry = [d for d in db if d['dataset'] == ds][0] - - h5_path = "os.path.join(ds_base_path, entry['filename'])" - with h5py.File(os.path.join(ds_base_path, entry['filename'])) as h5f: - dataset = h5f[entry['dets:detset']][:] - - # replace wafer_slot - merged = match.merged.as_array() - for i, row in enumerate(dataset): - idx = np.where(merged['readout_id'] == row['dets:readout_id'].astype(str))[0] - if idx.size != 0: - row[i]['dets:wafer_slot'] = merged[idx[0]]['wafer_slot'] - - # write entry back out to detset - write_dataset(core.metadata.ResultSet.from_friend(dataset), h5_path, ds, overwrite=True) - - def run_match_aman(runner: Runner, aman, detset, wafer_slot=None): stream_id = aman.det_info.stream_id[aman.det_info.detset == detset][0] @@ -347,8 +324,6 @@ def run_match(runner: Runner, detset: str) -> bool: match = run_match_aman(runner, aman, ds, wafer_slot=wafer_slot) fpath = os.path.join(runner.match_dir, f"{ds}.h5") match.save(fpath) - if wafer_slot == "ws.": - update_detset_wafer_slot(runner, ds, match) logger.info(f"Saved match to file: {fpath}") return True From 7ceae83ac2001c6fc6ef8f65d5a6ac720ce0b4c2 Mon Sep 17 00:00:00 2001 From: Michael McCrackan Date: Mon, 1 Jun 2026 07:56:32 -0700 Subject: [PATCH 10/15] fixes, rename ot to rx --- sotodlib/coords/det_match.py | 12 +++++++----- sotodlib/coords/det_match_solutions.py | 12 ++++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/sotodlib/coords/det_match.py b/sotodlib/coords/det_match.py index 2d97d0d31..aa4075e9e 100644 --- a/sotodlib/coords/det_match.py +++ b/sotodlib/coords/det_match.py @@ -65,6 +65,8 @@ class PointingConfig: ----- fp_file: str Path to focal-plane file that is used by the optics module. + rx_file: str + Path to optics tube YAML file that is used by the optics module. wafer_slot: str Wafer slot of the UFM. For example: "ws0" platform: str @@ -80,7 +82,7 @@ class PointingConfig: Either the tube name as a string or the tube number as an int. """ fp_file: str - ot_file: str + rx_file: str wafer_slot: str platform: str zemax_path: Optional[str] = None @@ -110,8 +112,8 @@ def __post_init__(self): self.tel_type, self.wafer_slot, self.fp_file ) - self.ot_pars = optics.get_fp_to_ot_pars( - self.tube_slot, self.ot_file) + self.rx_pars = optics.get_fp_to_rx_pars( + self.tube_slot, self.rx_file) def get_pointing(self, x, y, pol=0): if self.tel_type.upper() == 'SAT': @@ -125,7 +127,7 @@ def get_pointing(self, x, y, pol=0): tube_slot=self.tube_slot, wafer_slot=self.wafer_slot, ufm_to_fp_pars=self.fp_pars, - fp_to_ot_pars=self.ot_pars, + fp_to_rx_pars=self.rx_pars, ) elif self.tel_type.upper() == 'LAT': xi, eta, gamma = optics.get_focal_plane( @@ -138,7 +140,7 @@ def get_pointing(self, x, y, pol=0): tube_slot=self.tube_slot, wafer_slot=self.wafer_slot, ufm_to_fp_pars=self.fp_pars, - fp_to_ot_pars=self.ot_pars, + fp_to_rx_pars=self.rx_pars, zemax_path=self.zemax_path, ) return xi, eta, gamma diff --git a/sotodlib/coords/det_match_solutions.py b/sotodlib/coords/det_match_solutions.py index 1ce14e342..b7715578d 100644 --- a/sotodlib/coords/det_match_solutions.py +++ b/sotodlib/coords/det_match_solutions.py @@ -85,6 +85,8 @@ class SolutionsCfg: (xi_offset, eta_offset) where both are in radians. ufm_to_fp_path: str Path to file that maps wafer_slot to position on focal plane. + fp_to_rx_path: str + Path to file that maps tube slot to position on focal plane. freq_correct_by_muxband: bool If true, apply the same freq offset correction to all resonators in a mux-band. """ @@ -112,7 +114,7 @@ class SolutionsCfg: initial_pointing_offset: Tuple[float, float] = (0, 0) ufm_to_fp_path: Optional[str] = None - fp_to_ot_path: Optional[str] = None + fp_to_rx_path: Optional[str] = None imprinter_path: Optional[str] = None freq_correct_by_muxband: bool = True @@ -154,8 +156,8 @@ def __post_init__(self): self.ufm_to_fp_path = os.path.join( self.site_pipeline_cfg_dir, "shared/focalplane/ufm_to_fp.yaml" ) - if self.fp_to_ot_path is None: - self.fp_to_ot_path = os.path.join( + if self.fp_to_rx_path is None: + self.fp_to_rx_path = os.path.join( self.site_pipeline_cfg_dir, "shared/focalplane/optics_tubes.yaml" ) if self.imprinter_path is None: @@ -471,7 +473,7 @@ def match_wafer( # loop through and get pointing information for all wafers for ws in wafer_slots: pt_cfg = dm.PointingConfig( - fp_file=cfg.ufm_to_fp_path, ot_file=cfg.fp_to_ot_path, + fp_file=cfg.ufm_to_fp_path, rx_file=cfg.fp_to_rx_path, wafer_slot=ws, platform=cfg.platform, zemax_path=cfg.zemax_path, roll=np.deg2rad(am.obs_info.roll_center) if cfg.apply_roll else 0.0, @@ -510,6 +512,8 @@ def match_wafer( for field in merge_fields: dst[i][field] = dst_merged[idx][field] dst = dm.ResSet.from_array(dst) + else: + dst = dm.ResSet.from_array(dst_merged) # first match match_pars = dm.MatchParams( From e0300bd2ee65b225723370d6cce97ef6e2bbfa46 Mon Sep 17 00:00:00 2001 From: Michael McCrackan Date: Mon, 1 Jun 2026 08:00:24 -0700 Subject: [PATCH 11/15] remove variable --- sotodlib/site_pipeline/update_det_match.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sotodlib/site_pipeline/update_det_match.py b/sotodlib/site_pipeline/update_det_match.py index ca1876049..f1560f5dd 100644 --- a/sotodlib/site_pipeline/update_det_match.py +++ b/sotodlib/site_pipeline/update_det_match.py @@ -154,7 +154,6 @@ def __init__(self, cfg: UpdateDetMatchesConfig): self.cfg = cfg self.ctx = core.Context(cfg.context_path) self.detset_db = None - self.detset_db_path = None self.detcal_db = None with open(self.cfg.wafer_map_path, 'r') as f: self.wafer_map = yaml.safe_load(f) @@ -170,8 +169,7 @@ def __init__(self, cfg: UpdateDetMatchesConfig): else: continue if entry_name == cfg.detset_meta_name: - self.detset_db_path = d['db'] - self.detset_db = core.metadata.ManifestDb(self.detset_db_path) + self.detset_db = core.metadata.ManifestDb(d['db']) elif entry_name == cfg.detcal_meta_name: self.detcal_db = core.metadata.ManifestDb(d['db']) From 9cdf337f6d8f2c85b549c0dbea20da423208bd50 Mon Sep 17 00:00:00 2001 From: Michael McCrackan Date: Mon, 1 Jun 2026 08:11:52 -0700 Subject: [PATCH 12/15] fix optics file diffs --- sotodlib/coords/optics.py | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/sotodlib/coords/optics.py b/sotodlib/coords/optics.py index f88596b5f..dc5c51844 100644 --- a/sotodlib/coords/optics.py +++ b/sotodlib/coords/optics.py @@ -254,7 +254,6 @@ def get_fp_to_rx_pars(ot, config_path): return config[ot] - def fp_to_rx(aman, x=None, y=None, pol=None, phi=0, dx=0, dy=0): """ Transform from coords internal to focal plane to receiver coordinates. @@ -515,11 +514,11 @@ def LAT_focal_plane(aman, zemax_path, x=None, y=None, pol=None, roll=0, tube_slo If aman is provided then will be wrapped as aman.focal_plane.eta. """ if x is None: - x = aman.focal_plane.x_ot + x = aman.focal_plane.x_fp if y is None: - y = aman.focal_plane.y_ot + y = aman.focal_plane.y_fp if pol is None: - pol = aman.focal_plane.pol_ot + pol = aman.focal_plane.pol_fp sec2el, sec2xel = LAT_optics(zemax_path) array2secx, array2secy = LATR_optics(zemax_path, tube_slot) @@ -718,11 +717,6 @@ def get_focal_plane( Should be a dict where each key is an OT pointing to a dict with keys dx, dy, and phi. - ot_config_path: Path to the optics_tubes config file. - - fp_to_ot_pars: Loaded optics_tubes params. - If provided ot_config_path is is ignored. - zemax_path: Path to the data file from Zemax. Only used by the LAT. From 8546c2634ce4587e891750337faaa389f4eb1d7e Mon Sep 17 00:00:00 2001 From: Michael McCrackan Date: Mon, 1 Jun 2026 08:50:57 -0700 Subject: [PATCH 13/15] fix typo --- sotodlib/coords/det_match_solutions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sotodlib/coords/det_match_solutions.py b/sotodlib/coords/det_match_solutions.py index b7715578d..3af549d2f 100644 --- a/sotodlib/coords/det_match_solutions.py +++ b/sotodlib/coords/det_match_solutions.py @@ -506,7 +506,7 @@ def match_wafer( ).as_array() merge_fields = ["xi", "eta", "wafer_slot"] - for i, d in enumerate(dst_all): + for i, d in enumerate(dst_merged): idx = np.where(dst_merged['det_id'] == d['det_id'])[0][0] if idx: for field in merge_fields: From 8201b09dc1db4c83ed60ab1cb63db77b6c10ce44 Mon Sep 17 00:00:00 2001 From: Michael McCrackan Date: Mon, 1 Jun 2026 08:52:21 -0700 Subject: [PATCH 14/15] duplicate import --- sotodlib/coords/det_match_solutions.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sotodlib/coords/det_match_solutions.py b/sotodlib/coords/det_match_solutions.py index 3af549d2f..71c29552a 100644 --- a/sotodlib/coords/det_match_solutions.py +++ b/sotodlib/coords/det_match_solutions.py @@ -6,7 +6,6 @@ import os import numpy as np import yaml -from copy import deepcopy from scipy import interpolate from tqdm.auto import tqdm, trange from copy import deepcopy From e1f749a17960464e8e1ffb89c8feb76ae5414cd6 Mon Sep 17 00:00:00 2001 From: Michael McCrackan Date: Mon, 1 Jun 2026 11:20:19 -0700 Subject: [PATCH 15/15] fix indexing --- sotodlib/coords/det_match_solutions.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/sotodlib/coords/det_match_solutions.py b/sotodlib/coords/det_match_solutions.py index 71c29552a..b28061366 100644 --- a/sotodlib/coords/det_match_solutions.py +++ b/sotodlib/coords/det_match_solutions.py @@ -506,10 +506,10 @@ def match_wafer( merge_fields = ["xi", "eta", "wafer_slot"] for i, d in enumerate(dst_merged): - idx = np.where(dst_merged['det_id'] == d['det_id'])[0][0] + idx = np.where(dst['det_id'] == d['det_id'])[0][0] if idx: for field in merge_fields: - dst[i][field] = dst_merged[idx][field] + dst[idx][field] = dst_merged[i][field] dst = dm.ResSet.from_array(dst) else: dst = dm.ResSet.from_array(dst_merged) @@ -588,7 +588,6 @@ def match_wafer( match.match_pars.dist_width = np.deg2rad(cfg.match_pars["match2"]["dist_width"]) match._match() - match_iterations.append(deepcopy(match)) return MatchSolution(