From 9161cdcad811dbc493ab0a0bfc8bcb41b988c748 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 29 Apr 2026 10:08:03 +0200 Subject: [PATCH 01/10] make uncertainty trigger the use of weighted fit related to #4590 --- src/silx/math/fit/fitmanager.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/silx/math/fit/fitmanager.py b/src/silx/math/fit/fitmanager.py index 233010c09f..88d9f9bc15 100644 --- a/src/silx/math/fit/fitmanager.py +++ b/src/silx/math/fit/fitmanager.py @@ -777,10 +777,8 @@ def setdata(self, x, y, sigmay=None, xmin=None, xmax=None): numpy.sqrt(self.ydata) if self.fitconfig["WeightFlag"] else None ) else: - self.sigmay0 = numpy.array(sigmay) - self.sigmay = ( - numpy.array(sigmay) if self.fitconfig["WeightFlag"] else None - ) + self.fitconfig["WeightFlag"] = True + self.sigmay0 = self.sigmay = numpy.array(sigmay) # take the data between limits, using boolean array indexing if (xmin is not None or xmax is not None) and len(self.xdata): @@ -789,7 +787,10 @@ def setdata(self, x, y, sigmay=None, xmin=None, xmax=None): bool_array = (self.xdata >= xmin) & (self.xdata <= xmax) self.xdata = self.xdata[bool_array] self.ydata = self.ydata[bool_array] - self.sigmay = self.sigmay[bool_array] if sigmay is not None else None + if sigmay is None: + self.sigmay = self.sigmay[bool_array] if self.fitconfig["WeightFlag"] else None + else: + self.sigmay0 = self.sigmay = self.sigmay[bool_array] self._finite_mask = numpy.logical_and( numpy.all( From b0e74df69ba30a43805dfcdcc46abad5986cccd9 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Tue, 12 May 2026 13:16:47 +0200 Subject: [PATCH 02/10] implement file locking in ExternalResources --- package/debian13/control | 6 ++- pyproject.toml | 1 + src/silx/math/fit/fitmanager.py | 6 ++- src/silx/utils/ExternalResources.py | 68 +++++++++++++++++++++-------- 4 files changed, 61 insertions(+), 20 deletions(-) diff --git a/package/debian13/control b/package/debian13/control index 1d4b2a2441..6e1032de26 100644 --- a/package/debian13/control +++ b/package/debian13/control @@ -36,6 +36,7 @@ Build-Depends: cython3 (>= 0.23.2), python3-sphinx, python3-sphinx-copybutton, python3-sphinxcontrib.programoutput, + python3-filelock, xauth, xvfb Standards-Version: 4.1.3 @@ -74,7 +75,10 @@ Description: Toolbox for X-Ray data analysis - Executables Package: python3-silx Architecture: any Section: python -Depends: ${misc:Depends}, ${python3:Depends}, ${shlibs:Depends} +Depends: ${misc:Depends}, + ${python3:Depends}, + ${shlibs:Depends}, + python3-filelock Description: Toolbox for X-Ray data analysis - Python3 The silx project aims at providing a collection of Python packages to support the development of data assessment, reduction and analysis diff --git a/pyproject.toml b/pyproject.toml index 4b57e54503..45de5d6beb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,7 @@ dependencies = [ 'h5py >= 3', 'fabio', 'pydantic >= 2', + 'filelock', ] [build-system] diff --git a/src/silx/math/fit/fitmanager.py b/src/silx/math/fit/fitmanager.py index 88d9f9bc15..5cbfe8af56 100644 --- a/src/silx/math/fit/fitmanager.py +++ b/src/silx/math/fit/fitmanager.py @@ -788,7 +788,11 @@ def setdata(self, x, y, sigmay=None, xmin=None, xmax=None): self.xdata = self.xdata[bool_array] self.ydata = self.ydata[bool_array] if sigmay is None: - self.sigmay = self.sigmay[bool_array] if self.fitconfig["WeightFlag"] else None + self.sigmay = ( + self.sigmay[bool_array] + if self.fitconfig["WeightFlag"] + else None + ) else: self.sigmay0 = self.sigmay = self.sigmay[bool_array] diff --git a/src/silx/utils/ExternalResources.py b/src/silx/utils/ExternalResources.py index 258306f113..dc11b123f5 100644 --- a/src/silx/utils/ExternalResources.py +++ b/src/silx/utils/ExternalResources.py @@ -25,7 +25,7 @@ __authors__ = ["Thomas Vincent", "J. Kieffer"] __license__ = "MIT" -__date__ = "21/12/2021" +__date__ = "12/05/2026" import hashlib @@ -40,6 +40,7 @@ import urllib.request import urllib.error import zipfile +import filelock logger = logging.getLogger(__name__) @@ -74,6 +75,13 @@ def __init__(self, project, url_base, env_key=None, timeout=60, data_home=None): self.all_data = {} self.timeout = timeout self._data_home = data_home + self.testdata = "" + self.lock = None + + @property + def lockfile(self): + """Returns the lockfile path.""" + return self.testdata + ".lock" @property def data_home(self): @@ -101,8 +109,14 @@ def data_home(self): basename = f"{self.project}_testdata_{name}" data_home = os.path.join(tempfile.gettempdir(), basename) if not os.path.exists(data_home): - os.makedirs(data_home) + try: + os.makedirs(data_home) + except OSError as exc: + raise RuntimeError( + f"Unable to create data directory {data_home} ! ({exc})" + ) self._data_home = data_home + return data_home def get_hash(self, filename=None, data=None): @@ -126,15 +140,8 @@ def _initialize_data(self): with self.sem: if not self._initialized: self.testdata = os.path.join(self.data_home, "all_testdata.json") - if os.path.exists(self.testdata): - with open(self.testdata) as f: - jdata = json.load(f) - if isinstance(jdata, dict): - self.all_data = jdata - else: - # recalculate the hash only if the data was stored as a list - self.all_data = {k: self.get_hash(k) for k in jdata} - self.save_json() + self.lock = filelock.FileLock(self.lockfile, timeout=10) + self.all_data = self.load_json() self._initialized = True def getfile(self, filename): @@ -186,8 +193,8 @@ def getfile(self, filename): with open(fullfilename, mode="wb") as outfile: outfile.write(data) except OSError: - raise OSError("unable to write downloaded \ - data to disk at %s" % self.data_home) + raise OSError(f"unable to write downloaded \ + data to disk at {fullfilename}") if not os.path.isfile(fullfilename): raise RuntimeError("""Could not automatically download test files %s! @@ -212,15 +219,40 @@ def getfile(self, filename): return fullfilename + def load_json(self) -> dict: + """Loads the JSON file containing the list of files and their hashes""" + all_data = {} + if self.testdata and os.path.exists(self.testdata): + try: + with self.lock: + with open(self.testdata) as f: + jdata = json.load(f) + except filelock.Timeout: + logger.error("Unable to lock JSON file") + jdata = {} + if isinstance(jdata, dict): + all_data = jdata + else: + # recalculate the hash only if the data was stored as a list + self.all_data = {k: self.get_hash(k) for k in jdata} + return all_data + def save_json(self): - file_list = list(self.all_data.keys()) + """Saves the JSON file containing the list of files and their hashes""" + dico = self.load_json() + dico.update(self.all_data) + file_list = list(dico.keys()) file_list.sort() - dico = {i: self.all_data[i] for i in file_list} + dico = {i: dico[i] for i in file_list} # reorder items + try: - with open(self.testdata, "w") as fp: - json.dump(dico, fp, indent=4) + with self.lock: + with open(self.testdata, "w") as fp: + json.dump(dico, fp, indent=4) + except filelock.Timeout: + logger.error("Unable to lock JSON file") except OSError: - logger.info("Unable to save JSON dict") + logger.error("Unable to save JSON dict") def getdir(self, dirname): """Downloads the requested tarball from the server From 0fb973ab0d9a89c73059319a88c5844c6ce98378 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Tue, 12 May 2026 15:02:54 +0200 Subject: [PATCH 03/10] switch to dev mode --- src/silx/_version.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/silx/_version.py b/src/silx/_version.py index 6492dc9820..652e9025d6 100755 --- a/src/silx/_version.py +++ b/src/silx/_version.py @@ -71,8 +71,8 @@ MAJOR = 3 MINOR = 0 -MICRO = 1 -RELEV = "final" # <16 +MICRO = 2 +RELEV = "dev" # <16 SERIAL = 0 # <16 date = __date__ From 742bcca5fe44bc55cae1495f56ddd1a1a8a487e4 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Tue, 12 May 2026 16:37:50 +0200 Subject: [PATCH 04/10] reload dico if checksum not present --- src/silx/utils/ExternalResources.py | 32 ++++++++++++++++++----------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/src/silx/utils/ExternalResources.py b/src/silx/utils/ExternalResources.py index dc11b123f5..13dd03d431 100644 --- a/src/silx/utils/ExternalResources.py +++ b/src/silx/utils/ExternalResources.py @@ -158,7 +158,25 @@ def getfile(self, filename): fullfilename = os.path.abspath(os.path.join(self.data_home, filename)) - if not os.path.isfile(fullfilename): + if os.path.isfile(fullfilename): + h = self.hash() + with open(fullfilename, mode="rb") as fd: + h.update(fd.read()) + if filename not in self.all_data: + dico = self.load_json() + dico.update(self.all_data) + self.all_data = dico + if filename not in self.all_data: + logger.error( + f"Filename {filename} not present in all_data:{os.linesep}{self.all_data}" + ) + if h.hexdigest() != self.all_data[filename]: + logger.warning(f"Detected corruped file {fullfilename}") + self.all_data.pop(filename) + os.unlink(fullfilename) + return self.getfile(filename) + + else: logger.debug( "Trying to download file %s, timeout set to %ss", filename, @@ -207,16 +225,6 @@ def getfile(self, filename): self.all_data[filename] = self.get_hash(data=data) self.save_json() - else: - h = self.hash() - with open(fullfilename, mode="rb") as fd: - h.update(fd.read()) - if h.hexdigest() != self.all_data[filename]: - logger.warning(f"Detected corruped file {fullfilename}") - self.all_data.pop(filename) - os.unlink(fullfilename) - return self.getfile(filename) - return fullfilename def load_json(self) -> dict: @@ -234,7 +242,7 @@ def load_json(self) -> dict: all_data = jdata else: # recalculate the hash only if the data was stored as a list - self.all_data = {k: self.get_hash(k) for k in jdata} + all_data = {k: self.get_hash(k) for k in jdata} return all_data def save_json(self): From edc72ad3ab4f95dcec6ffd69d7b859c32c0bcec5 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Tue, 12 May 2026 17:20:41 +0200 Subject: [PATCH 05/10] manage unconsistent files --- src/silx/utils/ExternalResources.py | 45 ++++++++++++++++++----------- 1 file changed, 28 insertions(+), 17 deletions(-) diff --git a/src/silx/utils/ExternalResources.py b/src/silx/utils/ExternalResources.py index 13dd03d431..73028d29c5 100644 --- a/src/silx/utils/ExternalResources.py +++ b/src/silx/utils/ExternalResources.py @@ -51,6 +51,8 @@ class ExternalResources: """ + TIMEOUT = 10 + def __init__(self, project, url_base, env_key=None, timeout=60, data_home=None): """Constructor of the class @@ -140,7 +142,7 @@ def _initialize_data(self): with self.sem: if not self._initialized: self.testdata = os.path.join(self.data_home, "all_testdata.json") - self.lock = filelock.FileLock(self.lockfile, timeout=10) + self.lock = filelock.FileLock(self.lockfile, timeout=self.TIMEOUT) self.all_data = self.load_json() self._initialized = True @@ -159,23 +161,32 @@ def getfile(self, filename): fullfilename = os.path.abspath(os.path.join(self.data_home, filename)) if os.path.isfile(fullfilename): - h = self.hash() - with open(fullfilename, mode="rb") as fd: - h.update(fd.read()) if filename not in self.all_data: - dico = self.load_json() - dico.update(self.all_data) - self.all_data = dico - if filename not in self.all_data: + """File already exists but is not in the list of known files""" + time_out = time.time() + self.TIMEOUT + while time.time() < time_out: + dico = self.load_json() + if filename in dico: + dico.update(self.all_data) + self.all_data = dico + break + time.sleep(1) + else: logger.error( f"Filename {filename} not present in all_data:{os.linesep}{self.all_data}" ) + os.remove(fullfilename) + return self.getfile(filename) + + h = self.hash() + with open(fullfilename, mode="rb") as fd: + h.update(fd.read()) + if h.hexdigest() != self.all_data[filename]: logger.warning(f"Detected corruped file {fullfilename}") self.all_data.pop(filename) os.unlink(fullfilename) return self.getfile(filename) - else: logger.debug( "Trying to download file %s, timeout set to %ss", @@ -203,9 +214,10 @@ def getfile(self, filename): except urllib.error.URLError: raise unittest.SkipTest("network unreachable.") - if not os.path.isdir(os.path.dirname(fullfilename)): - # Create sub-directory if needed - os.makedirs(os.path.dirname(fullfilename)) + dirname = os.path.dirname(fullfilename) + if not os.path.isdir(dirname): + """Create sub-directory if needed""" + os.makedirs(dirname) try: with open(fullfilename, mode="wb") as outfile: @@ -214,17 +226,16 @@ def getfile(self, filename): raise OSError(f"unable to write downloaded \ data to disk at {fullfilename}") - if not os.path.isfile(fullfilename): + if os.path.isfile(fullfilename): + self.all_data[filename] = self.get_hash(data=data) + self.save_json() + else: raise RuntimeError("""Could not automatically download test files %s! If you are behind a firewall, please set both environment variable http_proxy and https_proxy. This even works under windows ! Otherwise please try to download the files manually from %s/%s""" % (filename, self.url_base, filename)) - else: - self.all_data[filename] = self.get_hash(data=data) - self.save_json() - return fullfilename def load_json(self) -> dict: From 60db9c01014dff4b550253821f0c32d865eecc22 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Tue, 12 May 2026 17:45:08 +0200 Subject: [PATCH 06/10] single timeout --- src/silx/utils/ExternalResources.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/silx/utils/ExternalResources.py b/src/silx/utils/ExternalResources.py index 73028d29c5..0044c28dea 100644 --- a/src/silx/utils/ExternalResources.py +++ b/src/silx/utils/ExternalResources.py @@ -34,8 +34,9 @@ import os import sys import tarfile -import threading import tempfile +import threading +import time import unittest import urllib.request import urllib.error @@ -51,8 +52,6 @@ class ExternalResources: """ - TIMEOUT = 10 - def __init__(self, project, url_base, env_key=None, timeout=60, data_home=None): """Constructor of the class @@ -142,7 +141,7 @@ def _initialize_data(self): with self.sem: if not self._initialized: self.testdata = os.path.join(self.data_home, "all_testdata.json") - self.lock = filelock.FileLock(self.lockfile, timeout=self.TIMEOUT) + self.lock = filelock.FileLock(self.lockfile, timeout=self.timeout) self.all_data = self.load_json() self._initialized = True @@ -163,7 +162,7 @@ def getfile(self, filename): if os.path.isfile(fullfilename): if filename not in self.all_data: """File already exists but is not in the list of known files""" - time_out = time.time() + self.TIMEOUT + time_out = time.time() + self.timeout while time.time() < time_out: dico = self.load_json() if filename in dico: From f1700c19085a2ebe596f9d31ce98bf303bfa712e Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Tue, 12 May 2026 17:46:40 +0200 Subject: [PATCH 07/10] change time -> perf_counter --- src/silx/utils/ExternalResources.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/silx/utils/ExternalResources.py b/src/silx/utils/ExternalResources.py index 0044c28dea..10cd8b91c3 100644 --- a/src/silx/utils/ExternalResources.py +++ b/src/silx/utils/ExternalResources.py @@ -162,8 +162,8 @@ def getfile(self, filename): if os.path.isfile(fullfilename): if filename not in self.all_data: """File already exists but is not in the list of known files""" - time_out = time.time() + self.timeout - while time.time() < time_out: + time_out = time.perf_counter() + self.timeout + while time.perf_counter() < time_out: dico = self.load_json() if filename in dico: dico.update(self.all_data) From 19be7914510daa318edaf6ad403d2376af690113 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Tue, 12 May 2026 18:05:43 +0200 Subject: [PATCH 08/10] lower logging level --- src/silx/utils/ExternalResources.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/silx/utils/ExternalResources.py b/src/silx/utils/ExternalResources.py index 10cd8b91c3..6a93ec04e5 100644 --- a/src/silx/utils/ExternalResources.py +++ b/src/silx/utils/ExternalResources.py @@ -171,8 +171,8 @@ def getfile(self, filename): break time.sleep(1) else: - logger.error( - f"Filename {filename} not present in all_data:{os.linesep}{self.all_data}" + logger.warning( + f"Timeout! Filename {filename} not present in all_data:{os.linesep}{json.dumps(self.all_data, indent=2)}" ) os.remove(fullfilename) return self.getfile(filename) @@ -182,7 +182,7 @@ def getfile(self, filename): h.update(fd.read()) if h.hexdigest() != self.all_data[filename]: - logger.warning(f"Detected corruped file {fullfilename}") + logger.warning(f"Detected corrupted file {fullfilename} !") self.all_data.pop(filename) os.unlink(fullfilename) return self.getfile(filename) @@ -209,7 +209,7 @@ def getfile(self, filename): data = opener( f"{self.url_base}/{filename}", data=None, timeout=self.timeout ).read() - logger.info("File %s successfully downloaded.", filename) + logger.info(f"File {filename} successfully downloaded.") except urllib.error.URLError: raise unittest.SkipTest("network unreachable.") From 1386929e49883c4a3bdfdf1573a2cb70102d0101 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 27 May 2026 17:56:51 +0200 Subject: [PATCH 09/10] revert modification in fitmanager --- src/silx/math/fit/fitmanager.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/silx/math/fit/fitmanager.py b/src/silx/math/fit/fitmanager.py index 5cbfe8af56..88d9f9bc15 100644 --- a/src/silx/math/fit/fitmanager.py +++ b/src/silx/math/fit/fitmanager.py @@ -788,11 +788,7 @@ def setdata(self, x, y, sigmay=None, xmin=None, xmax=None): self.xdata = self.xdata[bool_array] self.ydata = self.ydata[bool_array] if sigmay is None: - self.sigmay = ( - self.sigmay[bool_array] - if self.fitconfig["WeightFlag"] - else None - ) + self.sigmay = self.sigmay[bool_array] if self.fitconfig["WeightFlag"] else None else: self.sigmay0 = self.sigmay = self.sigmay[bool_array] From 5a7cad481d2fd148d7f9839c626822c60fb56093 Mon Sep 17 00:00:00 2001 From: Jerome Kieffer Date: Wed, 27 May 2026 17:59:08 +0200 Subject: [PATCH 10/10] revert fitmanager upgrade (black apply) --- src/silx/math/fit/fitmanager.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/silx/math/fit/fitmanager.py b/src/silx/math/fit/fitmanager.py index 88d9f9bc15..233010c09f 100644 --- a/src/silx/math/fit/fitmanager.py +++ b/src/silx/math/fit/fitmanager.py @@ -777,8 +777,10 @@ def setdata(self, x, y, sigmay=None, xmin=None, xmax=None): numpy.sqrt(self.ydata) if self.fitconfig["WeightFlag"] else None ) else: - self.fitconfig["WeightFlag"] = True - self.sigmay0 = self.sigmay = numpy.array(sigmay) + self.sigmay0 = numpy.array(sigmay) + self.sigmay = ( + numpy.array(sigmay) if self.fitconfig["WeightFlag"] else None + ) # take the data between limits, using boolean array indexing if (xmin is not None or xmax is not None) and len(self.xdata): @@ -787,10 +789,7 @@ def setdata(self, x, y, sigmay=None, xmin=None, xmax=None): bool_array = (self.xdata >= xmin) & (self.xdata <= xmax) self.xdata = self.xdata[bool_array] self.ydata = self.ydata[bool_array] - if sigmay is None: - self.sigmay = self.sigmay[bool_array] if self.fitconfig["WeightFlag"] else None - else: - self.sigmay0 = self.sigmay = self.sigmay[bool_array] + self.sigmay = self.sigmay[bool_array] if sigmay is not None else None self._finite_mask = numpy.logical_and( numpy.all(