From e8cc1aa0a45ad671d332f18b2be64d5d06f90be4 Mon Sep 17 00:00:00 2001 From: Romain Broucquart <romain.broucquart@synchrotron-soleil.fr> Date: Thu, 4 Apr 2024 15:11:38 +0200 Subject: [PATCH] fix: Crop max dim for huge spectrum * Sometimes max dim is well above what's used * Quick fix to crop it to 2048 elements, arbitrary --- ArchiveExtractor/Amenities.py | 6 +++++- ArchiveExtractor/Core.py | 3 ++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ArchiveExtractor/Amenities.py b/ArchiveExtractor/Amenities.py index fd48cd1..fc3117b 100644 --- a/ArchiveExtractor/Amenities.py +++ b/ArchiveExtractor/Amenities.py @@ -173,7 +173,11 @@ def _chunkerize(attribute, dateStart, dateStop, db, Nmax=100000): dx=int(info["max_dim_x"]) if dx > 1: logger.debug("Attribute is a vector with max dimension = %s"%dx) - N=N*dx + + # Quick fix : max dimension is not always used. Cap it to a few hundreds + N=N*min(dx,2048) + if dx > 2048: + logger.warning("Attribute vector has max dimension above 2048, assume no more than this limit for chunkerize.") # If data chunk is too much, we need to cut it if N > Nmax: diff --git a/ArchiveExtractor/Core.py b/ArchiveExtractor/Core.py index e193cf8..d1c0409 100644 --- a/ArchiveExtractor/Core.py +++ b/ArchiveExtractor/Core.py @@ -170,7 +170,8 @@ def _extract_vector(attribute, method, date1, date2, db): attrHist = ae._Extractors[{'H':0, 'T':1}[db]].attribute_history(name, N) # Transform to datetime - value arrays - _value = np.empty((N, int(info["max_dim_x"])), dtype=float) + mx = min(int(info["max_dim_x"]), 2048) # Quick fix: Crop dimension + _value = np.empty((N, mx), dtype=float) _value[:] = np.nan _date = np.empty(N, dtype=object) for i_h in range(N): -- GitLab