From e8cc1aa0a45ad671d332f18b2be64d5d06f90be4 Mon Sep 17 00:00:00 2001
From: Romain Broucquart <romain.broucquart@synchrotron-soleil.fr>
Date: Thu, 4 Apr 2024 15:11:38 +0200
Subject: [PATCH] fix: Crop max dim for huge spectrum

* Sometimes max dim is well above what's used
* Quick fix to crop it to 2048 elements, arbitrary
---
 ArchiveExtractor/Amenities.py | 6 +++++-
 ArchiveExtractor/Core.py      | 3 ++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/ArchiveExtractor/Amenities.py b/ArchiveExtractor/Amenities.py
index fd48cd1..fc3117b 100644
--- a/ArchiveExtractor/Amenities.py
+++ b/ArchiveExtractor/Amenities.py
@@ -173,7 +173,11 @@ def _chunkerize(attribute, dateStart, dateStop, db, Nmax=100000):
     dx=int(info["max_dim_x"])
     if dx > 1:
         logger.debug("Attribute is a vector with max dimension = %s"%dx)
-        N=N*dx
+
+        # Quick fix : max dimension is not always used. Cap it to a few hundreds
+        N=N*min(dx,2048)
+        if dx > 2048:
+            logger.warning("Attribute vector has max dimension above 2048, assume no more than this limit for chunkerize.")
 
     # If data chunk is too much, we need to cut it
     if N > Nmax:
diff --git a/ArchiveExtractor/Core.py b/ArchiveExtractor/Core.py
index e193cf8..d1c0409 100644
--- a/ArchiveExtractor/Core.py
+++ b/ArchiveExtractor/Core.py
@@ -170,7 +170,8 @@ def _extract_vector(attribute, method, date1, date2, db):
         attrHist = ae._Extractors[{'H':0, 'T':1}[db]].attribute_history(name, N)
 
         # Transform to datetime - value arrays
-        _value = np.empty((N, int(info["max_dim_x"])), dtype=float)
+        mx = min(int(info["max_dim_x"]), 2048) # Quick fix: Crop dimension
+        _value = np.empty((N, mx), dtype=float)
         _value[:] = np.nan
         _date = np.empty(N, dtype=object)
         for i_h in range(N):
-- 
GitLab