From 4cf2eb491843af489f0040d93b8cff4f2b78deaf Mon Sep 17 00:00:00 2001
From: Romain Broucquart <romain.broucquart@synchrotron-soleil.fr>
Date: Fri, 22 Apr 2022 18:56:13 +0200
Subject: [PATCH] Handle DevFailed tango exception

* retries 2 times before leaving with error
---
 core/ArchiveExtractor.py | 35 ++++++++++++++++++++++++-----------
 1 file changed, 24 insertions(+), 11 deletions(-)

diff --git a/core/ArchiveExtractor.py b/core/ArchiveExtractor.py
index 5fe1a52..4b59129 100755
--- a/core/ArchiveExtractor.py
+++ b/core/ArchiveExtractor.py
@@ -208,18 +208,31 @@ class ArchiveExtractor:
 
         # For each date chunk
         for i_d in range(len(cdates)-1):
-            # Make retrieval request
-            self.logger.debug("Perform ExtractBetweenDates (%s, %s, %s)"%(
-                attribute,
-                cdates[i_d].strftime(DBDFMT),
-                cdates[i_d+1].strftime(DBDFMT))
-                )
 
-            _date, _value = self.extractor.ExtractBetweenDates([
-                attribute,
-                cdates[i_d].strftime(DBDFMT),
-                cdates[i_d+1].strftime(DBDFMT)
-                ])
+            # 2 retries on DevFailed
+            for i in range(3):
+                # Make retrieval request
+                self.logger.debug("Perform ExtractBetweenDates (%s, %s, %s)"%(
+                    attribute,
+                    cdates[i_d].strftime(DBDFMT),
+                    cdates[i_d+1].strftime(DBDFMT))
+                    )
+
+                try:
+                    _date, _value = self.extractor.ExtractBetweenDates([
+                        attribute,
+                        cdates[i_d].strftime(DBDFMT),
+                        cdates[i_d+1].strftime(DBDFMT)
+                        ])
+                except tango.DevFailed as e:
+                    self.logger.warning("The extractor device returned the following error:")
+                    self.logger.warning(e)
+                    self.logger.warning("Retrying...")
+                    continue
+                break
+            if i==2:
+                logger.error("Could not extract this chunk. Check the device extractor")
+                return None
 
             # Transform to datetime - value arrays
             _value = np.asarray(_value, dtype=float)
-- 
GitLab