From a143ca61ecf8dbf96387b4d43330c61927592998 Mon Sep 17 00:00:00 2001
From: gwen-soleil <gwenaelle.abeille@gmail.com>
Date: Wed, 4 Jan 2023 09:11:46 +0100
Subject: [PATCH] Avoid Oracle DB error ORA-01795: maximum number of
expressions in a list is 1000 (Jira SOLEIL TANGOARCH-744)
---
.../archiving/snap/api/DataBaseAPI.java | 188 ++++++++++--------
1 file changed, 101 insertions(+), 87 deletions(-)
diff --git a/src/main/java/fr/soleil/archiving/snap/api/DataBaseAPI.java b/src/main/java/fr/soleil/archiving/snap/api/DataBaseAPI.java
index badafc8..0ca8314 100644
--- a/src/main/java/fr/soleil/archiving/snap/api/DataBaseAPI.java
+++ b/src/main/java/fr/soleil/archiving/snap/api/DataBaseAPI.java
@@ -172,6 +172,7 @@
//-======================================================================
package fr.soleil.archiving.snap.api;
+import com.google.common.collect.Lists;
import fr.esrf.Tango.AttrDataFormat;
import fr.esrf.Tango.AttrWriteType;
import fr.esrf.Tango.ErrSeverity;
@@ -227,6 +228,7 @@ import java.util.stream.Collectors;
public class DataBaseAPI {
private static final String ERROR_EXTRACTING_SNAPSHOT_DATA = "error extracting snapshot data";
+ public static final int MAX_SELECT_SIZE = 1000;
public static boolean useLog4JDBC = System.getProperty("log4jdbc.active") == null ? false : System.getProperty(
"log4jdbc.active").equalsIgnoreCase("true");
private final Logger logger = LoggerFactory.getLogger(DataBaseAPI.class);
@@ -278,15 +280,6 @@ public class DataBaseAPI {
return params.getName();
}
- /**
- * <b>Description : </b> Gets the database's schema name
- *
- * @return The database name
- */
- public String getDbSchema() {
- return params.getSchema();
- }
-
/**
* <b>Description : </b> Returns the connected database host identifier.
*
@@ -346,7 +339,6 @@ public class DataBaseAPI {
return arrayCount;
}
-
/**
* ************************************************************************
* <b>Description : </b> Closes the connection with the database
@@ -358,6 +350,22 @@ public class DataBaseAPI {
closeConnection(conn);
}
+ private void closeConnection(final Connection conn) {
+ if (conn == null) {
+ return;
+ }
+
+ try {
+ conn.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+
+ logger.error("ERROR !! " + "\r\n" + "\t Origin : \t " + "DataBaseAPI.closeConnection" + "\r\n"
+ + "\t Reason : \t " + getDbSchema().toUpperCase().trim() + "_FAILURE" + "\r\n"
+ + "\t Description : \t " + e.getMessage());
+ }
+ }
+
/*****************************************************************************
*
@@ -366,6 +374,33 @@ public class DataBaseAPI {
*
****************************************************************************/
+ /**
+ * <b>Description : </b> Gets the database's schema name
+ *
+ * @return The database name
+ */
+ public String getDbSchema() {
+ return params.getSchema();
+ }
+
+ /**
+ * <b>Description : </b> Checks if the attribute of the given name, is
+ * already registered in <I>SnapDb</I> (and more particularly in the table
+ * of the definitions).
+ *
+ * @param att_name The name of the attribute to check.
+ * @return boolean
+ * @throws SnapshotingException
+ */
+ public boolean isRegistered(final String att_name) throws SnapshotingException {
+ final int id = getAttID(att_name.trim());
+ if (id != 0) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
/**
* ************************************************************************
* <b>Description : </b> Gets for a specified attribute its ID as defined in
@@ -420,21 +455,35 @@ public class DataBaseAPI {
return attributesID;
}
- /**
- * <b>Description : </b> Checks if the attribute of the given name, is
- * already registered in <I>SnapDb</I> (and more particularly in the table
- * of the definitions).
- *
- * @param att_name The name of the attribute to check.
- * @return boolean
- * @throws SnapshotingException
- */
- public boolean isRegistered(final String att_name) throws SnapshotingException {
- final int id = getAttID(att_name.trim());
- if (id != 0) {
- return true;
- } else {
- return false;
+ public void closeResultSet(final ResultSet resultSet) {
+ if (resultSet == null) {
+ return;
+ }
+
+ try {
+ resultSet.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+
+ logger.error("ERROR !! " + "\r\n" + "\t Origin : \t " + "DataBaseAPI.closeResultSet" + "\r\n"
+ + "\t Reason : \t " + getDbSchema().toUpperCase().trim() + "_FAILURE" + "\r\n"
+ + "\t Description : \t " + e.getMessage());
+ }
+ }
+
+ public void closeStatement(final Statement preparedStatement) {
+ if (preparedStatement == null) {
+ return;
+ }
+
+ try {
+ preparedStatement.close();
+ } catch (final SQLException e) {
+ e.printStackTrace();
+
+ logger.error("ERROR !! " + "\r\n" + "\t Origin : \t " + "DataBaseAPI.closeStatement" + "\r\n"
+ + "\t Reason : \t " + getDbSchema().toUpperCase().trim() + "_FAILURE" + "\r\n"
+ + "\t Description : \t " + e.getMessage());
}
}
@@ -483,7 +532,6 @@ public class DataBaseAPI {
return res;
}
-
/**
* This method retrives from the the database, the list of all registered
* contexts (or 'snap-patterns) which subscribe to the clause and/or have
@@ -554,7 +602,6 @@ public class DataBaseAPI {
}
-
/**
* Retrieves the context identifier to wich the given snapshot identifier is
* associated.
@@ -632,7 +679,6 @@ public class DataBaseAPI {
}
}
-
private void selectRO(final List<SnapAttributeExtract> attributes, int snapID,
final String tableName) throws SnapshotingException, SQLException {
@@ -685,7 +731,7 @@ public class DataBaseAPI {
// t_sp_2val WHERE id_snap = ?
List<String> scpectrumRWAttId = attributes.stream().filter(s -> s.getDataFormat() == AttrDataFormat._SPECTRUM
- && (s.getWritable() == AttrWriteType._READ_WITH_WRITE || s.getWritable() == AttrWriteType._READ_WRITE))
+ && (s.getWritable() == AttrWriteType._READ_WITH_WRITE || s.getWritable() == AttrWriteType._READ_WRITE))
.map(s -> String.valueOf(s.getAttId())).collect(Collectors.toList());
if (scpectrumRWAttId.isEmpty()) {
return;
@@ -781,7 +827,6 @@ public class DataBaseAPI {
throws SnapshotingException {
// one sql request per table
-
List<SnapAttributeExtract> numericScalarRO = attributes.stream()
.filter(s -> s.getDataFormat() == AttrDataFormat._SCALAR
&& s.getDataType() != TangoConst.Tango_DEV_STRING
@@ -808,94 +853,64 @@ public class DataBaseAPI {
try {
// --- Get scalar numeric read only values
- selectScalarRO(numericScalarRO, snapID, SnapConst.T_SC_NUM_1VAL);
+
+ // Avoid Oracle DB error ORA-01795: maximum number of expressions in a list is 1000
+ final List<List<SnapAttributeExtract>> chunks = Lists.partition(numericScalarRO, MAX_SELECT_SIZE);
+ for (List<SnapAttributeExtract> chunk : chunks) {
+ selectScalarRO(chunk, snapID, SnapConst.T_SC_NUM_1VAL);
+ }
} catch (SQLException | SnapshotingException e) {
logger.error(ERROR_EXTRACTING_SNAPSHOT_DATA, e);
}
try {
// --- Get scalar numeric read write values
- selectScalarRW(numericScalarRW, snapID, SnapConst.T_SC_NUM_2VAL);
+ final List<List<SnapAttributeExtract>> chunks = Lists.partition(numericScalarRW, MAX_SELECT_SIZE);
+ for (List<SnapAttributeExtract> chunk : chunks) {
+ selectScalarRW(chunk, snapID, SnapConst.T_SC_NUM_2VAL);
+ }
} catch (SQLException | SnapshotingException e) {
logger.error(ERROR_EXTRACTING_SNAPSHOT_DATA, e);
}
try {
// --- Get scalar string read only values
- selectScalarRO(stringScalarRO, snapID, SnapConst.T_SC_STR_1VAL);
+ final List<List<SnapAttributeExtract>> chunks = Lists.partition(stringScalarRO, MAX_SELECT_SIZE);
+ for (List<SnapAttributeExtract> chunk : chunks) {
+ selectScalarRO(chunk, snapID, SnapConst.T_SC_STR_1VAL);
+ }
} catch (SQLException | SnapshotingException e) {
logger.error(ERROR_EXTRACTING_SNAPSHOT_DATA, e);
}
try {
// --- Get scalar numeric read write values
- selectScalarRW(stringScalarRW, snapID, SnapConst.T_SC_STR_2VAL);
+ final List<List<SnapAttributeExtract>> chunks = Lists.partition(stringScalarRW, MAX_SELECT_SIZE);
+ for (List<SnapAttributeExtract> chunk : chunks) {
+ selectScalarRW(chunk, snapID, SnapConst.T_SC_STR_2VAL);
+ }
} catch (SQLException | SnapshotingException e) {
logger.error(ERROR_EXTRACTING_SNAPSHOT_DATA, e);
}
try {
// --- Get spectrum read values
- selectRO(attributes, snapID, SnapConst.T_SP_1VAL);
+ final List<List<SnapAttributeExtract>> chunks = Lists.partition(attributes, MAX_SELECT_SIZE);
+ for (List<SnapAttributeExtract> chunk : chunks) {
+ selectRO(chunk, snapID, SnapConst.T_SP_1VAL);
+ }
} catch (SQLException | SnapshotingException e) {
logger.error(ERROR_EXTRACTING_SNAPSHOT_DATA, e);
}
try {
// --- Get spectrum read write values
- selectRW(attributes, snapID, SnapConst.T_SP_2VAL);
+ final List<List<SnapAttributeExtract>> chunks = Lists.partition(attributes, MAX_SELECT_SIZE);
+ for (List<SnapAttributeExtract> chunk : chunks) {
+ selectRW(chunk, snapID, SnapConst.T_SP_2VAL);
+ }
} catch (SQLException | SnapshotingException e) {
logger.error(ERROR_EXTRACTING_SNAPSHOT_DATA, e);
}
}
-
- public void closeResultSet(final ResultSet resultSet) {
- if (resultSet == null) {
- return;
- }
-
- try {
- resultSet.close();
- } catch (final SQLException e) {
- e.printStackTrace();
-
- logger.error("ERROR !! " + "\r\n" + "\t Origin : \t " + "DataBaseAPI.closeResultSet" + "\r\n"
- + "\t Reason : \t " + getDbSchema().toUpperCase().trim() + "_FAILURE" + "\r\n"
- + "\t Description : \t " + e.getMessage());
- }
- }
-
- public void closeStatement(final Statement preparedStatement) {
- if (preparedStatement == null) {
- return;
- }
-
- try {
- preparedStatement.close();
- } catch (final SQLException e) {
- e.printStackTrace();
-
- logger.error("ERROR !! " + "\r\n" + "\t Origin : \t " + "DataBaseAPI.closeStatement" + "\r\n"
- + "\t Reason : \t " + getDbSchema().toUpperCase().trim() + "_FAILURE" + "\r\n"
- + "\t Description : \t " + e.getMessage());
- }
- }
-
- private void closeConnection(final Connection conn) {
- if (conn == null) {
- return;
- }
-
- try {
- conn.close();
- } catch (final SQLException e) {
- e.printStackTrace();
-
- logger.error("ERROR !! " + "\r\n" + "\t Origin : \t " + "DataBaseAPI.closeConnection" + "\r\n"
- + "\t Reason : \t " + getDbSchema().toUpperCase().trim() + "_FAILURE" + "\r\n"
- + "\t Description : \t " + e.getMessage());
- }
- }
-
-
private NullableData<?> getSpectrumValue(final String readStr, final String writeStr, final int dataType,
final boolean returnAsReadWrite) {
int readSize = 0, writeSize = 0;
@@ -1367,7 +1382,6 @@ public class DataBaseAPI {
* @param id_context The given context's identifier
* @return The list of attributes associated to the given context
* @throws SnapshotingException
- * @see SnapAttributeLight
*/
public List<SnapAttributeExtract> getContextAssociatedAttributes(final int id_context) throws SnapshotingException {
final List<SnapAttributeExtract> attibutesList = new ArrayList<SnapAttributeExtract>();
--
GitLab