diff --git a/hdbtdbmonitor/src/main/java/org/tango/archiving/monitoring/database/MonitoringAgent.java b/hdbtdbmonitor/src/main/java/org/tango/archiving/monitoring/database/MonitoringAgent.java index f8619531a413149a8d3a88bd78b2c72de7756724..e3b4c6f212a4a6b5211f0dbef37e67b5061de27d 100644 --- a/hdbtdbmonitor/src/main/java/org/tango/archiving/monitoring/database/MonitoringAgent.java +++ b/hdbtdbmonitor/src/main/java/org/tango/archiving/monitoring/database/MonitoringAgent.java @@ -9,6 +9,7 @@ import org.tango.archiving.monitoring.database.domain.AttributeInsertionReport; import org.tango.archiving.monitoring.database.domain.AttributeInsertionStatus; import org.tango.archiving.monitoring.database.infra.IDatabaseAccess; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -22,19 +23,37 @@ public class MonitoringAgent { this.databaseAccess = databaseAccess; } - public Map<InsertionModes, InsertionStatus> getAttributes(final List<String> exclusions) { + public Map<InsertionModes, InsertionStatus> getAttributes(final List<String> exclusions, List<String> selections) { // Get started in DB Map<InsertionModes, InsertionStatus> runningAttributes = databaseAccess.getAllRunningAttributes(); + Map<InsertionModes, InsertionStatus> runningAttributesResult = new HashMap<>(runningAttributes); + if (!selections.isEmpty()) { + // Remove no selected attribute + getRegexList(selections).parallelStream().forEach(selection -> + runningAttributes.keySet().stream().filter( + runningAttribute -> !runningAttribute.getAttributeName().matches(selection)).forEach(runningAttributesResult::remove) + ); + } else if (!exclusions.isEmpty()) { + // Remove excluded list + getRegexList(exclusions).parallelStream().forEach(exclusion -> + runningAttributes.keySet().stream().filter( + runningAttribute -> runningAttribute.getAttributeName().matches(exclusion)).forEach(runningAttributesResult::remove) + ); + } + return runningAttributesResult; + } - // Remove excluded list - runningAttributes.keySet().forEach(runningAttribute -> { - exclusions.forEach(exclusion -> { - if (runningAttribute.getAttributeName().equalsIgnoreCase(exclusion)) { - runningAttributes.remove(runningAttribute); - } - }); - }); - return runningAttributes; + private List<String> getRegexList(List<String> list) { + List<String> regexList = new ArrayList<>(); + // format exclusion String to regex + list.forEach(exclusion -> + regexList.add( + exclusion.replace("*", ".*").replace("..*", ".*"). + replace("?", ".?").replace("..?", ".?") + ) + ); + LOGGER.info("regex list is {}", regexList); + return regexList; } public Map<String, InsertionModes> getInsertionsModes(final List<String> attributesNames) { @@ -55,24 +74,25 @@ public class MonitoringAgent { AttributeInsertionStatus attributeStatus; final AttributeValue<?> value = databaseAccess.getLast(attributeName); report.setLastValue(value); - LOGGER.info("{} last value is {}", attributeName, value); + LOGGER.debug("{} last value is {}", attributeName, value); if (value.getDataTime() != null) { int archivingPeriod = attribute.getKey().getPeriodPeriodic(); long delta = System.currentTimeMillis() - value.getDataTime().getTime(); if (delta > archivingPeriod + safetyPeriod) { // last insertion data is too old attributeStatus = AttributeInsertionStatus.CONTROL_KO; - LOGGER.info("{} KO, values has not been inserted since {}", attributeName, value.getDataTime()); + LOGGER.debug("{} KO, values has not been inserted since {}", attributeName, value.getDataTime()); } else if (value.getValueR() == null && value.getValueW() == null) { // Null value attributeStatus = AttributeInsertionStatus.CONTROL_NULL; - LOGGER.info("{} values are NULL", attributeName); + LOGGER.debug("{} values are NULL", attributeName); } else { attributeStatus = AttributeInsertionStatus.CONTROL_OK; - LOGGER.info("{} OK, values has not been inserted since {}", attributeName, value.getDataTime()); + LOGGER.debug("{} OK, values has not been inserted since {}", attributeName, value.getDataTime()); } } else { attributeStatus = AttributeInsertionStatus.CONTROL_KO; + LOGGER.debug("{} KO, no data in DB", attributeName); } report.setInsertionStatus(attributeStatus); diff --git a/hdbtdbmonitor/src/main/java/org/tango/archiving/monitoring/database/MonitoringProcess.java b/hdbtdbmonitor/src/main/java/org/tango/archiving/monitoring/database/MonitoringProcess.java index f0e733c9bdaf83ff244e5b8dc427f546b9e6f400..6a607061a659372bd1a889fa23001ca0a128c23f 100644 --- a/hdbtdbmonitor/src/main/java/org/tango/archiving/monitoring/database/MonitoringProcess.java +++ b/hdbtdbmonitor/src/main/java/org/tango/archiving/monitoring/database/MonitoringProcess.java @@ -8,11 +8,9 @@ import org.slf4j.LoggerFactory; import org.tango.archiving.monitoring.database.domain.AttributeInsertionReport; import org.tango.archiving.monitoring.database.domain.AttributeInsertionStatus; -import java.util.HashMap; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentLinkedQueue; public class MonitoringProcess implements Runnable { @@ -22,26 +20,27 @@ public class MonitoringProcess implements Runnable { private final MonitoringAgent monitoringAgent; private final List<String> excludedDevices; + private final List<String> selectedDevices; + private final int safetyPeriod; - private final Map<String, AttributeInsertionReport> attributeReportMap = new ConcurrentHashMap<>(); - private final Map<String, AttributeInsertionReport> attributeErrorReportMap = new ConcurrentHashMap<>(); - private ConcurrentLinkedQueue<String[]> attributeReportList = new ConcurrentLinkedQueue<>(); - private ConcurrentLinkedQueue<String[]> attributeErrorReportList = new ConcurrentLinkedQueue<>(); + private final int pauseBetweenAttributes; + private final ConcurrentLinkedQueue<String[]> attributeReportList = new ConcurrentLinkedQueue<>(); + private final ConcurrentLinkedQueue<String[]> attributeErrorReportList = new ConcurrentLinkedQueue<>(); private Map<InsertionModes, InsertionStatus> attributes; - public MonitoringProcess(final MonitoringAgent monitoringAgent, final List<String> excludedDevices, final int safetyPeriod) { + + public MonitoringProcess(final MonitoringAgent monitoringAgent, final List<String> excludedDevices, final List<String> selectedDevices, + final int safetyPeriod, final int pauseBetweenAttributes) { this.monitoringAgent = monitoringAgent; this.excludedDevices = excludedDevices; + this.selectedDevices = selectedDevices; this.safetyPeriod = safetyPeriod; + this.pauseBetweenAttributes = pauseBetweenAttributes; attributeReportList.add(new String[]{"attribute", "control", "last timestamp", "period", "archiver"}); attributeErrorReportList.add(new String[]{"attribute", "last timestamp", "period", "archiver", "last error"}); } - public Map<String, AttributeInsertionReport> getAttributeReportMap() { - return new HashMap<>(attributeReportMap); - } - public String[][] getErrorReport() { final LinkedList<String[]> tmp = new LinkedList<>(attributeErrorReportList); String[][] result = new String[tmp.size()][5]; @@ -60,14 +59,10 @@ public class MonitoringProcess implements Runnable { return result; } - public Map<String, AttributeInsertionReport> getAttributeErrorReportMap() { - return new HashMap<>(attributeErrorReportMap); - } - public void init() { LOGGER.info("Initializing monitoring"); - attributes = monitoringAgent.getAttributes(excludedDevices); - LOGGER.info("Will monitor for {} attributes", attributes.size()); + attributes = monitoringAgent.getAttributes(excludedDevices, selectedDevices); + LOGGER.info("Will monitor {} attributes", attributes.size()); } @Override @@ -75,24 +70,24 @@ public class MonitoringProcess implements Runnable { try { long tick = System.currentTimeMillis(); // 1 - Check attributes - int i=0; + int i = 0; for (Map.Entry<InsertionModes, InsertionStatus> attribute : attributes.entrySet()) { - LOGGER.info("Checking for {} / nr {}", attribute.getKey().getAttributeName(), i++); + LOGGER.info("Checking for {} nr {}", attribute.getKey().getAttributeName(), i++); try { final AttributeInsertionReport status = monitoringAgent.checkAttribute(attribute, safetyPeriod); - attributeReportMap.put(attribute.getKey().getAttributeName(), status); addReport(status); if (!status.getInsertionStatus().equals(AttributeInsertionStatus.CONTROL_OK)) { - attributeErrorReportMap.put(attribute.getKey().getAttributeName(), status); addErrorReport(status); } } catch (Throwable e) { LOGGER.error("Monitoring failed for attribute " + attribute, e); } - // Thread.sleep(1000); + if (pauseBetweenAttributes > 0) { + Thread.sleep(pauseBetweenAttributes); + } } long tock = System.currentTimeMillis(); - LOGGER.info("Diagnosis of {} attributes took {} s", attributes.size(), (tock-tick)/1000.0 ); + LOGGER.info("Diagnosis of {} attributes took {} s", attributes.size(), (tock - tick) / 1000.0); // 2- TODO Diagnosis } catch (Throwable e) { LOGGER.error("Monitoring failed ", e); diff --git a/hdbtdbmonitor/src/main/java/org/tango/archiving/server/archiver/ArchiversMonitor.java b/hdbtdbmonitor/src/main/java/org/tango/archiving/server/archiver/ArchiversMonitor.java index 832d66ca8efd449385ec7e409f85aeff091cbb05..b64421ea8f3afbf50a87a3e7f2045d4243c7869c 100644 --- a/hdbtdbmonitor/src/main/java/org/tango/archiving/server/archiver/ArchiversMonitor.java +++ b/hdbtdbmonitor/src/main/java/org/tango/archiving/server/archiver/ArchiversMonitor.java @@ -51,8 +51,7 @@ public class ArchiversMonitor { public static final String DEFAULT_ARCHIVER_CLASS = ConfigConst.HDB_CLASS_DEVICE; protected static String VERSION; protected final Logger logger = LoggerFactory.getLogger(this.getClass()); - ArchiversMonitorListeners listner; - ; + ArchiversMonitorListeners listeners; AttributeGroupReader task; AttributeGroupScheduler readScheduler; private String[] exportedArchivers = {"archiving/hdb-oracle/hdbarchiver.01"}; @@ -60,16 +59,22 @@ public class ArchiversMonitor { private String archiverClass = DEFAULT_ARCHIVER_CLASS; @DeviceProperty(description = "archiver polling period in milliseconds", defaultValue = "10000 milliseconds") private int period = 10 * 1000; - @DeviceProperty + @DeviceProperty(description = "database name, use only is enableDatabaseMonitoring==true") private String dbName; - @DeviceProperty + @DeviceProperty(description = "database host name, use only is enableDatabaseMonitoring==true") private String dbHost; - @DeviceProperty + @DeviceProperty(description = "database user name, use only is enableDatabaseMonitoring==true") private String dbUser; - @DeviceProperty + @DeviceProperty(description = "database password, use only is enableDatabaseMonitoring==true") private String dbPassword; - @DeviceProperty + @DeviceProperty(description = "Enable checks of last inserted values in database") private boolean enableDatabaseMonitoring = false; + @DeviceProperty(description = "Use for database monitoring: " + + "add an error margin in minutes on attribute insertion period to consider it in error") + private int errorMarginPeriod = 15; + @DeviceProperty(description = "Use for database monitoring: " + + "pause in ms between checking each attribute") + private int pauseBetweenAttributes; @DeviceManagement private DeviceManager device; @State @@ -81,12 +86,32 @@ public class ArchiversMonitor { private MonitoringProcess monitoringProcess; private ScheduledExecutorService executor; private ScheduledFuture<?> future; + @DeviceProperty(description = "List of attributes to exclude, can contain wildcar *") + private String[] excludedAttributes = new String[]{}; + @DeviceProperty(description = "List of attributes to select, can contain wildcar *") + private String[] selectedAttributes = new String[]{}; public static void main(final String[] args) { VERSION = ResourceBundle.getBundle("application").getString("project.version"); ServerManager.getInstance().start(args, ArchiversMonitor.class); } + public void setExcludedAttributes(final String[] excludedAttributes) { + this.excludedAttributes = excludedAttributes; + } + + public void setSelectedAttributes(final String[] selectedAttributes) { + this.selectedAttributes = selectedAttributes; + } + + public void setPauseBetweenAttributes(final int pauseBetweenAttributes) { + this.pauseBetweenAttributes = pauseBetweenAttributes; + } + + public void setErrorMarginPeriod(final int errorMarginPeriod) { + this.errorMarginPeriod = errorMarginPeriod; + } + public void setDbName(final String dbName) { this.dbName = dbName; } @@ -126,7 +151,7 @@ public class ArchiversMonitor { } else { tmpStatus = "No running archivers"; } - tmpStatus = tmpStatus + "\nDatabase processed attributes: " + monitoringProcess.getAttributeReportMap().size(); + tmpStatus = tmpStatus + "\nDatabase processed attributes: " + monitoringProcess.getErrorReport().length; return tmpStatus; } @@ -179,8 +204,8 @@ public class ArchiversMonitor { group = new Group(""); group.add(exportedArchivers); // start archivers monitoring - listner = new ArchiversMonitorListeners(exportedArchivers); - task = new AttributeGroupReader(listner, listner.getGroupAttributes(), false, true, false); + listeners = new ArchiversMonitorListeners(exportedArchivers); + task = new AttributeGroupReader(listeners, listeners.getGroupAttributes(), false, true, false); readScheduler = new AttributeGroupScheduler(); startMonitoringArchivers(); @@ -195,12 +220,8 @@ public class ArchiversMonitor { parameters.setPassword(dbPassword); databaseAccess.connectDatabase(parameters); MonitoringAgent agent = new MonitoringAgent(databaseAccess); - // TODO dev property excludedDeviceList - // TODO dev property safety period - // TODO delayBetweenAttributes - final List<String> excludedDevices = new ArrayList<>(); - final int safetyPeriod = 15 * 60000; // 15 minutes - monitoringProcess = new MonitoringProcess(agent, excludedDevices, safetyPeriod); + monitoringProcess = new MonitoringProcess(agent, Arrays.asList(excludedAttributes), Arrays.asList(selectedAttributes), + errorMarginPeriod * 60000, pauseBetweenAttributes); executor = Executors.newScheduledThreadPool(1); startMonitoringDatabase(); } @@ -258,7 +279,7 @@ public class ArchiversMonitor { public int getDataBaseErrors() { if (enableDatabaseMonitoring) { return monitoringProcess.getErrorReport().length; - }else{ + } else { return 0; } } @@ -266,13 +287,13 @@ public class ArchiversMonitor { @Attribute @AttributeProperties(description = "Report for read value of this attributes : ScalarCharge SpectrumCharge ImageCharge insertionRate memory state collectorErrorNr") public String[][] getArchiversReport() { - return listner.getReport(); + return listeners.getReport(); } @Attribute @AttributeProperties(description = "Errors found while reading report value") public String[][] getArchiversErrorReport() { - return listner.getReportError(); + return listeners.getReportError(); } @Attribute @@ -303,7 +324,7 @@ public class ArchiversMonitor { @Attribute @AttributeProperties(description = "Total number of archivers errors") public int getArchiversErrorTotal() { - return listner.getTotalErrors(); + return listeners.getTotalErrors(); } @Attribute @@ -331,43 +352,43 @@ public class ArchiversMonitor { @Attribute @AttributeProperties(description = "Total number of SCALAR attributes") public int getScalarTotalCharge() { - return listner.geTotalChargeScalar(); + return listeners.geTotalChargeScalar(); } @Attribute @AttributeProperties(description = "Total number of SPECTRUM attributes") public int getSpectrumTotalCharge() { - return listner.geTotalChargeSpectrum(); + return listeners.geTotalChargeSpectrum(); } @Attribute @AttributeProperties(description = "Total number of IMAGE attributes") public int getImageTotalCharge() { - return listner.getTotalChargeImage(); + return listeners.getTotalChargeImage(); } @Attribute @AttributeProperties(description = "Total number of SCALAR+SPECTRUM+IMAGE attributes") public int getArchiversTotalCharge() { - return listner.geTotalChargeScalar() + listner.geTotalChargeSpectrum() + listner.getTotalChargeImage(); + return listeners.geTotalChargeScalar() + listeners.geTotalChargeSpectrum() + listeners.getTotalChargeImage(); } @Attribute @AttributeProperties(description = "Total number of KO attributes") public int getArchiversKoTotal() { - return listner.getTotalKo(); + return listeners.getTotalKo(); } @Attribute @AttributeProperties(description = "Total insertion rate") public double getArchiversTotalInsertionRate() { - return listner.getTotalInsertionRate(); + return listeners.getTotalInsertionRate(); } @Attribute @AttributeProperties(description = "Total memory") public double getArchiversTotalMemory() { - return listner.getTotalMemory(); + return listeners.getTotalMemory(); } /**