Skip to content
Snippets Groups Projects
Commit 576c0c85 authored by gwen-soleil's avatar gwen-soleil
Browse files

Add configuration properties for database monitoring

parent 826929bd
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
hdbtdbmonitor/src/main/java/org/tango/archiving/monitoring/database/MonitoringAgent.java
+ 34
− 14
View file @ 576c0c85
...@@ -9,6 +9,7 @@ import org.tango.archiving.monitoring.database.domain.AttributeInsertionReport; ...@@ -9,6 +9,7 @@ import org.tango.archiving.monitoring.database.domain.AttributeInsertionReport;
import org.tango.archiving.monitoring.database.domain.AttributeInsertionStatus; import org.tango.archiving.monitoring.database.domain.AttributeInsertionStatus;
import org.tango.archiving.monitoring.database.infra.IDatabaseAccess; import org.tango.archiving.monitoring.database.infra.IDatabaseAccess;
import java.util.ArrayList;
import java.util.HashMap; import java.util.HashMap;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
...@@ -22,19 +23,37 @@ public class MonitoringAgent { ...@@ -22,19 +23,37 @@ public class MonitoringAgent {
this.databaseAccess = databaseAccess; this.databaseAccess = databaseAccess;
} }
public Map<InsertionModes, InsertionStatus> getAttributes(final List<String> exclusions) { public Map<InsertionModes, InsertionStatus> getAttributes(final List<String> exclusions, List<String> selections) {
// Get started in DB // Get started in DB
Map<InsertionModes, InsertionStatus> runningAttributes = databaseAccess.getAllRunningAttributes(); Map<InsertionModes, InsertionStatus> runningAttributes = databaseAccess.getAllRunningAttributes();
Map<InsertionModes, InsertionStatus> runningAttributesResult = new HashMap<>(runningAttributes);
if (!selections.isEmpty()) {
// Remove no selected attribute
getRegexList(selections).parallelStream().forEach(selection ->
runningAttributes.keySet().stream().filter(
runningAttribute -> !runningAttribute.getAttributeName().matches(selection)).forEach(runningAttributesResult::remove)
);
} else if (!exclusions.isEmpty()) {
// Remove excluded list // Remove excluded list
runningAttributes.keySet().forEach(runningAttribute -> { getRegexList(exclusions).parallelStream().forEach(exclusion ->
exclusions.forEach(exclusion -> { runningAttributes.keySet().stream().filter(
if (runningAttribute.getAttributeName().equalsIgnoreCase(exclusion)) { runningAttribute -> runningAttribute.getAttributeName().matches(exclusion)).forEach(runningAttributesResult::remove)
runningAttributes.remove(runningAttribute); );
} }
}); return runningAttributesResult;
}); }
return runningAttributes;
private List<String> getRegexList(List<String> list) {
List<String> regexList = new ArrayList<>();
// format exclusion String to regex
list.forEach(exclusion ->
regexList.add(
exclusion.replace("*", ".*").replace("..*", ".*").
replace("?", ".?").replace("..?", ".?")
)
);
LOGGER.info("regex list is {}", regexList);
return regexList;
} }
public Map<String, InsertionModes> getInsertionsModes(final List<String> attributesNames) { public Map<String, InsertionModes> getInsertionsModes(final List<String> attributesNames) {
...@@ -55,24 +74,25 @@ public class MonitoringAgent { ...@@ -55,24 +74,25 @@ public class MonitoringAgent {
AttributeInsertionStatus attributeStatus; AttributeInsertionStatus attributeStatus;
final AttributeValue<?> value = databaseAccess.getLast(attributeName); final AttributeValue<?> value = databaseAccess.getLast(attributeName);
report.setLastValue(value); report.setLastValue(value);
LOGGER.info("{} last value is {}", attributeName, value); LOGGER.debug("{} last value is {}", attributeName, value);
if (value.getDataTime() != null) { if (value.getDataTime() != null) {
int archivingPeriod = attribute.getKey().getPeriodPeriodic(); int archivingPeriod = attribute.getKey().getPeriodPeriodic();
long delta = System.currentTimeMillis() - value.getDataTime().getTime(); long delta = System.currentTimeMillis() - value.getDataTime().getTime();
if (delta > archivingPeriod + safetyPeriod) { if (delta > archivingPeriod + safetyPeriod) {
// last insertion data is too old // last insertion data is too old
attributeStatus = AttributeInsertionStatus.CONTROL_KO; attributeStatus = AttributeInsertionStatus.CONTROL_KO;
LOGGER.info("{} KO, values has not been inserted since {}", attributeName, value.getDataTime()); LOGGER.debug("{} KO, values has not been inserted since {}", attributeName, value.getDataTime());
} else if (value.getValueR() == null && value.getValueW() == null) { } else if (value.getValueR() == null && value.getValueW() == null) {
// Null value // Null value
attributeStatus = AttributeInsertionStatus.CONTROL_NULL; attributeStatus = AttributeInsertionStatus.CONTROL_NULL;
LOGGER.info("{} values are NULL", attributeName); LOGGER.debug("{} values are NULL", attributeName);
} else { } else {
attributeStatus = AttributeInsertionStatus.CONTROL_OK; attributeStatus = AttributeInsertionStatus.CONTROL_OK;
LOGGER.info("{} OK, values has not been inserted since {}", attributeName, value.getDataTime()); LOGGER.debug("{} OK, values has not been inserted since {}", attributeName, value.getDataTime());
} }
} else { } else {
attributeStatus = AttributeInsertionStatus.CONTROL_KO; attributeStatus = AttributeInsertionStatus.CONTROL_KO;
LOGGER.debug("{} KO, no data in DB", attributeName);
} }
report.setInsertionStatus(attributeStatus); report.setInsertionStatus(attributeStatus);
......
hdbtdbmonitor/src/main/java/org/tango/archiving/monitoring/database/MonitoringProcess.java
+ 18
− 23
View file @ 576c0c85
...@@ -8,11 +8,9 @@ import org.slf4j.LoggerFactory; ...@@ -8,11 +8,9 @@ import org.slf4j.LoggerFactory;
import org.tango.archiving.monitoring.database.domain.AttributeInsertionReport; import org.tango.archiving.monitoring.database.domain.AttributeInsertionReport;
import org.tango.archiving.monitoring.database.domain.AttributeInsertionStatus; import org.tango.archiving.monitoring.database.domain.AttributeInsertionStatus;
import java.util.HashMap;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Map; import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentLinkedQueue;
public class MonitoringProcess implements Runnable { public class MonitoringProcess implements Runnable {
...@@ -22,26 +20,27 @@ public class MonitoringProcess implements Runnable { ...@@ -22,26 +20,27 @@ public class MonitoringProcess implements Runnable {
private final MonitoringAgent monitoringAgent; private final MonitoringAgent monitoringAgent;
private final List<String> excludedDevices; private final List<String> excludedDevices;
private final List<String> selectedDevices;
private final int safetyPeriod; private final int safetyPeriod;
private final Map<String, AttributeInsertionReport> attributeReportMap = new ConcurrentHashMap<>(); private final int pauseBetweenAttributes;
private final Map<String, AttributeInsertionReport> attributeErrorReportMap = new ConcurrentHashMap<>(); private final ConcurrentLinkedQueue<String[]> attributeReportList = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<String[]> attributeReportList = new ConcurrentLinkedQueue<>(); private final ConcurrentLinkedQueue<String[]> attributeErrorReportList = new ConcurrentLinkedQueue<>();
private ConcurrentLinkedQueue<String[]> attributeErrorReportList = new ConcurrentLinkedQueue<>();
private Map<InsertionModes, InsertionStatus> attributes; private Map<InsertionModes, InsertionStatus> attributes;
public MonitoringProcess(final MonitoringAgent monitoringAgent, final List<String> excludedDevices, final int safetyPeriod) {
public MonitoringProcess(final MonitoringAgent monitoringAgent, final List<String> excludedDevices, final List<String> selectedDevices,
final int safetyPeriod, final int pauseBetweenAttributes) {
this.monitoringAgent = monitoringAgent; this.monitoringAgent = monitoringAgent;
this.excludedDevices = excludedDevices; this.excludedDevices = excludedDevices;
this.selectedDevices = selectedDevices;
this.safetyPeriod = safetyPeriod; this.safetyPeriod = safetyPeriod;
this.pauseBetweenAttributes = pauseBetweenAttributes;
attributeReportList.add(new String[]{"attribute", "control", "last timestamp", "period", "archiver"}); attributeReportList.add(new String[]{"attribute", "control", "last timestamp", "period", "archiver"});
attributeErrorReportList.add(new String[]{"attribute", "last timestamp", "period", "archiver", "last error"}); attributeErrorReportList.add(new String[]{"attribute", "last timestamp", "period", "archiver", "last error"});
} }
public Map<String, AttributeInsertionReport> getAttributeReportMap() {
return new HashMap<>(attributeReportMap);
}
public String[][] getErrorReport() { public String[][] getErrorReport() {
final LinkedList<String[]> tmp = new LinkedList<>(attributeErrorReportList); final LinkedList<String[]> tmp = new LinkedList<>(attributeErrorReportList);
String[][] result = new String[tmp.size()][5]; String[][] result = new String[tmp.size()][5];
...@@ -60,14 +59,10 @@ public class MonitoringProcess implements Runnable { ...@@ -60,14 +59,10 @@ public class MonitoringProcess implements Runnable {
return result; return result;
} }
public Map<String, AttributeInsertionReport> getAttributeErrorReportMap() {
return new HashMap<>(attributeErrorReportMap);
}
public void init() { public void init() {
LOGGER.info("Initializing monitoring"); LOGGER.info("Initializing monitoring");
attributes = monitoringAgent.getAttributes(excludedDevices); attributes = monitoringAgent.getAttributes(excludedDevices, selectedDevices);
LOGGER.info("Will monitor for {} attributes", attributes.size()); LOGGER.info("Will monitor {} attributes", attributes.size());
} }
@Override @Override
...@@ -77,19 +72,19 @@ public class MonitoringProcess implements Runnable { ...@@ -77,19 +72,19 @@ public class MonitoringProcess implements Runnable {
// 1 - Check attributes // 1 - Check attributes
int i = 0; int i = 0;
for (Map.Entry<InsertionModes, InsertionStatus> attribute : attributes.entrySet()) { for (Map.Entry<InsertionModes, InsertionStatus> attribute : attributes.entrySet()) {
LOGGER.info("Checking for {} / nr {}", attribute.getKey().getAttributeName(), i++); LOGGER.info("Checking for {} nr {}", attribute.getKey().getAttributeName(), i++);
try { try {
final AttributeInsertionReport status = monitoringAgent.checkAttribute(attribute, safetyPeriod); final AttributeInsertionReport status = monitoringAgent.checkAttribute(attribute, safetyPeriod);
attributeReportMap.put(attribute.getKey().getAttributeName(), status);
addReport(status); addReport(status);
if (!status.getInsertionStatus().equals(AttributeInsertionStatus.CONTROL_OK)) { if (!status.getInsertionStatus().equals(AttributeInsertionStatus.CONTROL_OK)) {
attributeErrorReportMap.put(attribute.getKey().getAttributeName(), status);
addErrorReport(status); addErrorReport(status);
} }
} catch (Throwable e) { } catch (Throwable e) {
LOGGER.error("Monitoring failed for attribute " + attribute, e); LOGGER.error("Monitoring failed for attribute " + attribute, e);
} }
// Thread.sleep(1000); if (pauseBetweenAttributes > 0) {
Thread.sleep(pauseBetweenAttributes);
}
} }
long tock = System.currentTimeMillis(); long tock = System.currentTimeMillis();
LOGGER.info("Diagnosis of {} attributes took {} s", attributes.size(), (tock - tick) / 1000.0); LOGGER.info("Diagnosis of {} attributes took {} s", attributes.size(), (tock - tick) / 1000.0);
......
hdbtdbmonitor/src/main/java/org/tango/archiving/server/archiver/ArchiversMonitor.java
+ 48
− 27
View file @ 576c0c85
...@@ -51,8 +51,7 @@ public class ArchiversMonitor { ...@@ -51,8 +51,7 @@ public class ArchiversMonitor {
public static final String DEFAULT_ARCHIVER_CLASS = ConfigConst.HDB_CLASS_DEVICE; public static final String DEFAULT_ARCHIVER_CLASS = ConfigConst.HDB_CLASS_DEVICE;
protected static String VERSION; protected static String VERSION;
protected final Logger logger = LoggerFactory.getLogger(this.getClass()); protected final Logger logger = LoggerFactory.getLogger(this.getClass());
ArchiversMonitorListeners listner; ArchiversMonitorListeners listeners;
;
AttributeGroupReader task; AttributeGroupReader task;
AttributeGroupScheduler readScheduler; AttributeGroupScheduler readScheduler;
private String[] exportedArchivers = {"archiving/hdb-oracle/hdbarchiver.01"}; private String[] exportedArchivers = {"archiving/hdb-oracle/hdbarchiver.01"};
...@@ -60,16 +59,22 @@ public class ArchiversMonitor { ...@@ -60,16 +59,22 @@ public class ArchiversMonitor {
private String archiverClass = DEFAULT_ARCHIVER_CLASS; private String archiverClass = DEFAULT_ARCHIVER_CLASS;
@DeviceProperty(description = "archiver polling period in milliseconds", defaultValue = "10000 milliseconds") @DeviceProperty(description = "archiver polling period in milliseconds", defaultValue = "10000 milliseconds")
private int period = 10 * 1000; private int period = 10 * 1000;
@DeviceProperty @DeviceProperty(description = "database name, use only is enableDatabaseMonitoring==true")
private String dbName; private String dbName;
@DeviceProperty @DeviceProperty(description = "database host name, use only is enableDatabaseMonitoring==true")
private String dbHost; private String dbHost;
@DeviceProperty @DeviceProperty(description = "database user name, use only is enableDatabaseMonitoring==true")
private String dbUser; private String dbUser;
@DeviceProperty @DeviceProperty(description = "database password, use only is enableDatabaseMonitoring==true")
private String dbPassword; private String dbPassword;
@DeviceProperty @DeviceProperty(description = "Enable checks of last inserted values in database")
private boolean enableDatabaseMonitoring = false; private boolean enableDatabaseMonitoring = false;
@DeviceProperty(description = "Use for database monitoring: " +
"add an error margin in minutes on attribute insertion period to consider it in error")
private int errorMarginPeriod = 15;
@DeviceProperty(description = "Use for database monitoring: " +
"pause in ms between checking each attribute")
private int pauseBetweenAttributes;
@DeviceManagement @DeviceManagement
private DeviceManager device; private DeviceManager device;
@State @State
...@@ -81,12 +86,32 @@ public class ArchiversMonitor { ...@@ -81,12 +86,32 @@ public class ArchiversMonitor {
private MonitoringProcess monitoringProcess; private MonitoringProcess monitoringProcess;
private ScheduledExecutorService executor; private ScheduledExecutorService executor;
private ScheduledFuture<?> future; private ScheduledFuture<?> future;
@DeviceProperty(description = "List of attributes to exclude, can contain wildcar *")
private String[] excludedAttributes = new String[]{};
@DeviceProperty(description = "List of attributes to select, can contain wildcar *")
private String[] selectedAttributes = new String[]{};
public static void main(final String[] args) { public static void main(final String[] args) {
VERSION = ResourceBundle.getBundle("application").getString("project.version"); VERSION = ResourceBundle.getBundle("application").getString("project.version");
ServerManager.getInstance().start(args, ArchiversMonitor.class); ServerManager.getInstance().start(args, ArchiversMonitor.class);
} }
public void setExcludedAttributes(final String[] excludedAttributes) {
this.excludedAttributes = excludedAttributes;
}
public void setSelectedAttributes(final String[] selectedAttributes) {
this.selectedAttributes = selectedAttributes;
}
public void setPauseBetweenAttributes(final int pauseBetweenAttributes) {
this.pauseBetweenAttributes = pauseBetweenAttributes;
}
public void setErrorMarginPeriod(final int errorMarginPeriod) {
this.errorMarginPeriod = errorMarginPeriod;
}
public void setDbName(final String dbName) { public void setDbName(final String dbName) {
this.dbName = dbName; this.dbName = dbName;
} }
...@@ -126,7 +151,7 @@ public class ArchiversMonitor { ...@@ -126,7 +151,7 @@ public class ArchiversMonitor {
} else { } else {
tmpStatus = "No running archivers"; tmpStatus = "No running archivers";
} }
tmpStatus = tmpStatus + "\nDatabase processed attributes: " + monitoringProcess.getAttributeReportMap().size(); tmpStatus = tmpStatus + "\nDatabase processed attributes: " + monitoringProcess.getErrorReport().length;
return tmpStatus; return tmpStatus;
} }
...@@ -179,8 +204,8 @@ public class ArchiversMonitor { ...@@ -179,8 +204,8 @@ public class ArchiversMonitor {
group = new Group(""); group = new Group("");
group.add(exportedArchivers); group.add(exportedArchivers);
// start archivers monitoring // start archivers monitoring
listner = new ArchiversMonitorListeners(exportedArchivers); listeners = new ArchiversMonitorListeners(exportedArchivers);
task = new AttributeGroupReader(listner, listner.getGroupAttributes(), false, true, false); task = new AttributeGroupReader(listeners, listeners.getGroupAttributes(), false, true, false);
readScheduler = new AttributeGroupScheduler(); readScheduler = new AttributeGroupScheduler();
startMonitoringArchivers(); startMonitoringArchivers();
...@@ -195,12 +220,8 @@ public class ArchiversMonitor { ...@@ -195,12 +220,8 @@ public class ArchiversMonitor {
parameters.setPassword(dbPassword); parameters.setPassword(dbPassword);
databaseAccess.connectDatabase(parameters); databaseAccess.connectDatabase(parameters);
MonitoringAgent agent = new MonitoringAgent(databaseAccess); MonitoringAgent agent = new MonitoringAgent(databaseAccess);
// TODO dev property excludedDeviceList monitoringProcess = new MonitoringProcess(agent, Arrays.asList(excludedAttributes), Arrays.asList(selectedAttributes),
// TODO dev property safety period errorMarginPeriod * 60000, pauseBetweenAttributes);
// TODO delayBetweenAttributes
final List<String> excludedDevices = new ArrayList<>();
final int safetyPeriod = 15 * 60000; // 15 minutes
monitoringProcess = new MonitoringProcess(agent, excludedDevices, safetyPeriod);
executor = Executors.newScheduledThreadPool(1); executor = Executors.newScheduledThreadPool(1);
startMonitoringDatabase(); startMonitoringDatabase();
} }
...@@ -266,13 +287,13 @@ public class ArchiversMonitor { ...@@ -266,13 +287,13 @@ public class ArchiversMonitor {
@Attribute @Attribute
@AttributeProperties(description = "Report for read value of this attributes : ScalarCharge SpectrumCharge ImageCharge insertionRate memory state collectorErrorNr") @AttributeProperties(description = "Report for read value of this attributes : ScalarCharge SpectrumCharge ImageCharge insertionRate memory state collectorErrorNr")
public String[][] getArchiversReport() { public String[][] getArchiversReport() {
return listner.getReport(); return listeners.getReport();
} }
@Attribute @Attribute
@AttributeProperties(description = "Errors found while reading report value") @AttributeProperties(description = "Errors found while reading report value")
public String[][] getArchiversErrorReport() { public String[][] getArchiversErrorReport() {
return listner.getReportError(); return listeners.getReportError();
} }
@Attribute @Attribute
...@@ -303,7 +324,7 @@ public class ArchiversMonitor { ...@@ -303,7 +324,7 @@ public class ArchiversMonitor {
@Attribute @Attribute
@AttributeProperties(description = "Total number of archivers errors") @AttributeProperties(description = "Total number of archivers errors")
public int getArchiversErrorTotal() { public int getArchiversErrorTotal() {
return listner.getTotalErrors(); return listeners.getTotalErrors();
} }
@Attribute @Attribute
...@@ -331,43 +352,43 @@ public class ArchiversMonitor { ...@@ -331,43 +352,43 @@ public class ArchiversMonitor {
@Attribute @Attribute
@AttributeProperties(description = "Total number of SCALAR attributes") @AttributeProperties(description = "Total number of SCALAR attributes")
public int getScalarTotalCharge() { public int getScalarTotalCharge() {
return listner.geTotalChargeScalar(); return listeners.geTotalChargeScalar();
} }
@Attribute @Attribute
@AttributeProperties(description = "Total number of SPECTRUM attributes") @AttributeProperties(description = "Total number of SPECTRUM attributes")
public int getSpectrumTotalCharge() { public int getSpectrumTotalCharge() {
return listner.geTotalChargeSpectrum(); return listeners.geTotalChargeSpectrum();
} }
@Attribute @Attribute
@AttributeProperties(description = "Total number of IMAGE attributes") @AttributeProperties(description = "Total number of IMAGE attributes")
public int getImageTotalCharge() { public int getImageTotalCharge() {
return listner.getTotalChargeImage(); return listeners.getTotalChargeImage();
} }
@Attribute @Attribute
@AttributeProperties(description = "Total number of SCALAR+SPECTRUM+IMAGE attributes") @AttributeProperties(description = "Total number of SCALAR+SPECTRUM+IMAGE attributes")
public int getArchiversTotalCharge() { public int getArchiversTotalCharge() {
return listner.geTotalChargeScalar() + listner.geTotalChargeSpectrum() + listner.getTotalChargeImage(); return listeners.geTotalChargeScalar() + listeners.geTotalChargeSpectrum() + listeners.getTotalChargeImage();
} }
@Attribute @Attribute
@AttributeProperties(description = "Total number of KO attributes") @AttributeProperties(description = "Total number of KO attributes")
public int getArchiversKoTotal() { public int getArchiversKoTotal() {
return listner.getTotalKo(); return listeners.getTotalKo();
} }
@Attribute @Attribute
@AttributeProperties(description = "Total insertion rate") @AttributeProperties(description = "Total insertion rate")
public double getArchiversTotalInsertionRate() { public double getArchiversTotalInsertionRate() {
return listner.getTotalInsertionRate(); return listeners.getTotalInsertionRate();
} }
@Attribute @Attribute
@AttributeProperties(description = "Total memory") @AttributeProperties(description = "Total memory")
public double getArchiversTotalMemory() { public double getArchiversTotalMemory() {
return listner.getTotalMemory(); return listeners.getTotalMemory();
} }
/** /**
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment

The place to be for your source code