Examples with MonitorDataBlock alma.TMCDB.MonitorDataBlock used on opensource projects

Example 11 with MonitorDataBlock

use of alma.TMCDB.MonitorDataBlock in project ACS by ACS-Community.

the class CorbaAnyExtractionTest method testExtractData_booleanSeqBlobDataSeq_singlevalued.

/**
	 * Test of extractData method for 'booleanSeqBlobDataSeq' data for a sequence property 
	 * that expands into multiple single-valued MPs.
	 */
@Test
public void testExtractData_booleanSeqBlobDataSeq_singlevalued() throws Exception {
    String propertyName = "SYSTEM_STATUS";
    Any any = create_any();
    boolean[] booleanData_time1 = { false, true, false };
    boolean[] booleanData_time2 = { true, true, true };
    boolean[][] booleanDataMatrix = { booleanData_time1, booleanData_time2 };
    booleanSeqBlobData[] booleanSeqBlobDataArray = createBooleanSeqBlobData(booleanDataMatrix);
    booleanSeqBlobDataSeqHelper.insert(any, booleanSeqBlobDataArray);
    monitorPointExpert.setMultivalued(propertyName, false);
    // Test the AnyExtractor stand-alone
    List<MonitorPointTimeSeries> extractedData = anyExtractor.extractData(any, propertyName);
    assertThat("Demultiplexing into several MonitorPointTimeSeries instances expected", extractedData, hasSize(booleanData_time1.length));
    for (int index = 0; index < extractedData.size(); index++) {
        // check one of the expanded logical properties at a time
        MonitorPointTimeSeries mpTs = extractedData.get(index);
        assertThat(mpTs.getCorbaTypeId(), equalTo("IDL:alma/TMCDB/booleanBlobDataSeq:1.0"));
        assertThat(mpTs.getMonitorPointIndex(), equalTo(index));
        List<MonitorPointValue> dataList = mpTs.getDataList();
        assertThat(dataList, hasSize(booleanDataMatrix.length));
        for (int i = 0; i < booleanDataMatrix.length; i++) {
            MonitorPointValue mpVal = dataList.get(i);
            assertThat(mpVal.getTime(), equalTo(BASE_TIME + i));
            // This should be the transpose of matrix booleanDataMatrix
            assertThat(mpVal.getData(), contains((Object) new Boolean(booleanDataMatrix[i][index])));
        }
    }
    logger.info("Validated booleanSeqBlobDataSeq interpreted as coming from a multiple single-valued MPs.");
    // As a variation we test also "BlobberWorker.createBlobData" which in real life surrounds the AnyExtractor call.
    // It includes generation of CLOB data.
    String componentName = "CONTROL/DV01/PSA";
    String serialNumber = "3456328928847";
    MonitorBlob blob = new MonitorBlob(false, (short) 0, new String[] {}, "wrong:" + propertyName, any);
    MonitorBlob[] blobs = new MonitorBlob[] { blob };
    long startTime = BASE_TIME + 100;
    long stopTime = BASE_TIME + 101;
    MonitorDataBlock block = new MonitorDataBlock(startTime, stopTime, componentName, serialNumber, blobs);
    String[] clobsExpected = new String[] { BASE_TIME + "|0|" + (BASE_TIME + 1) + "|1\n", BASE_TIME + "|1|" + (BASE_TIME + 1) + "|1\n", BASE_TIME + "|0|" + (BASE_TIME + 1) + "|1\n" };
    String[] statisticsExpected = { null, null, null };
    for (int i = 0; i < extractedData.size(); i++) {
        BlobData blobData = BlobberWorker.createBlobData(block, blob, extractedData.get(i), propertyName, serialNumber, logger);
        checkComponentData(blobData, clobsExpected[i], 2, componentName, propertyName, serialNumber, startTime, stopTime, i, statisticsExpected[i]);
    }
}

Example 12 with MonitorDataBlock

use of alma.TMCDB.MonitorDataBlock in project ACS by ACS-Community.

the class CorbaAnyExtractionTest method testExtractData_doubleSeqBlobDataSeq_singlevalued.

/**
	 * Test of extractData method for 'doubleSeqBlobDataSeq' data for a sequence property 
	 * that expands into multiple single-valued MPs.
	 */
@Test
public void testExtractData_doubleSeqBlobDataSeq_singlevalued() throws Exception {
    String propertyName = "SYSTEM_STATUS";
    Any any = create_any();
    double[] doubleData_time1 = { 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0 };
    double[] doubleData_time2 = { 11.1, 12.2, 13.3, 14.4, 15.5, 16.6, 17.7, 18.8, 19.9, 20.0 };
    double[] doubleData_time3 = { 21.0, 22.0, 23.0, 24.0, 25.0, 26.0, 27.0, 28.0, 29.0, 30.0 };
    double[][] doubleDataMatrix = { doubleData_time1, doubleData_time2, doubleData_time3 };
    doubleSeqBlobData[] doubleSeqBlobDataArray = createDoubleSeqBlobData(doubleDataMatrix);
    doubleSeqBlobDataSeqHelper.insert(any, doubleSeqBlobDataArray);
    monitorPointExpert.setMultivalued(propertyName, false);
    // Test the AnyExtractor stand-alone
    List<MonitorPointTimeSeries> extractedData = anyExtractor.extractData(any, propertyName);
    assertThat("Demultiplexing into several MonitorPointTimeSeries instances expected", extractedData, hasSize(doubleData_time1.length));
    for (int index = 0; index < extractedData.size(); index++) {
        // check one of the expanded logical properties at a time
        MonitorPointTimeSeries mpTs = extractedData.get(index);
        assertThat(mpTs.getCorbaTypeId(), equalTo("IDL:alma/TMCDB/doubleBlobDataSeq:1.0"));
        assertThat(mpTs.getMonitorPointIndex(), equalTo(index));
        List<MonitorPointValue> dataList = mpTs.getDataList();
        assertThat(dataList, hasSize(doubleDataMatrix.length));
        for (int i = 0; i < doubleDataMatrix.length; i++) {
            MonitorPointValue mpVal = dataList.get(i);
            assertThat(mpVal.getTime(), equalTo(BASE_TIME + i));
            // This should be the transpose of matrix doubleDataMatrix
            assertThat(mpVal.getData(), contains((Object) new Double(doubleDataMatrix[i][index])));
        }
    }
    logger.info("Validated doubleSeqBlobDataSeq interpreted as coming from a multiple single-valued MPs.");
    // As a variation we test also "BlobberWorker.createBlobData" which in real life surrounds the AnyExtractor call.
    // It includes generation of statistics and CLOB data.
    String componentName = "CONTROL/DV01/PSA";
    String serialNumber = "3456328928847";
    MonitorBlob blob = new MonitorBlob(false, (short) 0, new String[] {}, "wrong:" + propertyName, any);
    MonitorBlob[] blobs = new MonitorBlob[] { blob };
    long startTime = BASE_TIME + 100;
    long stopTime = BASE_TIME + 101;
    MonitorDataBlock block = new MonitorDataBlock(startTime, stopTime, componentName, serialNumber, blobs);
    String[] clobsExpected = new String[] { BASE_TIME + "|1.0|" + (BASE_TIME + 1) + "|11.1|" + (BASE_TIME + 2) + "|21.0\n", BASE_TIME + "|2.0|" + (BASE_TIME + 1) + "|12.2|" + (BASE_TIME + 2) + "|22.0\n", BASE_TIME + "|3.0|" + (BASE_TIME + 1) + "|13.3|" + (BASE_TIME + 2) + "|23.0\n", BASE_TIME + "|4.0|" + (BASE_TIME + 1) + "|14.4|" + (BASE_TIME + 2) + "|24.0\n", BASE_TIME + "|5.0|" + (BASE_TIME + 1) + "|15.5|" + (BASE_TIME + 2) + "|25.0\n", BASE_TIME + "|6.0|" + (BASE_TIME + 1) + "|16.6|" + (BASE_TIME + 2) + "|26.0\n", BASE_TIME + "|7.0|" + (BASE_TIME + 1) + "|17.7|" + (BASE_TIME + 2) + "|27.0\n", BASE_TIME + "|8.0|" + (BASE_TIME + 1) + "|18.8|" + (BASE_TIME + 2) + "|28.0\n", BASE_TIME + "|9.0|" + (BASE_TIME + 1) + "|19.9|" + (BASE_TIME + 2) + "|29.0\n", BASE_TIME + "|10.0|" + (BASE_TIME + 1) + "|20.0|" + (BASE_TIME + 2) + "|30.0\n" };
    String[] statisticsExpected = { "min: 1.0 max: 21.0 mean: 11.033333333333333 stdDev: 10.000166665277801\n", "min: 2.0 max: 22.0 mean: 12.066666666666666 stdDev: 10.000666644445925\n", "min: 3.0 max: 23.0 mean: 13.100000000000001 stdDev: 10.001499887516873\n", "min: 4.0 max: 24.0 mean: 14.133333333333333 stdDev: 10.002666311205894\n", "min: 5.0 max: 25.0 mean: 15.166666666666668 stdDev: 10.004165798972613\n", "min: 6.0 max: 26.0 mean: 16.2 stdDev: 10.00599820107919\n", "min: 7.0 max: 27.0 mean: 17.233333333333334 stdDev: 10.008163334665024\n", "min: 8.0 max: 28.0 mean: 18.266666666666666 stdDev: 10.010660983837848\n", "min: 9.0 max: 29.0 mean: 19.3 stdDev: 10.013490899781155\n", "min: 10.0 max: 30.0 mean: 20.0 stdDev: 10.0\n" };
    for (int i = 0; i < extractedData.size(); i++) {
        BlobData blobData = BlobberWorker.createBlobData(block, blob, extractedData.get(i), propertyName, serialNumber, logger);
        checkComponentData(blobData, clobsExpected[i], 3, componentName, propertyName, serialNumber, startTime, stopTime, i, statisticsExpected[i]);
    }
}

Example 13 with MonitorDataBlock

use of alma.TMCDB.MonitorDataBlock in project ACS by ACS-Community.

the class CorbaAnyExtractionTest method testExtractData_floatSeqBlobDataSeq_multivalued.

/**
	 * Test of extractData method for 'floatSeqBlobDataSeq' data for a multi-valued MP.
	 */
@Test
public void testExtractData_floatSeqBlobDataSeq_multivalued() throws Exception {
    String propertyName = "MODULE_MODE_STATUS";
    Any any = create_any();
    float[] floatData_time1 = { 1.111111f, 2f, 3f, 4f, 5f, 6f, 7f, 8f, 9f, 10f };
    float[] floatData_time2 = { 11, 12, 13, 14, 15, 16, 17, 18, 19, 20 };
    float[] floatData_time3 = { 21, 22, 23, 24, 25, 26, 27, 28, 29, 30 };
    float[][] floatDataMatrix = { floatData_time1, floatData_time2, floatData_time3 };
    floatSeqBlobData[] floatSeqBlobDataArray = createFloatSeqBlobData(floatDataMatrix);
    floatSeqBlobDataSeqHelper.insert(any, floatSeqBlobDataArray);
    monitorPointExpert.setMultivalued(propertyName, true);
    // Test the AnyExtractor stand-alone
    List<MonitorPointTimeSeries> extractedData = anyExtractor.extractData(any, propertyName);
    assertThat("No demultiplexing into several MonitorPointTimeSeries instances expected", extractedData, hasSize(1));
    // Check the raw data
    MonitorPointTimeSeries mpTs = extractedData.get(0);
    assertThat(mpTs.getCorbaTypeId(), equalTo("IDL:alma/TMCDB/floatSeqBlobDataSeq:1.0"));
    assertThat(mpTs.getMonitorPointIndex(), equalTo(0));
    List<MonitorPointValue> dataList = mpTs.getDataList();
    assertThat(dataList, hasSize(floatDataMatrix.length));
    for (int i = 0; i < floatDataMatrix.length; i++) {
        MonitorPointValue mpVal = dataList.get(i);
        assertThat(mpVal.getTime(), equalTo(BASE_TIME + i));
        List<Object> data = mpVal.getData();
        assertThat(data, hasSize(floatData_time1.length));
        for (int j = 0; j < floatData_time1.length; j++) {
            Object dataPart = data.get(j);
            assertThat(dataPart, instanceOf(Float.class));
            float dataPartExpected = floatDataMatrix[i][j];
            assertThat(((Float) dataPart).floatValue(), equalTo(dataPartExpected));
        }
    }
    logger.info("Validated floatSeqBlobDataSeq interpreted as coming from a multivalued MP.");
    // As a variation we test also "BlobberWorker.createBlobData" which in real life surrounds the AnyExtractor call 
    String componentName = "CONTROL/DV01/PSA";
    String serialNumber = "3456328928847";
    MonitorBlob blob = new MonitorBlob(false, (short) 0, new String[] {}, "wrong:" + propertyName, any);
    MonitorBlob[] blobs = new MonitorBlob[] { blob };
    long startTime = BASE_TIME + 100;
    long stopTime = BASE_TIME + 101;
    MonitorDataBlock block = new MonitorDataBlock(startTime, stopTime, componentName, serialNumber, blobs);
    BlobData blobData = BlobberWorker.createBlobData(block, blob, extractedData.get(0), propertyName, serialNumber, logger);
    String clobExpected = BASE_TIME + "|1.111111 2.0 3.0 4.0 5.0 6.0 7.0 8.0 9.0 10.0|" + (BASE_TIME + 1) + "|11.0 12.0 13.0 14.0 15.0 16.0 17.0 18.0 19.0 20.0|" + (BASE_TIME + 2) + "|21.0 22.0 23.0 24.0 25.0 26.0 27.0 28.0 29.0 30.0\n";
    checkComponentData(blobData, clobExpected, 3, componentName, propertyName, serialNumber, startTime, stopTime, 0, null);
}

Example 14 with MonitorDataBlock

use of alma.TMCDB.MonitorDataBlock in project ACS by ACS-Community.

the class BlobberWorker method harvestCollector.

/**
	 * Retrieves and processes the data from one monitor collector component, that is, from one container.
	 * <p>
	 * Storage details:
	 * <ul>
	 * <li>Uses {@link MonitorDAO} to control DB transactions and store the data.
	 * <li>Uses a total of one DB transaction (for the data from all properties of all devices, which comes in many
	 * {@link MonitorDataBlock}s).
	 * </ul>
	 * 
	 * @param collectorData
	 *            The collector ID / name, used for logging and stats.
	 * @param collector
	 *            Reference to the collector component
	 * @return Number of attempted or successful DB inserts, i.e., calls to
	 *         {@link alma.archive.tmcdb.DAO.MonitorDAO#store(alma.archive.tmcdb.DAO.ComponentData)}.
	 * @throws Exception
	 */
private int harvestCollector(CollectorData collectorData, MonitorCollectorOperations collector) throws Exception {
    int insertCount = 0;
    myLogger.fine("About to call " + collectorData.getCollectorId() + "#getMonitorData()");
    MonitorDataBlock[] dataBlocks = collector.getMonitorData();
    collectorData.setLastSuccessfulAccessTime(System.currentTimeMillis());
    if (dataBlocks != null) {
        // be the name.
        if (myLogger.isLoggable(Level.FINE)) {
            myLogger.fine("Received " + dataBlocks.length + " MonitorDataBlocks from collector " + collector.name());
        }
        if (isProfilingEnabled) {
            debugDataSender.sendUDPPacket("Received " + dataBlocks.length + " MonitorDataBlocks from collector " + collector.name(), cycleCount.longValue());
            Runtime runtime = Runtime.getRuntime();
            debugDataSender.sendUDPPacket("Used memory: " + Long.toString((runtime.totalMemory() - runtime.freeMemory())), cycleCount.longValue());
            debugDataSender.sendUDPPacket("Free memory: " + Long.toString(runtime.freeMemory()), cycleCount.longValue());
            debugDataSender.sendUDPPacket("Total memory: " + Long.toString(runtime.totalMemory()), cycleCount.longValue());
        }
        for (MonitorDAO monitorDAO : myMonitorDAOList) {
            // @TODO: Should we catch / log the possible Exception, or just let it fly?
            monitorDAO.openTransactionStore(Long.toString(cycleCount.longValue()) + "-" + collectorData.getCollectorId());
        }
        // iterate over devices
        for (MonitorDataBlock block : dataBlocks) {
            myLogger.log(AcsLogLevel.DEBUG, "MonitorDataBlock for device " + block.componentName + " contains " + block.monitorBlobs.length + " MonitorBlobs");
            // iterate over properties
            for (MonitorBlob blob : block.monitorBlobs) {
                BlobData blobData = null;
                try {
                    String propertyName = blob.propertyName;
                    String propertyNameSimple = propertyName.substring(propertyName.indexOf(':') + 1);
                    // TODO: Shouldn't we pass propertyName instead of propertyNameSimple?
                    List<MonitorPointTimeSeries> containerList = anyExtractor.extractData(blob.blobDataSeq, propertyNameSimple);
                    // iterate over expanded logical properties
                    for (int index = 0; index < containerList.size(); index++) {
                        MonitorPointTimeSeries container = containerList.get(index);
                        // The serial number by default comes from the device, 
                        // but it can be overwritten for a monitor point.
                        // In Alma this is done for correlator properties.
                        String serialNumber = block.deviceSerialNumber;
                        if (blob.propertySerialNumber != null) {
                            if (blob.propertySerialNumber.length == 1) {
                                // One SN per baci property, even if it expands to multiple monitor points.
                                serialNumber = blob.propertySerialNumber[0];
                            } else if (blob.propertySerialNumber.length > index) {
                                serialNumber = blob.propertySerialNumber[index];
                            } else if (blob.propertySerialNumber.length > 0) {
                                this.myLogger.warning("Underspecified MonitorBlob#propertySerialNumber for " + blob.propertyName);
                            }
                        }
                        myLogger.log(AcsLogLevel.DEBUG, "Handling data for property " + blob.propertyName);
                        blobData = createBlobData(block, blob, container, propertyNameSimple, serialNumber, myLogger);
                        if (blobData.getDataSize() > 0) {
                            insertCount++;
                            // hand over our blob data to the DAO(s)
                            storeData(blobData);
                        }
                    }
                } catch (Exception e) {
                    if (!this.loggedFailedStore.contains(blob.propertyName)) {
                        this.loggedFailedStore.add(blob.propertyName);
                        // @TODO check if http://jira.alma.cl/browse/COMP-4512 can be closed
                        String msg = "Problem when handling property [" + blob.propertyName + "]. " + "The data cache for this property in collector [" + collectorData.getCollectorId() + "] will be cleared, the data is NOT stored. ";
                        myLogger.log(Level.WARNING, msg, e);
                        // unclear if we want this log. If so, it should be included in the above log,
                        // to avoid spreading the info over many log records that won't be adjacent in jlog.
                        // see alma.archive.tmcdb.DAO.ComponentData.toString() where currently the clob data is excluded.
                        myLogger.log(Level.WARNING, "The data contained in the data cache: " + blobData);
                    } else {
                    // RK: Matthias suggested we log this too
                    // HSO commented it out again, because now instead we log this AcsJStoreFailureEx
                    // directly in alma.archive.tmcdb.DAO.MonitorDAOImpl.store(ComponentData) at level FINER
                    // myLogger.log(Level.WARNING,
                    // "Repeat of problem when handling property ["
                    // + blob.propertyName + "]", e);
                    }
                }
            }
        // end loop over properties of a single device
        }
        // close the transaction
        for (MonitorDAO monitorDAO : myMonitorDAOList) {
            try {
                myLogger.fine("myMonitorDAO.closeTransactionStore() for: " + dataBlocks.length + " MonitorDataBlocks from collector " + collector.name());
                monitorDAO.closeTransactionStore();
            } catch (Exception ex) {
                myLogger.log(Level.WARNING, "Exception caught. Some monitoring data couldn't be archived", ex);
            }
        }
    }
    return insertCount;
}

Example 15 with MonitorDataBlock

use of alma.TMCDB.MonitorDataBlock in project ACS by ACS-Community.

the class FileReaderCollectorImpl method getMonitorData.

@Override
public MonitorDataBlock[] getMonitorData() {
    // For each component, we read all its related files
    List<MonitorDataBlock> blocks = new ArrayList<MonitorDataBlock>();
    for (String componentName : m_files.keySet()) {
        MonitorDataBlock block = new MonitorDataBlock();
        List<MonitorBlob> blobList = new ArrayList<MonitorBlob>();
        // For each file (property), we read some values and wrap them
        for (FileBasedMonitorPoint mp : m_files.get(componentName)) {
            try {
                String componentDirPath = PREFIX + componentName.replaceAll("/", "_");
                mp.reader = new BufferedReader(new FileReader(new File(componentDirPath + File.separator + mp.propertyname + ".txt")));
                if (!mp.reader.ready())
                    break;
                mp.reader.skip(mp.readBytes);
            } catch (IOException e1) {
                break;
            }
            String line = null;
            MonitorBlob blob = new MonitorBlob();
            blob.propertyName = mp.propertyname;
            blob.blobDataSeq = m_containerServices.getAdvancedContainerServices().getORB().create_any();
            blob.blobDataSeq.type(doubleSeqBlobDataHelper.type());
            blob.propertySerialNumber = new String[0];
            blob.typeOfValue = doubleValueType.value;
            List<doubleBlobData> dataList = new ArrayList<doubleBlobData>();
            do {
                try {
                    mp.reader.mark(80);
                    line = mp.reader.readLine();
                    if (line == null)
                        break;
                    // +1 = '\n'
                    mp.readBytes += line.length() + 1;
                    doubleBlobData data = parseLine(line);
                    // We suppose we're being called every 1 minute here
                    m_logger.fine("Read: " + line + " for " + componentName + "/" + mp.propertyname);
                    if ((data.time - mp.lastTimestamp) >= 60 * 1000) {
                        m_logger.fine("Changing last timestamp from: " + mp.lastTimestamp + " for " + componentName + "/" + mp.propertyname);
                        mp.lastTimestamp = mp.lastTimestamp + 60 * 1000;
                        mp.reader.reset();
                        mp.readBytes -= (line.length() + 1);
                        break;
                    }
                    dataList.add(data);
                } catch (IOException e) {
                    e.printStackTrace();
                } catch (ParseException e) {
                    e.printStackTrace();
                }
            } while (line != null);
            try {
                mp.reader.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
            doubleBlobDataSeqHelper.insert(blob.blobDataSeq, dataList.toArray(new doubleBlobData[0]));
            blobList.add(blob);
        }
        block.componentName = componentName;
        block.deviceSerialNumber = "123456";
        block.startTime = new Date().getTime();
        block.stopTime = new Date().getTime();
        block.monitorBlobs = blobList.toArray(new MonitorBlob[blobList.size()]);
        blocks.add(block);
    }
    m_logger.fine("Returning " + blocks.size() + " MonitorDataBlocks");
    return blocks.toArray(new MonitorDataBlock[blocks.size()]);
}