Examples with Measurement org.opennms.newts.api.Measurement used on opensource projects

Example 1 with Measurement

use of org.opennms.newts.api.Measurement in project opennms by OpenNMS.

the class NewtsFetchStrategy method getMeasurementsForResourceCallable.

private Callable<Collection<Row<Measurement>>> getMeasurementsForResourceCallable(final String newtsResourceId, final List<Source> listOfSources, final Optional<Timestamp> start, final Optional<Timestamp> end, final LateAggregationParams lag) {
    return new Callable<Collection<Row<Measurement>>>() {

        @Override
        public Collection<Row<Measurement>> call() throws Exception {
            ResultDescriptor resultDescriptor = new ResultDescriptor(lag.getInterval());
            for (Source source : listOfSources) {
                final String metricName = source.getAttribute();
                final String name = source.getLabel();
                final AggregationFunction fn = toAggregationFunction(source.getAggregation());
                resultDescriptor.datasource(name, metricName, lag.getHeartbeat(), fn);
                resultDescriptor.export(name);
            }
            LOG.debug("Querying Newts for resource id {} with result descriptor: {}", newtsResourceId, resultDescriptor);
            Results<Measurement> results = m_sampleRepository.select(m_context, new Resource(newtsResourceId), start, end, resultDescriptor, Optional.of(Duration.millis(lag.getStep())), limitConcurrentAggregationsCallback);
            Collection<Row<Measurement>> rows = results.getRows();
            LOG.debug("Found {} rows.", rows.size());
            return rows;
        }
    };
}

Example 2 with Measurement

use of org.opennms.newts.api.Measurement in project opennms by OpenNMS.

the class NewtsFetchStrategy method fetch.

@Override
public FetchResults fetch(long start, long end, long step, int maxrows, Long interval, Long heartbeat, List<Source> sources, boolean relaxed) {
    final LateAggregationParams lag = getLagParams(step, interval, heartbeat);
    final Optional<Timestamp> startTs = Optional.of(Timestamp.fromEpochMillis(start));
    final Optional<Timestamp> endTs = Optional.of(Timestamp.fromEpochMillis(end));
    final Map<String, Object> constants = Maps.newHashMap();
    // Group the sources by resource id to avoid calling the ResourceDao
    // multiple times for the same resource
    Map<ResourceId, List<Source>> sourcesByResourceId = sources.stream().collect(Collectors.groupingBy((source) -> ResourceId.fromString(source.getResourceId())));
    // Lookup the OnmsResources in parallel
    Map<ResourceId, Future<OnmsResource>> resourceFuturesById = Maps.newHashMapWithExpectedSize(sourcesByResourceId.size());
    for (ResourceId resourceId : sourcesByResourceId.keySet()) {
        resourceFuturesById.put(resourceId, threadPool.submit(getResourceByIdCallable(resourceId)));
    }
    // Gather the results, fail if any of the resources were not found
    Map<OnmsResource, List<Source>> sourcesByResource = Maps.newHashMapWithExpectedSize(sourcesByResourceId.size());
    for (Entry<ResourceId, Future<OnmsResource>> entry : resourceFuturesById.entrySet()) {
        try {
            OnmsResource resource = entry.getValue().get();
            if (resource == null) {
                if (relaxed)
                    continue;
                LOG.error("No resource with id: {}", entry.getKey());
                return null;
            }
            sourcesByResource.put(resource, sourcesByResourceId.get(entry.getKey()));
        } catch (ExecutionException | InterruptedException e) {
            throw Throwables.propagate(e);
        }
    }
    // Now group the sources by Newts Resource ID, which differs from the OpenNMS Resource ID.
    Map<String, List<Source>> sourcesByNewtsResourceId = Maps.newHashMap();
    for (Entry<OnmsResource, List<Source>> entry : sourcesByResource.entrySet()) {
        final OnmsResource resource = entry.getKey();
        for (Source source : entry.getValue()) {
            // Gather the values from strings.properties
            Utils.convertStringAttributesToConstants(source.getLabel(), resource.getStringPropertyAttributes(), constants);
            // Grab the attribute that matches the source
            RrdGraphAttribute rrdGraphAttribute = resource.getRrdGraphAttributes().get(source.getAttribute());
            if (rrdGraphAttribute == null && !Strings.isNullOrEmpty(source.getFallbackAttribute())) {
                LOG.error("No attribute with name '{}', using fallback-attribute with name '{}'", source.getAttribute(), source.getFallbackAttribute());
                source.setAttribute(source.getFallbackAttribute());
                source.setFallbackAttribute(null);
                rrdGraphAttribute = resource.getRrdGraphAttributes().get(source.getAttribute());
            }
            if (rrdGraphAttribute == null) {
                if (relaxed)
                    continue;
                LOG.error("No attribute with name: {}", source.getAttribute());
                return null;
            }
            // The Newts Resource ID is stored in the rrdFile attribute
            String newtsResourceId = rrdGraphAttribute.getRrdRelativePath();
            // Remove the file separator prefix, added by the RrdGraphAttribute class
            if (newtsResourceId.startsWith(File.separator)) {
                newtsResourceId = newtsResourceId.substring(File.separator.length(), newtsResourceId.length());
            }
            List<Source> listOfSources = sourcesByNewtsResourceId.get(newtsResourceId);
            // Create the list if it doesn't exist
            if (listOfSources == null) {
                listOfSources = Lists.newLinkedList();
                sourcesByNewtsResourceId.put(newtsResourceId, listOfSources);
            }
            listOfSources.add(source);
        }
    }
    // The Newts API only allows us to perform a query using a single (Newts) Resource ID,
    // so we perform multiple queries in parallel, and aggregate the results.
    Map<String, Future<Collection<Row<Measurement>>>> measurementsByNewtsResourceId = Maps.newHashMapWithExpectedSize(sourcesByNewtsResourceId.size());
    for (Entry<String, List<Source>> entry : sourcesByNewtsResourceId.entrySet()) {
        measurementsByNewtsResourceId.put(entry.getKey(), threadPool.submit(getMeasurementsForResourceCallable(entry.getKey(), entry.getValue(), startTs, endTs, lag)));
    }
    long[] timestamps = null;
    Map<String, double[]> columns = Maps.newHashMap();
    for (Entry<String, Future<Collection<Row<Measurement>>>> entry : measurementsByNewtsResourceId.entrySet()) {
        Collection<Row<Measurement>> rows;
        try {
            rows = entry.getValue().get();
        } catch (InterruptedException | ExecutionException e) {
            throw Throwables.propagate(e);
        }
        final int N = rows.size();
        if (timestamps == null) {
            timestamps = new long[N];
            int k = 0;
            for (final Row<Measurement> row : rows) {
                timestamps[k] = row.getTimestamp().asMillis();
                k++;
            }
        }
        int k = 0;
        for (Row<Measurement> row : rows) {
            for (Measurement measurement : row.getElements()) {
                double[] column = columns.get(measurement.getName());
                if (column == null) {
                    column = new double[N];
                    columns.put(measurement.getName(), column);
                }
                column[k] = measurement.getValue();
            }
            k += 1;
        }
    }
    FetchResults fetchResults = new FetchResults(timestamps, columns, lag.getStep(), constants);
    if (relaxed) {
        Utils.fillMissingValues(fetchResults, sources);
    }
    LOG.trace("Fetch results: {}", fetchResults);
    return fetchResults;
}

Example 3 with Measurement

use of org.opennms.newts.api.Measurement in project opennms by OpenNMS.

the class NewtsConverterIT method execute.

private void execute(final boolean storeByGroup, final boolean storeByForeignSource, final boolean useRrdTool) throws Exception {
    final Path data = OPENNMS_HOME.resolve("share").resolve(useRrdTool ? "rrd" : "jrb").resolve(storeByGroup ? "sbg" : "sbm").resolve(storeByForeignSource ? "fs" : "id");
    assertTrue(Files.isDirectory(data));
    exit.expectSystemExitWithStatus(0);
    exit.checkAssertionAfterwards(() -> {
        assertThat(resourceStorageDao.exists(RESOURCE_PATH_SNMP, 0), is(true));
        assertThat(resourceStorageDao.getAttributes(RESOURCE_PATH_SNMP), hasItems(allOf(hasProperty("name", is("ifInOctets"))), allOf(hasProperty("name", is("ifSpeed")), hasProperty("value", is("1000")))));
        assertThat(resourceStorageDao.exists(RESOURCE_PATH_RESPONSE, 0), is(true));
        assertThat(resourceStorageDao.getAttributes(RESOURCE_PATH_RESPONSE), hasItems(allOf(hasProperty("name", is("icmp")))));
        final Results<Measurement> result = repository.select(Context.DEFAULT_CONTEXT, new Resource(NewtsUtils.toResourceId(ResourcePath.get(RESOURCE_PATH_SNMP, "mib2-interfaces"))), Optional.of(Timestamp.fromEpochSeconds(1414504800)), Optional.of(Timestamp.fromEpochSeconds(1417047045)), new ResultDescriptor(Duration.seconds(7200)).datasource("ifInOctets", StandardAggregationFunctions.AVERAGE).export("ifInOctets"), Optional.of(Duration.seconds(7200)));
        assertThat(result.getRows().size(), is(EXPECTED_DATA.length));
        int i = 0;
        for (Results.Row<Measurement> r : result) {
            final double deltaAbs = r.getElement("ifInOctets").getValue().doubleValue() - EXPECTED_DATA[i].value;
            final double deltaRel = deltaAbs / EXPECTED_DATA[i].value * 100.0;
            /* Use the following for debugging of non-matching entries...
                System.out.println(String.format(
                        "%4d: %-24s %11.2f %11.2f %11.2f (%6.2f) %36s|%-36s %f%%",
                        i,
                        r.getElement("ifInOctets").getTimestamp().asDate().toString(),
                        r.getElement("ifInOctets").getValue().doubleValue(),
                        EXPECTED_DATA[i].value,
                        deltaAbs,
                        deltaRel,
                        deltaAbs < -1.0 ? Strings.repeat("\u2592", (int) Math.abs(Math.log10(-deltaAbs) * 10.0)) : "",
                        deltaAbs >  1.0 ? Strings. repeat("\u2592", (int) Math.abs(Math.log10(deltaAbs) * 10.0)) : "",
                        Math.abs(deltaAbs / EXPECTED_DATA[i].value * 100.0)));
                 */
            assertThat(r.getTimestamp().asSeconds(), is(EXPECTED_DATA[i].timestamp));
            if (i != 270) {
                // We got some errors on the RRA boundaries - ignore them
                assertThat(r.getElement("ifInOctets").getValue().doubleValue(), is(anyOf(equalTo(EXPECTED_DATA[i].value), // Allow a relative error of 0.3%
                closeTo(EXPECTED_DATA[i].value, EXPECTED_DATA[i].value * 0.003))));
            }
            i++;
        }
    });
    NewtsConverter.main("-o", OPENNMS_HOME.toString(), "-r", data.toString(), "-t", Boolean.toString(useRrdTool), "-T", RRD_BINARY.toAbsolutePath().toString(), "-s", Boolean.toString(storeByGroup));
}

Example 4 with Measurement

use of org.opennms.newts.api.Measurement in project newts by OpenNMS.

the class Aggregation method next.

@Override
public Row<Measurement> next() {
    if (!hasNext())
        throw new NoSuchElementException();
    Multimap<String, Double> values = ArrayListMultimap.create();
    Map<String, Map<String, String>> aggregatedAttrs = Maps.newHashMap();
    while (inRange()) {
        // accumulate
        for (Datasource ds : getDatasources()) {
            Measurement metric = m_working.getElement(ds.getSource());
            values.put(ds.getLabel(), metric != null ? metric.getValue() : Double.NaN);
            Map<String, String> metricAttrs = aggregatedAttrs.get(ds.getLabel());
            if (metricAttrs == null) {
                metricAttrs = Maps.newHashMap();
                aggregatedAttrs.put(ds.getLabel(), metricAttrs);
            }
            if (metric != null && metric.getAttributes() != null) {
                metricAttrs.putAll(metric.getAttributes());
            }
        }
        m_working = nextWorking();
    }
    for (Datasource ds : getDatasources()) {
        Double v = aggregate(ds, values.get(ds.getLabel()));
        Map<String, String> attrs = aggregatedAttrs.get(ds.getLabel());
        m_nextOut.addElement(new Measurement(m_nextOut.getTimestamp(), m_resource, ds.getLabel(), v, attrs));
    }
    try {
        return m_nextOut;
    } finally {
        m_nextOut = m_timestamps.hasNext() ? new Row<Measurement>(m_timestamps.next(), m_resource) : null;
    }
}

Example 5 with Measurement

use of org.opennms.newts.api.Measurement in project newts by OpenNMS.

the class PrimaryData method next.

@Override
public Row<Measurement> next() {
    if (!hasNext())
        throw new NoSuchElementException();
    Timestamp intervalCeiling = m_timestamps.next();
    Row<Measurement> output = new Row<>(intervalCeiling, m_resource);
    for (Datasource ds : m_resultDescriptor.getDatasources().values()) {
        Accumulation accumulation = getOrCreateAccumulation(ds.getSource());
        accumulation.reset();
        int lastSampleIdx = 0;
        if (m_lastSampleIndex.containsKey(ds.getSource())) {
            lastSampleIdx = m_lastSampleIndex.get(ds.getSource());
        }
        Sample last = null;
        for (int sampleIdx = lastSampleIdx; sampleIdx < m_samples.size(); sampleIdx++) {
            Row<Sample> row = m_samples.get(sampleIdx);
            Sample current;
            current = row.getElement(ds.getSource());
            // Skip the row if it does not contain a sample for the current datasource
            if (current == null) {
                continue;
            }
            if (last == null) {
                last = current;
                lastSampleIdx = sampleIdx;
                continue;
            }
            // Accumulate nothing when samples are beyond this interval
            if (intervalCeiling.lt(last.getTimestamp())) {
                break;
            }
            Timestamp lowerBound = last.getTimestamp();
            if (lastIntervalCeiling != null && lastIntervalCeiling.gt(lowerBound)) {
                lowerBound = lastIntervalCeiling;
            }
            Timestamp upperBound = current.getTimestamp();
            if (intervalCeiling.lt(upperBound)) {
                upperBound = intervalCeiling;
            }
            if (lowerBound.gt(upperBound)) {
                lowerBound = upperBound;
            }
            Duration elapsedWithinInterval = upperBound.minus(lowerBound);
            Duration elapsedBetweenSamples = current.getTimestamp().minus(last.getTimestamp());
            m_lastSampleIndex.put(ds.getSource(), lastSampleIdx);
            accumulation.accumulateValue(elapsedWithinInterval, elapsedBetweenSamples, ds.getHeartbeat(), current.getValue()).accumlateAttrs(current.getAttributes());
            last = current;
            lastSampleIdx = sampleIdx;
        }
        // Add sample with accumulated value to output row
        output.addElement(new Measurement(output.getTimestamp(), output.getResource(), ds.getSource(), accumulation.getAverage(), accumulation.getAttributes()));
    }
    lastIntervalCeiling = intervalCeiling;
    return output;
}