Examples with Timestamp org.opennms.newts.api.Timestamp used on opensource projects

Example 11 with Timestamp

use of org.opennms.newts.api.Timestamp in project opennms by OpenNMS.

the class NewtsFetchStrategy method fetch.

@Override
public FetchResults fetch(long start, long end, long step, int maxrows, Long interval, Long heartbeat, List<Source> sources, boolean relaxed) {
    final LateAggregationParams lag = getLagParams(step, interval, heartbeat);
    final Optional<Timestamp> startTs = Optional.of(Timestamp.fromEpochMillis(start));
    final Optional<Timestamp> endTs = Optional.of(Timestamp.fromEpochMillis(end));
    final Map<String, Object> constants = Maps.newHashMap();
    // Group the sources by resource id to avoid calling the ResourceDao
    // multiple times for the same resource
    Map<ResourceId, List<Source>> sourcesByResourceId = sources.stream().collect(Collectors.groupingBy((source) -> ResourceId.fromString(source.getResourceId())));
    // Lookup the OnmsResources in parallel
    Map<ResourceId, Future<OnmsResource>> resourceFuturesById = Maps.newHashMapWithExpectedSize(sourcesByResourceId.size());
    for (ResourceId resourceId : sourcesByResourceId.keySet()) {
        resourceFuturesById.put(resourceId, threadPool.submit(getResourceByIdCallable(resourceId)));
    }
    // Gather the results, fail if any of the resources were not found
    Map<OnmsResource, List<Source>> sourcesByResource = Maps.newHashMapWithExpectedSize(sourcesByResourceId.size());
    for (Entry<ResourceId, Future<OnmsResource>> entry : resourceFuturesById.entrySet()) {
        try {
            OnmsResource resource = entry.getValue().get();
            if (resource == null) {
                if (relaxed)
                    continue;
                LOG.error("No resource with id: {}", entry.getKey());
                return null;
            }
            sourcesByResource.put(resource, sourcesByResourceId.get(entry.getKey()));
        } catch (ExecutionException | InterruptedException e) {
            throw Throwables.propagate(e);
        }
    }
    // Now group the sources by Newts Resource ID, which differs from the OpenNMS Resource ID.
    Map<String, List<Source>> sourcesByNewtsResourceId = Maps.newHashMap();
    for (Entry<OnmsResource, List<Source>> entry : sourcesByResource.entrySet()) {
        final OnmsResource resource = entry.getKey();
        for (Source source : entry.getValue()) {
            // Gather the values from strings.properties
            Utils.convertStringAttributesToConstants(source.getLabel(), resource.getStringPropertyAttributes(), constants);
            // Grab the attribute that matches the source
            RrdGraphAttribute rrdGraphAttribute = resource.getRrdGraphAttributes().get(source.getAttribute());
            if (rrdGraphAttribute == null && !Strings.isNullOrEmpty(source.getFallbackAttribute())) {
                LOG.error("No attribute with name '{}', using fallback-attribute with name '{}'", source.getAttribute(), source.getFallbackAttribute());
                source.setAttribute(source.getFallbackAttribute());
                source.setFallbackAttribute(null);
                rrdGraphAttribute = resource.getRrdGraphAttributes().get(source.getAttribute());
            }
            if (rrdGraphAttribute == null) {
                if (relaxed)
                    continue;
                LOG.error("No attribute with name: {}", source.getAttribute());
                return null;
            }
            // The Newts Resource ID is stored in the rrdFile attribute
            String newtsResourceId = rrdGraphAttribute.getRrdRelativePath();
            // Remove the file separator prefix, added by the RrdGraphAttribute class
            if (newtsResourceId.startsWith(File.separator)) {
                newtsResourceId = newtsResourceId.substring(File.separator.length(), newtsResourceId.length());
            }
            List<Source> listOfSources = sourcesByNewtsResourceId.get(newtsResourceId);
            // Create the list if it doesn't exist
            if (listOfSources == null) {
                listOfSources = Lists.newLinkedList();
                sourcesByNewtsResourceId.put(newtsResourceId, listOfSources);
            }
            listOfSources.add(source);
        }
    }
    // The Newts API only allows us to perform a query using a single (Newts) Resource ID,
    // so we perform multiple queries in parallel, and aggregate the results.
    Map<String, Future<Collection<Row<Measurement>>>> measurementsByNewtsResourceId = Maps.newHashMapWithExpectedSize(sourcesByNewtsResourceId.size());
    for (Entry<String, List<Source>> entry : sourcesByNewtsResourceId.entrySet()) {
        measurementsByNewtsResourceId.put(entry.getKey(), threadPool.submit(getMeasurementsForResourceCallable(entry.getKey(), entry.getValue(), startTs, endTs, lag)));
    }
    long[] timestamps = null;
    Map<String, double[]> columns = Maps.newHashMap();
    for (Entry<String, Future<Collection<Row<Measurement>>>> entry : measurementsByNewtsResourceId.entrySet()) {
        Collection<Row<Measurement>> rows;
        try {
            rows = entry.getValue().get();
        } catch (InterruptedException | ExecutionException e) {
            throw Throwables.propagate(e);
        }
        final int N = rows.size();
        if (timestamps == null) {
            timestamps = new long[N];
            int k = 0;
            for (final Row<Measurement> row : rows) {
                timestamps[k] = row.getTimestamp().asMillis();
                k++;
            }
        }
        int k = 0;
        for (Row<Measurement> row : rows) {
            for (Measurement measurement : row.getElements()) {
                double[] column = columns.get(measurement.getName());
                if (column == null) {
                    column = new double[N];
                    columns.put(measurement.getName(), column);
                }
                column[k] = measurement.getValue();
            }
            k += 1;
        }
    }
    FetchResults fetchResults = new FetchResults(timestamps, columns, lag.getStep(), constants);
    if (relaxed) {
        Utils.fillMissingValues(fetchResults, sources);
    }
    LOG.trace("Fetch results: {}", fetchResults);
    return fetchResults;
}

Example 12 with Timestamp

use of org.opennms.newts.api.Timestamp in project opennms by OpenNMS.

the class NewtsConverterTest method cantConvertOutOfRangeCounterToSample.

@Test(expected = IllegalArgumentException.class)
public void cantConvertOutOfRangeCounterToSample() {
    Resource resource = new Resource("resource", Optional.absent());
    DS ds = new DS();
    ds.setType(DSType.COUNTER);
    Timestamp timestamp = Timestamp.fromEpochSeconds(0);
    NewtsConverter.toSample(ds, resource, timestamp, Double.MAX_VALUE);
}

Example 13 with Timestamp

use of org.opennms.newts.api.Timestamp in project opennms by OpenNMS.

the class NewtsConverter method injectSamplesToNewts.

private void injectSamplesToNewts(final ResourcePath resourcePath, final String group, final List<? extends AbstractDS> dataSources, final SortedMap<Long, List<Double>> samples) {
    final ResourcePath groupPath = ResourcePath.get(resourcePath, group);
    // Create a resource ID from the resource path
    final String groupId = NewtsUtils.toResourceId(groupPath);
    // Build indexing attributes
    final Map<String, String> attributes = Maps.newHashMap();
    NewtsUtils.addIndicesToAttributes(groupPath, attributes);
    // Create the NewTS resource to insert
    final Resource resource = new Resource(groupId, Optional.of(attributes));
    // Transform the RRD samples into NewTS samples
    List<Sample> batch = new ArrayList<>(this.batchSize);
    for (final Map.Entry<Long, List<Double>> s : samples.entrySet()) {
        for (int i = 0; i < dataSources.size(); i++) {
            final double value = s.getValue().get(i);
            if (Double.isNaN(value)) {
                continue;
            }
            final AbstractDS ds = dataSources.get(i);
            final Timestamp timestamp = Timestamp.fromEpochSeconds(s.getKey());
            try {
                batch.add(toSample(ds, resource, timestamp, value));
            } catch (IllegalArgumentException e) {
                // type i.e. negative for a counter, so we silently skip these
                continue;
            }
            if (batch.size() >= this.batchSize) {
                this.repository.insert(batch, true);
                this.processedSamples.getAndAdd(batch.size());
                batch = new ArrayList<>(this.batchSize);
            }
        }
    }
    if (!batch.isEmpty()) {
        this.repository.insert(batch, true);
        this.processedSamples.getAndAdd(batch.size());
    }
    this.processedMetrics.getAndAdd(dataSources.size());
    LOG.trace("Stats: {} / {}", this.processedMetrics, this.processedSamples);
}

Example 14 with Timestamp

use of org.opennms.newts.api.Timestamp in project opennms by OpenNMS.

the class NewtsPersistOperationBuilder method getSamplesToInsert.

public List<Sample> getSamplesToInsert() {
    final List<Sample> samples = Lists.newLinkedList();
    ResourcePath path = ResourceTypeUtils.getResourcePathWithRepository(m_repository, ResourcePath.get(m_resource.getPath(), m_name));
    // Add extra attributes that can be used to walk the resource tree.
    NewtsUtils.addIndicesToAttributes(path, m_metaData);
    Resource resource = new Resource(NewtsUtils.toResourceId(path), Optional.of(m_metaData));
    // Convert numeric attributes to samples
    Timestamp timestamp = Timestamp.fromEpochMillis(m_timeKeeper.getCurrentTime());
    for (Entry<CollectionAttributeType, Number> entry : m_declarations.entrySet()) {
        CollectionAttributeType attrType = entry.getKey();
        MetricType type = mapType(attrType.getType());
        if (type == null) {
            // Skip attributes with no type
            continue;
        }
        Number value = entry.getValue();
        if (value == null) {
            // Skip attributes with no value (see NMS-8103)
            continue;
        }
        samples.add(new Sample(timestamp, m_context, resource, attrType.getName(), type, ValueType.compose(entry.getValue(), type)));
    }
    return samples;
}

Example 15 with Timestamp

use of org.opennms.newts.api.Timestamp in project newts by OpenNMS.

the class RateTest method testMissing.

@Test
public void testMissing() {
    Results<Sample> input = new Results<>();
    Timestamp start = Timestamp.fromEpochMillis(1000);
    Duration step = Duration.seconds(1);
    // row_1
    input.addElement(new Sample(start, m_resource, m_metrics[0], COUNTER, new Counter(0)));
    input.addElement(new Sample(start, m_resource, m_metrics[1], COUNTER, new Counter(0)));
    // row_2
    input.addElement(new Sample(start.plus(step), m_resource, m_metrics[0], COUNTER, new Counter(100)));
    input.addElement(new Sample(start.plus(step), m_resource, m_metrics[1], COUNTER, new Counter(100)));
    // row_3 (sample for m_metrics[0] missing)
    input.addElement(new Sample(start.plus(step.times(2)), m_resource, m_metrics[1], COUNTER, new Counter(200)));
    // row_4
    input.addElement(new Sample(start.plus(step.times(3)), m_resource, m_metrics[0], COUNTER, new Counter(300)));
    input.addElement(new Sample(start.plus(step.times(3)), m_resource, m_metrics[1], COUNTER, new Counter(300)));
    Iterator<Results.Row<Sample>> output = new Rate(input.iterator(), getMetrics(2)).iterator();
    // result_1 is always null
    assertTrue(output.hasNext());
    assertEquals(new Gauge(Double.NaN), output.next().getElement(m_metrics[0]).getValue());
    // result_2, rate 100
    assertTrue(output.hasNext());
    assertEquals(100.0d, output.next().getElement(m_metrics[0]).getValue().doubleValue(), 0.0d);
    // result_3, missing because sample in row_3 is missing
    assertTrue(output.hasNext());
    assertNull(output.next().getElement(m_metrics[0]));
    // result_4, rate of 100 calculated between row_4 and row_2
    assertTrue(output.hasNext());
    assertEquals(100.0d, output.next().getElement(m_metrics[0]).getValue().doubleValue(), 0.0d);
}