Examples with Row org.opennms.newts.api.Results.Row used on opensource projects

Example 6 with Row

use of org.opennms.newts.api.Results.Row in project newts by OpenNMS.

the class InsertSelectMeasurementsITCase method testWithCounter.

@Test
public void testWithCounter() {
    Iterator<Row<Sample>> testSamples = new SampleRowsBuilder(new Resource("localhost"), MetricType.COUNTER).row(900000000).element("mCounter", // Thu Jul  9 11:00:00 CDT 1998
    0).row(900000300).element("mCounter", 300).row(900000600).element("mCounter", 600).row(900000900).element("mCounter", 900).row(900001200).element("mCounter", 1200).row(900001500).element("mCounter", 1500).row(900001800).element("mCounter", 1800, mapFor("a", "1")).row(900002100).element("mCounter", 2100).row(900002400).element("mCounter", 2400, mapFor("b", "2")).row(900002700).element("mCounter", 2700).row(900003000).element("mCounter", 3000).row(900003300).element("mCounter", 3300).row(900003600).element("mCounter", 3600).row(900003900).element("mCounter", 3900).row(900004200).element("mCounter", 4200).row(900004500).element("mCounter", 4500).row(900004800).element("mCounter", 4800).row(900005100).element("mCounter", 5100, mapFor("c", "3")).row(900005400).element("mCounter", 5400).row(900005700).element("mCounter", 5700, mapFor("d", "4")).row(900006000).element("mCounter", 6000).row(900006300).element("mCounter", 6300).row(900006600).element("mCounter", 6600).row(900006900).element("mCounter", 6900).row(900007200).element("mCounter", // Thu Jul  9 13:00:00 CDT 1998
    7200).build();
    ResultDescriptor rDescriptor = new ResultDescriptor(Duration.seconds(300)).datasource("mCounter-avg", "mCounter", Duration.seconds(600), AVERAGE).export("mCounter-avg");
    Iterator<Row<Measurement>> expected = new MeasurementRowsBuilder(new Resource("localhost")).row(900003600).element("mCounter-avg", 1).row(900007200).element("mCounter-avg", 1).build();
    writeSamples(testSamples);
    Results<Measurement> results = getRepository().select(Context.DEFAULT_CONTEXT, new Resource("localhost"), Optional.of(Timestamp.fromEpochSeconds(900003600)), Optional.of(Timestamp.fromEpochSeconds(900007200)), rDescriptor, Optional.of(Duration.minutes(60)));
    // Validate results
    assertRowsEqual(expected, results.iterator());
    // Validate merged attributes
    Iterator<Row<Measurement>> rows = results.iterator();
    assertAttributes(rows.next().getElement("mCounter-avg"), mapFor("a", "1", "b", "2"));
    assertAttributes(rows.next().getElement("mCounter-avg"), mapFor("c", "3", "d", "4"));
}

Example 7 with Row

use of org.opennms.newts.api.Results.Row in project newts by OpenNMS.

the class InsertSelectSamplesITCase method test.

@Test
public void test() {
    List<Sample> samples = Lists.newArrayList();
    int rows = 10, cols = 3;
    Resource resource = new Resource("r");
    for (int i = 1; i <= rows; i++) {
        Timestamp ts = Timestamp.fromEpochMillis(i * 1000);
        for (int j = 1; j <= cols; j++) {
            samples.add(new Sample(ts, resource, "m" + j, GAUGE, new Gauge((i + 1) * j)));
        }
    }
    // Override the shard period to ensure we test query concurrency
    m_contextConfigurations.addContextConfig(Context.DEFAULT_CONTEXT, Duration.seconds(1), ConsistencyLevel.ALL, ConsistencyLevel.ALL);
    getRepository().insert(samples);
    Timestamp start = Timestamp.fromEpochMillis(0), end = Timestamp.fromEpochMillis(rows * 1000);
    Iterator<Row<Sample>> results = getRepository().select(Context.DEFAULT_CONTEXT, resource, Optional.of(start), Optional.of(end)).iterator();
    for (int i = 1; i <= rows; i++) {
        assertTrue("Insufficient number of results", results.hasNext());
        Timestamp timestamp = Timestamp.fromEpochMillis(i * 1000);
        Row<Sample> row = results.next();
        assertEquals("Unexpected timestamp for row " + i, timestamp, row.getTimestamp());
        assertEquals("Unexpected resource name", resource, row.getResource());
        for (int j = 1; j <= cols; j++) {
            assertNotNull("Missing sample: m" + j, row.getElement("m" + j));
            Sample sample = row.getElement("m" + j);
            assertEquals("Unexpected timestamp for metric m" + j, timestamp, sample.getTimestamp());
            assertEquals("Unexpected resource name", resource, sample.getResource());
            assertEquals("Unexpected metric name", "m" + j, sample.getName());
            assertEquals("Unexpected metric type", GAUGE, sample.getType());
            assertEquals((double) ((i + 1) * j), sample.getValue().doubleValue(), 0.0d);
        }
    }
}

Example 8 with Row

use of org.opennms.newts.api.Results.Row in project opennms by OpenNMS.

the class NewtsFetchStrategy method fetch.

@Override
public FetchResults fetch(long start, long end, long step, int maxrows, Long interval, Long heartbeat, List<Source> sources, boolean relaxed) {
    final LateAggregationParams lag = getLagParams(step, interval, heartbeat);
    final Optional<Timestamp> startTs = Optional.of(Timestamp.fromEpochMillis(start));
    final Optional<Timestamp> endTs = Optional.of(Timestamp.fromEpochMillis(end));
    final Map<String, Object> constants = Maps.newHashMap();
    // Group the sources by resource id to avoid calling the ResourceDao
    // multiple times for the same resource
    Map<ResourceId, List<Source>> sourcesByResourceId = sources.stream().collect(Collectors.groupingBy((source) -> ResourceId.fromString(source.getResourceId())));
    // Lookup the OnmsResources in parallel
    Map<ResourceId, Future<OnmsResource>> resourceFuturesById = Maps.newHashMapWithExpectedSize(sourcesByResourceId.size());
    for (ResourceId resourceId : sourcesByResourceId.keySet()) {
        resourceFuturesById.put(resourceId, threadPool.submit(getResourceByIdCallable(resourceId)));
    }
    // Gather the results, fail if any of the resources were not found
    Map<OnmsResource, List<Source>> sourcesByResource = Maps.newHashMapWithExpectedSize(sourcesByResourceId.size());
    for (Entry<ResourceId, Future<OnmsResource>> entry : resourceFuturesById.entrySet()) {
        try {
            OnmsResource resource = entry.getValue().get();
            if (resource == null) {
                if (relaxed)
                    continue;
                LOG.error("No resource with id: {}", entry.getKey());
                return null;
            }
            sourcesByResource.put(resource, sourcesByResourceId.get(entry.getKey()));
        } catch (ExecutionException | InterruptedException e) {
            throw Throwables.propagate(e);
        }
    }
    // Now group the sources by Newts Resource ID, which differs from the OpenNMS Resource ID.
    Map<String, List<Source>> sourcesByNewtsResourceId = Maps.newHashMap();
    for (Entry<OnmsResource, List<Source>> entry : sourcesByResource.entrySet()) {
        final OnmsResource resource = entry.getKey();
        for (Source source : entry.getValue()) {
            // Gather the values from strings.properties
            Utils.convertStringAttributesToConstants(source.getLabel(), resource.getStringPropertyAttributes(), constants);
            // Grab the attribute that matches the source
            RrdGraphAttribute rrdGraphAttribute = resource.getRrdGraphAttributes().get(source.getAttribute());
            if (rrdGraphAttribute == null && !Strings.isNullOrEmpty(source.getFallbackAttribute())) {
                LOG.error("No attribute with name '{}', using fallback-attribute with name '{}'", source.getAttribute(), source.getFallbackAttribute());
                source.setAttribute(source.getFallbackAttribute());
                source.setFallbackAttribute(null);
                rrdGraphAttribute = resource.getRrdGraphAttributes().get(source.getAttribute());
            }
            if (rrdGraphAttribute == null) {
                if (relaxed)
                    continue;
                LOG.error("No attribute with name: {}", source.getAttribute());
                return null;
            }
            // The Newts Resource ID is stored in the rrdFile attribute
            String newtsResourceId = rrdGraphAttribute.getRrdRelativePath();
            // Remove the file separator prefix, added by the RrdGraphAttribute class
            if (newtsResourceId.startsWith(File.separator)) {
                newtsResourceId = newtsResourceId.substring(File.separator.length(), newtsResourceId.length());
            }
            List<Source> listOfSources = sourcesByNewtsResourceId.get(newtsResourceId);
            // Create the list if it doesn't exist
            if (listOfSources == null) {
                listOfSources = Lists.newLinkedList();
                sourcesByNewtsResourceId.put(newtsResourceId, listOfSources);
            }
            listOfSources.add(source);
        }
    }
    // The Newts API only allows us to perform a query using a single (Newts) Resource ID,
    // so we perform multiple queries in parallel, and aggregate the results.
    Map<String, Future<Collection<Row<Measurement>>>> measurementsByNewtsResourceId = Maps.newHashMapWithExpectedSize(sourcesByNewtsResourceId.size());
    for (Entry<String, List<Source>> entry : sourcesByNewtsResourceId.entrySet()) {
        measurementsByNewtsResourceId.put(entry.getKey(), threadPool.submit(getMeasurementsForResourceCallable(entry.getKey(), entry.getValue(), startTs, endTs, lag)));
    }
    long[] timestamps = null;
    Map<String, double[]> columns = Maps.newHashMap();
    for (Entry<String, Future<Collection<Row<Measurement>>>> entry : measurementsByNewtsResourceId.entrySet()) {
        Collection<Row<Measurement>> rows;
        try {
            rows = entry.getValue().get();
        } catch (InterruptedException | ExecutionException e) {
            throw Throwables.propagate(e);
        }
        final int N = rows.size();
        if (timestamps == null) {
            timestamps = new long[N];
            int k = 0;
            for (final Row<Measurement> row : rows) {
                timestamps[k] = row.getTimestamp().asMillis();
                k++;
            }
        }
        int k = 0;
        for (Row<Measurement> row : rows) {
            for (Measurement measurement : row.getElements()) {
                double[] column = columns.get(measurement.getName());
                if (column == null) {
                    column = new double[N];
                    columns.put(measurement.getName(), column);
                }
                column[k] = measurement.getValue();
            }
            k += 1;
        }
    }
    FetchResults fetchResults = new FetchResults(timestamps, columns, lag.getStep(), constants);
    if (relaxed) {
        Utils.fillMissingValues(fetchResults, sources);
    }
    LOG.trace("Fetch results: {}", fetchResults);
    return fetchResults;
}

Example 9 with Row

use of org.opennms.newts.api.Results.Row in project opennms by OpenNMS.

the class NewtsFetchStrategy method getMeasurementsForResourceCallable.

private Callable<Collection<Row<Measurement>>> getMeasurementsForResourceCallable(final String newtsResourceId, final List<Source> listOfSources, final Optional<Timestamp> start, final Optional<Timestamp> end, final LateAggregationParams lag) {
    return new Callable<Collection<Row<Measurement>>>() {

        @Override
        public Collection<Row<Measurement>> call() throws Exception {
            ResultDescriptor resultDescriptor = new ResultDescriptor(lag.getInterval());
            for (Source source : listOfSources) {
                final String metricName = source.getAttribute();
                final String name = source.getLabel();
                final AggregationFunction fn = toAggregationFunction(source.getAggregation());
                resultDescriptor.datasource(name, metricName, lag.getHeartbeat(), fn);
                resultDescriptor.export(name);
            }
            LOG.debug("Querying Newts for resource id {} with result descriptor: {}", newtsResourceId, resultDescriptor);
            Results<Measurement> results = m_sampleRepository.select(m_context, new Resource(newtsResourceId), start, end, resultDescriptor, Optional.of(Duration.millis(lag.getStep())), limitConcurrentAggregationsCallback);
            Collection<Row<Measurement>> rows = results.getRows();
            LOG.debug("Found {} rows.", rows.size());
            return rows;
        }
    };
}

Example 10 with Row

use of org.opennms.newts.api.Results.Row in project newts by OpenNMS.

the class InsertSelectMeasurementsITCase method testWithAbsolute.

@Test
public void testWithAbsolute() {
    Iterator<Row<Sample>> testSamples = new SampleRowsBuilder(new Resource("localhost"), MetricType.ABSOLUTE).row(900000000).element("mAbsolute", // Thu Jul  9 11:00:00 CDT 1998
    300).row(900000300).element("mAbsolute", 300).row(900000600).element("mAbsolute", 300).row(900000900).element("mAbsolute", 300).row(900001200).element("mAbsolute", 300).row(900001500).element("mAbsolute", 300).row(900001800).element("mAbsolute", 300, mapFor("a", "1")).row(900002100).element("mAbsolute", 300).row(900002400).element("mAbsolute", 300, mapFor("b", "2")).row(900002700).element("mAbsolute", 300).row(900003000).element("mAbsolute", 300).row(900003300).element("mAbsolute", 300).row(900003600).element("mAbsolute", 300).row(900003900).element("mAbsolute", 300).row(900004200).element("mAbsolute", 300).row(900004500).element("mAbsolute", 300).row(900004800).element("mAbsolute", 300).row(900005100).element("mAbsolute", 300, mapFor("c", "3")).row(900005400).element("mAbsolute", 300).row(900005700).element("mAbsolute", 300, mapFor("d", "4")).row(900006000).element("mAbsolute", 300).row(900006300).element("mAbsolute", 300).row(900006600).element("mAbsolute", 300).row(900006900).element("mAbsolute", 300).row(900007200).element("mAbsolute", // Thu Jul  9 13:00:00 CDT 1998
    300).build();
    ResultDescriptor rDescriptor = new ResultDescriptor(Duration.seconds(300)).datasource("mAbsolute-avg", "mAbsolute", Duration.seconds(600), AVERAGE).export("mAbsolute-avg");
    Iterator<Row<Measurement>> expected = new MeasurementRowsBuilder(new Resource("localhost")).row(900003600).element("mAbsolute-avg", 1).row(900007200).element("mAbsolute-avg", 1).build();
    writeSamples(testSamples);
    Results<Measurement> results = getRepository().select(Context.DEFAULT_CONTEXT, new Resource("localhost"), Optional.of(Timestamp.fromEpochSeconds(900003600)), Optional.of(Timestamp.fromEpochSeconds(900007200)), rDescriptor, Optional.of(Duration.minutes(60)));
    // Validate results
    assertRowsEqual(expected, results.iterator());
    // Validate merged attributes
    Iterator<Row<Measurement>> rows = results.iterator();
    assertAttributes(rows.next().getElement("mAbsolute-avg"), mapFor("a", "1", "b", "2"));
    assertAttributes(rows.next().getElement("mAbsolute-avg"), mapFor("c", "3", "d", "4"));
}