Examples with Sample org.opennms.newts.api.Sample used on opensource projects

Example 11 with Sample

use of org.opennms.newts.api.Sample in project newts by OpenNMS.

the class ImportRunner method execute.

public void execute(String... args) throws Exception {
    CmdLineParser parser = new CmdLineParser(this);
    try {
        parser.parseArgument(args);
    } catch (CmdLineException e) {
        // handling of wrong arguments
        System.err.println(e.getMessage());
        parser.printUsage(System.err);
        return;
    }
    // Setup the slf4j metrics reporter
    MetricRegistry metrics = new MetricRegistry();
    final long start = System.currentTimeMillis();
    metrics.register("elapsed-seconds", new Gauge<Double>() {

        @Override
        public Double getValue() {
            return (System.currentTimeMillis() - start) / 1000.0;
        }
    });
    final ConsoleReporter reporter = ConsoleReporter.forRegistry(metrics).outputTo(System.err).convertRatesTo(SECONDS).convertDurationsTo(MILLISECONDS).build();
    reporter.start(10, SECONDS);
    if (m_restUrl == null) {
        // we are using a direct importer so use a NewtsReporter for storing metrics
        NewtsReporter newtsReporter = NewtsReporter.forRegistry(metrics).name("importer").convertRatesTo(SECONDS).convertDurationsTo(MILLISECONDS).build(repository());
        newtsReporter.start(1, SECONDS);
    }
    LOG.debug("Scanning {} for GSOD data files...", m_source);
    // walk the files in the directory given
    Observable<Sample> samples = fileTreeWalker(m_source.toPath()).subscribeOn(Schedulers.io()).map(meter(metrics.meter("files"), Path.class)).map(reportFile()).mergeMap(lines()).filter(exclude("YEARMODA")).mergeMap(samples()).map(adjustTime()).map(meter(metrics.meter("samples"), Sample.class));
    Observable<List<Sample>> batches = samples.buffer(m_samplesPerBatch);
    Observable<Boolean> doImport = m_restUrl != null ? restPoster(batches, metrics) : directPoster(batches, metrics);
    System.err.println("doImport = " + doImport);
    // GO!!!
    final AtomicReference<Subscription> subscription = new AtomicReference<>();
    final AtomicBoolean failed = new AtomicBoolean(false);
    final CountDownLatch latch = new CountDownLatch(1);
    Subscription s = doImport.subscribe(new Observer<Boolean>() {

        @Override
        public void onCompleted() {
            System.err.println("Finished Importing Everything!");
            reporter.report();
            latch.countDown();
            System.exit(0);
        }

        @Override
        public void onError(Throwable e) {
            failed.set(true);
            System.err.println("Error importing!");
            e.printStackTrace();
            try {
                //latch.await();
                Subscription s = subscription.get();
                if (s != null)
                    s.unsubscribe();
            } catch (Exception ex) {
                System.err.println("Failed to close httpClient!");
                ex.printStackTrace();
            } finally {
            //dumpThreads();
            }
        }

        @Override
        public void onNext(Boolean t) {
            System.err.println("Received a boolen: " + t);
        }
    });
    subscription.set(s);
    if (failed.get()) {
        s.unsubscribe();
    }
    //latch.countDown();
    System.err.println("Return from Subscribe!");
    latch.await();
//dumpThreads();
}

Example 12 with Sample

use of org.opennms.newts.api.Sample in project newts by OpenNMS.

the class ImportRunner method directPoster.

private Observable<Boolean> directPoster(Observable<List<Sample>> samples, MetricRegistry metrics) {
    final SampleRepository repository = repository();
    final Timer timer = metrics.timer("writes");
    final Meter completions = metrics.meter("samples-completed");
    Func1<List<Sample>, Boolean> insert = new Func1<List<Sample>, Boolean>() {

        @Override
        public Boolean call(List<Sample> s) {
            int sz = s.size();
            try (Context timerCtx = timer.time()) {
                repository.insert(s);
                return true;
            } finally {
                completions.mark(sz);
            }
        }
    };
    return (m_threadCount == 1 ? samples.map(insert) : parMap(samples, metrics, insert)).all(Functions.<Boolean>identity());
}

Example 13 with Sample

use of org.opennms.newts.api.Sample in project newts by OpenNMS.

the class LineParser method parseLine.

List<Sample> parseLine(String line) throws ParseException {
    LOG.trace("Parsing {}", line);
    List<Sample> samples = Lists.newArrayList();
    Resource station = new Resource(stringAt(line, 0));
    String wban = stringAt(line, 7);
    String dateYMD = stringAt(line, 14);
    Date date = getDateFormat().parse(dateYMD);
    Timestamp ts = new Timestamp(date.getTime(), TimeUnit.MILLISECONDS);
    double meanTemp = doubleAt(line, 24);
    samples.add(new Sample(ts, station, "meanTemperature", GAUGE, LineParser.valueFor(meanTemp, 9999.9)));
    double dewpoint = doubleAt(line, 35);
    samples.add(new Sample(ts, station, "dewPoint", GAUGE, LineParser.valueFor(dewpoint, 9999.9)));
    double slp = doubleAt(line, 46);
    Gauge seaLevelPressure = LineParser.valueFor(slp, 9999.9);
    samples.add(new Sample(ts, station, "seaLevelPressure", GAUGE, seaLevelPressure));
    double stp = doubleAt(line, 57);
    Gauge stationPressure = LineParser.valueFor(stp, 9999.9);
    samples.add(new Sample(ts, station, "stationPressure", GAUGE, stationPressure));
    double vis = doubleAt(line, 68);
    Gauge visibility = LineParser.valueFor(vis, 999.9);
    samples.add(new Sample(ts, station, "visibility", GAUGE, visibility));
    double speed = doubleAt(line, 78);
    Gauge meanWindSpeed = new Gauge(speed);
    samples.add(new Sample(ts, station, "meanWindSpeed", GAUGE, meanWindSpeed));
    double maxSpeed = doubleAt(line, 88);
    Gauge maxWindSpeed = LineParser.valueFor(maxSpeed, 999.9);
    samples.add(new Sample(ts, station, "maxWindSpeed", GAUGE, maxWindSpeed));
    double maxGust = doubleAt(line, 95);
    Gauge maxWindGust = LineParser.valueFor(maxGust, 999.9);
    samples.add(new Sample(ts, station, "maxWindGust", GAUGE, maxWindGust));
    double maxTemp = doubleAt(line, 102);
    Gauge maxTemperature = LineParser.valueFor(maxTemp, 9999.9);
    samples.add(new Sample(ts, station, "maxTemperature", GAUGE, maxTemperature));
    double minTemp = doubleAt(line, 110);
    Gauge minTemperature = LineParser.valueFor(minTemp, 9999.9);
    samples.add(new Sample(ts, station, "minTemperature", GAUGE, minTemperature));
    LOG.trace("Station number {}, WBAN {}, date {}, Max Temp {}...", station, wban, dateYMD, maxTemperature);
    return samples;
}

Example 14 with Sample

use of org.opennms.newts.api.Sample in project newts by OpenNMS.

the class InsertSelectSamplesITCase method test.

@Test
public void test() {
    List<Sample> samples = Lists.newArrayList();
    int rows = 10, cols = 3;
    Resource resource = new Resource("r");
    for (int i = 1; i <= rows; i++) {
        Timestamp ts = Timestamp.fromEpochMillis(i * 1000);
        for (int j = 1; j <= cols; j++) {
            samples.add(new Sample(ts, resource, "m" + j, GAUGE, new Gauge((i + 1) * j)));
        }
    }
    // Override the shard period to ensure we test query concurrency
    m_contextConfigurations.addContextConfig(Context.DEFAULT_CONTEXT, Duration.seconds(1), ConsistencyLevel.ALL, ConsistencyLevel.ALL);
    getRepository().insert(samples);
    Timestamp start = Timestamp.fromEpochMillis(0), end = Timestamp.fromEpochMillis(rows * 1000);
    Iterator<Row<Sample>> results = getRepository().select(Context.DEFAULT_CONTEXT, resource, Optional.of(start), Optional.of(end)).iterator();
    for (int i = 1; i <= rows; i++) {
        assertTrue("Insufficient number of results", results.hasNext());
        Timestamp timestamp = Timestamp.fromEpochMillis(i * 1000);
        Row<Sample> row = results.next();
        assertEquals("Unexpected timestamp for row " + i, timestamp, row.getTimestamp());
        assertEquals("Unexpected resource name", resource, row.getResource());
        for (int j = 1; j <= cols; j++) {
            assertNotNull("Missing sample: m" + j, row.getElement("m" + j));
            Sample sample = row.getElement("m" + j);
            assertEquals("Unexpected timestamp for metric m" + j, timestamp, sample.getTimestamp());
            assertEquals("Unexpected resource name", resource, sample.getResource());
            assertEquals("Unexpected metric name", "m" + j, sample.getName());
            assertEquals("Unexpected metric type", GAUGE, sample.getType());
            assertEquals((double) ((i + 1) * j), sample.getValue().doubleValue(), 0.0d);
        }
    }
}

Example 15 with Sample

use of org.opennms.newts.api.Sample in project opennms by OpenNMS.

the class NewtsConverter method injectStringPropertiesToNewts.

private void injectStringPropertiesToNewts(final ResourcePath resourcePath, final Map<String, String> stringProperties) {
    final Resource resource = new Resource(NewtsUtils.toResourceId(resourcePath), Optional.of(stringProperties));
    final Sample sample = new Sample(EPOCH, resource, "strings", MetricType.GAUGE, ZERO);
    indexer.update(Lists.newArrayList(sample));
}