Examples with Observation org.spf4j.tsdb2.avro.Observation used on opensource projects

Example 1 with Observation

use of org.spf4j.tsdb2.avro.Observation in project spf4j by zolyfarkas.

the class AvroMeasurementStoreReader method getObservations.

@Override
public AvroCloseableIterable<Observation> getObservations() throws IOException {
    Schema oSchema = Observation.getClassSchema();
    if (dataFiles.length == 0) {
        return AvroCloseableIterable.from(Collections.emptyList(), () -> {
        }, oSchema);
    }
    SpecificDatumReader<Observation> specificDatumReader = new SpecificDatumReader<>(Observation.class);
    Iterable<Observation>[] streams = new Iterable[dataFiles.length];
    Closeable[] closeables = new Closeable[dataFiles.length];
    for (int i = 0; i < dataFiles.length; i++) {
        Path dataFile = dataFiles[i];
        DataFileStream<Observation> ds;
        try {
            ds = new DataFileStream<Observation>(Files.newInputStream(dataFile), specificDatumReader);
        } catch (IOException ex) {
            IOException ex2 = Closeables.closeAll(closeables, 0, i);
            if (ex2 != null) {
                ex2.addSuppressed(ex);
                throw ex2;
            }
            throw ex;
        }
        long fileTimeRef = ds.getMetaLong("timeRef");
        streams[i] = Iterables.transform(ds, new TimeCalibrate(fileTimeRef));
        closeables[i] = ds;
    }
    Iterable<Observation> stream = Iterables.concat(streams);
    return AvroCloseableIterable.from(stream, () -> {
        IOException ex = Closeables.closeAll(closeables);
        if (ex != null) {
            throw new UncheckedIOException(ex);
        }
    }, oSchema);
}

Example 2 with Observation

use of org.spf4j.tsdb2.avro.Observation in project spf4j by zolyfarkas.

the class Charts2 method readToTs.

// false positive.
@SuppressFBWarnings("RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE")
public static TimeSeries readToTs(final MeasurementStoreQuery query, final Schema table, final long startTime, final long endTime, final int aggTimeMillis) throws IOException {
    TLongList ts = new TLongArrayList();
    List<long[]> values = new ArrayList<>();
    try (AvroCloseableIterable<Observation> data = aggTimeMillis <= 0 ? query.getObservations(table, Instant.ofEpochMilli(startTime), Instant.ofEpochMilli(endTime)) : query.getAggregatedObservations(table, Instant.ofEpochMilli(startTime), Instant.ofEpochMilli(endTime), aggTimeMillis, TimeUnit.MILLISECONDS)) {
        for (Observation rec : data) {
            ts.add(rec.getRelTimeStamp());
            values.add(Longs.toArray(rec.getData()));
        }
    }
    return new TimeSeries(ts.toArray(), values.toArray(new long[values.size()][]));
}

Example 3 with Observation

use of org.spf4j.tsdb2.avro.Observation in project spf4j by zolyfarkas.

the class RecorderFactoryTest method assertData.

@SuppressFBWarnings("CLI_CONSTANT_LIST_INDEX")
public static void assertData(final String forWhat, final long expectedValue) throws IOException {
    MeasurementStore store = ProcessMeasurementStore.getMeasurementStore();
    store.flush();
    MeasurementStoreQuery query = store.query();
    Collection<Schema> schemas = query.getMeasurements((x) -> forWhat.equals(x));
    Schema schema = schemas.iterator().next();
    try (AvroCloseableIterable<Observation> observations = query.getObservations(schema, Instant.EPOCH, Instant.ofEpochMilli(Long.MAX_VALUE))) {
        long sum = 0;
        for (Observation o : observations) {
            sum += o.getData().get(0);
        }
        Assert.assertEquals(expectedValue, sum);
    }
}

Example 4 with Observation

use of org.spf4j.tsdb2.avro.Observation in project spf4j by zolyfarkas.

the class TimeSeriesRecord method accumulateObservations.

@Beta
static void accumulateObservations(final Schema recSchema, final Observation r1, final Observation r2) {
    r1.setRelTimeStamp(r2.getRelTimeStamp());
    r1.setTableDefId(-1L);
    Iterator<Schema.Field> it = recSchema.getFields().iterator();
    it.next();
    List<Long> r1d = r1.getData();
    List<Long> r2d = r2.getData();
    while (it.hasNext()) {
        Schema.Field nf = it.next();
        int pos = nf.pos();
        Aggregation agg;
        String prop = nf.schema().getProp(AGGREGATION_TYPE_PROP);
        if (prop != null) {
            agg = Aggregation.valueOf(prop);
        } else {
            agg = inferAggregationFromName(nf, recSchema);
        }
        int apos = pos - 1;
        switch(agg) {
            case SUM:
                r1d.set(apos, r1d.get(apos) + r2d.get(apos));
                break;
            case MIN:
                r1d.set(apos, Math.min(r1d.get(apos), r2d.get(apos)));
                break;
            case MAX:
                r1d.set(apos, Math.max(r1d.get(apos), r2d.get(apos)));
                break;
            case FIRST:
                break;
            case LAST:
            case UNKNOWN:
                r1d.set(apos, r2d.get(apos));
                break;
            default:
                throw new UnsupportedOperationException("Unsupported aggregation: " + agg);
        }
    }
}

Example 5 with Observation

use of org.spf4j.tsdb2.avro.Observation in project spf4j by zolyfarkas.

the class FileMonitorAspectTest method testFileIOMonitoring.

/**
 * Test of nioReadLong method, of class FileMonitorAspect.
 */
@Test
public void testFileIOMonitoring() throws Exception {
    System.setProperty("spf4j.perf.file.sampleTimeMillis", "1000");
    File tempFile = File.createTempFile("test", ".tmp");
    tempFile.deleteOnExit();
    try (Writer fw = new OutputStreamWriter(Files.newOutputStream(tempFile.toPath()), StandardCharsets.UTF_8)) {
        for (int i = 0; i < 10; i++) {
            fw.write("bla bla test\n");
            Thread.sleep(500);
        }
    }
    Thread.sleep(1000);
    try (BufferedReader fr = new BufferedReader(new InputStreamReader(Files.newInputStream(tempFile.toPath()), StandardCharsets.UTF_8))) {
        String line = fr.readLine();
        while (line != null) {
            LOG.debug("Read: {}", line);
            line = fr.readLine();
            Thread.sleep(500);
        }
    }
    RecorderFactory.MEASUREMENT_STORE.flush();
    MeasurementStoreQuery query = RecorderFactory.MEASUREMENT_STORE.query();
    Collection<Schema> measurements = query.getMeasurements((x) -> true);
    LOG.debug("Tables {}", measurements);
    THashSet<String> collect = measurements.stream().map((x) -> x.getProp(TimeSeriesRecord.RAW_NAME)).collect(Collectors.toCollection(() -> new THashSet<String>(measurements.size())));
    Assert.assertThat(collect, (Matcher) Matchers.hasItem("file-write,org.spf4j.perf.aspects.FileMonitorAspectTest"));
    Assert.assertThat(collect, (Matcher) Matchers.hasItem("file-read,org.spf4j.perf.aspects.FileMonitorAspectTest"));
    Collection<Schema> fmw = query.getMeasurements((x) -> "file_write_org_spf4j_perf_aspects_FileMonitorAspectTest".equals(x));
    try (AvroCloseableIterable<Observation> obs = query.getObservations(fmw.iterator().next(), Instant.EPOCH, Instant.now())) {
        Assert.assertTrue(obs.iterator().hasNext());
    }
    Assert.assertTrue(OperatingSystem.getOpenFileDescriptorCount() > 0);
}