Examples with DataPoint com.srotya.sidewinder.core.storage.DataPoint used on opensource projects

Example 46 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestPersistentMeasurement method testCompaction.

@Test
public void testCompaction() throws IOException {
    final long ts = 1484788896586L;
    MiscUtils.delete(new File("target/db45/"));
    List<String> tags = Arrays.asList("test=1", "test=2");
    PersistentMeasurement m = new PersistentMeasurement();
    Map<String, String> map = new HashMap<>();
    map.put("disk.compression.class", ByzantineWriter.class.getName());
    map.put("malloc.file.max", String.valueOf(2 * 1024 * 1024));
    map.put("malloc.ptrfile.increment", String.valueOf(1024));
    map.put("compaction.ratio", "1.2");
    map.put("compaction.enabled", "true");
    m.configure(map, null, DBNAME, "m1", "target/db45/index", "target/db45/data", metadata, bgTaskPool);
    int LIMIT = 7000;
    for (int i = 0; i < LIMIT; i++) {
        TimeSeries t = m.getOrCreateTimeSeries("value1", tags, 1024, false, map);
        t.addDataPoint(TimeUnit.MILLISECONDS, ts + i, i * 1.2);
    }
    assertEquals(1, m.getTimeSeries().size());
    TimeSeries series = m.getTimeSeries().iterator().next();
    assertEquals(1, series.getBucketRawMap().size());
    assertEquals(3, series.getBucketCount());
    assertEquals(3, series.getBucketRawMap().entrySet().iterator().next().getValue().size());
    assertEquals(1, series.getCompactionSet().size());
    int maxDp = series.getBucketRawMap().values().stream().flatMap(v -> v.stream()).mapToInt(l -> l.getCount()).max().getAsInt();
    // check and read datapoint count before
    List<DataPoint> queryDataPoints = series.queryDataPoints("", ts, ts + LIMIT + 1, null);
    assertEquals(LIMIT, queryDataPoints.size());
    for (int i = 0; i < LIMIT; i++) {
        DataPoint dp = queryDataPoints.get(i);
        assertEquals(ts + i, dp.getTimestamp());
        assertEquals(i * 1.2, dp.getValue(), 0.01);
    }
    m.compact();
    assertEquals(2, series.getBucketCount());
    assertEquals(2, series.getBucketRawMap().entrySet().iterator().next().getValue().size());
    assertEquals(0, series.getCompactionSet().size());
    assertTrue(maxDp <= series.getBucketRawMap().values().stream().flatMap(v -> v.stream()).mapToInt(l -> l.getCount()).max().getAsInt());
    // validate query after compaction
    queryDataPoints = series.queryDataPoints("", ts, ts + LIMIT + 1, null);
    assertEquals(LIMIT, queryDataPoints.size());
    for (int i = 0; i < LIMIT; i++) {
        DataPoint dp = queryDataPoints.get(i);
        assertEquals(ts + i, dp.getTimestamp());
        assertEquals(i * 1.2, dp.getValue(), 0.01);
    }
    // test buffer recovery after compaction, validate count
    m = new PersistentMeasurement();
    m.configure(map, null, DBNAME, "m1", "target/db45/index", "target/db45/data", metadata, bgTaskPool);
    series = m.getTimeSeries().iterator().next();
    queryDataPoints = series.queryDataPoints("", ts, ts + LIMIT + 1, null);
    assertEquals(LIMIT, queryDataPoints.size());
    for (int i = 0; i < LIMIT; i++) {
        DataPoint dp = queryDataPoints.get(i);
        assertEquals(ts + i, dp.getTimestamp());
        assertEquals(i * 1.2, dp.getValue(), 0.01);
    }
    for (int i = 0; i < LIMIT; i++) {
        TimeSeries t = m.getOrCreateTimeSeries("value1", tags, 1024, false, map);
        t.addDataPoint(TimeUnit.MILLISECONDS, LIMIT + ts + i, i * 1.2);
    }
    series.getBucketRawMap().entrySet().iterator().next().getValue().stream().map(v -> "" + v.getCount() + ":" + v.isReadOnly() + ":" + (int) v.getRawBytes().get(1)).forEach(System.out::println);
    // test recovery again
    m = new PersistentMeasurement();
    m.configure(map, null, DBNAME, "m1", "target/db45/index", "target/db45/data", metadata, bgTaskPool);
    series = m.getTimeSeries().iterator().next();
    queryDataPoints = series.queryDataPoints("", ts - 1, ts + 2 + (LIMIT * 2), null);
    assertEquals(LIMIT * 2, queryDataPoints.size());
    for (int i = 0; i < LIMIT * 2; i++) {
        DataPoint dp = queryDataPoints.get(i);
        assertEquals("Error:" + i + " " + (dp.getTimestamp() - ts - i), ts + i, dp.getTimestamp());
    }
}

Example 47 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestMemStorageEngine method testCompaction.

@Test
public void testCompaction() throws IOException, InterruptedException {
    MemStorageEngine engine = new MemStorageEngine();
    HashMap<String, String> conf2 = new HashMap<>();
    conf2.put("default.bucket.size", "409600");
    conf2.put("compaction.enabled", "true");
    conf2.put("use.query.pool", "false");
    conf2.put("compaction.codec", "gorilla");
    conf2.put("compaction.delay", "1");
    conf2.put("compaction.frequency", "1");
    engine.configure(conf2, bgTasks);
    final long curr = 1497720652566L;
    String dbName = "test";
    String measurementName = "cpu";
    String valueFieldName = "value";
    String tag = "host=123123";
    List<String> tags = Arrays.asList(tag);
    for (int i = 1; i <= 10000; i++) {
        engine.writeDataPoint(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, tags, curr + i * 1000, i * 1.1));
    }
    long ts = System.nanoTime();
    List<Series> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, curr - 1000, curr + 10000 * 1000 + 1, null, null);
    ts = System.nanoTime() - ts;
    System.out.println("Before compaction:" + ts / 1000 + "us");
    assertEquals(1, queryDataPoints.size());
    assertEquals(10000, queryDataPoints.iterator().next().getDataPoints().size());
    List<DataPoint> dataPoints = queryDataPoints.iterator().next().getDataPoints();
    for (int i = 1; i <= 10000; i++) {
        DataPoint dp = dataPoints.get(i - 1);
        assertEquals("Bad ts:" + i, curr + i * 1000, dp.getTimestamp());
        assertEquals(dp.getValue(), i * 1.1, 0.001);
    }
    TimeSeries series = engine.getOrCreateTimeSeries(dbName, measurementName, valueFieldName, tags, 409600, false);
    SortedMap<String, List<Writer>> bucketRawMap = series.getBucketRawMap();
    assertEquals(1, bucketRawMap.size());
    int size = bucketRawMap.values().iterator().next().size();
    assertTrue(series.getCompactionSet().size() < size);
    assertTrue(size > 2);
    Thread.sleep(2000);
    ts = System.nanoTime();
    queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, curr - 1, curr + 20000 * 1000 + 1, null, null);
    ts = System.nanoTime() - ts;
    System.out.println("After compaction:" + ts / 1000 + "us");
    bucketRawMap = series.getBucketRawMap();
    assertEquals(2, bucketRawMap.values().iterator().next().size());
    assertEquals(10000, queryDataPoints.iterator().next().getDataPoints().size());
    dataPoints = queryDataPoints.iterator().next().getDataPoints();
    for (int i = 1; i <= 10000; i++) {
        DataPoint dp = dataPoints.get(i - 1);
        assertEquals("Bad ts:" + i, curr + i * 1000, dp.getTimestamp());
        assertEquals(dp.getValue(), i * 1.1, 0.001);
    }
}

Example 48 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestMemStorageEngine method testAddAndReadDataPointsWithTagFilters.

@Test
public void testAddAndReadDataPointsWithTagFilters() throws Exception {
    MemStorageEngine engine = new MemStorageEngine();
    engine.configure(new HashMap<>(), bgTasks);
    long curr = System.currentTimeMillis();
    String dbName = "test";
    String measurementName = "cpu";
    String valueFieldName = "value";
    String tag = "host=123123";
    for (int i = 1; i <= 3; i++) {
        engine.writeDataPoint(dbName, measurementName, valueFieldName, Arrays.asList(tag + i, tag + (i + 1)), curr + i, 2 * i);
    }
    assertEquals(1, engine.getAllMeasurementsForDb(dbName).size());
    SimpleTagFilter filter1 = new SimpleTagFilter(FilterType.EQUALS, "host", "1231231");
    SimpleTagFilter filter2 = new SimpleTagFilter(FilterType.EQUALS, "host", "1231232");
    List<Series> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, curr - 1, curr + 4, new ComplexTagFilter(ComplexFilterType.OR, Arrays.asList(filter1, filter2)), null, null);
    assertEquals(2, queryDataPoints.size());
    assertEquals(1, queryDataPoints.iterator().next().getDataPoints().size());
    Collections.sort(queryDataPoints);
    int i = 1;
    for (Series list : queryDataPoints) {
        List<DataPoint> dataPoints = list.getDataPoints();
        DataPoint dataPoint = dataPoints.get(0);
        assertEquals(i + curr, dataPoint.getTimestamp());
        i++;
    }
    Set<String> tagKeys = engine.getTagKeysForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList("host")), tagKeys);
    assertEquals(new HashSet<>(Arrays.asList("1231231", "1231232", "1231233", "1231234")), engine.getTagValuesForMeasurement(dbName, measurementName, "host"));
    Set<String> fieldsForMeasurement = engine.getFieldsForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList(valueFieldName)), fieldsForMeasurement);
    try {
        engine.getTagKeysForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getTagKeysForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
}

Example 49 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestMemStorageEngine method testAddAndReaderDataPoints.

@Test
public void testAddAndReaderDataPoints() throws Exception {
    MemStorageEngine engine = new MemStorageEngine();
    engine.configure(new HashMap<>(), bgTasks);
    long curr = System.currentTimeMillis();
    String dbName = "test";
    String measurementName = "cpu";
    String valueFieldName = "value";
    try {
        engine.writeDataPoint(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, null, curr, 2.2 * 0));
        fail("Must reject the above datapoint due to missing tags");
    } catch (Exception e) {
    }
    for (int i = 1; i <= 3; i++) {
        engine.writeDataPoint(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, Arrays.asList(dbName + "=2"), curr + i, 2.2 * i));
    }
    assertEquals(1, engine.getAllMeasurementsForDb(dbName).size());
    LinkedHashMap<Reader, Boolean> readers = engine.queryReaders(dbName, measurementName, valueFieldName, curr, curr + 3);
    int count = 0;
    for (Entry<Reader, Boolean> entry : readers.entrySet()) {
        assertTrue(entry.getValue());
        while (true) {
            try {
                DataPoint readPair = entry.getKey().readPair();
                assertEquals(2.2 * (count + 1), readPair.getValue(), 0.01);
                count++;
            } catch (RejectException e) {
                break;
            }
        }
    }
    assertEquals(3, count);
    assertTrue(engine.checkIfExists(dbName, measurementName));
    try {
        engine.checkIfExists(dbName + "1");
    } catch (Exception e) {
    }
    engine.dropMeasurement(dbName, measurementName);
    assertEquals(0, engine.getAllMeasurementsForDb(dbName).size());
}

Example 50 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestTransformFunctions method testLog.

@Test
public void testLog() {
    Function f = new LogFunction();
    List<Series> series = new ArrayList<>();
    Series s = new Series(Arrays.asList(new DataPoint(1L, 100), new DataPoint(1L, 1000)));
    series.add(s);
    List<Series> apply = f.apply(series);
    assertEquals(2, apply.get(0).getDataPoints().size());
    assertEquals(4.0, apply.get(0).getDataPoints().get(0).getLongValue(), 0.1);
    assertEquals(6.0, apply.get(0).getDataPoints().get(1).getLongValue(), 0.1);
}