Examples with DataPoint com.srotya.sidewinder.core.storage.DataPoint used on opensource projects

Example 21 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestDiskStorageEngine method testSeriesToDataPointConversion.

@Test
public void testSeriesToDataPointConversion() throws IOException {
    List<DataPoint> points = new ArrayList<>();
    long headerTimestamp = System.currentTimeMillis();
    HashMap<String, String> map = new HashMap<>();
    map.put("metadata.dir", "target/db1/mdq");
    map.put("index.dir", "target/db1/index");
    map.put("data.dir", "target/db1/data");
    map.put(StorageEngine.PERSISTENCE_DISK, "true");
    ByteBuffer buf = ByteBuffer.allocate(100);
    Writer timeSeries = new ByzantineWriter();
    timeSeries.configure(map, buf, true, 1, false);
    timeSeries.setHeaderTimestamp(headerTimestamp);
    timeSeries.addValue(headerTimestamp, 1L);
    TimeSeries.seriesToDataPoints("value", Arrays.asList("test=1"), points, timeSeries, null, null, false);
    assertEquals(1, points.size());
    points.clear();
    Predicate timepredicate = new BetweenPredicate(Long.MAX_VALUE, Long.MAX_VALUE);
    TimeSeries.seriesToDataPoints("value", Arrays.asList("test=1"), points, timeSeries, timepredicate, null, false);
    assertEquals(0, points.size());
}

Example 22 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestPersistentMeasurement method testMeasurementRecovery.

@Test
public void testMeasurementRecovery() throws IOException {
    MiscUtils.delete(new File("target/db141/"));
    PersistentMeasurement m = new PersistentMeasurement();
    m.configure(conf, engine, DBNAME, "m1", "target/db141/index", "target/db141/data", metadata, bgTaskPool);
    TimeSeries ts = m.getOrCreateTimeSeries("vf1", Arrays.asList("t=1", "t=2"), 4096, false, conf);
    long t = System.currentTimeMillis();
    for (int i = 0; i < 100; i++) {
        ts.addDataPoint(TimeUnit.MILLISECONDS, t + i * 1000, i);
    }
    List<DataPoint> dps = ts.queryDataPoints("vf1", t, t + 1000 * 100, null);
    assertEquals(100, dps.size());
    for (int i = 0; i < 100; i++) {
        DataPoint dp = dps.get(i);
        assertEquals(t + i * 1000, dp.getTimestamp());
        assertEquals(i, dp.getLongValue());
    }
    List<Series> resultMap = new ArrayList<>();
    m.queryDataPoints("vf1", t, t + 1000 * 100, null, null, resultMap);
    assertEquals(1, resultMap.size());
    Series next = resultMap.iterator().next();
    for (int i = 0; i < next.getDataPoints().size(); i++) {
        DataPoint dp = next.getDataPoints().get(i);
        assertEquals(t + i * 1000, dp.getTimestamp());
        assertEquals(i, dp.getLongValue());
    }
    LinkedHashMap<Reader, Boolean> readers = new LinkedHashMap<>();
    m.queryReaders("vf1", t, t + 1000 * 100, readers);
    for (Reader reader : readers.keySet()) {
        assertEquals(100, reader.getPairCount());
    }
    m.close();
}

Example 23 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestPersistentMeasurement method testLinearizabilityWithRollOverBucket.

@Test
public void testLinearizabilityWithRollOverBucket() throws IOException, InterruptedException {
    for (int p = 0; p < 2; p++) {
        final int LIMIT = 10000;
        MiscUtils.delete(new File("target/db135/"));
        final long t1 = 1497720452566L;
        Measurement m = new PersistentMeasurement();
        m.configure(conf, engine, DBNAME, "m1", "target/db135/index", "target/db135/data", metadata, bgTaskPool);
        ExecutorService es = Executors.newFixedThreadPool(2, new BackgrounThreadFactory("tlinear2"));
        AtomicBoolean wait = new AtomicBoolean(false);
        for (int i = 0; i < 2; i++) {
            final int th = i;
            es.submit(() -> {
                while (!wait.get()) {
                    try {
                        Thread.sleep(1);
                    } catch (InterruptedException e) {
                    }
                }
                long t = t1 + th * 3;
                for (int j = 0; j < LIMIT; j++) {
                    try {
                        TimeSeries ts = m.getOrCreateTimeSeries("vf1", Arrays.asList("t=1", "t=2"), 4096, false, conf);
                        long timestamp = t + j * 1000;
                        ts.addDataPoint(TimeUnit.MILLISECONDS, timestamp, j);
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            });
        }
        es.shutdown();
        wait.set(true);
        es.awaitTermination(10, TimeUnit.SECONDS);
        TimeSeries ts = m.getOrCreateTimeSeries("vf1", Arrays.asList("t=1", "t=2"), 4096, false, conf);
        List<DataPoint> dps = ts.queryDataPoints("vf1", t1 - 100, t1 + 1000_0000, null);
        assertEquals(LIMIT * 2, dps.size(), 10);
        m.close();
    }
}

Example 24 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestPersistentMeasurement method testDataPointsQuery.

@Test
public void testDataPointsQuery() throws Exception {
    long ts = System.currentTimeMillis();
    MiscUtils.delete(new File("target/db41/"));
    List<String> tags = Arrays.asList("test=1", "test=2");
    PersistentMeasurement m = new PersistentMeasurement();
    Map<String, String> map = new HashMap<>();
    map.put("disk.compression.class", ByzantineWriter.class.getName());
    map.put("malloc.file.max", String.valueOf(2 * 1024 * 1024));
    m.configure(map, null, DBNAME, "m1", "target/db41/index", "target/db41/data", metadata, bgTaskPool);
    int LIMIT = 1000;
    for (int i = 0; i < LIMIT; i++) {
        TimeSeries t = m.getOrCreateTimeSeries("value1", tags, 4096, false, map);
        t.addDataPoint(TimeUnit.MILLISECONDS, ts + i * 1000, 1L);
    }
    for (int i = 0; i < LIMIT; i++) {
        TimeSeries t = m.getOrCreateTimeSeries("value2", tags, 4096, false, map);
        t.addDataPoint(TimeUnit.MILLISECONDS, ts + i * 1000, 1L);
    }
    List<Series> resultMap = new ArrayList<>();
    m.queryDataPoints("value.*$", ts, ts + 1000 * LIMIT, null, null, resultMap);
    assertEquals(2, resultMap.size());
    for (Series s : resultMap) {
        for (int i = 0; i < s.getDataPoints().size(); i++) {
            DataPoint dataPoint = s.getDataPoints().get(i);
            assertEquals(ts + i * 1000, dataPoint.getTimestamp());
            assertEquals(1L, dataPoint.getLongValue());
        }
    }
    List<List<Tag>> tagsResult = m.getTagsForMeasurement();
    Collections.sort(tags);
    for (List<Tag> list : tagsResult) {
        Set<Tag> hashSet = new HashSet<>(list);
        for (int i = 0; i < tags.size(); i++) {
            String tag = tags.get(i);
            String[] split = tag.split("=");
            assertTrue(hashSet.contains(new Tag(split[0], split[1])));
        }
    }
    try {
        tagsResult = m.getTagsForMeasurement();
    } catch (IOException e) {
    }
    m.close();
}

Example 25 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestPersistentMeasurement method testMemoryMapFree.

@Test
public void testMemoryMapFree() throws IOException, InterruptedException {
    final long ts = 1484788896586L;
    DiskMalloc.debug = true;
    MiscUtils.delete(new File("target/db46/"));
    List<String> tags = Arrays.asList("test=1", "test=2");
    PersistentMeasurement m = new PersistentMeasurement();
    Map<String, String> map = new HashMap<>();
    map.put("compression.class", "byzantine");
    map.put("compaction.class", "byzantine");
    map.put("malloc.file.max", String.valueOf(512 * 1024));
    map.put("malloc.file.increment", String.valueOf(256 * 1024));
    map.put("malloc.buf.increment", String.valueOf(1024));
    map.put("default.series.retention.hours", String.valueOf(2));
    map.put("compaction.ratio", "1.2");
    map.put("compaction.enabled", "true");
    m.configure(map, null, DBNAME, "m1", "target/db46/index", "target/db46/data", metadata, bgTaskPool);
    int LIMIT = 20000;
    for (int i = 0; i < LIMIT; i++) {
        for (int k = 0; k < 2; k++) {
            TimeSeries t = m.getOrCreateTimeSeries("value" + k, tags, 512, false, map);
            t.addDataPoint(TimeUnit.MILLISECONDS, ts + i * 10000, i * 1.2);
        }
    }
    System.out.println(m.getOrCreateTimeSeries("value0", tags, 512, false, map).getBucketRawMap().size());
    m.collectGarbage(null);
    for (int k = 0; k < 2; k++) {
        TimeSeries t = m.getOrCreateTimeSeries("value" + k, tags, 512, false, map);
        List<DataPoint> dps = t.queryDataPoints("", ts, ts + LIMIT * 10000, null);
        assertEquals(10032, dps.size());
    }
    System.gc();
    Thread.sleep(200);
    for (int k = 0; k < 2; k++) {
        TimeSeries t = m.getOrCreateTimeSeries("value" + k, tags, 512, false, map);
        List<DataPoint> dps = t.queryDataPoints("", ts, ts + LIMIT * 10000, null);
        assertEquals(10032, dps.size());
    }
    m.close();
}