Search in sources :

Example 21 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestDiskStorageEngine method testSeriesToDataPointConversion.

@Test
public void testSeriesToDataPointConversion() throws IOException {
    List<DataPoint> points = new ArrayList<>();
    long headerTimestamp = System.currentTimeMillis();
    HashMap<String, String> map = new HashMap<>();
    map.put("metadata.dir", "target/db1/mdq");
    map.put("index.dir", "target/db1/index");
    map.put("data.dir", "target/db1/data");
    map.put(StorageEngine.PERSISTENCE_DISK, "true");
    ByteBuffer buf = ByteBuffer.allocate(100);
    Writer timeSeries = new ByzantineWriter();
    timeSeries.configure(map, buf, true, 1, false);
    timeSeries.setHeaderTimestamp(headerTimestamp);
    timeSeries.addValue(headerTimestamp, 1L);
    TimeSeries.seriesToDataPoints("value", Arrays.asList("test=1"), points, timeSeries, null, null, false);
    assertEquals(1, points.size());
    points.clear();
    Predicate timepredicate = new BetweenPredicate(Long.MAX_VALUE, Long.MAX_VALUE);
    TimeSeries.seriesToDataPoints("value", Arrays.asList("test=1"), points, timeSeries, timepredicate, null, false);
    assertEquals(0, points.size());
}
Also used : BetweenPredicate(com.srotya.sidewinder.core.predicates.BetweenPredicate) DataPoint(com.srotya.sidewinder.core.storage.DataPoint) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) ByzantineWriter(com.srotya.sidewinder.core.storage.compression.byzantine.ByzantineWriter) ArrayList(java.util.ArrayList) ByteBuffer(java.nio.ByteBuffer) Writer(com.srotya.sidewinder.core.storage.compression.Writer) ByzantineWriter(com.srotya.sidewinder.core.storage.compression.byzantine.ByzantineWriter) BetweenPredicate(com.srotya.sidewinder.core.predicates.BetweenPredicate) Predicate(com.srotya.sidewinder.core.predicates.Predicate) Test(org.junit.Test)

Example 22 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestPersistentMeasurement method testMeasurementRecovery.

@Test
public void testMeasurementRecovery() throws IOException {
    MiscUtils.delete(new File("target/db141/"));
    PersistentMeasurement m = new PersistentMeasurement();
    m.configure(conf, engine, DBNAME, "m1", "target/db141/index", "target/db141/data", metadata, bgTaskPool);
    TimeSeries ts = m.getOrCreateTimeSeries("vf1", Arrays.asList("t=1", "t=2"), 4096, false, conf);
    long t = System.currentTimeMillis();
    for (int i = 0; i < 100; i++) {
        ts.addDataPoint(TimeUnit.MILLISECONDS, t + i * 1000, i);
    }
    List<DataPoint> dps = ts.queryDataPoints("vf1", t, t + 1000 * 100, null);
    assertEquals(100, dps.size());
    for (int i = 0; i < 100; i++) {
        DataPoint dp = dps.get(i);
        assertEquals(t + i * 1000, dp.getTimestamp());
        assertEquals(i, dp.getLongValue());
    }
    List<Series> resultMap = new ArrayList<>();
    m.queryDataPoints("vf1", t, t + 1000 * 100, null, null, resultMap);
    assertEquals(1, resultMap.size());
    Series next = resultMap.iterator().next();
    for (int i = 0; i < next.getDataPoints().size(); i++) {
        DataPoint dp = next.getDataPoints().get(i);
        assertEquals(t + i * 1000, dp.getTimestamp());
        assertEquals(i, dp.getLongValue());
    }
    LinkedHashMap<Reader, Boolean> readers = new LinkedHashMap<>();
    m.queryReaders("vf1", t, t + 1000 * 100, readers);
    for (Reader reader : readers.keySet()) {
        assertEquals(100, reader.getPairCount());
    }
    m.close();
}
Also used : TimeSeries(com.srotya.sidewinder.core.storage.TimeSeries) ArrayList(java.util.ArrayList) Reader(com.srotya.sidewinder.core.storage.compression.Reader) DataPoint(com.srotya.sidewinder.core.storage.DataPoint) LinkedHashMap(java.util.LinkedHashMap) TimeSeries(com.srotya.sidewinder.core.storage.TimeSeries) Series(com.srotya.sidewinder.core.storage.Series) DataPoint(com.srotya.sidewinder.core.storage.DataPoint) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) File(java.io.File) Test(org.junit.Test)

Example 23 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestPersistentMeasurement method testLinearizabilityWithRollOverBucket.

@Test
public void testLinearizabilityWithRollOverBucket() throws IOException, InterruptedException {
    for (int p = 0; p < 2; p++) {
        final int LIMIT = 10000;
        MiscUtils.delete(new File("target/db135/"));
        final long t1 = 1497720452566L;
        Measurement m = new PersistentMeasurement();
        m.configure(conf, engine, DBNAME, "m1", "target/db135/index", "target/db135/data", metadata, bgTaskPool);
        ExecutorService es = Executors.newFixedThreadPool(2, new BackgrounThreadFactory("tlinear2"));
        AtomicBoolean wait = new AtomicBoolean(false);
        for (int i = 0; i < 2; i++) {
            final int th = i;
            es.submit(() -> {
                while (!wait.get()) {
                    try {
                        Thread.sleep(1);
                    } catch (InterruptedException e) {
                    }
                }
                long t = t1 + th * 3;
                for (int j = 0; j < LIMIT; j++) {
                    try {
                        TimeSeries ts = m.getOrCreateTimeSeries("vf1", Arrays.asList("t=1", "t=2"), 4096, false, conf);
                        long timestamp = t + j * 1000;
                        ts.addDataPoint(TimeUnit.MILLISECONDS, timestamp, j);
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            });
        }
        es.shutdown();
        wait.set(true);
        es.awaitTermination(10, TimeUnit.SECONDS);
        TimeSeries ts = m.getOrCreateTimeSeries("vf1", Arrays.asList("t=1", "t=2"), 4096, false, conf);
        List<DataPoint> dps = ts.queryDataPoints("vf1", t1 - 100, t1 + 1000_0000, null);
        assertEquals(LIMIT * 2, dps.size(), 10);
        m.close();
    }
}
Also used : Measurement(com.srotya.sidewinder.core.storage.Measurement) BackgrounThreadFactory(com.srotya.sidewinder.core.utils.BackgrounThreadFactory) TimeSeries(com.srotya.sidewinder.core.storage.TimeSeries) DataPoint(com.srotya.sidewinder.core.storage.DataPoint) IOException(java.io.IOException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) DataPoint(com.srotya.sidewinder.core.storage.DataPoint) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ExecutorService(java.util.concurrent.ExecutorService) File(java.io.File) Test(org.junit.Test)

Example 24 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestPersistentMeasurement method testDataPointsQuery.

@Test
public void testDataPointsQuery() throws Exception {
    long ts = System.currentTimeMillis();
    MiscUtils.delete(new File("target/db41/"));
    List<String> tags = Arrays.asList("test=1", "test=2");
    PersistentMeasurement m = new PersistentMeasurement();
    Map<String, String> map = new HashMap<>();
    map.put("disk.compression.class", ByzantineWriter.class.getName());
    map.put("malloc.file.max", String.valueOf(2 * 1024 * 1024));
    m.configure(map, null, DBNAME, "m1", "target/db41/index", "target/db41/data", metadata, bgTaskPool);
    int LIMIT = 1000;
    for (int i = 0; i < LIMIT; i++) {
        TimeSeries t = m.getOrCreateTimeSeries("value1", tags, 4096, false, map);
        t.addDataPoint(TimeUnit.MILLISECONDS, ts + i * 1000, 1L);
    }
    for (int i = 0; i < LIMIT; i++) {
        TimeSeries t = m.getOrCreateTimeSeries("value2", tags, 4096, false, map);
        t.addDataPoint(TimeUnit.MILLISECONDS, ts + i * 1000, 1L);
    }
    List<Series> resultMap = new ArrayList<>();
    m.queryDataPoints("value.*$", ts, ts + 1000 * LIMIT, null, null, resultMap);
    assertEquals(2, resultMap.size());
    for (Series s : resultMap) {
        for (int i = 0; i < s.getDataPoints().size(); i++) {
            DataPoint dataPoint = s.getDataPoints().get(i);
            assertEquals(ts + i * 1000, dataPoint.getTimestamp());
            assertEquals(1L, dataPoint.getLongValue());
        }
    }
    List<List<Tag>> tagsResult = m.getTagsForMeasurement();
    Collections.sort(tags);
    for (List<Tag> list : tagsResult) {
        Set<Tag> hashSet = new HashSet<>(list);
        for (int i = 0; i < tags.size(); i++) {
            String tag = tags.get(i);
            String[] split = tag.split("=");
            assertTrue(hashSet.contains(new Tag(split[0], split[1])));
        }
    }
    try {
        tagsResult = m.getTagsForMeasurement();
    } catch (IOException e) {
    }
    m.close();
}
Also used : TimeSeries(com.srotya.sidewinder.core.storage.TimeSeries) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) ByzantineWriter(com.srotya.sidewinder.core.storage.compression.byzantine.ByzantineWriter) ArrayList(java.util.ArrayList) IOException(java.io.IOException) DataPoint(com.srotya.sidewinder.core.storage.DataPoint) TimeSeries(com.srotya.sidewinder.core.storage.TimeSeries) Series(com.srotya.sidewinder.core.storage.Series) DataPoint(com.srotya.sidewinder.core.storage.DataPoint) ArrayList(java.util.ArrayList) List(java.util.List) Tag(com.srotya.sidewinder.core.filters.Tag) File(java.io.File) HashSet(java.util.HashSet) Test(org.junit.Test)

Example 25 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestPersistentMeasurement method testMemoryMapFree.

@Test
public void testMemoryMapFree() throws IOException, InterruptedException {
    final long ts = 1484788896586L;
    DiskMalloc.debug = true;
    MiscUtils.delete(new File("target/db46/"));
    List<String> tags = Arrays.asList("test=1", "test=2");
    PersistentMeasurement m = new PersistentMeasurement();
    Map<String, String> map = new HashMap<>();
    map.put("compression.class", "byzantine");
    map.put("compaction.class", "byzantine");
    map.put("malloc.file.max", String.valueOf(512 * 1024));
    map.put("malloc.file.increment", String.valueOf(256 * 1024));
    map.put("malloc.buf.increment", String.valueOf(1024));
    map.put("default.series.retention.hours", String.valueOf(2));
    map.put("compaction.ratio", "1.2");
    map.put("compaction.enabled", "true");
    m.configure(map, null, DBNAME, "m1", "target/db46/index", "target/db46/data", metadata, bgTaskPool);
    int LIMIT = 20000;
    for (int i = 0; i < LIMIT; i++) {
        for (int k = 0; k < 2; k++) {
            TimeSeries t = m.getOrCreateTimeSeries("value" + k, tags, 512, false, map);
            t.addDataPoint(TimeUnit.MILLISECONDS, ts + i * 10000, i * 1.2);
        }
    }
    System.out.println(m.getOrCreateTimeSeries("value0", tags, 512, false, map).getBucketRawMap().size());
    m.collectGarbage(null);
    for (int k = 0; k < 2; k++) {
        TimeSeries t = m.getOrCreateTimeSeries("value" + k, tags, 512, false, map);
        List<DataPoint> dps = t.queryDataPoints("", ts, ts + LIMIT * 10000, null);
        assertEquals(10032, dps.size());
    }
    System.gc();
    Thread.sleep(200);
    for (int k = 0; k < 2; k++) {
        TimeSeries t = m.getOrCreateTimeSeries("value" + k, tags, 512, false, map);
        List<DataPoint> dps = t.queryDataPoints("", ts, ts + LIMIT * 10000, null);
        assertEquals(10032, dps.size());
    }
    m.close();
}
Also used : TimeSeries(com.srotya.sidewinder.core.storage.TimeSeries) HashMap(java.util.HashMap) LinkedHashMap(java.util.LinkedHashMap) DataPoint(com.srotya.sidewinder.core.storage.DataPoint) File(java.io.File) DataPoint(com.srotya.sidewinder.core.storage.DataPoint) Test(org.junit.Test)

Aggregations

DataPoint (com.srotya.sidewinder.core.storage.DataPoint)67 Test (org.junit.Test)52 Series (com.srotya.sidewinder.core.storage.Series)40 ArrayList (java.util.ArrayList)39 IOException (java.io.IOException)16 Reader (com.srotya.sidewinder.core.storage.compression.Reader)13 TimeSeries (com.srotya.sidewinder.core.storage.TimeSeries)12 ByteBuffer (java.nio.ByteBuffer)11 ReducingWindowedAggregator (com.srotya.sidewinder.core.functions.windowed.ReducingWindowedAggregator)10 HashMap (java.util.HashMap)10 LinkedHashMap (java.util.LinkedHashMap)10 RejectException (com.srotya.sidewinder.core.storage.RejectException)9 File (java.io.File)9 Point (com.srotya.sidewinder.core.rpc.Point)8 ItemNotFoundException (com.srotya.sidewinder.core.storage.ItemNotFoundException)8 List (java.util.List)8 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)6 Tag (com.srotya.sidewinder.core.filters.Tag)5 Writer (com.srotya.sidewinder.core.storage.compression.Writer)5 ByzantineWriter (com.srotya.sidewinder.core.storage.compression.byzantine.ByzantineWriter)4