Examples with Series com.srotya.sidewinder.core.storage.Series used on opensource projects

Example 11 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestDiskStorageEngine method testGarbageCollector.

@Test
public void testGarbageCollector() throws Exception {
    StorageEngine engine = new DiskStorageEngine();
    MiscUtils.delete(new File("target/dbgc/"));
    HashMap<String, String> map = new HashMap<>();
    map.put("index.dir", "target/dbgc/index");
    map.put("data.dir", "target/dbgc/data");
    map.put(StorageEngine.GC_DELAY, "1");
    map.put(StorageEngine.GC_FREQUENCY, "10");
    map.put(StorageEngine.PERSISTENCE_DISK, "true");
    map.put(StorageEngine.DEFAULT_BUCKET_SIZE, "4096");
    map.put(DiskMalloc.CONF_MEASUREMENT_INCREMENT_SIZE, "4096");
    map.put(DiskMalloc.CONF_MEASUREMENT_FILE_INCREMENT, "10240");
    map.put(DiskMalloc.CONF_MEASUREMENT_FILE_MAX, String.valueOf(1024 * 100));
    map.put(StorageEngine.RETENTION_HOURS, "28");
    engine.configure(map, bgTasks);
    long base = 1497720452566L;
    long ts = base;
    for (int i = 320; i >= 0; i--) {
        engine.writeDataPoint(MiscUtils.buildDataPoint("test", "cpu2", "value", Arrays.asList("test=1"), base - (3600_000 * i), 2L));
    }
    engine.getMeasurementMap().get("test").get("cpu2").collectGarbage(null);
    List<Series> queryDataPoints = engine.queryDataPoints("test", "cpu2", "value", ts - (3600_000 * 320), ts, null, null);
    assertEquals(27, queryDataPoints.iterator().next().getDataPoints().size());
    assertTrue(!engine.isMeasurementFieldFP("test", "cpu2", "value"));
}

Example 12 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestPersistentMeasurement method testMeasurementRecovery.

@Test
public void testMeasurementRecovery() throws IOException {
    MiscUtils.delete(new File("target/db141/"));
    PersistentMeasurement m = new PersistentMeasurement();
    m.configure(conf, engine, DBNAME, "m1", "target/db141/index", "target/db141/data", metadata, bgTaskPool);
    TimeSeries ts = m.getOrCreateTimeSeries("vf1", Arrays.asList("t=1", "t=2"), 4096, false, conf);
    long t = System.currentTimeMillis();
    for (int i = 0; i < 100; i++) {
        ts.addDataPoint(TimeUnit.MILLISECONDS, t + i * 1000, i);
    }
    List<DataPoint> dps = ts.queryDataPoints("vf1", t, t + 1000 * 100, null);
    assertEquals(100, dps.size());
    for (int i = 0; i < 100; i++) {
        DataPoint dp = dps.get(i);
        assertEquals(t + i * 1000, dp.getTimestamp());
        assertEquals(i, dp.getLongValue());
    }
    List<Series> resultMap = new ArrayList<>();
    m.queryDataPoints("vf1", t, t + 1000 * 100, null, null, resultMap);
    assertEquals(1, resultMap.size());
    Series next = resultMap.iterator().next();
    for (int i = 0; i < next.getDataPoints().size(); i++) {
        DataPoint dp = next.getDataPoints().get(i);
        assertEquals(t + i * 1000, dp.getTimestamp());
        assertEquals(i, dp.getLongValue());
    }
    LinkedHashMap<Reader, Boolean> readers = new LinkedHashMap<>();
    m.queryReaders("vf1", t, t + 1000 * 100, readers);
    for (Reader reader : readers.keySet()) {
        assertEquals(100, reader.getPairCount());
    }
    m.close();
}

Example 13 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestPersistentMeasurement method testDataPointsQuery.

@Test
public void testDataPointsQuery() throws Exception {
    long ts = System.currentTimeMillis();
    MiscUtils.delete(new File("target/db41/"));
    List<String> tags = Arrays.asList("test=1", "test=2");
    PersistentMeasurement m = new PersistentMeasurement();
    Map<String, String> map = new HashMap<>();
    map.put("disk.compression.class", ByzantineWriter.class.getName());
    map.put("malloc.file.max", String.valueOf(2 * 1024 * 1024));
    m.configure(map, null, DBNAME, "m1", "target/db41/index", "target/db41/data", metadata, bgTaskPool);
    int LIMIT = 1000;
    for (int i = 0; i < LIMIT; i++) {
        TimeSeries t = m.getOrCreateTimeSeries("value1", tags, 4096, false, map);
        t.addDataPoint(TimeUnit.MILLISECONDS, ts + i * 1000, 1L);
    }
    for (int i = 0; i < LIMIT; i++) {
        TimeSeries t = m.getOrCreateTimeSeries("value2", tags, 4096, false, map);
        t.addDataPoint(TimeUnit.MILLISECONDS, ts + i * 1000, 1L);
    }
    List<Series> resultMap = new ArrayList<>();
    m.queryDataPoints("value.*$", ts, ts + 1000 * LIMIT, null, null, resultMap);
    assertEquals(2, resultMap.size());
    for (Series s : resultMap) {
        for (int i = 0; i < s.getDataPoints().size(); i++) {
            DataPoint dataPoint = s.getDataPoints().get(i);
            assertEquals(ts + i * 1000, dataPoint.getTimestamp());
            assertEquals(1L, dataPoint.getLongValue());
        }
    }
    List<List<Tag>> tagsResult = m.getTagsForMeasurement();
    Collections.sort(tags);
    for (List<Tag> list : tagsResult) {
        Set<Tag> hashSet = new HashSet<>(list);
        for (int i = 0; i < tags.size(); i++) {
            String tag = tags.get(i);
            String[] split = tag.split("=");
            assertTrue(hashSet.contains(new Tag(split[0], split[1])));
        }
    }
    try {
        tagsResult = m.getTagsForMeasurement();
    } catch (IOException e) {
    }
    m.close();
}

Example 14 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestMemStorageEngine method testSeriesBucketLookups.

@Test
public void testSeriesBucketLookups() throws IOException, ItemNotFoundException {
    MemStorageEngine engine = new MemStorageEngine();
    engine.configure(new HashMap<>(), bgTasks);
    engine.connect();
    String dbName = "test1";
    String measurementName = "cpu";
    List<String> tags = Arrays.asList("test=1212");
    long ts = 1483923600000L;
    System.out.println("Base timestamp=" + new Date(ts));
    for (int i = 0; i < 100; i++) {
        engine.writeDataPoint(MiscUtils.buildDataPoint(dbName, measurementName, "value", tags, ts + (i * 60000), 2.2));
    }
    System.out.println("Buckets:" + engine.getMeasurement(dbName, measurementName).getTimeSeries().size());
    long endTs = ts + 99 * 60000;
    // validate all points are returned with a full range query
    List<Series> points = engine.queryDataPoints(dbName, measurementName, "value", ts, endTs, null);
    assertEquals(ts, points.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals(endTs, points.iterator().next().getDataPoints().get(points.iterator().next().getDataPoints().size() - 1).getTimestamp());
    // validate ts-1 yields the same result
    points = engine.queryDataPoints(dbName, measurementName, "value", ts - 1, endTs, null);
    assertEquals(ts, points.iterator().next().getDataPoints().get(0).getTimestamp());
    System.out.println("Value count:" + points.iterator().next().getDataPoints().size());
    assertEquals(endTs, points.iterator().next().getDataPoints().get(points.iterator().next().getDataPoints().size() - 1).getTimestamp());
    // validate ts+1 yields correct result
    points = engine.queryDataPoints(dbName, measurementName, "value", ts + 1, endTs, null);
    assertEquals(ts + 60000, points.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals(endTs, points.iterator().next().getDataPoints().get(points.iterator().next().getDataPoints().size() - 1).getTimestamp());
    // validate that points have been written to 2 different buckets
    assertTrue(TimeUtils.getTimeBucket(TimeUnit.MILLISECONDS, ts, 4096) != TimeUtils.getTimeBucket(TimeUnit.MILLISECONDS, endTs, 4096));
    // calculate base timestamp for the second bucket
    long baseTs2 = ((long) TimeUtils.getTimeBucket(TimeUnit.MILLISECONDS, endTs, 4096)) * 1000;
    System.out.println("Bucket2 base timestamp=" + new Date(baseTs2));
    // validate random seek with deliberate time offset
    points = engine.queryDataPoints(dbName, measurementName, "value", ts, baseTs2, null);
    assertEquals("Invalid first entry:" + new Date(points.iterator().next().getDataPoints().get(0).getTimestamp()), ts, points.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals("Invalid first entry:" + (baseTs2 - ts), (baseTs2 / 60000) * 60000, points.iterator().next().getDataPoints().get(points.iterator().next().getDataPoints().size() - 1).getTimestamp());
    points = engine.queryDataPoints(dbName, measurementName, "value", baseTs2, endTs, null);
    assertEquals("Invalid first entry:" + new Date(points.iterator().next().getDataPoints().get(0).getTimestamp()), (baseTs2 - ts), (baseTs2 / 60000) * 60000, points.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals("Invalid first entry:" + endTs, endTs, points.iterator().next().getDataPoints().get(points.iterator().next().getDataPoints().size() - 1).getTimestamp());
    // validate correct results when time range is incorrectly swapped i.e.
    // end time is smaller than start time
    points = engine.queryDataPoints(dbName, measurementName, "value", endTs - 1, baseTs2, null);
    assertEquals("Invalid first entry:" + new Date(points.iterator().next().getDataPoints().get(0).getTimestamp()), (baseTs2 - ts), (baseTs2 / 60000) * 60000, points.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals("Invalid first entry:" + endTs, endTs - 60000, points.iterator().next().getDataPoints().get(points.iterator().next().getDataPoints().size() - 1).getTimestamp());
}

Example 15 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestMemStorageEngine method testAddAndReadDataPoints.

@Test
public void testAddAndReadDataPoints() throws Exception {
    MemStorageEngine engine = new MemStorageEngine();
    engine.configure(new HashMap<>(), bgTasks);
    long curr = System.currentTimeMillis();
    String dbName = "test";
    String measurementName = "cpu";
    String valueFieldName = "value";
    try {
        engine.writeDataPoint(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, null, curr, 2 * 0));
        fail("Must reject the above datapoint due to missing tags");
    } catch (Exception e) {
    }
    String tag = "host=123123";
    for (int i = 1; i <= 3; i++) {
        engine.writeDataPoint(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, Arrays.asList(tag), curr + i, 2 * i));
    }
    assertEquals(1, engine.getAllMeasurementsForDb(dbName).size());
    List<Series> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, curr, curr + 3, null, null);
    assertEquals(1, queryDataPoints.size());
    int i = 1;
    assertEquals(3, queryDataPoints.iterator().next().getDataPoints().size());
    for (Series list : queryDataPoints) {
        for (DataPoint dataPoint : list.getDataPoints()) {
            assertEquals(curr + i, dataPoint.getTimestamp());
            i++;
        }
    }
    Set<String> tags = engine.getTagKeysForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList("host")), tags);
    Set<String> fieldsForMeasurement = engine.getFieldsForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList(valueFieldName)), fieldsForMeasurement);
    try {
        engine.getTagKeysForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getTagKeysForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    assertEquals(1, engine.getFieldsForMeasurement(dbName, "c.*").size());
    assertEquals(1, engine.getFieldsForMeasurement(dbName, ".*").size());
    try {
        engine.getFieldsForMeasurement(dbName, "d.*");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
}