Examples with Series com.srotya.sidewinder.core.storage.Series used on opensource projects

Example 31 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class ReduceFunction method apply.

@Override
public Series apply(Series dataPoints) {
    Series output = new Series(dataPoints.getMeasurementName(), dataPoints.getValueFieldName(), dataPoints.getTags());
    output.setFp(dataPoints.isFp());
    DataPoint single = new DataPoint();
    single.setTimestamp(dataPoints.getDataPoints().get(0).getTimestamp());
    aggregateToSingle(dataPoints.getDataPoints(), single, dataPoints.isFp());
    output.setDataPoints(Arrays.asList(single));
    return output;
}

Example 32 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestDiskStorageEngine method testConfigureTimeBuckets.

@Test
public void testConfigureTimeBuckets() throws ItemNotFoundException, IOException {
    StorageEngine engine = new DiskStorageEngine();
    HashMap<String, String> map = new HashMap<>();
    MiscUtils.delete(new File("targer/db101/"));
    map.put("index.dir", "target/db101/index");
    map.put("data.dir", "target/db101/data");
    map.put(StorageEngine.PERSISTENCE_DISK, "true");
    long ts = System.currentTimeMillis();
    map.put(StorageEngine.DEFAULT_BUCKET_SIZE, String.valueOf(4096 * 10));
    try {
        engine.configure(map, bgTasks);
    } catch (IOException e) {
        fail("No IOException should be thrown");
    }
    try {
        for (int i = 0; i < 10; i++) {
            engine.writeDataPoint(MiscUtils.buildDataPoint("test", "ss", "value", Arrays.asList("t=e"), ts + (i * 4096 * 1000), 2.2));
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail("Engine is initialized, no IO Exception should be thrown:" + e.getMessage());
    }
    List<Series> queryDataPoints = engine.queryDataPoints("test", "ss", "value", ts, ts + (4096 * 100 * 1000) + 1, new SimpleTagFilter(FilterType.EQUALS, "t", "e"));
    assertTrue(queryDataPoints.size() >= 1);
}

Example 33 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestDiskStorageEngine method testConcurrentOperations.

@Test
public void testConcurrentOperations() throws Exception {
    MiscUtils.delete(new File("target/dst-6/data"));
    final StorageEngine engine = new DiskStorageEngine();
    Map<String, String> conf = new HashMap<>();
    conf.put("data.dir", "target/dst-6/data");
    conf.put("index.dir", "target/dst-6/index");
    engine.configure(conf, bgTasks);
    final long ts = System.currentTimeMillis();
    ExecutorService es = Executors.newFixedThreadPool(2, new BackgrounThreadFactory("wr1"));
    String measurementName = "mmm2";
    String valueFieldName = "v1";
    String dbName = "db9";
    String tag = "h=1";
    for (int k = 0; k < 2; k++) {
        final int p = k;
        es.submit(() -> {
            long t = ts + p;
            for (int i = 0; i < 100; i++) {
                Point dp = MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, Arrays.asList(tag), t + i * 1000, i);
                try {
                    engine.writeDataPoint(dp);
                } catch (Exception e) {
                    e.printStackTrace();
                    throw new RuntimeException(e);
                }
            }
            System.err.println("Completed writes:" + 100 + " data points");
        });
    }
    es.shutdown();
    es.awaitTermination(100, TimeUnit.SECONDS);
    assertEquals(1, engine.getAllMeasurementsForDb(dbName).size());
    assertEquals(1, engine.getMeasurementMap().size());
    try {
        TimeSeries timeSeries = engine.getTimeSeries(dbName, measurementName, valueFieldName, Arrays.asList(tag));
        assertNotNull(timeSeries);
    } catch (ItemNotFoundException e) {
        fail("Time series must exist");
    }
    List<Series> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, ts, ts + 220 * 1000, null);
    assertEquals(1, queryDataPoints.size());
    Series next = queryDataPoints.iterator().next();
    assertEquals(200, next.getDataPoints().size());
}

Example 34 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestDiskStorageEngine method testQueryDataPoints.

@Test
public void testQueryDataPoints() throws IOException, ItemNotFoundException {
    StorageEngine engine = new DiskStorageEngine();
    MiscUtils.delete(new File("target/db15/"));
    HashMap<String, String> map = new HashMap<>();
    map.put("metadata.dir", "target/db15/mdq");
    map.put("index.dir", "target/db15/index");
    map.put("data.dir", "target/db15/data");
    map.put(StorageEngine.PERSISTENCE_DISK, "true");
    map.put("disk.compression.class", ByzantineWriter.class.getName());
    engine.configure(map, bgTasks);
    long ts = System.currentTimeMillis();
    Map<String, Measurement> db = engine.getOrCreateDatabase("test3", 24);
    assertEquals(0, db.size());
    engine.writeDataPoint(MiscUtils.buildDataPoint("test3", "cpu", "value", Arrays.asList("test=1"), ts, 1));
    engine.writeDataPoint(MiscUtils.buildDataPoint("test3", "cpu", "value", Arrays.asList("test=1"), ts + (400 * 60000), 4));
    assertEquals(1, engine.getOrCreateMeasurement("test3", "cpu").getSeriesKeys().size());
    List<Series> queryDataPoints = null;
    try {
        queryDataPoints = engine.queryDataPoints("test3", "cpu", "value", ts, ts + (400 * 60000), null, null);
    } catch (Exception e) {
    }
    try {
        engine.queryDataPoints("test123", "cpu", "value", ts, ts + (400 * 60000), null, null);
    } catch (ItemNotFoundException e) {
    }
    try {
        engine.queryDataPoints("test3", "123cpu", "value", ts, ts + (400 * 60000), null, null);
    } catch (ItemNotFoundException e) {
    }
    assertTrue(!engine.isMeasurementFieldFP("test3", "cpu", "value"));
    try {
        engine.isMeasurementFieldFP("test3", "test", "test");
        fail("Measurement should not exist");
    } catch (Exception e) {
    }
    assertEquals(2, queryDataPoints.iterator().next().getDataPoints().size());
    assertEquals(ts, queryDataPoints.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals(ts + (400 * 60000), queryDataPoints.iterator().next().getDataPoints().get(1).getTimestamp());
    try {
        engine.dropDatabase("test3");
    } catch (Exception e) {
    }
    assertEquals(0, engine.getOrCreateMeasurement("test3", "cpu").getSeriesKeys().size());
    engine.disconnect();
}

Example 35 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestDiskStorageEngine method testQueryDataPointsRecovery.

@Test
public void testQueryDataPointsRecovery() throws Exception {
    try {
        DiskStorageEngine engine = new DiskStorageEngine();
        File file = new File("target/db201/");
        MiscUtils.delete(file);
        MiscUtils.ls(file);
        Map<String, String> map = new HashMap<>();
        map.put("index.dir", "target/db201/index");
        map.put("data.dir", "target/db201/data");
        map.put("disk.compression.class", ByzantineWriter.class.getName());
        engine.configure(map, bgTasks);
        long ts = System.currentTimeMillis();
        Map<String, Measurement> db = engine.getOrCreateDatabase("test3", 24);
        assertEquals(0, db.size());
        engine.writeDataPoint(MiscUtils.buildDataPoint("test3", "cpu", "value", Arrays.asList("test=1"), ts, 1));
        engine.writeDataPoint(MiscUtils.buildDataPoint("test3", "cpu", "value", Arrays.asList("test=1"), ts + (400 * 60000), 4));
        Measurement measurement = engine.getOrCreateMeasurement("test3", "cpu");
        assertEquals(1, measurement.getSeriesKeys().size());
        MiscUtils.ls(file);
        engine = new DiskStorageEngine();
        engine.configure(map, bgTasks);
        assertTrue(!engine.isMeasurementFieldFP("test3", "cpu", "value"));
        List<Series> queryDataPoints = engine.queryDataPoints("test3", "cpu", "value", ts, ts + (400 * 60000), null, null);
        try {
            engine.isMeasurementFieldFP("test3", "test", "test");
            fail("Measurement should not exist");
        } catch (Exception e) {
        }
        assertEquals(1, queryDataPoints.size());
        assertEquals(2, queryDataPoints.iterator().next().getDataPoints().size());
        assertEquals(ts, queryDataPoints.iterator().next().getDataPoints().get(0).getTimestamp());
        assertEquals(ts + (400 * 60000), queryDataPoints.iterator().next().getDataPoints().get(1).getTimestamp());
        try {
            engine.dropDatabase("test3");
        } catch (Exception e) {
            e.printStackTrace();
            fail("Database delete must succeed");
        }
        assertTrue(!new File("target/db201/data/test3").exists());
        assertEquals(0, engine.getOrCreateMeasurement("test3", "cpu").getSeriesKeys().size());
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
}