Examples with StorageEngine com.srotya.sidewinder.core.storage.StorageEngine used on opensource projects

Example 6 with StorageEngine

use of com.srotya.sidewinder.core.storage.StorageEngine in project sidewinder by srotya.

the class TestDiskStorageEngine method testMetadataOperations.

@Test
public void testMetadataOperations() throws Exception {
    MiscUtils.delete(new File("target/dst-3/"));
    StorageEngine engine = new DiskStorageEngine();
    Map<String, String> conf = new HashMap<>();
    conf.put("data.dir", "target/dst-3/data");
    conf.put("index.dir", "target/dst-3/index");
    engine.configure(conf, bgTasks);
    engine.getOrCreateTimeSeries("db1", "m1", "vf1", Arrays.asList("t=1"), 4096, false);
    assertEquals(1, engine.getAllMeasurementsForDb("db1").size());
    assertEquals(1, engine.getTagKeysForMeasurement("db1", "m1").size());
    assertEquals(1, engine.getTagsForMeasurement("db1", "m1", "vf1").size());
    try {
        engine.getAllMeasurementsForDb("db2");
        fail("Exception must be thrown");
    } catch (ItemNotFoundException e) {
    }
    try {
        engine.getTagKeysForMeasurement("db1", "m2");
        fail("Exception must be thrown");
    } catch (ItemNotFoundException e) {
    }
    assertEquals(1, engine.getTagsForMeasurement("db1", "m1", "vf2").size());
}

Example 7 with StorageEngine

use of com.srotya.sidewinder.core.storage.StorageEngine in project sidewinder by srotya.

the class TestDiskStorageEngine method testConfigureTimeBuckets.

@Test
public void testConfigureTimeBuckets() throws ItemNotFoundException, IOException {
    StorageEngine engine = new DiskStorageEngine();
    HashMap<String, String> map = new HashMap<>();
    MiscUtils.delete(new File("targer/db101/"));
    map.put("index.dir", "target/db101/index");
    map.put("data.dir", "target/db101/data");
    map.put(StorageEngine.PERSISTENCE_DISK, "true");
    long ts = System.currentTimeMillis();
    map.put(StorageEngine.DEFAULT_BUCKET_SIZE, String.valueOf(4096 * 10));
    try {
        engine.configure(map, bgTasks);
    } catch (IOException e) {
        fail("No IOException should be thrown");
    }
    try {
        for (int i = 0; i < 10; i++) {
            engine.writeDataPoint(MiscUtils.buildDataPoint("test", "ss", "value", Arrays.asList("t=e"), ts + (i * 4096 * 1000), 2.2));
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail("Engine is initialized, no IO Exception should be thrown:" + e.getMessage());
    }
    List<Series> queryDataPoints = engine.queryDataPoints("test", "ss", "value", ts, ts + (4096 * 100 * 1000) + 1, new SimpleTagFilter(FilterType.EQUALS, "t", "e"));
    assertTrue(queryDataPoints.size() >= 1);
}

Example 8 with StorageEngine

use of com.srotya.sidewinder.core.storage.StorageEngine in project sidewinder by srotya.

the class TestDiskStorageEngine method testConcurrentOperations.

@Test
public void testConcurrentOperations() throws Exception {
    MiscUtils.delete(new File("target/dst-6/data"));
    final StorageEngine engine = new DiskStorageEngine();
    Map<String, String> conf = new HashMap<>();
    conf.put("data.dir", "target/dst-6/data");
    conf.put("index.dir", "target/dst-6/index");
    engine.configure(conf, bgTasks);
    final long ts = System.currentTimeMillis();
    ExecutorService es = Executors.newFixedThreadPool(2, new BackgrounThreadFactory("wr1"));
    String measurementName = "mmm2";
    String valueFieldName = "v1";
    String dbName = "db9";
    String tag = "h=1";
    for (int k = 0; k < 2; k++) {
        final int p = k;
        es.submit(() -> {
            long t = ts + p;
            for (int i = 0; i < 100; i++) {
                Point dp = MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, Arrays.asList(tag), t + i * 1000, i);
                try {
                    engine.writeDataPoint(dp);
                } catch (Exception e) {
                    e.printStackTrace();
                    throw new RuntimeException(e);
                }
            }
            System.err.println("Completed writes:" + 100 + " data points");
        });
    }
    es.shutdown();
    es.awaitTermination(100, TimeUnit.SECONDS);
    assertEquals(1, engine.getAllMeasurementsForDb(dbName).size());
    assertEquals(1, engine.getMeasurementMap().size());
    try {
        TimeSeries timeSeries = engine.getTimeSeries(dbName, measurementName, valueFieldName, Arrays.asList(tag));
        assertNotNull(timeSeries);
    } catch (ItemNotFoundException e) {
        fail("Time series must exist");
    }
    List<Series> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, ts, ts + 220 * 1000, null);
    assertEquals(1, queryDataPoints.size());
    Series next = queryDataPoints.iterator().next();
    assertEquals(200, next.getDataPoints().size());
}

Example 9 with StorageEngine

use of com.srotya.sidewinder.core.storage.StorageEngine in project sidewinder by srotya.

the class TestDiskStorageEngine method testQueryDataPoints.

@Test
public void testQueryDataPoints() throws IOException, ItemNotFoundException {
    StorageEngine engine = new DiskStorageEngine();
    MiscUtils.delete(new File("target/db15/"));
    HashMap<String, String> map = new HashMap<>();
    map.put("metadata.dir", "target/db15/mdq");
    map.put("index.dir", "target/db15/index");
    map.put("data.dir", "target/db15/data");
    map.put(StorageEngine.PERSISTENCE_DISK, "true");
    map.put("disk.compression.class", ByzantineWriter.class.getName());
    engine.configure(map, bgTasks);
    long ts = System.currentTimeMillis();
    Map<String, Measurement> db = engine.getOrCreateDatabase("test3", 24);
    assertEquals(0, db.size());
    engine.writeDataPoint(MiscUtils.buildDataPoint("test3", "cpu", "value", Arrays.asList("test=1"), ts, 1));
    engine.writeDataPoint(MiscUtils.buildDataPoint("test3", "cpu", "value", Arrays.asList("test=1"), ts + (400 * 60000), 4));
    assertEquals(1, engine.getOrCreateMeasurement("test3", "cpu").getSeriesKeys().size());
    List<Series> queryDataPoints = null;
    try {
        queryDataPoints = engine.queryDataPoints("test3", "cpu", "value", ts, ts + (400 * 60000), null, null);
    } catch (Exception e) {
    }
    try {
        engine.queryDataPoints("test123", "cpu", "value", ts, ts + (400 * 60000), null, null);
    } catch (ItemNotFoundException e) {
    }
    try {
        engine.queryDataPoints("test3", "123cpu", "value", ts, ts + (400 * 60000), null, null);
    } catch (ItemNotFoundException e) {
    }
    assertTrue(!engine.isMeasurementFieldFP("test3", "cpu", "value"));
    try {
        engine.isMeasurementFieldFP("test3", "test", "test");
        fail("Measurement should not exist");
    } catch (Exception e) {
    }
    assertEquals(2, queryDataPoints.iterator().next().getDataPoints().size());
    assertEquals(ts, queryDataPoints.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals(ts + (400 * 60000), queryDataPoints.iterator().next().getDataPoints().get(1).getTimestamp());
    try {
        engine.dropDatabase("test3");
    } catch (Exception e) {
    }
    assertEquals(0, engine.getOrCreateMeasurement("test3", "cpu").getSeriesKeys().size());
    engine.disconnect();
}

Example 10 with StorageEngine

use of com.srotya.sidewinder.core.storage.StorageEngine in project sidewinder by srotya.

the class TestDiskStorageEngine method testMultipleDrives.

@Test
public void testMultipleDrives() throws ItemNotFoundException, IOException {
    StorageEngine engine = new DiskStorageEngine();
    HashMap<String, String> map = new HashMap<>();
    MiscUtils.delete(new File("targer/db10221/"));
    map.put("index.dir", "target/db10221/index");
    map.put("data.dir", "target/db10221/data1, target/db10221/data2");
    map.put(StorageEngine.PERSISTENCE_DISK, "true");
    try {
        engine.configure(map, bgTasks);
    } catch (IOException e) {
        e.printStackTrace();
        fail("No IOException should be thrown");
    }
    long ts = System.currentTimeMillis();
    try {
        for (int i = 0; i < 10; i++) {
            engine.writeDataPoint(MiscUtils.buildDataPoint("test" + i, "ss", "value", Arrays.asList("t=e"), ts, 2.2));
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail("Engine is initialized, no IO Exception should be thrown:" + e.getMessage());
    }
    assertEquals(5, new File("target/db10221/data1").listFiles().length);
    assertEquals(5, new File("target/db10221/data2").listFiles().length);
}