Examples with Measurement com.srotya.sidewinder.core.storage.Measurement used on opensource projects

Example 11 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class MemStorageEngine method getOrCreateTimeSeries.

@Override
public TimeSeries getOrCreateTimeSeries(String dbName, String measurementName, String valueFieldName, List<String> tags, int timeBucketSize, boolean fp) throws IOException {
    // check and create database map
    Map<String, Measurement> dbMap = getOrCreateDatabase(dbName);
    // check and create measurement map
    Measurement measurement = getOrCreateMeasurement(dbMap, dbName, measurementName);
    // check and create timeseries
    return measurement.getOrCreateTimeSeries(valueFieldName, tags, timeBucketSize, fp, conf);
}

Example 12 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class TestDiskStorageEngine method testQueryDataPoints.

@Test
public void testQueryDataPoints() throws IOException, ItemNotFoundException {
    StorageEngine engine = new DiskStorageEngine();
    MiscUtils.delete(new File("target/db15/"));
    HashMap<String, String> map = new HashMap<>();
    map.put("metadata.dir", "target/db15/mdq");
    map.put("index.dir", "target/db15/index");
    map.put("data.dir", "target/db15/data");
    map.put(StorageEngine.PERSISTENCE_DISK, "true");
    map.put("disk.compression.class", ByzantineWriter.class.getName());
    engine.configure(map, bgTasks);
    long ts = System.currentTimeMillis();
    Map<String, Measurement> db = engine.getOrCreateDatabase("test3", 24);
    assertEquals(0, db.size());
    engine.writeDataPoint(MiscUtils.buildDataPoint("test3", "cpu", "value", Arrays.asList("test=1"), ts, 1));
    engine.writeDataPoint(MiscUtils.buildDataPoint("test3", "cpu", "value", Arrays.asList("test=1"), ts + (400 * 60000), 4));
    assertEquals(1, engine.getOrCreateMeasurement("test3", "cpu").getSeriesKeys().size());
    List<Series> queryDataPoints = null;
    try {
        queryDataPoints = engine.queryDataPoints("test3", "cpu", "value", ts, ts + (400 * 60000), null, null);
    } catch (Exception e) {
    }
    try {
        engine.queryDataPoints("test123", "cpu", "value", ts, ts + (400 * 60000), null, null);
    } catch (ItemNotFoundException e) {
    }
    try {
        engine.queryDataPoints("test3", "123cpu", "value", ts, ts + (400 * 60000), null, null);
    } catch (ItemNotFoundException e) {
    }
    assertTrue(!engine.isMeasurementFieldFP("test3", "cpu", "value"));
    try {
        engine.isMeasurementFieldFP("test3", "test", "test");
        fail("Measurement should not exist");
    } catch (Exception e) {
    }
    assertEquals(2, queryDataPoints.iterator().next().getDataPoints().size());
    assertEquals(ts, queryDataPoints.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals(ts + (400 * 60000), queryDataPoints.iterator().next().getDataPoints().get(1).getTimestamp());
    try {
        engine.dropDatabase("test3");
    } catch (Exception e) {
    }
    assertEquals(0, engine.getOrCreateMeasurement("test3", "cpu").getSeriesKeys().size());
    engine.disconnect();
}

Example 13 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class TestDiskStorageEngine method testQueryDataPointsRecovery.

@Test
public void testQueryDataPointsRecovery() throws Exception {
    try {
        DiskStorageEngine engine = new DiskStorageEngine();
        File file = new File("target/db201/");
        MiscUtils.delete(file);
        MiscUtils.ls(file);
        Map<String, String> map = new HashMap<>();
        map.put("index.dir", "target/db201/index");
        map.put("data.dir", "target/db201/data");
        map.put("disk.compression.class", ByzantineWriter.class.getName());
        engine.configure(map, bgTasks);
        long ts = System.currentTimeMillis();
        Map<String, Measurement> db = engine.getOrCreateDatabase("test3", 24);
        assertEquals(0, db.size());
        engine.writeDataPoint(MiscUtils.buildDataPoint("test3", "cpu", "value", Arrays.asList("test=1"), ts, 1));
        engine.writeDataPoint(MiscUtils.buildDataPoint("test3", "cpu", "value", Arrays.asList("test=1"), ts + (400 * 60000), 4));
        Measurement measurement = engine.getOrCreateMeasurement("test3", "cpu");
        assertEquals(1, measurement.getSeriesKeys().size());
        MiscUtils.ls(file);
        engine = new DiskStorageEngine();
        engine.configure(map, bgTasks);
        assertTrue(!engine.isMeasurementFieldFP("test3", "cpu", "value"));
        List<Series> queryDataPoints = engine.queryDataPoints("test3", "cpu", "value", ts, ts + (400 * 60000), null, null);
        try {
            engine.isMeasurementFieldFP("test3", "test", "test");
            fail("Measurement should not exist");
        } catch (Exception e) {
        }
        assertEquals(1, queryDataPoints.size());
        assertEquals(2, queryDataPoints.iterator().next().getDataPoints().size());
        assertEquals(ts, queryDataPoints.iterator().next().getDataPoints().get(0).getTimestamp());
        assertEquals(ts + (400 * 60000), queryDataPoints.iterator().next().getDataPoints().get(1).getTimestamp());
        try {
            engine.dropDatabase("test3");
        } catch (Exception e) {
            e.printStackTrace();
            fail("Database delete must succeed");
        }
        assertTrue(!new File("target/db201/data/test3").exists());
        assertEquals(0, engine.getOrCreateMeasurement("test3", "cpu").getSeriesKeys().size());
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
}

Example 14 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class TestMappedSetTagIndex method testTagIndexPerformance.

// @Test
public void testTagIndexPerformance() throws IOException, InterruptedException {
    MiscUtils.delete(new File("target/perf/index-dir"));
    MiscUtils.delete(new File("target/perf/data-dir"));
    DiskStorageEngine engine = new DiskStorageEngine();
    HashMap<String, String> conf = new HashMap<>();
    conf.put("index.dir", "target/perf/index-dir");
    conf.put("data.dir", "target/perf/data-dir");
    engine.configure(conf, Executors.newScheduledThreadPool(1, new BackgrounThreadFactory("bgt")));
    final long ms = System.currentTimeMillis();
    ExecutorService es = Executors.newCachedThreadPool();
    for (int k = 0; k < 6; k++) {
        es.submit(() -> {
            for (int i = 0; i < 200_000_000; i++) {
                try {
                    engine.getOrCreateTimeSeries("db1", "m1", "v10", Arrays.asList(String.valueOf(i % 10_000), "test=" + String.valueOf(i % 5), "test2=" + String.valueOf(i % 5), "goliath=" + String.valueOf(i % 10_000), "goliath2=" + String.valueOf(i % 1_500)), 4096, true);
                } catch (IOException e) {
                    e.printStackTrace();
                }
                if (i % 1_000_000 == 0) {
                    System.out.println(i + "\t" + (System.currentTimeMillis() - ms) / 1000);
                }
            }
        });
    }
    es.shutdown();
    es.awaitTermination(1000, TimeUnit.SECONDS);
    System.err.println("Index time:" + (System.currentTimeMillis() - ms));
    Map<String, Map<String, Measurement>> index = engine.getMeasurementMap();
    assertEquals(1, index.size());
    Entry<String, Map<String, Measurement>> next = index.entrySet().iterator().next();
    assertEquals("db1", next.getKey());
    Entry<String, Measurement> itr = next.getValue().entrySet().iterator().next();
    assertEquals("m1", itr.getKey());
    MappedSetTagIndex value = (MappedSetTagIndex) itr.getValue().getTagIndex();
    assertEquals(20000 + 10 + 1500, value.getTagKeys().size());
}

Example 15 with Measurement

use of com.srotya.sidewinder.core.storage.Measurement in project sidewinder by srotya.

the class TestPersistentMeasurement method testLinearizability.

@Test
public void testLinearizability() throws IOException, InterruptedException {
    for (int p = 0; p < 100; p++) {
        MiscUtils.delete(new File("target/db134/"));
        final long t1 = 1497720452566L;
        Measurement m = new PersistentMeasurement();
        m.configure(conf, engine, DBNAME, "m1", "target/db134/index", "target/db134/data", metadata, bgTaskPool);
        ExecutorService es = Executors.newFixedThreadPool(2, new BackgrounThreadFactory("tlinear"));
        AtomicBoolean wait = new AtomicBoolean(false);
        for (int i = 0; i < 2; i++) {
            final int th = i;
            es.submit(() -> {
                while (!wait.get()) {
                    try {
                        Thread.sleep(1);
                    } catch (InterruptedException e) {
                    }
                }
                long t = t1 + th * 3;
                for (int j = 0; j < 100; j++) {
                    try {
                        TimeSeries ts = m.getOrCreateTimeSeries("vf1", Arrays.asList("t=1", "t=2"), 4096, false, conf);
                        long timestamp = t + j * 1000;
                        ts.addDataPoint(TimeUnit.MILLISECONDS, timestamp, j);
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            });
        }
        es.shutdown();
        wait.set(true);
        es.awaitTermination(100, TimeUnit.SECONDS);
        TimeSeries ts = m.getOrCreateTimeSeries("vf1", Arrays.asList("t=1", "t=2"), 4096, false, conf);
        List<DataPoint> dps = ts.queryDataPoints("vf1", t1 - 120, t1 + 1000_000, null);
        assertEquals(200, dps.size());
        assertEquals(1, ts.getBucketCount());
        m.close();
    }
}