Examples with Series com.srotya.sidewinder.core.storage.Series used on opensource projects

Example 16 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestMemStorageEngine method testCompactionThreadSafety.

@Test
public void testCompactionThreadSafety() throws IOException, InterruptedException {
    MemStorageEngine engine = new MemStorageEngine();
    HashMap<String, String> conf2 = new HashMap<>();
    conf2.put("default.bucket.size", "409600");
    conf2.put("compaction.enabled", "true");
    conf2.put("use.query.pool", "false");
    engine.configure(conf2, bgTasks);
    final long curr = 1497720652566L;
    String dbName = "test";
    String measurementName = "cpu";
    String valueFieldName = "value";
    String tag = "host=123123";
    List<String> tags = Arrays.asList(tag);
    for (int i = 1; i <= 10000; i++) {
        engine.writeDataPoint(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, tags, curr + i * 1000, i * 1.1));
    }
    long ts = System.nanoTime();
    List<Series> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, curr - 1000, curr + 10000 * 1000 + 1, null, null);
    ts = System.nanoTime() - ts;
    System.out.println("Before compaction:" + ts / 1000 + "us");
    assertEquals(1, queryDataPoints.size());
    assertEquals(10000, queryDataPoints.iterator().next().getDataPoints().size());
    List<DataPoint> dataPoints = queryDataPoints.iterator().next().getDataPoints();
    for (int i = 1; i <= 10000; i++) {
        DataPoint dp = dataPoints.get(i - 1);
        assertEquals("Bad ts:" + i, curr + i * 1000, dp.getTimestamp());
        assertEquals(dp.getValue(), i * 1.1, 0.001);
    }
    final TimeSeries series = engine.getOrCreateTimeSeries(dbName, measurementName, valueFieldName, tags, 409600, false);
    SortedMap<String, List<Writer>> bucketRawMap = series.getBucketRawMap();
    assertEquals(1, bucketRawMap.size());
    int size = bucketRawMap.values().iterator().next().size();
    assertTrue(series.getCompactionSet().size() < size);
    assertTrue(size > 2);
    final AtomicBoolean bool = new AtomicBoolean(false);
    bgTasks.execute(() -> {
        while (!bool.get()) {
            try {
                Thread.sleep(100);
            } catch (InterruptedException e) {
                break;
            }
        }
        try {
            series.addDataPoint(TimeUnit.MILLISECONDS, curr + 1000 * 10001, 1.11);
            bool.set(false);
        } catch (IOException e) {
            e.printStackTrace();
            return;
        }
    });
    series.compact(l -> {
        bool.set(true);
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        if (!bool.get()) {
            throw new RuntimeException("Synchronized block failed");
        }
    });
    Thread.sleep(100);
    assertTrue(!bool.get());
}

Example 17 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestMemStorageEngine method testQueryDataPoints.

@Test
public void testQueryDataPoints() throws IOException, ItemNotFoundException {
    StorageEngine engine = new MemStorageEngine();
    engine.configure(conf, bgTasks);
    long ts = System.currentTimeMillis();
    Map<String, Measurement> db = engine.getOrCreateDatabase("test", 24);
    assertEquals(0, db.size());
    engine.writeDataPoint(MiscUtils.buildDataPoint("test", "cpu", "value", Arrays.asList("test=1"), ts, 1));
    engine.writeDataPoint(MiscUtils.buildDataPoint("test", "cpu", "value", Arrays.asList("test=1"), ts + (400 * 60000), 4));
    assertEquals(1, engine.getOrCreateMeasurement("test", "cpu").getSeriesKeys().size());
    List<Series> queryDataPoints = engine.queryDataPoints("test", "cpu", "value", ts, ts + (400 * 60000), null, null);
    assertEquals(2, queryDataPoints.iterator().next().getDataPoints().size());
    assertEquals(ts, queryDataPoints.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals(ts + (400 * 60000), queryDataPoints.iterator().next().getDataPoints().get(1).getTimestamp());
    assertTrue(!engine.isMeasurementFieldFP("test", "cpu", "value"));
    try {
        engine.isMeasurementFieldFP("test", "test", "test");
        fail("Measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.dropDatabase("test");
    } catch (Exception e) {
    }
    assertEquals(0, engine.getOrCreateMeasurement("test", "cpu").getTimeSeries().size());
}

Example 18 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestMemStorageEngine method testWritePerformance.

public void testWritePerformance() throws Exception {
    final MemStorageEngine engine = new MemStorageEngine();
    engine.configure(new HashMap<>(), bgTasks);
    long timeMillis = System.currentTimeMillis();
    int tcount = 12;
    ExecutorService es = Executors.newCachedThreadPool();
    int count = 1000000;
    final int modulator = 100;
    final AtomicInteger rejects = new AtomicInteger(0);
    for (int k = 0; k < tcount; k++) {
        final int j = k;
        es.submit(new Thread() {

            @Override
            public void run() {
                long ts = System.currentTimeMillis();
                for (int i = 0; i < count; i++) {
                    if (isInterrupted()) {
                        break;
                    }
                    try {
                        Point dp = MiscUtils.buildDataPoint("test" + j, "cpu" + (i % modulator), "value", Arrays.asList("test2"), ts + i, i * 1.1);
                        engine.writeDataPoint(dp);
                    } catch (IOException e) {
                        rejects.incrementAndGet();
                    }
                }
            }
        });
    }
    es.shutdown();
    es.awaitTermination(120, TimeUnit.SECONDS);
    int backOff = 30;
    while (!es.isTerminated()) {
        backOff = backOff * 2;
        Thread.sleep(backOff);
    }
    System.out.println("Write throughput direct " + tcount + "x" + count + ":" + (System.currentTimeMillis() - timeMillis) + "ms with " + rejects.get() + " rejects using " + tcount + "\nWriting " + tcount + " each with " + (modulator) + " measurements");
    assertEquals(tcount, engine.getDatabases().size());
    for (int i = 0; i < tcount; i++) {
        String dbName = "test" + i;
        Set<String> allMeasurementsForDb = engine.getAllMeasurementsForDb(dbName);
        assertEquals(modulator, allMeasurementsForDb.size());
        for (String measurementName : allMeasurementsForDb) {
            List<Series> queryDataPoints = engine.queryDataPoints(dbName, measurementName, "value", timeMillis - (3600_000 * 2000), timeMillis + (3600_000 * 2000), null);
            assertEquals(1, queryDataPoints.size());
            assertEquals(count / modulator, queryDataPoints.iterator().next().getDataPoints().size());
        }
    }
}

Example 19 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestTransformFunctions method testCubeRoot.

@Test
public void testCubeRoot() {
    Function f = new CbrtFunction();
    List<Series> series = new ArrayList<>();
    Series s = new Series(Arrays.asList(new DataPoint(1L, 27), new DataPoint(1L, 64)));
    series.add(s);
    List<Series> apply = f.apply(series);
    assertEquals(2, apply.get(0).getDataPoints().size());
    assertEquals(3, apply.get(0).getDataPoints().get(0).getLongValue());
    assertEquals(4, apply.get(0).getDataPoints().get(1).getLongValue());
}

Example 20 with Series

use of com.srotya.sidewinder.core.storage.Series in project sidewinder by srotya.

the class TestTransformFunctions method testCube.

@Test
public void testCube() {
    Function f = new CubeFunction();
    List<Series> series = new ArrayList<>();
    Series s = new Series(Arrays.asList(new DataPoint(1L, 3), new DataPoint(1L, 4)));
    series.add(s);
    List<Series> apply = f.apply(series);
    assertEquals(2, apply.get(0).getDataPoints().size());
    assertEquals(27, apply.get(0).getDataPoints().get(0).getLongValue());
    assertEquals(64, apply.get(0).getDataPoints().get(1).getLongValue());
}