Examples with Tag com.srotya.sidewinder.core.rpc.Tag used on opensource projects

Example 21 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestPersistentMeasurement method testDataPointsRecoveryPTR.

@Test
public void testDataPointsRecoveryPTR() throws Exception {
    long ts = System.currentTimeMillis();
    List<Tag> tags = Arrays.asList(Tag.newBuilder().setTagKey("test").setTagValue("1").build(), Tag.newBuilder().setTagKey("test").setTagValue("2").build());
    Map<String, String> conf = new HashMap<>();
    conf.put(DiskMalloc.CONF_MEASUREMENT_FILE_MAX, String.valueOf(2 * 1024 * 1024));
    conf.put("malloc.ptrfile.increment", String.valueOf(2 * 1024));
    measurement.configure(conf, null, 4096, DBNAME, "m1", indexDir, dataDir, metadata, bgTaskPool);
    int LIMIT = 100;
    for (int i = 0; i < LIMIT; i++) {
        measurement.addPointWithLocking(MiscUtils.buildDataPoint(DBNAME, "m1", "value" + i, tags, ts, 1L), true);
    }
    measurement.close();
    measurement.configure(conf, null, 4096, DBNAME, "m1", indexDir, dataDir, metadata, bgTaskPool);
    List<SeriesOutput> resultMap = new ArrayList<>();
    measurement.queryDataPoints("value.*", ts, ts + 1000, null, null, resultMap, null);
    assertEquals(LIMIT, resultMap.size());
    measurement.close();
}

Example 22 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestPersistentMeasurement method testDataPointsRecovery.

@Test
public void testDataPointsRecovery() throws Exception {
    long ts = System.currentTimeMillis();
    List<Tag> tags = Arrays.asList(Tag.newBuilder().setTagKey("test").setTagValue("1").build(), Tag.newBuilder().setTagKey("test").setTagValue("2").build());
    conf.put(DiskMalloc.CONF_MEASUREMENT_FILE_MAX, String.valueOf(1024 * 1024));
    try {
        measurement.configure(conf, null, 4096, DBNAME, "m1", indexDir, dataDir, metadata, bgTaskPool);
        fail("Must throw invalid file max size exception");
    } catch (Exception e) {
    }
    conf.put(DiskMalloc.CONF_MEASUREMENT_FILE_MAX, String.valueOf(2 * 1024 * 1024));
    measurement.configure(conf, null, 4096, DBNAME, "m1", indexDir, dataDir, metadata, bgTaskPool);
    int LIMIT = 100000;
    for (int i = 0; i < LIMIT; i++) {
        measurement.addPointWithLocking(TestMeasurement.build("value", tags, ts + i * 1000, 1L), false);
    }
    measurement.close();
    measurement.configure(conf, null, 4096, DBNAME, "m1", indexDir, dataDir, metadata, bgTaskPool);
    List<SeriesOutput> resultMap = new ArrayList<>();
    measurement.queryDataPoints("value", ts, ts + 1000 * LIMIT, null, null, resultMap, null);
    Iterator<SeriesOutput> iterator = resultMap.iterator();
    assertEquals(LIMIT, iterator.next().getDataPoints().size());
    resultMap.clear();
    TagFilter filter = MiscUtils.buildTagFilter("test=1&test=2");
    measurement.queryDataPoints("value", ts, ts + 1000 * LIMIT, filter, null, resultMap, null);
    iterator = resultMap.iterator();
    assertEquals(LIMIT, iterator.next().getDataPoints().size());
    measurement.close();
}

Example 23 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestPersistentMeasurement method testCompaction.

@Test
public void testCompaction() throws IOException {
    final long ts = 1484788896586L;
    List<Tag> tags = Arrays.asList(Tag.newBuilder().setTagKey("test").setTagValue("1").build(), Tag.newBuilder().setTagKey("test2").setTagValue("2").build());
    int defaultTimeBucketSize = 4096;
    conf.put(DiskMalloc.CONF_MEASUREMENT_FILE_MAX, String.valueOf(2 * defaultTimeBucketSize * defaultTimeBucketSize));
    conf.put("buffer.size", String.valueOf(32768));
    conf.put("malloc.ptrfile.increment", String.valueOf(defaultTimeBucketSize));
    TimeField.compactionRatio = 1.2;
    ValueField.compactionRatio = 1.2;
    String measurementName = "m2";
    measurement.configure(conf, null, defaultTimeBucketSize, DBNAME, measurementName, indexDir, dataDir, metadata, bgTaskPool);
    int LIMIT = 8000;
    String valueFieldName = "value1";
    for (int i = 0; i < LIMIT; i++) {
        measurement.addPointWithLocking(TestMeasurement.build(valueFieldName, tags, ts + i, 1.2 * i), false);
    }
    assertEquals(1, measurement.getSeriesList().size());
    Series series = measurement.getSeriesList().iterator().next();
    System.out.println("Series:" + series);
    assertEquals(1, series.getBucketMap().size());
    assertEquals(4, MiscUtils.bucketCounter(series));
    int maxDp = series.getBucketMap().values().stream().flatMap(v -> v.values().stream()).flatMap(f -> f.getWriters().stream()).mapToInt(l -> l.getCount()).max().getAsInt();
    // check and read datapoint count before
    Series seriesField = measurement.getSeriesField(tags);
    List<DataPoint> queryDataPoints = seriesField.queryDataPoints(measurement, Arrays.asList(valueFieldName), ts, ts + LIMIT + 1, null).get(valueFieldName);
    assertEquals(LIMIT, queryDataPoints.size());
    for (int i = 0; i < LIMIT; i++) {
        DataPoint dp = queryDataPoints.get(i);
        assertEquals(ts + i, dp.getTimestamp());
        assertEquals(i * 1.2, dp.getValue(), 0.01);
    }
    measurement.compact();
    assertEquals(3, MiscUtils.bucketCounter(series));
    assertTrue(maxDp <= series.getBucketMap().values().stream().flatMap(v -> v.values().stream()).flatMap(f -> f.getWriters().stream()).mapToInt(l -> l.getCount()).max().getAsInt());
    // validate query after compaction
    queryDataPoints = seriesField.queryDataPoints(measurement, Arrays.asList(valueFieldName), ts, ts + LIMIT + 1, null).get(valueFieldName);
    assertEquals(LIMIT, queryDataPoints.size());
    for (int i = 0; i < LIMIT; i++) {
        DataPoint dp = queryDataPoints.get(i);
        assertEquals(ts + i, dp.getTimestamp());
        assertEquals(i * 1.2, dp.getValue(), 0.01);
    }
    measurement.close();
    // test buffer recovery after compaction, validate count
    measurement.configure(conf, null, defaultTimeBucketSize, DBNAME, measurementName, indexDir, dataDir, metadata, bgTaskPool);
    series = measurement.getSeriesList().iterator().next();
    queryDataPoints = seriesField.queryDataPoints(measurement, Arrays.asList(valueFieldName), ts, ts + LIMIT + 1, null).get(valueFieldName);
    assertEquals(LIMIT, queryDataPoints.size());
    for (int i = 0; i < LIMIT; i++) {
        DataPoint dp = queryDataPoints.get(i);
        assertEquals(ts + i, dp.getTimestamp());
        assertEquals(i * 1.2, dp.getValue(), 0.01);
    }
    for (int i = 0; i < LIMIT; i++) {
        measurement.addPointWithLocking(TestMeasurement.build(valueFieldName, tags, LIMIT + ts + i, 1.2), false);
    }
    series.getBucketMap().entrySet().iterator().next().getValue().values().stream().flatMap(f -> f.getWriters().stream()).map(v -> "" + v.getCount() + ":" + v.isReadOnly() + ":" + (int) v.getRawBytes().get(1)).forEach(System.out::println);
    measurement.close();
    // test recovery again
    measurement.configure(conf, null, defaultTimeBucketSize, DBNAME, measurementName, indexDir, dataDir, metadata, bgTaskPool);
    series = measurement.getSeriesList().iterator().next();
    seriesField = measurement.getSeriesField(tags);
    queryDataPoints = seriesField.queryDataPoints(measurement, Arrays.asList(valueFieldName), ts - 100, ts + 2 + (LIMIT * 2), null).get(valueFieldName);
    assertEquals(LIMIT * 2, queryDataPoints.size());
    for (int i = 0; i < LIMIT * 2; i++) {
        DataPoint dp = queryDataPoints.get(i);
        assertEquals("Error:" + i + " " + (dp.getTimestamp() - ts - i), ts + i, dp.getTimestamp());
    }
}

Example 24 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestMeasurement method testOptimizationsLambdaInvoke.

@Test
public void testOptimizationsLambdaInvoke() throws IOException {
    long ts = System.currentTimeMillis();
    List<Tag> tags = Arrays.asList(Tag.newBuilder().setTagKey("test").setTagValue("1").build(), Tag.newBuilder().setTagKey("test").setTagValue("2").build());
    conf.put(DiskMalloc.CONF_MEASUREMENT_FILE_MAX, String.valueOf(2 * 1024 * 1024));
    measurement.configure(conf, null, 4096, DBNAME, "m1", indexDir, dataDir, metadata, bgTaskPool);
    int LIMIT = 1000;
    for (int i = 0; i < LIMIT; i++) {
        measurement.addPointWithLocking(build("value1", tags, ts + i * 1000, 1L), false);
    }
    measurement.runCleanupOperation("print", s -> {
        // don't cleanup anything
        return new ArrayList<>();
    });
}

Example 25 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestMeasurement method testLinearizabilityWithRollOverBucket.

@Test
public void testLinearizabilityWithRollOverBucket() throws IOException, InterruptedException {
    for (int p = 0; p < 2; p++) {
        MiscUtils.delete(new File("target/measurement-common"));
        final int LIMIT = 10000;
        final long t1 = 1497720452566L;
        final List<Tag> tags = Arrays.asList(Tag.newBuilder().setTagKey("t").setTagValue("1").build(), Tag.newBuilder().setTagKey("t").setTagValue("2").build());
        measurement.configure(conf, engine, 4096, DBNAME, "m1", indexDir, dataDir, metadata, bgTaskPool);
        ExecutorService es = Executors.newFixedThreadPool(2, new BackgrounThreadFactory("tlinear2"));
        AtomicBoolean wait = new AtomicBoolean(false);
        for (int i = 0; i < 2; i++) {
            final int th = i;
            es.submit(() -> {
                while (!wait.get()) {
                    try {
                        Thread.sleep(1);
                    } catch (InterruptedException e) {
                    }
                }
                long t = t1 + th * 3;
                for (int j = 0; j < LIMIT; j++) {
                    try {
                        long timestamp = t + j * 1000;
                        measurement.addPointWithLocking(build("vf1", tags, timestamp, j), false);
                    } catch (Exception e) {
                        e.printStackTrace();
                    }
                }
            });
        }
        es.shutdown();
        wait.set(true);
        es.awaitTermination(10, TimeUnit.SECONDS);
        Series s = measurement.getOrCreateSeries(tags, false);
        List<DataPoint> dps = s.queryDataPoints(measurement, Arrays.asList("vf1"), t1 - 100, t1 + 1000_0000, null).get("vf1");
        assertEquals(LIMIT * 2, dps.size(), 10);
        measurement.close();
    }
}