Examples with Tag com.srotya.sidewinder.core.rpc.Tag used on opensource projects

Example 26 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestStorageEngine method testGarbageCollector.

@Test
public void testGarbageCollector() throws Exception {
    conf.put(StorageEngine.GC_DELAY, "1");
    conf.put(StorageEngine.GC_FREQUENCY, "10");
    conf.put(StorageEngine.DEFAULT_BUCKET_SIZE, "4096");
    conf.put(DiskMalloc.CONF_MEASUREMENT_BUF_INCREMENT_SIZE, "4096");
    conf.put(DiskMalloc.CONF_MEASUREMENT_FILE_INCREMENT, "10240");
    conf.put(DiskMalloc.CONF_MEASUREMENT_FILE_MAX, String.valueOf(1024 * 100));
    conf.put(StorageEngine.RETENTION_HOURS, "28");
    engine.configure(conf, bgTasks);
    long base = 1497720452566L;
    long ts = base;
    List<Tag> tagd = Arrays.asList(Tag.newBuilder().setTagKey("test").setTagValue("1").build());
    for (int i = 320; i >= 0; i--) {
        engine.writeDataPointWithLock(MiscUtils.buildDataPoint("test", "cpu2", "value", tagd, base - (3600_000 * i), 2L), false);
    }
    engine.getMeasurementMap().get("test").get("cpu2").collectGarbage(null);
    List<SeriesOutput> queryDataPoints = engine.queryDataPoints("test", "cpu2", "value", ts - (3600_000 * 320), ts, null, null);
    assertEquals(27, queryDataPoints.iterator().next().getDataPoints().size());
    assertTrue(!engine.isMeasurementFieldFP("test", "cpu2", "value"));
}

Example 27 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestStorageEngine method testCompactionThreadSafety.

@SuppressWarnings("unchecked")
@Test
public void testCompactionThreadSafety() throws IOException, InterruptedException {
    conf.put("default.bucket.size", "409600");
    conf.put("use.query.pool", "false");
    engine.configure(conf, bgTasks);
    final long curr = 1497720652566L;
    String dbName = "test";
    String measurementName = "cpu";
    String valueFieldName = "value";
    List<Tag> tags = Arrays.asList(Tag.newBuilder().setTagKey("host").setTagValue("123123").build());
    for (int i = 1; i <= 10000; i++) {
        engine.writeDataPointWithLock(MiscUtils.buildDataPoint(dbName, measurementName, Arrays.asList(valueFieldName), tags, curr + i * 1000, Arrays.asList(Double.doubleToLongBits(i * 1.1)), Arrays.asList(true)), false);
    }
    long ts = System.nanoTime();
    List<SeriesOutput> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, curr - 1000, curr + 10000 * 1000 + 1, null, null);
    ts = System.nanoTime() - ts;
    System.out.println("Before compaction:" + ts / 1000 + "us");
    assertEquals(1, queryDataPoints.size());
    assertEquals(10000, queryDataPoints.iterator().next().getDataPoints().size());
    List<DataPoint> dataPoints = queryDataPoints.iterator().next().getDataPoints();
    for (int i = 1; i <= 10000; i++) {
        DataPoint dp = dataPoints.get(i - 1);
        assertEquals("Bad ts:" + i, curr + i * 1000, dp.getTimestamp());
        assertEquals(dp.getValue(), i * 1.1, 0.001);
    }
    Measurement m = engine.getOrCreateMeasurement(dbName, measurementName);
    Series series = m.getOrCreateSeries(tags, false);
    SortedMap<Integer, Map<String, Field>> bucketRawMap = series.getBucketMap();
    assertEquals(1, bucketRawMap.size());
    int size = (int) MiscUtils.bucketCounter(series);
    assertTrue(size > 2);
    final AtomicBoolean bool = new AtomicBoolean(false);
    bgTasks.execute(() -> {
        while (!bool.get()) {
            try {
                Thread.sleep(100);
            } catch (InterruptedException e) {
                break;
            }
        }
        try {
            series.addPoint(Point.newBuilder().setTimestamp(curr + 1000 * 10001).addValueFieldName("vf1").addFp(true).addValue(Double.doubleToLongBits(1.11)).build(), m);
            bool.set(false);
        } catch (IOException e) {
            e.printStackTrace();
            return;
        }
    });
    series.compact(m, l -> {
        bool.set(true);
        try {
            Thread.sleep(1000);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
        if (!bool.get()) {
            throw new RuntimeException("Synchronized block failed");
        }
    });
    Thread.sleep(100);
    assertTrue(!bool.get());
}

Example 28 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestStorageEngine method testBaseTimeSeriesWrites.

@Test
public void testBaseTimeSeriesWrites() throws Exception {
    engine.configure(conf, bgTasks);
    engine.startup();
    final List<Tag> tagd = Arrays.asList(Tag.newBuilder().setTagKey("p").setTagValue("1").build());
    final long ts1 = System.currentTimeMillis();
    ExecutorService es = Executors.newCachedThreadPool();
    for (int k = 0; k < 10; k++) {
        final int p = k;
        es.submit(() -> {
            long ts = System.currentTimeMillis();
            for (int i = 0; i < 1000; i++) {
                try {
                    engine.writeDataPointWithLock(MiscUtils.buildDataPoint("test", "helo" + p, "value", tagd, ts + i * 60, ts + i), false);
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        });
    }
    es.shutdown();
    es.awaitTermination(10, TimeUnit.SECONDS);
    System.out.println("Write time:" + (System.currentTimeMillis() - ts1) + "\tms");
}

Example 29 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestStorageEngine method testAddAndReadDataPoints.

@Test
public void testAddAndReadDataPoints() throws Exception {
    engine.configure(conf, bgTasks);
    long curr = System.currentTimeMillis();
    String dbName = "test";
    String measurementName = "cpu";
    String valueFieldName = "value";
    try {
        engine.writeDataPointWithLock(build(dbName, measurementName, valueFieldName, null, curr, 2 * 0), false);
        fail("Must reject the above datapoint due to missing tags");
    } catch (Exception e) {
    }
    Tag tag = Tag.newBuilder().setTagKey("host").setTagValue("123123").build();
    for (int i = 1; i <= 3; i++) {
        engine.writeDataPointWithLock(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, Arrays.asList(tag), curr + i, 2 * i), false);
    }
    assertEquals(1, engine.getAllMeasurementsForDb(dbName).size());
    List<SeriesOutput> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, curr, curr + 3, null);
    assertEquals(1, queryDataPoints.size());
    int i = 1;
    assertEquals(3, queryDataPoints.iterator().next().getDataPoints().size());
    List<List<DataPoint>> output = new ArrayList<>();
    for (SeriesOutput series : queryDataPoints) {
        output.add(series.getDataPoints());
    }
    for (List<DataPoint> list : output) {
        for (DataPoint dataPoint : list) {
            assertEquals(curr + i, dataPoint.getTimestamp());
            i++;
        }
    }
    Set<String> tags = engine.getTagKeysForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList("host")), tags);
    Set<String> fieldsForMeasurement = engine.getFieldsForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList(valueFieldName, Series.TS)), fieldsForMeasurement);
    try {
        engine.getTagKeysForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getTagKeysForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    engine.shutdown();
}

Example 30 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestStorageEngine method testTagFiltering.

@Test
public void testTagFiltering() throws Exception {
    engine.configure(conf, bgTasks);
    long curr = 1497720452566L;
    String dbName = "test";
    String measurementName = "cpu";
    String valueFieldName = "value";
    for (int i = 1; i <= 3; i++) {
        List<Tag> tagd = Arrays.asList(Tag.newBuilder().setTagKey("p").setTagValue(String.valueOf(i)).build(), Tag.newBuilder().setTagKey("k").setTagValue(String.valueOf(i + 7)).build());
        engine.writeDataPointWithLock(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, tagd, curr, 2 * i), false);
    }
    for (int i = 1; i <= 3; i++) {
        List<Tag> tagd = Arrays.asList(Tag.newBuilder().setTagKey("p").setTagValue(String.valueOf(i)).build(), Tag.newBuilder().setTagKey("k").setTagValue(String.valueOf(i + 12)).build());
        engine.writeDataPointWithLock(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName + "2", tagd, curr, 2 * i), false);
    }
    Set<String> tags = engine.getTagKeysForMeasurement(dbName, measurementName);
    System.out.println("Tags:" + tags);
    assertEquals(2, tags.size());
    // Set<String> series = engine.getSeriesIdsWhereTags(dbName, measurementName,
    // Arrays.asList("p=" + String.valueOf(1)));
    // assertEquals(2, series.size());
    TagFilter tagFilterTree = new ComplexTagFilter(ComplexFilterType.OR, Arrays.asList(new SimpleTagFilter(FilterType.EQUALS, "p", "1"), new SimpleTagFilter(FilterType.EQUALS, "p", "2")));
    Set<String> series = engine.getTagFilteredRowKeys(dbName, measurementName, tagFilterTree);
    assertEquals(4, series.size());
    System.out.println(engine.getTagKeysForMeasurement(dbName, measurementName));
    tagFilterTree = new ComplexTagFilter(ComplexFilterType.AND, Arrays.asList(new SimpleTagFilter(FilterType.EQUALS, "p", "1"), new SimpleTagFilter(FilterType.EQUALS, "k", "8")));
    series = engine.getTagFilteredRowKeys(dbName, measurementName, tagFilterTree);
    System.out.println("Series::" + series);
    assertEquals(1, series.size());
    engine.shutdown();
}