Examples with Tag com.srotya.sidewinder.core.rpc.Tag used on opensource projects

Example 6 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestStorageEngine method testSeriesBucketLookups.

// @Test
// public void testSeriesToDataPointConversion() throws IOException {
// List<DataPoint> points = new ArrayList<>();
// long headerTimestamp = System.currentTimeMillis();
// ByteBuffer buf = ByteBuffer.allocate(100);
// TimeWriter timeSeries = new ByzantineTimestampWriter();
// timeSeries.configure(buf, true, 1);
// timeSeries.setHeaderTimestamp(headerTimestamp);
// timeSeries.add(headerTimestamp);
// Series.seriesToDataPoints(Arrays.asList("test=1"), points, timeSeries, null,
// null, false);
// assertEquals(1, points.size());
// points.clear();
// 
// Predicate timepredicate = new BetweenPredicate(Long.MAX_VALUE,
// Long.MAX_VALUE);
// Series.seriesToDataPoints(Arrays.asList("test=1"), points, timeSeries,
// timepredicate, null, false);
// assertEquals(0, points.size());
// }
@Test
public void testSeriesBucketLookups() throws IOException, ItemNotFoundException {
    engine.configure(conf, bgTasks);
    engine.startup();
    String dbName = "test1";
    String measurementName = "cpu";
    List<Tag> tagd = Arrays.asList(Tag.newBuilder().setTagKey("test").setTagValue("1").build());
    long ts = 1483923600000L;
    System.out.println("Base timestamp=" + new Date(ts));
    for (int i = 0; i < 100; i++) {
        engine.writeDataPointWithLock(MiscUtils.buildDataPoint(dbName, measurementName, "value", tagd, ts + (i * 60000), 2.2), false);
    }
    long endTs = ts + 99 * 60000;
    // validate all points are returned with a full range query
    List<SeriesOutput> points = engine.queryDataPoints(dbName, measurementName, "value", ts, endTs, new SimpleTagFilter(FilterType.EQUALS, "test", "1"));
    assertEquals(ts, points.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals(endTs, points.iterator().next().getDataPoints().get(points.iterator().next().getDataPoints().size() - 1).getTimestamp());
    // validate ts-1 yields the same result
    points = engine.queryDataPoints(dbName, measurementName, "value", ts - 1, endTs, new SimpleTagFilter(FilterType.EQUALS, "test", "1"));
    assertEquals(ts, points.iterator().next().getDataPoints().get(0).getTimestamp());
    System.out.println("Value count:" + points.iterator().next().getDataPoints().size());
    assertEquals(endTs, points.iterator().next().getDataPoints().get(points.iterator().next().getDataPoints().size() - 1).getTimestamp());
    // validate ts+1 yields correct result
    points = engine.queryDataPoints(dbName, measurementName, "value", ts + 1, endTs, new SimpleTagFilter(FilterType.EQUALS, "test", "1"));
    assertEquals(ts + 60000, points.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals(endTs, points.iterator().next().getDataPoints().get(points.iterator().next().getDataPoints().size() - 1).getTimestamp());
    // validate that points have been written to 2 different buckets
    assertTrue(TimeUtils.getTimeBucket(TimeUnit.MILLISECONDS, ts, 4096) != TimeUtils.getTimeBucket(TimeUnit.MILLISECONDS, endTs, 4096));
    // calculate base timestamp for the second bucket
    long baseTs2 = ((long) TimeUtils.getTimeBucket(TimeUnit.MILLISECONDS, endTs, 4096)) * 1000;
    System.out.println("Bucket2 base timestamp=" + new Date(baseTs2));
    // validate random seek with deliberate time offset
    points = engine.queryDataPoints(dbName, measurementName, "value", ts, baseTs2, new SimpleTagFilter(FilterType.EQUALS, "test", "1"));
    assertEquals("Invalid first entry:" + new Date(points.iterator().next().getDataPoints().get(0).getTimestamp()), ts, points.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals("Invalid first entry:" + (baseTs2 - ts), (baseTs2 / 60000) * 60000, points.iterator().next().getDataPoints().get(points.iterator().next().getDataPoints().size() - 1).getTimestamp());
    points = engine.queryDataPoints(dbName, measurementName, "value", baseTs2, endTs, new SimpleTagFilter(FilterType.EQUALS, "test", "1"));
    assertEquals("Invalid first entry:" + new Date(points.iterator().next().getDataPoints().get(0).getTimestamp()), (baseTs2 - ts), (baseTs2 / 60000) * 60000, points.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals("Invalid first entry:" + endTs, endTs, points.iterator().next().getDataPoints().get(points.iterator().next().getDataPoints().size() - 1).getTimestamp());
    // validate correct results when time range is incorrectly swapped i.e.
    // end time is smaller than start time
    points = engine.queryDataPoints(dbName, measurementName, "value", endTs - 1, baseTs2, new SimpleTagFilter(FilterType.EQUALS, "test", "1"));
    assertEquals("Invalid first entry:" + new Date(points.iterator().next().getDataPoints().get(0).getTimestamp()), (baseTs2 - ts), (baseTs2 / 60000) * 60000, points.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals("Invalid first entry:" + endTs, endTs - 60000, points.iterator().next().getDataPoints().get(points.iterator().next().getDataPoints().size() - 1).getTimestamp());
    engine.shutdown();
}

Example 7 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestStorageEngine method testAddAndReaderDataPoints.

@Test
public void testAddAndReaderDataPoints() throws Exception {
    File file = new File("target/db8/");
    if (file.exists()) {
        MiscUtils.delete(file);
    }
    engine.configure(conf, bgTasks);
    long curr = 1497720452566L;
    String dbName = "test";
    String measurementName = "cpu";
    String valueFieldName = "value";
    try {
        engine.writeDataPointWithLock(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, null, curr, 2.2 * 0), false);
        fail("Must reject the above datapoint due to missing tags");
    } catch (Exception e) {
    }
    List<Tag> tagd = Arrays.asList(Tag.newBuilder().setTagKey(dbName).setTagValue("1").build());
    for (int i = 1; i <= 3; i++) {
        engine.writeDataPointWithLock(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, tagd, curr + i, 2.2 * i), false);
    }
    assertEquals(1, engine.getAllMeasurementsForDb(dbName).size());
    Map<ByteString, FieldReaderIterator[]> queryReaders = engine.queryReaders(dbName, measurementName, Arrays.asList(valueFieldName), false, curr, curr + 3);
    int count = 0;
    for (Entry<ByteString, FieldReaderIterator[]> entry : queryReaders.entrySet()) {
        assertEquals(2, entry.getValue().length);
        while (true) {
            try {
                long[] extracted = FieldReaderIterator.extracted(entry.getValue());
                assertEquals(2.2 * (count + 1), Double.longBitsToDouble(extracted[0]), 0.01);
                count++;
            } catch (RejectException e) {
                break;
            }
        }
    }
    assertEquals(3, count);
    assertTrue(engine.checkIfExists(dbName, measurementName));
    try {
        engine.checkIfExists(dbName + "1");
    } catch (Exception e) {
    }
    engine.dropMeasurement(dbName, measurementName);
    assertEquals(0, engine.getAllMeasurementsForDb(dbName).size());
    engine.shutdown();
}

Example 8 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestStorageEngine method testQueryDataPoints.

@Test
public void testQueryDataPoints() throws IOException, ItemNotFoundException {
    List<Tag> tagd = Arrays.asList(Tag.newBuilder().setTagKey("test").setTagValue("e").build());
    MiscUtils.delete(new File("target/db15/"));
    HashMap<String, String> map = new HashMap<>();
    map.put("index.dir", "target/db15/index");
    map.put("data.dir", "target/db15/data");
    engine.configure(map, bgTasks);
    long ts = System.currentTimeMillis();
    Map<String, Measurement> db = engine.getOrCreateDatabase("test3", 24, map);
    assertEquals(0, db.size());
    engine.writeDataPointWithLock(MiscUtils.buildDataPoint("test3", "cpu", "value", tagd, ts, 1), false);
    engine.writeDataPointWithLock(MiscUtils.buildDataPoint("test3", "cpu", "value", tagd, ts + (400 * 60000), 4), false);
    assertEquals(1, engine.getOrCreateMeasurement("test3", "cpu").getSeriesKeys().size());
    List<SeriesOutput> queryDataPoints = null;
    try {
        queryDataPoints = engine.queryDataPoints("test3", "cpu", "value", ts, ts + (400 * 60000), null, null);
    } catch (Exception e) {
    }
    try {
        engine.queryDataPoints("test123", "cpu", "value", ts, ts + (400 * 60000), null, null);
    } catch (ItemNotFoundException e) {
    }
    try {
        engine.queryDataPoints("test3", "123cpu", "value", ts, ts + (400 * 60000), null, null);
    } catch (ItemNotFoundException e) {
    }
    assertTrue(!engine.isMeasurementFieldFP("test3", "cpu", "value"));
    try {
        engine.isMeasurementFieldFP("test3", "test", "test");
        fail("Measurement should not exist");
    } catch (Exception e) {
    }
    assertEquals(2, queryDataPoints.iterator().next().getDataPoints().size());
    assertEquals(ts, queryDataPoints.iterator().next().getDataPoints().get(0).getTimestamp());
    assertEquals(ts + (400 * 60000), queryDataPoints.iterator().next().getDataPoints().get(1).getTimestamp());
    try {
        engine.dropDatabase("test3");
    } catch (Exception e) {
    }
    assertEquals(0, engine.getOrCreateMeasurement("test3", "cpu").getSeriesKeys().size());
    engine.shutdown();
}

Example 9 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestStorageEngine method testAddAndReadDataPointsWithTagFilters.

@Test
public void testAddAndReadDataPointsWithTagFilters() throws Exception {
    engine.configure(conf, bgTasks);
    long curr = 1497720452566L;
    String dbName = "test";
    String measurementName = "cpu";
    String valueFieldName = "value";
    String tag = "host123123";
    for (int i = 1; i <= 3; i++) {
        List<Tag> tagd = Arrays.asList(Tag.newBuilder().setTagKey(tag).setTagValue(String.valueOf(i)).build(), Tag.newBuilder().setTagKey(tag).setTagValue(String.valueOf(i + 1)).build());
        engine.writeDataPointWithLock(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, tagd, curr + i, 2 * i), false);
    }
    assertEquals(1, engine.getAllMeasurementsForDb(dbName).size());
    SimpleTagFilter filter1 = new SimpleTagFilter(FilterType.EQUALS, "host123123", "1");
    SimpleTagFilter filter2 = new SimpleTagFilter(FilterType.EQUALS, "host123123", "2");
    List<SeriesOutput> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, curr, curr + 3, new ComplexTagFilter(ComplexFilterType.OR, Arrays.asList(filter1, filter2)), null, null);
    assertEquals(2, queryDataPoints.size());
    int i = 1;
    assertEquals(1, queryDataPoints.iterator().next().getDataPoints().size());
    queryDataPoints.sort(new Comparator<SeriesOutput>() {

        @Override
        public int compare(SeriesOutput o1, SeriesOutput o2) {
            return o1.getTags().toString().compareTo(o2.getTags().toString());
        }
    });
    for (SeriesOutput list : queryDataPoints) {
        for (DataPoint dataPoint : list.getDataPoints()) {
            assertEquals(curr + i, dataPoint.getTimestamp());
            i++;
        }
    }
    Set<String> tags = engine.getTagKeysForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList(tag)), tags);
    assertEquals(new HashSet<>(Arrays.asList("1", "2", "3", "4")), engine.getTagValuesForMeasurement(dbName, measurementName, tag));
    Set<String> fieldsForMeasurement = engine.getFieldsForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList(valueFieldName, Series.TS)), fieldsForMeasurement);
    try {
        engine.getTagKeysForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getTagKeysForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    engine.shutdown();
}

Example 10 with Tag

use of com.srotya.sidewinder.core.rpc.Tag in project sidewinder by srotya.

the class TestPersistentMeasurement method testHyperThreading.

@Test
public void testHyperThreading() throws Exception {
    measurement.configure(conf, engine, 4096, DBNAME, "m1", indexDir, dataDir, metadata, bgTaskPool);
    int nThreads = 20;
    ExecutorService es = Executors.newFixedThreadPool(nThreads, new BackgrounThreadFactory("tlinear2"));
    final List<Tag> tags = Arrays.asList(Tag.newBuilder().setTagKey("host").setTagValue("ll1.eew.wwe.com").build(), Tag.newBuilder().setTagKey("dc").setTagValue("2").build());
    final int LIMIT = 1000;
    final long t1 = 1497720452566L;
    for (int i = 0; i < nThreads * 2; i++) {
        final int th = i;
        es.submit(() -> {
            long t = t1 + th * 4096;
            for (int j = 0; j < LIMIT; j++) {
                try {
                    long timestamp = t + j * 1000;
                    measurement.addPointWithLocking(TestMeasurement.build("vf1", tags, timestamp, j), false);
                } catch (Exception e) {
                    e.printStackTrace();
                }
            }
        });
    }
    es.shutdown();
    es.awaitTermination(10, TimeUnit.SECONDS);
    measurement.configure(conf, engine, 4096, DBNAME, "m1", indexDir, dataDir, metadata, bgTaskPool);
    Series s = measurement.getOrCreateSeries(tags, false);
    List<DataPoint> dps = s.queryDataPoints(measurement, Arrays.asList("vf1"), t1 - 100, t1 + 1000_0000, null).get("vf1");
    assertEquals(LIMIT * nThreads * 2, dps.size(), 10);
    measurement.close();
}