Examples with DataPoint com.srotya.sidewinder.core.storage.DataPoint used on opensource projects

Example 16 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestByzantineReadWrite method testBootstrapDiskRecovery.

@Test
public void testBootstrapDiskRecovery() throws IOException, InterruptedException {
    ByteBuffer buf = ByteBuffer.allocateDirect(1024 * 1024 * 10);
    ByzantineWriter writer = new ByzantineWriter();
    writer.configure(new HashMap<>(), buf, true, startOffset, true);
    long ots = System.currentTimeMillis();
    writer.setHeaderTimestamp(ots);
    int limit = 1_000_000;
    for (int i = 0; i < limit; i++) {
        writer.addValue(ots + i * 1000, i);
    }
    Reader reader = writer.getReader();
    for (int i = 0; i < limit; i++) {
        DataPoint pair = reader.readPair();
        assertEquals("Iteration:" + i, ots + i * 1000, pair.getTimestamp());
        assertEquals("Iteration:" + i, i, pair.getLongValue());
    }
    ByteBuffer rawBytes = writer.getRawBytes();
    try {
        writer = new ByzantineWriter();
        writer.configure(new HashMap<>(), buf, false, startOffset, true);
        writer.bootstrap(rawBytes);
        reader = writer.getReader();
        for (int i = 0; i < limit; i++) {
            DataPoint pair = reader.readPair();
            assertEquals(ots + i * 1000, pair.getTimestamp());
            assertEquals(i, pair.getLongValue());
        }
        reader = writer.getReader();
        for (int i = 0; i < limit; i++) {
            DataPoint pair = reader.readPair();
            assertEquals(ots + i * 1000, pair.getTimestamp());
            assertEquals(i, pair.getLongValue());
        }
    } catch (Exception e) {
        throw e;
    }
}

Example 17 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestGorillaCompression method testCompressUncompressFloating.

@Test
public void testCompressUncompressFloating() throws IOException {
    ByteBuffer buf = ByteBuffer.allocate(1024);
    GorillaWriter writer = new GorillaWriter();
    writer.configure(new HashMap<>(), buf, true, 0, false);
    long ts = System.currentTimeMillis();
    writer.setHeaderTimestamp(ts);
    for (int i = 0; i < 100; i++) {
        writer.addValue(ts + i * 100, i * 1.1);
    }
    writer.makeReadOnly();
    Reader reader = writer.getReader();
    assertEquals(100, reader.getPairCount());
    for (int i = 0; i < 100; i++) {
        DataPoint pair = reader.readPair();
        assertEquals(ts + i * 100, pair.getTimestamp());
        assertEquals(i * 1.1, Double.longBitsToDouble(pair.getLongValue()), 0.01);
    }
}

Example 18 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestGorillaCompression method testRecovery.

@Test
public void testRecovery() throws IOException {
    ByteBuffer buf = ByteBuffer.allocate(1024);
    GorillaWriter writer = new GorillaWriter();
    writer.configure(new HashMap<>(), buf, true, 0, false);
    long ts = System.currentTimeMillis();
    writer.setHeaderTimestamp(ts);
    for (int i = 0; i < 100; i++) {
        writer.addValue(ts + i * 100, i * 1.1);
    }
    writer.makeReadOnly();
    ByteBuffer rawBytes = writer.getRawBytes();
    writer = new GorillaWriter();
    writer.configure(new HashMap<>(), rawBytes, false, 0, false);
    Reader reader = writer.getReader();
    assertEquals(100, reader.getPairCount());
    for (int i = 0; i < 100; i++) {
        DataPoint pair = reader.readPair();
        assertEquals(ts + i * 100, pair.getTimestamp());
        assertEquals(i * 1.1, Double.longBitsToDouble(pair.getLongValue()), 0.01);
    }
}

Example 19 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestDiskStorageEngine method testAddAndReadDataPointsWithTagFilters.

@Test
public void testAddAndReadDataPointsWithTagFilters() throws Exception {
    DiskStorageEngine engine = new DiskStorageEngine();
    MiscUtils.delete(new File("target/db5/"));
    HashMap<String, String> map = new HashMap<>();
    map.put("metadata.dir", "target/db5/mdq");
    map.put("index.dir", "target/db5/index");
    map.put("data.dir", "target/db5/data");
    map.put(StorageEngine.PERSISTENCE_DISK, "true");
    engine.configure(map, bgTasks);
    long curr = 1497720452566L;
    String dbName = "test";
    String measurementName = "cpu";
    String valueFieldName = "value";
    String tag = "host123123";
    for (int i = 1; i <= 3; i++) {
        engine.writeDataPoint(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, Arrays.asList(tag + "=" + i, tag + "=" + (i + 1)), curr + i, 2 * i));
    }
    assertEquals(1, engine.getAllMeasurementsForDb(dbName).size());
    SimpleTagFilter filter1 = new SimpleTagFilter(FilterType.EQUALS, "host123123", "1");
    SimpleTagFilter filter2 = new SimpleTagFilter(FilterType.EQUALS, "host123123", "2");
    List<Series> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, curr, curr + 3, new ComplexTagFilter(ComplexFilterType.OR, Arrays.asList(filter1, filter2)), null, null);
    assertEquals(2, queryDataPoints.size());
    int i = 1;
    assertEquals(1, queryDataPoints.iterator().next().getDataPoints().size());
    queryDataPoints.sort(new Comparator<Series>() {

        @Override
        public int compare(Series o1, Series o2) {
            return o1.getTags().toString().compareTo(o2.getTags().toString());
        }
    });
    for (Series list : queryDataPoints) {
        for (DataPoint dataPoint : list.getDataPoints()) {
            assertEquals(curr + i, dataPoint.getTimestamp());
            i++;
        }
    }
    Set<String> tags = engine.getTagKeysForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList(tag)), tags);
    assertEquals(new HashSet<>(Arrays.asList("1", "2", "3", "4")), engine.getTagValuesForMeasurement(dbName, measurementName, tag));
    Set<String> fieldsForMeasurement = engine.getFieldsForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList(valueFieldName)), fieldsForMeasurement);
    try {
        engine.getTagKeysForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getTagKeysForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    engine.disconnect();
}

Example 20 with DataPoint

use of com.srotya.sidewinder.core.storage.DataPoint in project sidewinder by srotya.

the class TestDiskStorageEngine method testAddAndReadDataPoints.

@Test
public void testAddAndReadDataPoints() throws Exception {
    DiskStorageEngine engine = new DiskStorageEngine();
    MiscUtils.delete(new File("target/db19/"));
    HashMap<String, String> map = new HashMap<>();
    map.put("metadata.dir", "target/db19/mdq");
    map.put("index.dir", "target/db19/index");
    map.put("data.dir", "target/db19/data");
    map.put(StorageEngine.PERSISTENCE_DISK, "true");
    engine.configure(map, bgTasks);
    long curr = System.currentTimeMillis();
    String dbName = "test";
    String measurementName = "cpu";
    String valueFieldName = "value";
    try {
        engine.writeDataPoint(dbName, measurementName, valueFieldName, null, curr, 2 * 0);
        fail("Must reject the above datapoint due to missing tags");
    } catch (Exception e) {
    }
    String tag = "host=123123";
    for (int i = 1; i <= 3; i++) {
        engine.writeDataPoint(MiscUtils.buildDataPoint(dbName, measurementName, valueFieldName, Arrays.asList(tag), curr + i, 2 * i));
    }
    assertEquals(1, engine.getAllMeasurementsForDb(dbName).size());
    List<Series> queryDataPoints = engine.queryDataPoints(dbName, measurementName, valueFieldName, curr, curr + 3, null);
    assertEquals(1, queryDataPoints.size());
    int i = 1;
    assertEquals(3, queryDataPoints.iterator().next().getDataPoints().size());
    List<List<DataPoint>> output = new ArrayList<>();
    for (Series series : queryDataPoints) {
        output.add(series.getDataPoints());
    }
    for (List<DataPoint> list : output) {
        for (DataPoint dataPoint : list) {
            assertEquals(curr + i, dataPoint.getTimestamp());
            i++;
        }
    }
    Set<String> tags = engine.getTagKeysForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList("host")), tags);
    Set<String> fieldsForMeasurement = engine.getFieldsForMeasurement(dbName, measurementName);
    assertEquals(new HashSet<>(Arrays.asList(valueFieldName)), fieldsForMeasurement);
    try {
        engine.getTagKeysForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getTagKeysForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName + "1", measurementName);
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    try {
        engine.getFieldsForMeasurement(dbName, measurementName + "1");
        fail("This measurement should not exist");
    } catch (Exception e) {
    }
    engine.disconnect();
}