Examples with Reader com.srotya.sidewinder.core.storage.compression.Reader used on opensource projects

Example 1 with Reader

use of com.srotya.sidewinder.core.storage.compression.Reader in project sidewinder by srotya.

the class TestByzantineReadWrite method testPredicateFilter.

@Test
public void testPredicateFilter() throws IOException {
    ByteBuffer buf = ByteBuffer.allocateDirect(1024 * 1024 * 10);
    ByzantineWriter writer = new ByzantineWriter();
    writer.configure(new HashMap<>(), buf, true, startOffset, true);
    long ots = System.currentTimeMillis();
    writer.setHeaderTimestamp(ots);
    int limit = 1_000_000;
    for (int i = 0; i < limit; i++) {
        writer.addValue(ots + i * 1000, i);
    }
    Reader reader = writer.getReader();
    reader.setTimePredicate(new GreaterThanEqualsPredicate(ots + 1000));
    int c = 0;
    assertEquals(limit, reader.getPairCount());
    try {
        for (int i = 0; i < limit; i++) {
            DataPoint pair = reader.readPair();
            if (pair != null) {
                assertEquals(ots + i * 1000, pair.getTimestamp());
                assertEquals(i, pair.getLongValue());
                c++;
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
    assertEquals(limit - 1, c);
    reader = writer.getReader();
    reader.setValuePredicate(new GreaterThanEqualsPredicate(1000));
    c = 0;
    assertEquals(limit, reader.getPairCount());
    try {
        for (int i = 0; i < limit; i++) {
            DataPoint pair = reader.readPair();
            if (pair != null) {
                assertEquals(ots + i * 1000, pair.getTimestamp());
                assertEquals(i, pair.getLongValue());
                c++;
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
    try {
        reader.readPair();
        fail("Must throw end of stream exception");
    } catch (RejectException e) {
    }
    try {
        reader.read();
        fail("Must throw end of stream exception");
    } catch (RejectException e) {
    }
    assertEquals(limit - 1000, c);
    assertEquals(limit, reader.getCounter());
    reader = writer.getReader();
    reader.setTimePredicate(new GreaterThanEqualsPredicate(ots + 1000));
    c = 0;
    assertEquals(limit, reader.getPairCount());
    try {
        for (int i = 0; i < limit; i++) {
            long[] pair = reader.read();
            if (pair != null) {
                assertEquals(ots + i * 1000, pair[0]);
                assertEquals(i, pair[1]);
                c++;
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
    assertEquals(limit - 1, c);
    reader = writer.getReader();
    reader.setValuePredicate(new GreaterThanEqualsPredicate(1000));
    c = 0;
    assertEquals(limit, reader.getPairCount());
    try {
        for (int i = 0; i < limit; i++) {
            long[] pair = reader.read();
            if (pair != null) {
                assertEquals(ots + i * 1000, pair[0]);
                assertEquals(i, pair[1]);
                c++;
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        throw e;
    }
    assertEquals(limit - 1000, c);
}

Example 2 with Reader

use of com.srotya.sidewinder.core.storage.compression.Reader in project sidewinder by srotya.

the class TestByzantineReadWrite method testReadWriteDataPoints.

@Test
public void testReadWriteDataPoints() throws IOException {
    ByteBuffer buf = ByteBuffer.allocateDirect(1024);
    Writer writer = new ByzantineWriter();
    writer.configure(new HashMap<>(), buf, true, startOffset, false);
    long ts = System.currentTimeMillis();
    writer.setHeaderTimestamp(ts);
    for (int i = 0; i < 100; i++) {
        writer.addValue(ts + i * 10, i);
    }
    Reader reader = writer.getReader();
    for (int i = 0; i < 100; i++) {
        DataPoint pair = reader.readPair();
        assertEquals(ts + i * 10, pair.getTimestamp());
        assertEquals(i, pair.getLongValue());
    }
    for (int i = 0; i < 100; i++) {
        DataPoint dp = new DataPoint(1000 + ts + i * 10, i);
        writer.write(dp);
    }
    reader = writer.getReader();
    for (int i = 0; i < 200; i++) {
        DataPoint pair = reader.readPair();
        assertEquals(ts + i * 10, pair.getTimestamp());
        assertEquals(i % 100, pair.getLongValue());
    }
    System.out.println("Compression Ratio:" + writer.getCompressionRatio());
}

Example 3 with Reader

use of com.srotya.sidewinder.core.storage.compression.Reader in project sidewinder by srotya.

the class TestByzantineReadWrite method testBootstrapDiskRecovery.

@Test
public void testBootstrapDiskRecovery() throws IOException, InterruptedException {
    ByteBuffer buf = ByteBuffer.allocateDirect(1024 * 1024 * 10);
    ByzantineWriter writer = new ByzantineWriter();
    writer.configure(new HashMap<>(), buf, true, startOffset, true);
    long ots = System.currentTimeMillis();
    writer.setHeaderTimestamp(ots);
    int limit = 1_000_000;
    for (int i = 0; i < limit; i++) {
        writer.addValue(ots + i * 1000, i);
    }
    Reader reader = writer.getReader();
    for (int i = 0; i < limit; i++) {
        DataPoint pair = reader.readPair();
        assertEquals("Iteration:" + i, ots + i * 1000, pair.getTimestamp());
        assertEquals("Iteration:" + i, i, pair.getLongValue());
    }
    ByteBuffer rawBytes = writer.getRawBytes();
    try {
        writer = new ByzantineWriter();
        writer.configure(new HashMap<>(), buf, false, startOffset, true);
        writer.bootstrap(rawBytes);
        reader = writer.getReader();
        for (int i = 0; i < limit; i++) {
            DataPoint pair = reader.readPair();
            assertEquals(ots + i * 1000, pair.getTimestamp());
            assertEquals(i, pair.getLongValue());
        }
        reader = writer.getReader();
        for (int i = 0; i < limit; i++) {
            DataPoint pair = reader.readPair();
            assertEquals(ots + i * 1000, pair.getTimestamp());
            assertEquals(i, pair.getLongValue());
        }
    } catch (Exception e) {
        throw e;
    }
}

Example 4 with Reader

use of com.srotya.sidewinder.core.storage.compression.Reader in project sidewinder by srotya.

the class TestByzantineReadWrite method testDiskWrites.

/*
	 * 100M Read/Writer performance test
	 * 
	 * Gorilla Write time:5631 Gorilla Read time:3505 data size:450000219
	 * 
	 * DoD Write time:100x Byzantine data size: 40x Byzantine
	 * 
	 * Byzantine Write time:2543 Byzantine Read time:476 data size:300787360
	 * 
	 * Disk Byzantine Write time:2723 Disk Byzantine Read time:476
	 * 
	 */
@Test
public void testDiskWrites() throws IOException, InterruptedException {
    ByteBuffer buf = ByteBuffer.allocateDirect(1024 * 1024 * 10);
    ByzantineWriter writer = new ByzantineWriter();
    writer.configure(new HashMap<>(), buf, true, startOffset, true);
    long ts = System.currentTimeMillis();
    writer.setHeaderTimestamp(ts);
    int limit = 1_000_000;
    for (int i = 0; i < limit; i++) {
        writer.addValue(ts + i * 1000, i);
    }
    ts = (System.currentTimeMillis() - ts);
    System.out.println("Byzantine Write time:" + ts + " data size:" + writer.getBuf().position());
    ts = System.currentTimeMillis();
    int i = 0;
    try {
        Reader reader = writer.getReader();
        for (i = 0; i < limit; i++) {
            reader.readPair();
        }
        ts = (System.currentTimeMillis() - ts);
        System.out.println("Byzantine Read time:" + ts);
    } catch (Exception e) {
        e.printStackTrace();
        System.err.println("EOF:" + i);
        throw e;
    }
}

Example 5 with Reader

use of com.srotya.sidewinder.core.storage.compression.Reader in project sidewinder by srotya.

the class TestGorillaCompression method testCompressUncompressFloating.

@Test
public void testCompressUncompressFloating() throws IOException {
    ByteBuffer buf = ByteBuffer.allocate(1024);
    GorillaWriter writer = new GorillaWriter();
    writer.configure(new HashMap<>(), buf, true, 0, false);
    long ts = System.currentTimeMillis();
    writer.setHeaderTimestamp(ts);
    for (int i = 0; i < 100; i++) {
        writer.addValue(ts + i * 100, i * 1.1);
    }
    writer.makeReadOnly();
    Reader reader = writer.getReader();
    assertEquals(100, reader.getPairCount());
    for (int i = 0; i < 100; i++) {
        DataPoint pair = reader.readPair();
        assertEquals(ts + i * 100, pair.getTimestamp());
        assertEquals(i * 1.1, Double.longBitsToDouble(pair.getLongValue()), 0.01);
    }
}