Examples with ByteBufferInputStream org.apache.druid.io.ByteBufferInputStream used on opensource projects

Example 1 with ByteBufferInputStream

use of org.apache.druid.io.ByteBufferInputStream in project druid by druid-io.

the class ByteBufferWriteOutBytes method asInputStream.

@Override
public InputStream asInputStream() throws IOException {
    checkOpen();
    Function<ByteBuffer, ByteSource> byteBufferToByteSource = buf -> new ByteSource() {

        @Override
        public InputStream openStream() {
            ByteBuffer inputBuf = buf.duplicate();
            inputBuf.flip();
            return new ByteBufferInputStream(inputBuf);
        }
    };
    return ByteSource.concat(buffers.stream().map(byteBufferToByteSource).collect(Collectors.toList())).openStream();
}

Example 2 with ByteBufferInputStream

use of org.apache.druid.io.ByteBufferInputStream in project druid by druid-io.

the class BloomKFilterTest method testBloomKFilterLong.

@Test
public void testBloomKFilterLong() throws IOException {
    BloomKFilter bf = new BloomKFilter(10000);
    ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
    BloomKFilter.serialize(bytesOut, bf);
    byte[] bfBytes = bytesOut.toByteArray();
    ByteBuffer buffer = ByteBuffer.wrap(bfBytes);
    long val = Long.MIN_VALUE;
    long val1 = 1;
    long val2 = 2;
    long val3 = Long.MAX_VALUE;
    BloomKFilter.addLong(buffer, val);
    BloomKFilter rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.testLong(val));
    Assert.assertFalse(rehydrated.testLong(val1));
    Assert.assertFalse(rehydrated.testLong(val2));
    Assert.assertFalse(rehydrated.testLong(val3));
    BloomKFilter.addLong(buffer, val1);
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.testLong(val));
    Assert.assertTrue(rehydrated.testLong(val1));
    Assert.assertFalse(rehydrated.testLong(val2));
    Assert.assertFalse(rehydrated.testLong(val3));
    BloomKFilter.addLong(buffer, val2);
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.testLong(val));
    Assert.assertTrue(rehydrated.testLong(val1));
    Assert.assertTrue(rehydrated.testLong(val2));
    Assert.assertFalse(rehydrated.testLong(val3));
    BloomKFilter.addLong(buffer, val3);
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.testLong(val));
    Assert.assertTrue(rehydrated.testLong(val1));
    Assert.assertTrue(rehydrated.testLong(val2));
    Assert.assertTrue(rehydrated.testLong(val3));
    int randVal = 0;
    for (int i = 0; i < COUNT; i++) {
        randVal = rand.nextInt();
        BloomKFilter.addLong(buffer, randVal);
    }
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    // last value should be present
    Assert.assertTrue(rehydrated.testLong(randVal));
    // most likely this value should not exist
    Assert.assertFalse(rehydrated.testLong(-120));
    Assert.assertEquals(7808, rehydrated.sizeInBytes());
}

Example 3 with ByteBufferInputStream

use of org.apache.druid.io.ByteBufferInputStream in project druid by druid-io.

the class BloomKFilterTest method testBloomKFilterByte.

@Test
public void testBloomKFilterByte() throws IOException {
    BloomKFilter bf = new BloomKFilter(10000);
    ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
    BloomKFilter.serialize(bytesOut, bf);
    byte[] bfBytes = bytesOut.toByteArray();
    ByteBuffer buffer = ByteBuffer.wrap(bfBytes);
    byte val = Byte.MIN_VALUE;
    byte val1 = 1;
    byte val2 = 2;
    byte val3 = Byte.MAX_VALUE;
    BloomKFilter.addLong(buffer, val);
    BloomKFilter rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.testLong(val));
    Assert.assertFalse(rehydrated.testLong(val1));
    Assert.assertFalse(rehydrated.testLong(val2));
    Assert.assertFalse(rehydrated.testLong(val3));
    BloomKFilter.addLong(buffer, val1);
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.testLong(val));
    Assert.assertTrue(rehydrated.testLong(val1));
    Assert.assertFalse(rehydrated.testLong(val2));
    Assert.assertFalse(rehydrated.testLong(val3));
    BloomKFilter.addLong(buffer, val2);
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.testLong(val));
    Assert.assertTrue(rehydrated.testLong(val1));
    Assert.assertTrue(rehydrated.testLong(val2));
    Assert.assertFalse(rehydrated.testLong(val3));
    BloomKFilter.addLong(buffer, val3);
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.testLong(val));
    Assert.assertTrue(rehydrated.testLong(val1));
    Assert.assertTrue(rehydrated.testLong(val2));
    Assert.assertTrue(rehydrated.testLong(val3));
    byte randVal = 0;
    for (int i = 0; i < COUNT; i++) {
        randVal = (byte) rand.nextInt(Byte.MAX_VALUE);
        BloomKFilter.addLong(buffer, randVal);
    }
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    // last value should be present
    Assert.assertTrue(rehydrated.testLong(randVal));
    // most likely this value should not exist
    Assert.assertFalse(rehydrated.testLong((byte) -120));
    Assert.assertEquals(7808, rehydrated.sizeInBytes());
}

Example 4 with ByteBufferInputStream

use of org.apache.druid.io.ByteBufferInputStream in project druid by druid-io.

the class BloomKFilterTest method testBloomKFilterBytes.

@Test
public void testBloomKFilterBytes() throws IOException {
    BloomKFilter bf = new BloomKFilter(10000);
    ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
    BloomKFilter.serialize(bytesOut, bf);
    byte[] bfBytes = bytesOut.toByteArray();
    ByteBuffer buffer = ByteBuffer.wrap(bfBytes);
    byte[] val = new byte[] { 1, 2, 3 };
    byte[] val1 = new byte[] { 1, 2, 3, 4 };
    byte[] val2 = new byte[] { 1, 2, 3, 4, 5 };
    byte[] val3 = new byte[] { 1, 2, 3, 4, 5, 6 };
    bf.add(val);
    BloomKFilter.add(buffer, val);
    BloomKFilter rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.test(val));
    Assert.assertFalse(rehydrated.test(val1));
    Assert.assertFalse(rehydrated.test(val2));
    Assert.assertFalse(rehydrated.test(val3));
    BloomKFilter.add(buffer, val1);
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.test(val));
    Assert.assertTrue(rehydrated.test(val1));
    Assert.assertFalse(rehydrated.test(val2));
    Assert.assertFalse(rehydrated.test(val3));
    BloomKFilter.add(buffer, val2);
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.test(val));
    Assert.assertTrue(rehydrated.test(val1));
    Assert.assertTrue(rehydrated.test(val2));
    Assert.assertFalse(rehydrated.test(val3));
    BloomKFilter.add(buffer, val3);
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.test(val));
    Assert.assertTrue(rehydrated.test(val1));
    Assert.assertTrue(rehydrated.test(val2));
    Assert.assertTrue(rehydrated.test(val3));
    byte[] randVal = new byte[COUNT];
    for (int i = 0; i < COUNT; i++) {
        rand.nextBytes(randVal);
        BloomKFilter.add(buffer, randVal);
    }
    // last value should be present
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    Assert.assertTrue(rehydrated.test(randVal));
    // most likely this value should not exist
    randVal[0] = 0;
    randVal[1] = 0;
    randVal[2] = 0;
    randVal[3] = 0;
    randVal[4] = 0;
    Assert.assertFalse(rehydrated.test(randVal));
    Assert.assertEquals(7808, rehydrated.sizeInBytes());
}

Example 5 with ByteBufferInputStream

use of org.apache.druid.io.ByteBufferInputStream in project druid by druid-io.

the class BloomKFilterTest method testBloomKFilterString.

@Test
public void testBloomKFilterString() throws IOException {
    BloomKFilter bf = new BloomKFilter(100000);
    ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
    BloomKFilter.serialize(bytesOut, bf);
    byte[] bfBytes = bytesOut.toByteArray();
    ByteBuffer buffer = ByteBuffer.wrap(bfBytes);
    String val = "bloo";
    String val1 = "bloom fil";
    String val2 = "bloom filter";
    String val3 = "cuckoo filter";
    BloomKFilter.addString(buffer, val);
    BloomKFilter rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.testString(val));
    Assert.assertFalse(rehydrated.testString(val1));
    Assert.assertFalse(rehydrated.testString(val2));
    Assert.assertFalse(rehydrated.testString(val3));
    BloomKFilter.addString(buffer, val1);
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.testString(val));
    Assert.assertTrue(rehydrated.testString(val1));
    Assert.assertFalse(rehydrated.testString(val2));
    Assert.assertFalse(rehydrated.testString(val3));
    BloomKFilter.addString(buffer, val2);
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.testString(val));
    Assert.assertTrue(rehydrated.testString(val1));
    Assert.assertTrue(rehydrated.testString(val2));
    Assert.assertFalse(rehydrated.testString(val3));
    BloomKFilter.addString(buffer, val3);
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    buffer.position(0);
    Assert.assertTrue(rehydrated.testString(val));
    Assert.assertTrue(rehydrated.testString(val1));
    Assert.assertTrue(rehydrated.testString(val2));
    Assert.assertTrue(rehydrated.testString(val3));
    long randVal = 0;
    for (int i = 0; i < COUNT; i++) {
        randVal = rand.nextLong();
        BloomKFilter.addString(buffer, Long.toString(randVal));
    }
    rehydrated = BloomKFilter.deserialize(new ByteBufferInputStream(buffer));
    // last value should be present
    Assert.assertTrue(rehydrated.testString(Long.toString(randVal)));
    // most likely this value should not exist
    Assert.assertFalse(rehydrated.testString(Long.toString(-120)));
    Assert.assertEquals(77952, rehydrated.sizeInBytes());
}