Examples with ResourceHolder io.druid.collections.ResourceHolder used on opensource projects

Example 1 with ResourceHolder

use of io.druid.collections.ResourceHolder in project druid by druid-io.

the class CompressedIntsIndexedSupplier method fromIntBuffer.

public static CompressedIntsIndexedSupplier fromIntBuffer(final IntBuffer buffer, final int chunkFactor, final ByteOrder byteOrder, CompressedObjectStrategy.CompressionStrategy compression) {
    Preconditions.checkArgument(chunkFactor <= MAX_INTS_IN_BUFFER, "Chunks must be <= 64k bytes. chunkFactor was[%s]", chunkFactor);
    return new CompressedIntsIndexedSupplier(buffer.remaining(), chunkFactor, GenericIndexed.fromIterable(new Iterable<ResourceHolder<IntBuffer>>() {

        @Override
        public Iterator<ResourceHolder<IntBuffer>> iterator() {
            return new Iterator<ResourceHolder<IntBuffer>>() {

                IntBuffer myBuffer = buffer.asReadOnlyBuffer();

                @Override
                public boolean hasNext() {
                    return myBuffer.hasRemaining();
                }

                @Override
                public ResourceHolder<IntBuffer> next() {
                    IntBuffer retVal = myBuffer.asReadOnlyBuffer();
                    if (chunkFactor < myBuffer.remaining()) {
                        retVal.limit(retVal.position() + chunkFactor);
                    }
                    myBuffer.position(myBuffer.position() + retVal.remaining());
                    return StupidResourceHolder.create(retVal);
                }

                @Override
                public void remove() {
                    throw new UnsupportedOperationException();
                }
            };
        }
    }, CompressedIntBufferObjectStrategy.getBufferForOrder(byteOrder, compression, chunkFactor)), compression);
}

Example 2 with ResourceHolder

use of io.druid.collections.ResourceHolder in project druid by druid-io.

the class CompressedObjectStrategy method fromByteBuffer.

@Override
public ResourceHolder<T> fromByteBuffer(ByteBuffer buffer, int numBytes) {
    final ResourceHolder<ByteBuffer> bufHolder = CompressedPools.getByteBuf(order);
    final ByteBuffer buf = bufHolder.get();
    buf.position(0);
    buf.limit(buf.capacity());
    decompress(buffer, numBytes, buf);
    return new ResourceHolder<T>() {

        @Override
        public T get() {
            return converter.convert(buf);
        }

        @Override
        public void close() {
            bufHolder.close();
        }
    };
}

Example 3 with ResourceHolder

use of io.druid.collections.ResourceHolder in project druid by druid-io.

the class CompressedVSizeIntsIndexedSupplier method fromList.

public static CompressedVSizeIntsIndexedSupplier fromList(final List<Integer> list, final int maxValue, final int chunkFactor, final ByteOrder byteOrder, CompressedObjectStrategy.CompressionStrategy compression) {
    final int numBytes = VSizeIndexedInts.getNumBytesForMax(maxValue);
    final int chunkBytes = chunkFactor * numBytes + bufferPadding(numBytes);
    Preconditions.checkArgument(chunkFactor <= maxIntsInBufferForBytes(numBytes), "Chunks must be <= 64k bytes. chunkFactor was[%s]", chunkFactor);
    return new CompressedVSizeIntsIndexedSupplier(list.size(), chunkFactor, numBytes, GenericIndexed.fromIterable(new Iterable<ResourceHolder<ByteBuffer>>() {

        @Override
        public Iterator<ResourceHolder<ByteBuffer>> iterator() {
            return new Iterator<ResourceHolder<ByteBuffer>>() {

                int position = 0;

                @Override
                public boolean hasNext() {
                    return position < list.size();
                }

                @Override
                public ResourceHolder<ByteBuffer> next() {
                    ByteBuffer retVal = ByteBuffer.allocate(chunkBytes).order(byteOrder);
                    if (chunkFactor > list.size() - position) {
                        retVal.limit((list.size() - position) * numBytes);
                    } else {
                        retVal.limit(chunkFactor * numBytes);
                    }
                    final List<Integer> ints = list.subList(position, position + retVal.remaining() / numBytes);
                    final ByteBuffer buf = ByteBuffer.allocate(Ints.BYTES).order(byteOrder);
                    final boolean bigEndian = byteOrder.equals(ByteOrder.BIG_ENDIAN);
                    for (int value : ints) {
                        buf.putInt(0, value);
                        if (bigEndian) {
                            retVal.put(buf.array(), Ints.BYTES - numBytes, numBytes);
                        } else {
                            retVal.put(buf.array(), 0, numBytes);
                        }
                    }
                    retVal.rewind();
                    position += retVal.remaining() / numBytes;
                    return StupidResourceHolder.create(retVal);
                }

                @Override
                public void remove() {
                    throw new UnsupportedOperationException();
                }
            };
        }
    }, CompressedByteBufferObjectStrategy.getBufferForOrder(byteOrder, compression, chunkBytes)), compression);
}

Example 4 with ResourceHolder

use of io.druid.collections.ResourceHolder in project druid by druid-io.

the class MemcachedCacheBenchmark method setUp.

@Override
protected void setUp() throws Exception {
    SerializingTranscoder transcoder = new SerializingTranscoder(// 50 MB
    50 * 1024 * 1024);
    // disable compression
    transcoder.setCompressionThreshold(Integer.MAX_VALUE);
    client = new MemcachedClient(new ConnectionFactoryBuilder().setProtocol(ConnectionFactoryBuilder.Protocol.BINARY).setHashAlg(DefaultHashAlgorithm.FNV1A_64_HASH).setLocatorType(ConnectionFactoryBuilder.Locator.CONSISTENT).setDaemon(true).setFailureMode(FailureMode.Retry).setTranscoder(transcoder).setShouldOptimize(true).build(), AddrUtil.getAddresses(hosts));
    cache = new MemcachedCache(Suppliers.<ResourceHolder<MemcachedClientIF>>ofInstance(StupidResourceHolder.create(client)), new MemcachedCacheConfig() {

        @Override
        public String getMemcachedPrefix() {
            return "druid-memcached-benchmark";
        }

        @Override
        public int getTimeout() {
            return 30000;
        }

        @Override
        public int getExpiration() {
            return 3600;
        }
    }, MemcachedCacheTest.NOOP_MONITOR);
    randBytes = new byte[objectSize * 1024];
    new Random(0).nextBytes(randBytes);
}

Example 5 with ResourceHolder

use of io.druid.collections.ResourceHolder in project druid by druid-io.

the class CompressedIntsIndexedSupplier method fromList.

public static CompressedIntsIndexedSupplier fromList(final List<Integer> list, final int chunkFactor, final ByteOrder byteOrder, CompressedObjectStrategy.CompressionStrategy compression) {
    Preconditions.checkArgument(chunkFactor <= MAX_INTS_IN_BUFFER, "Chunks must be <= 64k bytes. chunkFactor was[%s]", chunkFactor);
    return new CompressedIntsIndexedSupplier(list.size(), chunkFactor, GenericIndexed.fromIterable(new Iterable<ResourceHolder<IntBuffer>>() {

        @Override
        public Iterator<ResourceHolder<IntBuffer>> iterator() {
            return new Iterator<ResourceHolder<IntBuffer>>() {

                int position = 0;

                @Override
                public boolean hasNext() {
                    return position < list.size();
                }

                @Override
                public ResourceHolder<IntBuffer> next() {
                    IntBuffer retVal = IntBuffer.allocate(chunkFactor);
                    if (chunkFactor > list.size() - position) {
                        retVal.limit(list.size() - position);
                    }
                    final List<Integer> ints = list.subList(position, position + retVal.remaining());
                    for (int value : ints) {
                        retVal.put(value);
                    }
                    retVal.rewind();
                    position += retVal.remaining();
                    return StupidResourceHolder.create(retVal);
                }

                @Override
                public void remove() {
                    throw new UnsupportedOperationException();
                }
            };
        }
    }, CompressedIntBufferObjectStrategy.getBufferForOrder(byteOrder, compression, chunkFactor)), compression);
}