Examples with WritableByteChannel java.nio.channels.WritableByteChannel used on opensource projects

Example 66 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project druid by druid-io.

the class CompressedVSizeIndexedV3WriterTest method checkSerializedSizeAndData.

private void checkSerializedSizeAndData(int offsetChunkFactor, int valueChunkFactor) throws Exception {
    FileSmoosher smoosher = new FileSmoosher(FileUtils.getTempDirectory());
    final IOPeon ioPeon = new TmpFileIOPeon();
    final IndexedMultivalue<IndexedInts> indexedMultivalue;
    try {
        int maxValue = vals.size() > 0 ? getMaxValue(vals) : 0;
        CompressedIntsIndexedWriter offsetWriter = new CompressedIntsIndexedWriter(ioPeon, "offset", offsetChunkFactor, byteOrder, compressionStrategy);
        CompressedVSizeIntsIndexedWriter valueWriter = new CompressedVSizeIntsIndexedWriter(ioPeon, "value", maxValue, valueChunkFactor, byteOrder, compressionStrategy);
        CompressedVSizeIndexedV3Writer writer = new CompressedVSizeIndexedV3Writer(offsetWriter, valueWriter);
        CompressedVSizeIndexedV3Supplier supplierFromIterable = CompressedVSizeIndexedV3Supplier.fromIterable(Iterables.transform(vals, new Function<int[], IndexedInts>() {

            @Nullable
            @Override
            public IndexedInts apply(@Nullable final int[] input) {
                return ArrayBasedIndexedInts.of(input);
            }
        }), offsetChunkFactor, maxValue, byteOrder, compressionStrategy);
        writer.open();
        for (int[] val : vals) {
            writer.add(val);
        }
        writer.close();
        long writtenLength = writer.getSerializedSize();
        final WritableByteChannel outputChannel = Channels.newChannel(ioPeon.makeOutputStream("output"));
        writer.writeToChannel(outputChannel, smoosher);
        outputChannel.close();
        smoosher.close();
        assertEquals(writtenLength, supplierFromIterable.getSerializedSize());
        // read from ByteBuffer and check values
        CompressedVSizeIndexedV3Supplier supplierFromByteBuffer = CompressedVSizeIndexedV3Supplier.fromByteBuffer(ByteBuffer.wrap(IOUtils.toByteArray(ioPeon.makeInputStream("output"))), byteOrder, null);
        indexedMultivalue = supplierFromByteBuffer.get();
        assertEquals(indexedMultivalue.size(), vals.size());
        for (int i = 0; i < vals.size(); ++i) {
            IndexedInts subVals = indexedMultivalue.get(i);
            assertEquals(subVals.size(), vals.get(i).length);
            for (int j = 0; j < subVals.size(); ++j) {
                assertEquals(subVals.get(j), vals.get(i)[j]);
            }
        }
        CloseQuietly.close(indexedMultivalue);
    } finally {
        ioPeon.close();
    }
}

Example 67 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project druid by druid-io.

the class CompressedVSizeIntsIndexedWriterTest method checkSerializedSizeAndData.

private void checkSerializedSizeAndData(int chunkSize) throws Exception {
    FileSmoosher smoosher = new FileSmoosher(FileUtils.getTempDirectory());
    CompressedVSizeIntsIndexedWriter writer = new CompressedVSizeIntsIndexedWriter(ioPeon, "test", vals.length > 0 ? Ints.max(vals) : 0, chunkSize, byteOrder, compressionStrategy);
    CompressedVSizeIntsIndexedSupplier supplierFromList = CompressedVSizeIntsIndexedSupplier.fromList(Ints.asList(vals), vals.length > 0 ? Ints.max(vals) : 0, chunkSize, byteOrder, compressionStrategy);
    writer.open();
    for (int val : vals) {
        writer.add(val);
    }
    writer.close();
    long writtenLength = writer.getSerializedSize();
    final WritableByteChannel outputChannel = Channels.newChannel(ioPeon.makeOutputStream("output"));
    writer.writeToChannel(outputChannel, smoosher);
    outputChannel.close();
    smoosher.close();
    assertEquals(writtenLength, supplierFromList.getSerializedSize());
    // read from ByteBuffer and check values
    CompressedVSizeIntsIndexedSupplier supplierFromByteBuffer = CompressedVSizeIntsIndexedSupplier.fromByteBuffer(ByteBuffer.wrap(IOUtils.toByteArray(ioPeon.makeInputStream("output"))), byteOrder, null);
    IndexedInts indexedInts = supplierFromByteBuffer.get();
    for (int i = 0; i < vals.length; ++i) {
        assertEquals(vals[i], indexedInts.get(i));
    }
    CloseQuietly.close(indexedInts);
}

Example 68 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project druid by druid-io.

the class VSizeIndexedIntsWriterTest method checkSerializedSizeAndData.

private void checkSerializedSizeAndData() throws Exception {
    int maxValue = vals.length == 0 ? 0 : Ints.max(vals);
    VSizeIndexedIntsWriter writer = new VSizeIndexedIntsWriter(ioPeon, "test", maxValue);
    VSizeIndexedInts intsFromList = VSizeIndexedInts.fromList(Ints.asList(vals), maxValue);
    writer.open();
    for (int val : vals) {
        writer.add(val);
    }
    writer.close();
    long writtenLength = writer.getSerializedSize();
    final WritableByteChannel outputChannel = Channels.newChannel(ioPeon.makeOutputStream("output"));
    writer.writeToChannel(outputChannel, null);
    outputChannel.close();
    assertEquals(writtenLength, intsFromList.getSerializedSize());
    // read from ByteBuffer and check values
    VSizeIndexedInts intsFromByteBuffer = VSizeIndexedInts.readFromByteBuffer(ByteBuffer.wrap(IOUtils.toByteArray(ioPeon.makeInputStream("output"))));
    assertEquals(vals.length, intsFromByteBuffer.size());
    for (int i = 0; i < vals.length; ++i) {
        assertEquals(vals[i], intsFromByteBuffer.get(i));
    }
}

Example 69 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project druid by druid-io.

the class GenericIndexedTest method serializeAndDeserialize.

private GenericIndexed<String> serializeAndDeserialize(GenericIndexed<String> indexed) throws IOException {
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final WritableByteChannel channel = Channels.newChannel(baos);
    indexed.writeToChannel(channel);
    channel.close();
    final ByteBuffer byteBuffer = ByteBuffer.wrap(baos.toByteArray());
    Assert.assertEquals(indexed.getSerializedSize(), byteBuffer.remaining());
    GenericIndexed<String> deserialized = GenericIndexed.read(byteBuffer, GenericIndexed.STRING_STRATEGY);
    Assert.assertEquals(0, byteBuffer.remaining());
    return deserialized;
}

Example 70 with WritableByteChannel

use of java.nio.channels.WritableByteChannel in project cassandra by apache.

the class OutboundTcpConnection method connect.

@SuppressWarnings("resource")
private boolean connect() {
    logger.debug("Attempting to connect to {}", poolReference.endPoint());
    long start = System.nanoTime();
    long timeout = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getRpcTimeout());
    while (System.nanoTime() - start < timeout) {
        targetVersion = MessagingService.instance().getVersion(poolReference.endPoint());
        try {
            socket = poolReference.newSocket();
            socket.setKeepAlive(true);
            if (isLocalDC(poolReference.endPoint())) {
                socket.setTcpNoDelay(INTRADC_TCP_NODELAY);
            } else {
                socket.setTcpNoDelay(DatabaseDescriptor.getInterDCTcpNoDelay());
            }
            if (DatabaseDescriptor.getInternodeSendBufferSize() > 0) {
                try {
                    socket.setSendBufferSize(DatabaseDescriptor.getInternodeSendBufferSize());
                } catch (SocketException se) {
                    logger.warn("Failed to set send buffer size on internode socket.", se);
                }
            }
            // SocketChannel may be null when using SSL
            WritableByteChannel ch = socket.getChannel();
            out = new BufferedDataOutputStreamPlus(ch != null ? ch : Channels.newChannel(socket.getOutputStream()), BUFFER_SIZE);
            out.writeInt(MessagingService.PROTOCOL_MAGIC);
            writeHeader(out, targetVersion, shouldCompressConnection());
            out.flush();
            DataInputStream in = new DataInputStream(socket.getInputStream());
            int maxTargetVersion = handshakeVersion(in);
            if (maxTargetVersion == NO_VERSION) {
                // no version is returned, so disconnect an try again
                logger.trace("Target max version is {}; no version information yet, will retry", maxTargetVersion);
                disconnect();
                continue;
            } else {
                MessagingService.instance().setVersion(poolReference.endPoint(), maxTargetVersion);
            }
            if (targetVersion > maxTargetVersion) {
                logger.trace("Target max version is {}; will reconnect with that version", maxTargetVersion);
                try {
                    if (DatabaseDescriptor.getSeeds().contains(poolReference.endPoint()))
                        logger.warn("Seed gossip version is {}; will not connect with that version", maxTargetVersion);
                } catch (Throwable e) {
                    // If invalid yaml has been added to the config since startup, getSeeds() will throw an AssertionError
                    // Additionally, third party seed providers may throw exceptions if network is flakey
                    // Regardless of what's thrown, we must catch it, disconnect, and try again
                    JVMStabilityInspector.inspectThrowable(e);
                    logger.warn("Configuration error prevented outbound connection: {}", e.getLocalizedMessage());
                } finally {
                    disconnect();
                    return false;
                }
            }
            if (targetVersion < maxTargetVersion && targetVersion < MessagingService.current_version) {
                logger.trace("Detected higher max version {} (using {}); will reconnect when queued messages are done", maxTargetVersion, targetVersion);
                softCloseSocket();
            }
            out.writeInt(MessagingService.current_version);
            CompactEndpointSerializationHelper.serialize(FBUtilities.getBroadcastAddress(), out);
            if (shouldCompressConnection()) {
                out.flush();
                logger.trace("Upgrading OutputStream to {} to be compressed", poolReference.endPoint());
                // TODO: custom LZ4 OS that supports BB write methods
                LZ4Compressor compressor = LZ4Factory.fastestInstance().fastCompressor();
                Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(LZ4_HASH_SEED).asChecksum();
                out = new WrappedDataOutputStreamPlus(new LZ4BlockOutputStream(socket.getOutputStream(), // 16k block size
                1 << 14, compressor, checksum, // no async flushing
                true));
            }
            logger.debug("Done connecting to {}", poolReference.endPoint());
            return true;
        } catch (SSLHandshakeException e) {
            logger.error("SSL handshake error for outbound connection to " + socket, e);
            socket = null;
            // SSL errors won't be recoverable within timeout period so we'll just abort
            return false;
        } catch (IOException e) {
            socket = null;
            logger.debug("Unable to connect to {}", poolReference.endPoint(), e);
            Uninterruptibles.sleepUninterruptibly(OPEN_RETRY_DELAY, TimeUnit.MILLISECONDS);
        }
    }
    return false;
}