Examples with LZ4BlockOutputStream net.jpountz.lz4.LZ4BlockOutputStream used on opensource projects

Example 1 with LZ4BlockOutputStream

use of net.jpountz.lz4.LZ4BlockOutputStream in project sonarqube by SonarSource.

the class FileSourceDto method encodeSourceData.

/**
   * Serialize and compress protobuf message {@link org.sonar.db.protobuf.DbFileSources.Data}
   * in the column BINARY_DATA.
   */
public static byte[] encodeSourceData(DbFileSources.Data data) {
    ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
    LZ4BlockOutputStream compressedOutput = new LZ4BlockOutputStream(byteOutput);
    try {
        data.writeTo(compressedOutput);
        compressedOutput.close();
        return byteOutput.toByteArray();
    } catch (IOException e) {
        throw new IllegalStateException("Fail to serialize and compress source data", e);
    } finally {
        IOUtils.closeQuietly(compressedOutput);
    }
}

Example 2 with LZ4BlockOutputStream

use of net.jpountz.lz4.LZ4BlockOutputStream in project cassandra by apache.

the class OutboundTcpConnection method connect.

@SuppressWarnings("resource")
private boolean connect() {
    logger.debug("Attempting to connect to {}", poolReference.endPoint());
    long start = System.nanoTime();
    long timeout = TimeUnit.MILLISECONDS.toNanos(DatabaseDescriptor.getRpcTimeout());
    while (System.nanoTime() - start < timeout) {
        targetVersion = MessagingService.instance().getVersion(poolReference.endPoint());
        try {
            socket = poolReference.newSocket();
            socket.setKeepAlive(true);
            if (isLocalDC(poolReference.endPoint())) {
                socket.setTcpNoDelay(INTRADC_TCP_NODELAY);
            } else {
                socket.setTcpNoDelay(DatabaseDescriptor.getInterDCTcpNoDelay());
            }
            if (DatabaseDescriptor.getInternodeSendBufferSize() > 0) {
                try {
                    socket.setSendBufferSize(DatabaseDescriptor.getInternodeSendBufferSize());
                } catch (SocketException se) {
                    logger.warn("Failed to set send buffer size on internode socket.", se);
                }
            }
            // SocketChannel may be null when using SSL
            WritableByteChannel ch = socket.getChannel();
            out = new BufferedDataOutputStreamPlus(ch != null ? ch : Channels.newChannel(socket.getOutputStream()), BUFFER_SIZE);
            out.writeInt(MessagingService.PROTOCOL_MAGIC);
            writeHeader(out, targetVersion, shouldCompressConnection());
            out.flush();
            DataInputStream in = new DataInputStream(socket.getInputStream());
            int maxTargetVersion = handshakeVersion(in);
            if (maxTargetVersion == NO_VERSION) {
                // no version is returned, so disconnect an try again
                logger.trace("Target max version is {}; no version information yet, will retry", maxTargetVersion);
                disconnect();
                continue;
            } else {
                MessagingService.instance().setVersion(poolReference.endPoint(), maxTargetVersion);
            }
            if (targetVersion > maxTargetVersion) {
                logger.trace("Target max version is {}; will reconnect with that version", maxTargetVersion);
                try {
                    if (DatabaseDescriptor.getSeeds().contains(poolReference.endPoint()))
                        logger.warn("Seed gossip version is {}; will not connect with that version", maxTargetVersion);
                } catch (Throwable e) {
                    // If invalid yaml has been added to the config since startup, getSeeds() will throw an AssertionError
                    // Additionally, third party seed providers may throw exceptions if network is flakey
                    // Regardless of what's thrown, we must catch it, disconnect, and try again
                    JVMStabilityInspector.inspectThrowable(e);
                    logger.warn("Configuration error prevented outbound connection: {}", e.getLocalizedMessage());
                } finally {
                    disconnect();
                    return false;
                }
            }
            if (targetVersion < maxTargetVersion && targetVersion < MessagingService.current_version) {
                logger.trace("Detected higher max version {} (using {}); will reconnect when queued messages are done", maxTargetVersion, targetVersion);
                softCloseSocket();
            }
            out.writeInt(MessagingService.current_version);
            CompactEndpointSerializationHelper.serialize(FBUtilities.getBroadcastAddress(), out);
            if (shouldCompressConnection()) {
                out.flush();
                logger.trace("Upgrading OutputStream to {} to be compressed", poolReference.endPoint());
                // TODO: custom LZ4 OS that supports BB write methods
                LZ4Compressor compressor = LZ4Factory.fastestInstance().fastCompressor();
                Checksum checksum = XXHashFactory.fastestInstance().newStreamingHash32(LZ4_HASH_SEED).asChecksum();
                out = new WrappedDataOutputStreamPlus(new LZ4BlockOutputStream(socket.getOutputStream(), // 16k block size
                1 << 14, compressor, checksum, // no async flushing
                true));
            }
            logger.debug("Done connecting to {}", poolReference.endPoint());
            return true;
        } catch (SSLHandshakeException e) {
            logger.error("SSL handshake error for outbound connection to " + socket, e);
            socket = null;
            // SSL errors won't be recoverable within timeout period so we'll just abort
            return false;
        } catch (IOException e) {
            socket = null;
            logger.debug("Unable to connect to {}", poolReference.endPoint(), e);
            Uninterruptibles.sleepUninterruptibly(OPEN_RETRY_DELAY, TimeUnit.MILLISECONDS);
        }
    }
    return false;
}

Example 3 with LZ4BlockOutputStream

use of net.jpountz.lz4.LZ4BlockOutputStream in project druid by druid-io.

the class SpillingGrouper method spill.

private void spill() throws IOException {
    final File outFile;
    try (final LimitedTemporaryStorage.LimitedOutputStream out = temporaryStorage.createFile();
        final LZ4BlockOutputStream compressedOut = new LZ4BlockOutputStream(out);
        final JsonGenerator jsonGenerator = spillMapper.getFactory().createGenerator(compressedOut)) {
        outFile = out.getFile();
        final Iterator<Entry<KeyType>> it = grouper.iterator(true);
        while (it.hasNext()) {
            BaseQuery.checkInterrupted();
            jsonGenerator.writeObject(it.next());
        }
    }
    files.add(outFile);
    grouper.reset();
}

Example 4 with LZ4BlockOutputStream

use of net.jpountz.lz4.LZ4BlockOutputStream in project netty by netty.

the class Lz4FrameDecoderTest method compress.

@Override
protected byte[] compress(byte[] data) throws Exception {
    ByteArrayOutputStream os = new ByteArrayOutputStream();
    int size = MAX_BLOCK_SIZE + 1;
    LZ4BlockOutputStream lz4Os = new LZ4BlockOutputStream(os, rand.nextInt(size - MIN_BLOCK_SIZE) + MIN_BLOCK_SIZE);
    lz4Os.write(data);
    lz4Os.close();
    return os.toByteArray();
}

Example 5 with LZ4BlockOutputStream

use of net.jpountz.lz4.LZ4BlockOutputStream in project sonarqube by SonarSource.

the class FileSourceDto method encodeTestData.

/**
   * Serialize and compress protobuf message {@link org.sonar.db.protobuf.DbFileSources.Data}
   * in the column BINARY_DATA.
   */
public static byte[] encodeTestData(List<DbFileSources.Test> tests) {
    ByteArrayOutputStream byteOutput = new ByteArrayOutputStream();
    LZ4BlockOutputStream compressedOutput = new LZ4BlockOutputStream(byteOutput);
    try {
        for (DbFileSources.Test test : tests) {
            test.writeDelimitedTo(compressedOutput);
        }
        compressedOutput.close();
        return byteOutput.toByteArray();
    } catch (IOException e) {
        throw new IllegalStateException("Fail to serialize and compress source tests", e);
    } finally {
        IOUtils.closeQuietly(compressedOutput);
    }
}