Examples with CompressionOutputStream org.apache.nifi.remote.io.CompressionOutputStream used on opensource projects

Example 1 with CompressionOutputStream

use of org.apache.nifi.remote.io.CompressionOutputStream in project nifi by apache.

the class AbstractTransaction method send.

@Override
public final void send(final DataPacket dataPacket) throws IOException {
    try {
        try {
            if (state != TransactionState.DATA_EXCHANGED && state != TransactionState.TRANSACTION_STARTED) {
                throw new IllegalStateException("Cannot send data to " + peer + " because Transaction State is " + state);
            }
            if (direction == TransferDirection.RECEIVE) {
                throw new IllegalStateException("Attempting to send data to " + peer + " but started a RECEIVE Transaction");
            }
            if (transfers > 0) {
                writeTransactionResponse(ResponseCode.CONTINUE_TRANSACTION);
            }
            logger.debug("{} Sending data to {}", this, peer);
            final OutputStream os = peer.getCommunicationsSession().getOutput().getOutputStream();
            final OutputStream dataOut = compress ? new CompressionOutputStream(os) : os;
            final OutputStream out = new CheckedOutputStream(dataOut, crc);
            codec.encode(dataPacket, out);
            // (CompressionOutputStream will not close the underlying stream when it's closed)
            if (compress) {
                out.close();
            }
            transfers++;
            contentBytes += dataPacket.getSize();
            this.state = TransactionState.DATA_EXCHANGED;
        } catch (final IOException ioe) {
            throw new IOException("Failed to send data to " + peer + " due to " + ioe, ioe);
        }
    } catch (final Exception e) {
        error();
        throw e;
    }
}

Example 2 with CompressionOutputStream

use of org.apache.nifi.remote.io.CompressionOutputStream in project nifi by apache.

the class TestCompressionInputOutputStreams method testSendingMultipleFilesBackToBackOnSameStream.

@Test
public void testSendingMultipleFilesBackToBackOnSameStream() throws IOException {
    final String str = "The quick brown fox jumps over the lazy dog\r\n\n\n\r";
    final byte[] data = str.getBytes("UTF-8");
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final CompressionOutputStream cos = new CompressionOutputStream(baos, 8192);
    for (int i = 0; i < 512; i++) {
        cos.write(data);
        cos.flush();
    }
    cos.close();
    final CompressionOutputStream cos2 = new CompressionOutputStream(baos, 8192);
    for (int i = 0; i < 512; i++) {
        cos2.write(data);
        cos2.flush();
    }
    cos2.close();
    final byte[] data512;
    final StringBuilder sb = new StringBuilder();
    for (int i = 0; i < 512; i++) {
        sb.append(str);
    }
    data512 = sb.toString().getBytes("UTF-8");
    final byte[] compressedBytes = baos.toByteArray();
    final ByteArrayInputStream bais = new ByteArrayInputStream(compressedBytes);
    final CompressionInputStream cis = new CompressionInputStream(bais);
    final byte[] decompressed = readFully(cis);
    assertTrue(Arrays.equals(data512, decompressed));
    final CompressionInputStream cis2 = new CompressionInputStream(bais);
    final byte[] decompressed2 = readFully(cis2);
    assertTrue(Arrays.equals(data512, decompressed2));
}

Example 3 with CompressionOutputStream

use of org.apache.nifi.remote.io.CompressionOutputStream in project nifi by apache.

the class AbstractFlowFileServerProtocol method transferFlowFiles.

@Override
public int transferFlowFiles(final Peer peer, final ProcessContext context, final ProcessSession session, final FlowFileCodec codec) throws IOException, ProtocolException {
    if (!handshakeCompleted) {
        throw new IllegalStateException("Handshake has not been completed");
    }
    if (shutdown) {
        throw new IllegalStateException("Protocol is shutdown");
    }
    logger.debug("{} Sending FlowFiles to {}", this, peer);
    final CommunicationsSession commsSession = peer.getCommunicationsSession();
    String remoteDn = commsSession.getUserDn();
    if (remoteDn == null) {
        remoteDn = "none";
    }
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        // we have no data to send. Notify the peer.
        logger.debug("{} No data to send to {}", this, peer);
        writeTransactionResponse(true, ResponseCode.NO_MORE_DATA, commsSession);
        return 0;
    }
    // we have data to send.
    logger.debug("{} Data is available to send to {}", this, peer);
    writeTransactionResponse(true, ResponseCode.MORE_DATA, commsSession);
    final StopWatch stopWatch = new StopWatch(true);
    long bytesSent = 0L;
    final Set<FlowFile> flowFilesSent = new HashSet<>();
    final CRC32 crc = new CRC32();
    // send data until we reach some batch size
    boolean continueTransaction = true;
    final long startNanos = System.nanoTime();
    String calculatedCRC = "";
    OutputStream os = new DataOutputStream(commsSession.getOutput().getOutputStream());
    while (continueTransaction) {
        final boolean useGzip = handshakeProperties.isUseGzip();
        final OutputStream flowFileOutputStream = useGzip ? new CompressionOutputStream(os) : os;
        logger.debug("{} Sending {} to {}", new Object[] { this, flowFile, peer });
        final CheckedOutputStream checkedOutputStream = new CheckedOutputStream(flowFileOutputStream, crc);
        final StopWatch transferWatch = new StopWatch(true);
        final FlowFile toSend = flowFile;
        session.read(flowFile, new InputStreamCallback() {

            @Override
            public void process(final InputStream in) throws IOException {
                final DataPacket dataPacket = new StandardDataPacket(toSend.getAttributes(), in, toSend.getSize());
                codec.encode(dataPacket, checkedOutputStream);
            }
        });
        final long transmissionMillis = transferWatch.getElapsed(TimeUnit.MILLISECONDS);
        // (CompressionOutputStream will not close the underlying stream when it's closed)
        if (useGzip) {
            checkedOutputStream.close();
        }
        flowFilesSent.add(flowFile);
        bytesSent += flowFile.getSize();
        final String transitUri = createTransitUri(peer, flowFile.getAttribute(CoreAttributes.UUID.key()));
        session.getProvenanceReporter().send(flowFile, transitUri, "Remote Host=" + peer.getHost() + ", Remote DN=" + remoteDn, transmissionMillis, false);
        session.remove(flowFile);
        // determine if we should check for more data on queue.
        final long sendingNanos = System.nanoTime() - startNanos;
        boolean poll = true;
        double batchDurationNanos = handshakeProperties.getBatchDurationNanos();
        if (sendingNanos >= batchDurationNanos && batchDurationNanos > 0L) {
            poll = false;
        }
        double batchBytes = handshakeProperties.getBatchBytes();
        if (bytesSent >= batchBytes && batchBytes > 0L) {
            poll = false;
        }
        double batchCount = handshakeProperties.getBatchCount();
        if (flowFilesSent.size() >= batchCount && batchCount > 0) {
            poll = false;
        }
        if (batchDurationNanos == 0 && batchBytes == 0 && batchCount == 0) {
            poll = (sendingNanos < DEFAULT_BATCH_NANOS);
        }
        if (poll) {
            // we've not elapsed the requested sending duration, so get more data.
            flowFile = session.get();
        } else {
            flowFile = null;
        }
        continueTransaction = (flowFile != null);
        if (continueTransaction) {
            logger.debug("{} Sending ContinueTransaction indicator to {}", this, peer);
            writeTransactionResponse(true, ResponseCode.CONTINUE_TRANSACTION, commsSession);
        } else {
            logger.debug("{} Sending FinishTransaction indicator to {}", this, peer);
            writeTransactionResponse(true, ResponseCode.FINISH_TRANSACTION, commsSession);
            calculatedCRC = String.valueOf(checkedOutputStream.getChecksum().getValue());
        }
    }
    FlowFileTransaction transaction = new FlowFileTransaction(session, context, stopWatch, bytesSent, flowFilesSent, calculatedCRC);
    return commitTransferTransaction(peer, transaction);
}

Example 4 with CompressionOutputStream

use of org.apache.nifi.remote.io.CompressionOutputStream in project nifi by apache.

the class TestCompressionInputOutputStreams method testDataLargerThanBufferWhileFlushing.

@Test
public void testDataLargerThanBufferWhileFlushing() throws IOException {
    final String str = "The quick brown fox jumps over the lazy dog\r\n\n\n\r";
    final byte[] data = str.getBytes("UTF-8");
    final StringBuilder sb = new StringBuilder();
    final byte[] data1024;
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final CompressionOutputStream cos = new CompressionOutputStream(baos, 8192);
    for (int i = 0; i < 1024; i++) {
        cos.write(data);
        cos.flush();
        sb.append(str);
    }
    cos.close();
    data1024 = sb.toString().getBytes("UTF-8");
    final byte[] compressedBytes = baos.toByteArray();
    final CompressionInputStream cis = new CompressionInputStream(new ByteArrayInputStream(compressedBytes));
    final byte[] decompressed = readFully(cis);
    assertTrue(Arrays.equals(data1024, decompressed));
}

Example 5 with CompressionOutputStream

use of org.apache.nifi.remote.io.CompressionOutputStream in project nifi by apache.

the class TestCompressionInputOutputStreams method testSimple.

@Test
public void testSimple() throws IOException {
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final byte[] data = "Hello, World!".getBytes("UTF-8");
    final CompressionOutputStream cos = new CompressionOutputStream(baos);
    cos.write(data);
    cos.flush();
    cos.close();
    final byte[] compressedBytes = baos.toByteArray();
    final CompressionInputStream cis = new CompressionInputStream(new ByteArrayInputStream(compressedBytes));
    final byte[] decompressed = readFully(cis);
    assertTrue(Arrays.equals(data, decompressed));
}