Examples with CompressionInputStream org.apache.nifi.remote.io.CompressionInputStream used on opensource projects

Example 1 with CompressionInputStream

use of org.apache.nifi.remote.io.CompressionInputStream in project nifi by apache.

the class TestHttpClient method readIncomingPacket.

private static DataPacket readIncomingPacket(HttpServletRequest req) throws IOException {
    final StandardFlowFileCodec codec = new StandardFlowFileCodec();
    InputStream inputStream = req.getInputStream();
    if (Boolean.valueOf(req.getHeader(HttpHeaders.HANDSHAKE_PROPERTY_USE_COMPRESSION))) {
        inputStream = new CompressionInputStream(inputStream);
    }
    return codec.decode(inputStream);
}

Example 2 with CompressionInputStream

use of org.apache.nifi.remote.io.CompressionInputStream in project nifi by apache.

the class TestCompressionInputOutputStreams method testSendingMultipleFilesBackToBackOnSameStream.

@Test
public void testSendingMultipleFilesBackToBackOnSameStream() throws IOException {
    final String str = "The quick brown fox jumps over the lazy dog\r\n\n\n\r";
    final byte[] data = str.getBytes("UTF-8");
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final CompressionOutputStream cos = new CompressionOutputStream(baos, 8192);
    for (int i = 0; i < 512; i++) {
        cos.write(data);
        cos.flush();
    }
    cos.close();
    final CompressionOutputStream cos2 = new CompressionOutputStream(baos, 8192);
    for (int i = 0; i < 512; i++) {
        cos2.write(data);
        cos2.flush();
    }
    cos2.close();
    final byte[] data512;
    final StringBuilder sb = new StringBuilder();
    for (int i = 0; i < 512; i++) {
        sb.append(str);
    }
    data512 = sb.toString().getBytes("UTF-8");
    final byte[] compressedBytes = baos.toByteArray();
    final ByteArrayInputStream bais = new ByteArrayInputStream(compressedBytes);
    final CompressionInputStream cis = new CompressionInputStream(bais);
    final byte[] decompressed = readFully(cis);
    assertTrue(Arrays.equals(data512, decompressed));
    final CompressionInputStream cis2 = new CompressionInputStream(bais);
    final byte[] decompressed2 = readFully(cis2);
    assertTrue(Arrays.equals(data512, decompressed2));
}

Example 3 with CompressionInputStream

use of org.apache.nifi.remote.io.CompressionInputStream in project nifi by apache.

the class AbstractFlowFileServerProtocol method receiveFlowFiles.

@Override
public int receiveFlowFiles(final Peer peer, final ProcessContext context, final ProcessSession session, final FlowFileCodec codec) throws IOException, ProtocolException {
    if (!handshakeCompleted) {
        throw new IllegalStateException("Handshake has not been completed");
    }
    if (shutdown) {
        throw new IllegalStateException("Protocol is shutdown");
    }
    logger.debug("{} receiving FlowFiles from {}", this, peer);
    final CommunicationsSession commsSession = peer.getCommunicationsSession();
    final DataInputStream dis = new DataInputStream(commsSession.getInput().getInputStream());
    String remoteDn = commsSession.getUserDn();
    if (remoteDn == null) {
        remoteDn = "none";
    }
    final StopWatch stopWatch = new StopWatch(true);
    final CRC32 crc = new CRC32();
    // Peer has data. Otherwise, we would not have been called, because they would not have sent
    // a SEND_FLOWFILES request to use. Just decode the bytes into FlowFiles until peer says he's
    // finished sending data.
    final Set<FlowFile> flowFilesReceived = new HashSet<>();
    long bytesReceived = 0L;
    boolean continueTransaction = true;
    while (continueTransaction) {
        final long startNanos = System.nanoTime();
        final InputStream flowFileInputStream = handshakeProperties.isUseGzip() ? new CompressionInputStream(dis) : dis;
        final CheckedInputStream checkedInputStream = new CheckedInputStream(flowFileInputStream, crc);
        final DataPacket dataPacket = codec.decode(checkedInputStream);
        if (dataPacket == null) {
            logger.debug("{} Received null dataPacket indicating the end of transaction from {}", this, peer);
            break;
        }
        FlowFile flowFile = session.create();
        flowFile = session.importFrom(dataPacket.getData(), flowFile);
        flowFile = session.putAllAttributes(flowFile, dataPacket.getAttributes());
        if (handshakeProperties.isUseGzip()) {
            // Close CompressionInputStream to free acquired memory, without closing underlying stream.
            checkedInputStream.close();
        }
        final long transferNanos = System.nanoTime() - startNanos;
        final long transferMillis = TimeUnit.MILLISECONDS.convert(transferNanos, TimeUnit.NANOSECONDS);
        final String sourceSystemFlowFileUuid = dataPacket.getAttributes().get(CoreAttributes.UUID.key());
        final String host = StringUtils.isEmpty(peer.getHost()) ? "unknown" : peer.getHost();
        final String port = peer.getPort() <= 0 ? "unknown" : String.valueOf(peer.getPort());
        final Map<String, String> attributes = new HashMap<>(4);
        attributes.put(CoreAttributes.UUID.key(), UUID.randomUUID().toString());
        attributes.put(SiteToSiteAttributes.S2S_HOST.key(), host);
        attributes.put(SiteToSiteAttributes.S2S_ADDRESS.key(), host + ":" + port);
        flowFile = session.putAllAttributes(flowFile, attributes);
        final String transitUri = createTransitUri(peer, sourceSystemFlowFileUuid);
        session.getProvenanceReporter().receive(flowFile, transitUri, sourceSystemFlowFileUuid == null ? null : "urn:nifi:" + sourceSystemFlowFileUuid, "Remote Host=" + peer.getHost() + ", Remote DN=" + remoteDn, transferMillis);
        session.transfer(flowFile, Relationship.ANONYMOUS);
        flowFilesReceived.add(flowFile);
        bytesReceived += flowFile.getSize();
        final Response transactionResponse = readTransactionResponse(false, commsSession);
        switch(transactionResponse.getCode()) {
            case CONTINUE_TRANSACTION:
                logger.debug("{} Received ContinueTransaction indicator from {}", this, peer);
                break;
            case FINISH_TRANSACTION:
                logger.debug("{} Received FinishTransaction indicator from {}", this, peer);
                continueTransaction = false;
                break;
            case CANCEL_TRANSACTION:
                logger.info("{} Received CancelTransaction indicator from {} with explanation {}", this, peer, transactionResponse.getMessage());
                session.rollback();
                return 0;
            default:
                throw new ProtocolException("Received unexpected response from peer: when expecting Continue Transaction or Finish Transaction, received" + transactionResponse);
        }
    }
    // we received a FINISH_TRANSACTION indicator. Send back a CONFIRM_TRANSACTION message
    // to peer so that we can verify that the connection is still open. This is a two-phase commit,
    // which helps to prevent the chances of data duplication. Without doing this, we may commit the
    // session and then when we send the response back to the peer, the peer may have timed out and may not
    // be listening. As a result, it will re-send the data. By doing this two-phase commit, we narrow the
    // Critical Section involved in this transaction so that rather than the Critical Section being the
    // time window involved in the entire transaction, it is reduced to a simple round-trip conversation.
    logger.debug("{} Sending CONFIRM_TRANSACTION Response Code to {}", this, peer);
    String calculatedCRC = String.valueOf(crc.getValue());
    writeTransactionResponse(false, ResponseCode.CONFIRM_TRANSACTION, commsSession, calculatedCRC);
    FlowFileTransaction transaction = new FlowFileTransaction(session, context, stopWatch, bytesReceived, flowFilesReceived, calculatedCRC);
    return commitReceiveTransaction(peer, transaction);
}

Example 4 with CompressionInputStream

use of org.apache.nifi.remote.io.CompressionInputStream in project nifi by apache.

the class AbstractTransaction method receive.

@Override
public final DataPacket receive() throws IOException {
    try {
        try {
            if (state != TransactionState.DATA_EXCHANGED && state != TransactionState.TRANSACTION_STARTED) {
                throw new IllegalStateException("Cannot receive data from " + peer + " because Transaction State is " + state);
            }
            if (direction == TransferDirection.SEND) {
                throw new IllegalStateException("Attempting to receive data from " + peer + " but started a SEND Transaction");
            }
            // if we already know there's no data, just return null
            if (!dataAvailable) {
                return null;
            }
            // if we have already received a packet, check if another is available.
            if (transfers > 0) {
                // Determine if Peer will send us data or has no data to send us
                final Response dataAvailableCode = readTransactionResponse();
                switch(dataAvailableCode.getCode()) {
                    case CONTINUE_TRANSACTION:
                        logger.debug("{} {} Indicates Transaction should continue", this, peer);
                        this.dataAvailable = true;
                        break;
                    case FINISH_TRANSACTION:
                        logger.debug("{} {} Indicates Transaction should finish", this, peer);
                        this.dataAvailable = false;
                        break;
                    default:
                        throw new ProtocolException("Got unexpected response from " + peer + " when asking for data: " + dataAvailableCode);
                }
            }
            // if no data available, return null
            if (!dataAvailable) {
                return null;
            }
            logger.debug("{} Receiving data from {}", this, peer);
            final InputStream is = peer.getCommunicationsSession().getInput().getInputStream();
            final InputStream dataIn = compress ? new CompressionInputStream(is) : is;
            final DataPacket packet = codec.decode(new CheckedInputStream(dataIn, crc));
            if (packet == null) {
                this.dataAvailable = false;
            } else {
                transfers++;
                contentBytes += packet.getSize();
            }
            this.state = TransactionState.DATA_EXCHANGED;
            return packet;
        } catch (final IOException ioe) {
            throw new IOException("Failed to receive data from " + peer + " due to " + ioe, ioe);
        }
    } catch (final Exception e) {
        error();
        throw e;
    }
}

Example 5 with CompressionInputStream

use of org.apache.nifi.remote.io.CompressionInputStream in project nifi by apache.

the class TestCompressionInputOutputStreams method testDataLargerThanBufferWhileFlushing.

@Test
public void testDataLargerThanBufferWhileFlushing() throws IOException {
    final String str = "The quick brown fox jumps over the lazy dog\r\n\n\n\r";
    final byte[] data = str.getBytes("UTF-8");
    final StringBuilder sb = new StringBuilder();
    final byte[] data1024;
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    final CompressionOutputStream cos = new CompressionOutputStream(baos, 8192);
    for (int i = 0; i < 1024; i++) {
        cos.write(data);
        cos.flush();
        sb.append(str);
    }
    cos.close();
    data1024 = sb.toString().getBytes("UTF-8");
    final byte[] compressedBytes = baos.toByteArray();
    final CompressionInputStream cis = new CompressionInputStream(new ByteArrayInputStream(compressedBytes));
    final byte[] decompressed = readFully(cis);
    assertTrue(Arrays.equals(data1024, decompressed));
}