Examples with FlowFile org.apache.nifi.flowfile.FlowFile used on opensource projects

Example 31 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class AbstractFlowFileServerProtocol method commitTransferTransaction.

protected int commitTransferTransaction(Peer peer, FlowFileTransaction transaction) throws IOException {
    ProcessSession session = transaction.getSession();
    Set<FlowFile> flowFilesSent = transaction.getFlowFilesSent();
    // we've sent a FINISH_TRANSACTION. Now we'll wait for the peer to send a 'Confirm Transaction' response
    CommunicationsSession commsSession = peer.getCommunicationsSession();
    final Response transactionConfirmationResponse = readTransactionResponse(true, commsSession);
    if (transactionConfirmationResponse.getCode() == ResponseCode.CONFIRM_TRANSACTION) {
        // Confirm Checksum and echo back the confirmation.
        logger.debug("{} Received {}  from {}", this, transactionConfirmationResponse, peer);
        final String receivedCRC = transactionConfirmationResponse.getMessage();
        if (getVersionNegotiator().getVersion() > 3) {
            String calculatedCRC = transaction.getCalculatedCRC();
            if (!receivedCRC.equals(calculatedCRC)) {
                writeTransactionResponse(true, ResponseCode.BAD_CHECKSUM, commsSession);
                session.rollback();
                throw new IOException(this + " Sent data to peer " + peer + " but calculated CRC32 Checksum as " + calculatedCRC + " while peer calculated CRC32 Checksum as " + receivedCRC + "; canceling transaction and rolling back session");
            }
        }
        writeTransactionResponse(true, ResponseCode.CONFIRM_TRANSACTION, commsSession, "");
    } else {
        throw new ProtocolException("Expected to receive 'Confirm Transaction' response from peer " + peer + " but received " + transactionConfirmationResponse);
    }
    final String flowFileDescription = flowFilesSent.size() < 20 ? flowFilesSent.toString() : flowFilesSent.size() + " FlowFiles";
    final Response transactionResponse;
    try {
        transactionResponse = readTransactionResponse(true, commsSession);
    } catch (final IOException e) {
        logger.error("{} Failed to receive a response from {} when expecting a TransactionFinished Indicator." + " It is unknown whether or not the peer successfully received/processed the data." + " Therefore, {} will be rolled back, possibly resulting in data duplication of {}", this, peer, session, flowFileDescription);
        session.rollback();
        throw e;
    }
    logger.debug("{} received {} from {}", new Object[] { this, transactionResponse, peer });
    if (transactionResponse.getCode() == ResponseCode.TRANSACTION_FINISHED_BUT_DESTINATION_FULL) {
        peer.penalize(port.getIdentifier(), port.getYieldPeriod(TimeUnit.MILLISECONDS));
    } else if (transactionResponse.getCode() != ResponseCode.TRANSACTION_FINISHED) {
        throw new ProtocolException("After sending data, expected TRANSACTION_FINISHED response but got " + transactionResponse);
    }
    session.commit();
    StopWatch stopWatch = transaction.getStopWatch();
    long bytesSent = transaction.getBytesSent();
    stopWatch.stop();
    final String uploadDataRate = stopWatch.calculateDataRate(bytesSent);
    final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    final String dataSize = FormatUtils.formatDataSize(bytesSent);
    logger.info("{} Successfully sent {} ({}) to {} in {} milliseconds at a rate of {}", new Object[] { this, flowFileDescription, dataSize, peer, uploadMillis, uploadDataRate });
    return flowFilesSent.size();
}

Example 32 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class AbstractFlowFileServerProtocol method commitReceiveTransaction.

protected int commitReceiveTransaction(Peer peer, FlowFileTransaction transaction) throws IOException {
    CommunicationsSession commsSession = peer.getCommunicationsSession();
    ProcessSession session = transaction.getSession();
    final Response confirmTransactionResponse = readTransactionResponse(false, commsSession);
    logger.debug("{} Received {} from {}", this, confirmTransactionResponse, peer);
    switch(confirmTransactionResponse.getCode()) {
        case CONFIRM_TRANSACTION:
            break;
        case BAD_CHECKSUM:
            session.rollback();
            throw new IOException(this + " Received a BadChecksum response from peer " + peer);
        default:
            throw new ProtocolException(this + " Received unexpected Response Code from peer " + peer + " : " + confirmTransactionResponse + "; expected 'Confirm Transaction' Response Code");
    }
    // Commit the session so that we have persisted the data
    session.commit();
    if (transaction.getContext().getAvailableRelationships().isEmpty()) {
        // Confirm that we received the data and the peer can now discard it but that the peer should not
        // send any more data for a bit
        logger.debug("{} Sending TRANSACTION_FINISHED_BUT_DESTINATION_FULL to {}", this, peer);
        writeTransactionResponse(false, ResponseCode.TRANSACTION_FINISHED_BUT_DESTINATION_FULL, commsSession);
    } else {
        // Confirm that we received the data and the peer can now discard it
        logger.debug("{} Sending TRANSACTION_FINISHED to {}", this, peer);
        writeTransactionResponse(false, ResponseCode.TRANSACTION_FINISHED, commsSession);
    }
    Set<FlowFile> flowFilesReceived = transaction.getFlowFilesSent();
    long bytesReceived = transaction.getBytesSent();
    StopWatch stopWatch = transaction.getStopWatch();
    stopWatch.stop();
    final String flowFileDescription = flowFilesReceived.size() < 20 ? flowFilesReceived.toString() : flowFilesReceived.size() + " FlowFiles";
    final String uploadDataRate = stopWatch.calculateDataRate(bytesReceived);
    final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    final String dataSize = FormatUtils.formatDataSize(bytesReceived);
    logger.info("{} Successfully received {} ({}) from {} in {} milliseconds at a rate of {}", new Object[] { this, flowFileDescription, dataSize, peer, uploadMillis, uploadDataRate });
    return flowFilesReceived.size();
}

Example 33 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class TestStandardProcessSession method testReadFromInputStreamWithoutClosingThenRemove.

@Test
public void testReadFromInputStreamWithoutClosingThenRemove() throws IOException {
    FlowFile flowFile = session.create();
    flowFile = session.write(flowFile, new OutputStreamCallback() {

        @Override
        public void process(final OutputStream out) throws IOException {
            out.write("hello, world".getBytes());
        }
    });
    InputStream in = session.read(flowFile);
    final byte[] buffer = new byte[12];
    StreamUtils.fillBuffer(in, buffer);
    assertEquals("hello, world", new String(buffer));
    try {
        session.remove(flowFile);
        Assert.fail("Was able to remove FlowFile while an InputStream is open for it");
    } catch (final IllegalStateException e) {
    // expected
    }
    in.close();
    session.remove(flowFile);
    // This should generate a WARN log message. We can't really test this in a unit test but can verify manually.
    session.commit();
}

Example 34 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class TestStandardProcessSession method testWriteToOutputStream.

@Test
public void testWriteToOutputStream() throws IOException {
    final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder().addAttribute("uuid", "12345678-1234-1234-1234-123456789012").entryDate(System.currentTimeMillis()).size(12L).build();
    flowFileQueue.put(flowFileRecord);
    FlowFile flowFile = session.get();
    try (final OutputStream out = session.write(flowFile)) {
        out.write("hello, world".getBytes());
    }
    // Call putAllAttributes, because this will return to us the most recent version
    // of the FlowFile. In a Processor, we wouldn't need this, but for testing purposes
    // we need it in order to get the Content Claim.
    flowFile = session.putAllAttributes(flowFile, Collections.emptyMap());
    assertEquals(12L, flowFile.getSize());
    final byte[] buffer = new byte[(int) flowFile.getSize()];
    try (final InputStream in = session.read(flowFile)) {
        StreamUtils.fillBuffer(in, buffer);
    }
    assertEquals(new String(buffer), "hello, world");
}

Example 35 with FlowFile

use of org.apache.nifi.flowfile.FlowFile in project nifi by apache.

the class TestStandardProcessSession method testModifyContentWithOutputStreamCallbackHasCorrectSize.

@Test
public void testModifyContentWithOutputStreamCallbackHasCorrectSize() throws IOException {
    final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder().id(1000L).addAttribute("uuid", "12345678-1234-1234-1234-123456789012").entryDate(System.currentTimeMillis()).build();
    flowFileQueue.put(flowFileRecord);
    FlowFile original = session.get();
    assertNotNull(original);
    FlowFile child = session.write(original, out -> out.write("hello".getBytes()));
    session.transfer(child);
    session.commit();
    final FlowFileRecord onQueue = flowFileQueue.poll(Collections.emptySet());
    assertEquals(5, onQueue.getSize());
}