Examples with ProcessSession org.apache.nifi.processor.ProcessSession used on opensource projects

Example 1 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class StandardRootGroupPort method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSessionFactory sessionFactory) {
    final FlowFileRequest flowFileRequest;
    try {
        flowFileRequest = requestQueue.poll(100, TimeUnit.MILLISECONDS);
    } catch (final InterruptedException ie) {
        return;
    }
    if (flowFileRequest == null) {
        return;
    }
    flowFileRequest.setServiceBegin();
    requestLock.lock();
    try {
        if (shutdown) {
            final CommunicationsSession commsSession = flowFileRequest.getPeer().getCommunicationsSession();
            if (commsSession != null) {
                commsSession.interrupt();
            }
        }
        activeRequests.add(flowFileRequest);
    } finally {
        requestLock.unlock();
    }
    final ProcessSession session = sessionFactory.createSession();
    try {
        onTrigger(context, session, flowFileRequest);
    // we leave the session open, because we send it back to the caller of #receiveFlowFile or #transmitFlowFile,
    // so that they can perform appropriate actions to commit or rollback the transaction.
    } catch (final TransmissionDisabledException e) {
        session.rollback();
    } catch (final Exception e) {
        logger.error("{} Failed to process data due to {}", new Object[] { this, e });
        if (logger.isDebugEnabled()) {
            logger.error("", e);
        }
        session.rollback();
    } finally {
        requestLock.lock();
        try {
            activeRequests.remove(flowFileRequest);
        } finally {
            requestLock.unlock();
        }
    }
}

Example 2 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class AbstractFlowFileServerProtocol method commitTransferTransaction.

protected int commitTransferTransaction(Peer peer, FlowFileTransaction transaction) throws IOException {
    ProcessSession session = transaction.getSession();
    Set<FlowFile> flowFilesSent = transaction.getFlowFilesSent();
    // we've sent a FINISH_TRANSACTION. Now we'll wait for the peer to send a 'Confirm Transaction' response
    CommunicationsSession commsSession = peer.getCommunicationsSession();
    final Response transactionConfirmationResponse = readTransactionResponse(true, commsSession);
    if (transactionConfirmationResponse.getCode() == ResponseCode.CONFIRM_TRANSACTION) {
        // Confirm Checksum and echo back the confirmation.
        logger.debug("{} Received {}  from {}", this, transactionConfirmationResponse, peer);
        final String receivedCRC = transactionConfirmationResponse.getMessage();
        if (getVersionNegotiator().getVersion() > 3) {
            String calculatedCRC = transaction.getCalculatedCRC();
            if (!receivedCRC.equals(calculatedCRC)) {
                writeTransactionResponse(true, ResponseCode.BAD_CHECKSUM, commsSession);
                session.rollback();
                throw new IOException(this + " Sent data to peer " + peer + " but calculated CRC32 Checksum as " + calculatedCRC + " while peer calculated CRC32 Checksum as " + receivedCRC + "; canceling transaction and rolling back session");
            }
        }
        writeTransactionResponse(true, ResponseCode.CONFIRM_TRANSACTION, commsSession, "");
    } else {
        throw new ProtocolException("Expected to receive 'Confirm Transaction' response from peer " + peer + " but received " + transactionConfirmationResponse);
    }
    final String flowFileDescription = flowFilesSent.size() < 20 ? flowFilesSent.toString() : flowFilesSent.size() + " FlowFiles";
    final Response transactionResponse;
    try {
        transactionResponse = readTransactionResponse(true, commsSession);
    } catch (final IOException e) {
        logger.error("{} Failed to receive a response from {} when expecting a TransactionFinished Indicator." + " It is unknown whether or not the peer successfully received/processed the data." + " Therefore, {} will be rolled back, possibly resulting in data duplication of {}", this, peer, session, flowFileDescription);
        session.rollback();
        throw e;
    }
    logger.debug("{} received {} from {}", new Object[] { this, transactionResponse, peer });
    if (transactionResponse.getCode() == ResponseCode.TRANSACTION_FINISHED_BUT_DESTINATION_FULL) {
        peer.penalize(port.getIdentifier(), port.getYieldPeriod(TimeUnit.MILLISECONDS));
    } else if (transactionResponse.getCode() != ResponseCode.TRANSACTION_FINISHED) {
        throw new ProtocolException("After sending data, expected TRANSACTION_FINISHED response but got " + transactionResponse);
    }
    session.commit();
    StopWatch stopWatch = transaction.getStopWatch();
    long bytesSent = transaction.getBytesSent();
    stopWatch.stop();
    final String uploadDataRate = stopWatch.calculateDataRate(bytesSent);
    final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    final String dataSize = FormatUtils.formatDataSize(bytesSent);
    logger.info("{} Successfully sent {} ({}) to {} in {} milliseconds at a rate of {}", new Object[] { this, flowFileDescription, dataSize, peer, uploadMillis, uploadDataRate });
    return flowFilesSent.size();
}

Example 3 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class AbstractFlowFileServerProtocol method commitReceiveTransaction.

protected int commitReceiveTransaction(Peer peer, FlowFileTransaction transaction) throws IOException {
    CommunicationsSession commsSession = peer.getCommunicationsSession();
    ProcessSession session = transaction.getSession();
    final Response confirmTransactionResponse = readTransactionResponse(false, commsSession);
    logger.debug("{} Received {} from {}", this, confirmTransactionResponse, peer);
    switch(confirmTransactionResponse.getCode()) {
        case CONFIRM_TRANSACTION:
            break;
        case BAD_CHECKSUM:
            session.rollback();
            throw new IOException(this + " Received a BadChecksum response from peer " + peer);
        default:
            throw new ProtocolException(this + " Received unexpected Response Code from peer " + peer + " : " + confirmTransactionResponse + "; expected 'Confirm Transaction' Response Code");
    }
    // Commit the session so that we have persisted the data
    session.commit();
    if (transaction.getContext().getAvailableRelationships().isEmpty()) {
        // Confirm that we received the data and the peer can now discard it but that the peer should not
        // send any more data for a bit
        logger.debug("{} Sending TRANSACTION_FINISHED_BUT_DESTINATION_FULL to {}", this, peer);
        writeTransactionResponse(false, ResponseCode.TRANSACTION_FINISHED_BUT_DESTINATION_FULL, commsSession);
    } else {
        // Confirm that we received the data and the peer can now discard it
        logger.debug("{} Sending TRANSACTION_FINISHED to {}", this, peer);
        writeTransactionResponse(false, ResponseCode.TRANSACTION_FINISHED, commsSession);
    }
    Set<FlowFile> flowFilesReceived = transaction.getFlowFilesSent();
    long bytesReceived = transaction.getBytesSent();
    StopWatch stopWatch = transaction.getStopWatch();
    stopWatch.stop();
    final String flowFileDescription = flowFilesReceived.size() < 20 ? flowFilesReceived.toString() : flowFilesReceived.size() + " FlowFiles";
    final String uploadDataRate = stopWatch.calculateDataRate(bytesReceived);
    final long uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    final String dataSize = FormatUtils.formatDataSize(bytesReceived);
    logger.info("{} Successfully received {} ({}) from {} in {} milliseconds at a rate of {}", new Object[] { this, flowFileDescription, dataSize, peer, uploadMillis, uploadDataRate });
    return flowFilesReceived.size();
}

Example 4 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class TestHttpRemoteSiteListener method testNormalTransactionProgress.

@Test
public void testNormalTransactionProgress() {
    HttpRemoteSiteListener transactionManager = HttpRemoteSiteListener.getInstance(NiFiProperties.createBasicNiFiProperties(null, null));
    String transactionId = transactionManager.createTransaction();
    assertTrue("Transaction should be active.", transactionManager.isTransactionActive(transactionId));
    ProcessSession processSession = Mockito.mock(ProcessSession.class);
    FlowFileTransaction transaction = new FlowFileTransaction(processSession, null, null, 0, null, null);
    transactionManager.holdTransaction(transactionId, transaction, new HandshakeProperties());
    assertNotNull(transactionManager.getHandshakenProperties(transactionId));
    transaction = transactionManager.finalizeTransaction(transactionId);
    assertNotNull(transaction);
    assertFalse("Transaction should not be active anymore.", transactionManager.isTransactionActive(transactionId));
}

Example 5 with ProcessSession

use of org.apache.nifi.processor.ProcessSession in project nifi by apache.

the class TestHttpFlowFileServerProtocol method testReceiveZeroFile.

@Test
public void testReceiveZeroFile() throws Exception {
    final HttpFlowFileServerProtocol serverProtocol = getDefaultHttpFlowFileServerProtocol();
    final Peer peer = getDefaultPeer("testReceiveZeroFile");
    final HttpServerCommunicationsSession commsSession = (HttpServerCommunicationsSession) peer.getCommunicationsSession();
    commsSession.setUserDn("unit-test");
    serverProtocol.handshake(peer);
    assertTrue(serverProtocol.isHandshakeSuccessful());
    final FlowFileCodec negotiatedCoded = serverProtocol.negotiateCodec(peer);
    final ProcessContext context = null;
    final ProcessSession processSession = mock(ProcessSession.class);
    final InputStream httpInputStream = new ByteArrayInputStream(new byte[] {});
    ((HttpInput) commsSession.getInput()).setInputStream(httpInputStream);
    // Execute test using mock
    final int flowFileReceived = serverProtocol.receiveFlowFiles(peer, context, processSession, negotiatedCoded);
    assertEquals(0, flowFileReceived);
}