Examples with HttpCommunicationsSession org.apache.nifi.remote.io.http.HttpCommunicationsSession used on opensource projects

Example 1 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class TestStandardRemoteGroupPort method testSendHttp.

@Test
public void testSendHttp() throws Exception {
    setupMock(SiteToSiteTransportProtocol.HTTP, TransferDirection.SEND);
    setupMockProcessSession();
    final String peerUrl = "https://node1.example.com:8080/nifi";
    final PeerDescription peerDescription = new PeerDescription("node1.example.com", 8080, true);
    final HttpCommunicationsSession commsSession = new HttpCommunicationsSession();
    commsSession.setUserDn("nifi.node1.example.com");
    final Peer peer = new Peer(peerDescription, commsSession, peerUrl, REMOTE_CLUSTER_URL);
    final String flowFileEndpointUri = "https://node1.example.com:8080/nifi-api/output-ports/port-id/transactions/transaction-id/flow-files";
    doReturn(peer).when(transaction).getCommunicant();
    commsSession.setDataTransferUrl(flowFileEndpointUri);
    final MockFlowFile flowFile = processSession.createFlowFile("0123456789".getBytes());
    sessionState.getFlowFileQueue().offer(flowFile);
    port.onTrigger(processContext, processSession);
    // Assert provenance.
    final List<ProvenanceEventRecord> provenanceEvents = sessionState.getProvenanceEvents();
    assertEquals(1, provenanceEvents.size());
    final ProvenanceEventRecord provenanceEvent = provenanceEvents.get(0);
    assertEquals(ProvenanceEventType.SEND, provenanceEvent.getEventType());
    assertEquals(flowFileEndpointUri, provenanceEvent.getTransitUri());
    assertEquals("Remote DN=nifi.node1.example.com", provenanceEvent.getDetails());
}

Example 2 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class HttpClient method createTransaction.

@Override
public Transaction createTransaction(final TransferDirection direction) throws HandshakeException, PortNotRunningException, ProtocolException, UnknownPortException, IOException {
    final int timeoutMillis = (int) config.getTimeout(TimeUnit.MILLISECONDS);
    PeerStatus peerStatus;
    while ((peerStatus = peerSelector.getNextPeerStatus(direction)) != null) {
        logger.debug("peerStatus={}", peerStatus);
        final CommunicationsSession commSession = new HttpCommunicationsSession();
        final String nodeApiUrl = resolveNodeApiUrl(peerStatus.getPeerDescription());
        final StringBuilder clusterUrls = new StringBuilder();
        config.getUrls().forEach(url -> {
            if (clusterUrls.length() > 0) {
                clusterUrls.append(",");
                clusterUrls.append(url);
            }
        });
        final Peer peer = new Peer(peerStatus.getPeerDescription(), commSession, nodeApiUrl, clusterUrls.toString());
        final int penaltyMillis = (int) config.getPenalizationPeriod(TimeUnit.MILLISECONDS);
        String portId = config.getPortIdentifier();
        if (StringUtils.isEmpty(portId)) {
            portId = siteInfoProvider.getPortIdentifier(config.getPortName(), direction);
            if (StringUtils.isEmpty(portId)) {
                peer.close();
                throw new IOException("Failed to determine the identifier of port " + config.getPortName());
            }
        }
        final SiteToSiteRestApiClient apiClient = new SiteToSiteRestApiClient(config.getSslContext(), config.getHttpProxy(), config.getEventReporter());
        apiClient.setBaseUrl(peer.getUrl());
        apiClient.setConnectTimeoutMillis(timeoutMillis);
        apiClient.setReadTimeoutMillis(timeoutMillis);
        apiClient.setCacheExpirationMillis(config.getCacheExpiration(TimeUnit.MILLISECONDS));
        apiClient.setLocalAddress(config.getLocalAddress());
        apiClient.setCompress(config.isUseCompression());
        apiClient.setRequestExpirationMillis(config.getIdleConnectionExpiration(TimeUnit.MILLISECONDS));
        apiClient.setBatchCount(config.getPreferredBatchCount());
        apiClient.setBatchSize(config.getPreferredBatchSize());
        apiClient.setBatchDurationMillis(config.getPreferredBatchDuration(TimeUnit.MILLISECONDS));
        final String transactionUrl;
        try {
            transactionUrl = apiClient.initiateTransaction(direction, portId);
            commSession.setUserDn(apiClient.getTrustedPeerDn());
        } catch (final Exception e) {
            apiClient.close();
            logger.warn("Penalizing a peer {} due to {}", peer, e.toString());
            peerSelector.penalize(peer, penaltyMillis);
            // Following exceptions will be thrown even if we tried other peers, so throw it.
            if (e instanceof UnknownPortException || e instanceof PortNotRunningException || e instanceof HandshakeException) {
                throw e;
            }
            logger.debug("Continue trying other peers...");
            continue;
        }
        // We found a valid peer to communicate with.
        final Integer transactionProtocolVersion = apiClient.getTransactionProtocolVersion();
        final HttpClientTransaction transaction = new HttpClientTransaction(transactionProtocolVersion, peer, direction, config.isUseCompression(), portId, penaltyMillis, config.getEventReporter()) {

            @Override
            protected void close() throws IOException {
                try {
                    super.close();
                } finally {
                    activeTransactions.remove(this);
                }
            }
        };
        try {
            transaction.initialize(apiClient, transactionUrl);
        } catch (final Exception e) {
            transaction.error();
            throw e;
        }
        activeTransactions.add(transaction);
        return transaction;
    }
    logger.info("Couldn't find a valid peer to communicate with.");
    return null;
}

Example 3 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class HttpClientTransaction method writeTransactionResponse.

@Override
protected void writeTransactionResponse(ResponseCode response, String explanation) throws IOException {
    HttpCommunicationsSession commSession = (HttpCommunicationsSession) peer.getCommunicationsSession();
    if (TransferDirection.RECEIVE.equals(direction)) {
        switch(response) {
            case CONFIRM_TRANSACTION:
                logger.debug("{} Confirming transaction. checksum={}", this, explanation);
                commSession.setChecksum(explanation);
                break;
            case TRANSACTION_FINISHED:
                logger.debug("{} Finishing transaction.", this);
                break;
            case CANCEL_TRANSACTION:
                logger.debug("{} Canceling transaction. explanation={}", this, explanation);
                TransactionResultEntity resultEntity = apiClient.commitReceivingFlowFiles(transactionUrl, ResponseCode.CANCEL_TRANSACTION, null);
                ResponseCode cancelResponse = ResponseCode.fromCode(resultEntity.getResponseCode());
                switch(cancelResponse) {
                    case CANCEL_TRANSACTION:
                        logger.debug("{} CANCEL_TRANSACTION, The transaction is canceled on server properly.", this);
                        break;
                    default:
                        logger.warn("{} CANCEL_TRANSACTION, Expected the transaction is canceled on server, but received {}.", this, cancelResponse);
                        break;
                }
                break;
        }
    } else {
        switch(response) {
            case FINISH_TRANSACTION:
                // The actual HTTP request will be sent in readTransactionResponse.
                logger.debug("{} Finished sending flow files.", this);
                break;
            case BAD_CHECKSUM:
                {
                    TransactionResultEntity resultEntity = apiClient.commitTransferFlowFiles(transactionUrl, ResponseCode.BAD_CHECKSUM);
                    ResponseCode badChecksumCancelResponse = ResponseCode.fromCode(resultEntity.getResponseCode());
                    switch(badChecksumCancelResponse) {
                        case CANCEL_TRANSACTION:
                            logger.debug("{} BAD_CHECKSUM, The transaction is canceled on server properly.", this);
                            break;
                        default:
                            logger.warn("{} BAD_CHECKSUM, Expected the transaction is canceled on server, but received {}.", this, badChecksumCancelResponse);
                            break;
                    }
                }
                break;
            case CONFIRM_TRANSACTION:
                // The actual HTTP request will be sent in readTransactionResponse.
                logger.debug("{} Transaction is confirmed.", this);
                break;
            case CANCEL_TRANSACTION:
                {
                    logger.debug("{} Canceling transaction.", this);
                    TransactionResultEntity resultEntity = apiClient.commitTransferFlowFiles(transactionUrl, ResponseCode.CANCEL_TRANSACTION);
                    ResponseCode cancelResponse = ResponseCode.fromCode(resultEntity.getResponseCode());
                    switch(cancelResponse) {
                        case CANCEL_TRANSACTION:
                            logger.debug("{} CANCEL_TRANSACTION, The transaction is canceled on server properly.", this);
                            break;
                        default:
                            logger.warn("{} CANCEL_TRANSACTION, Expected the transaction is canceled on server, but received {}.", this, cancelResponse);
                            break;
                    }
                }
                break;
        }
    }
}

Example 4 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class SiteToSiteRestApiClient method finishTransferFlowFiles.

public void finishTransferFlowFiles(final CommunicationsSession commSession) throws IOException {
    if (postResult == null) {
        new IllegalStateException("Data transfer has not started yet.");
    }
    // No more data can be sent.
    // Close PipedOutputStream so that dataPacketChannel doesn't blocked.
    // If we don't close this output stream, then PipedInputStream loops infinitely at read().
    commSession.getOutput().getOutputStream().close();
    logger.debug("{} FinishTransferFlowFiles no more data can be sent", this);
    try {
        if (!transferDataLatch.await(requestExpirationMillis, TimeUnit.MILLISECONDS)) {
            throw new IOException("Awaiting transferDataLatch has been timeout.");
        }
    } catch (final InterruptedException e) {
        throw new IOException("Awaiting transferDataLatch has been interrupted.", e);
    }
    stopExtendingTtl();
    final HttpResponse response;
    try {
        response = postResult.get(readTimeoutMillis, TimeUnit.MILLISECONDS);
    } catch (final ExecutionException e) {
        logger.debug("Something has happened at sending data thread. {}", e.getMessage());
        throw toIOException(e);
    } catch (TimeoutException | InterruptedException e) {
        throw new IOException(e);
    }
    final int responseCode = response.getStatusLine().getStatusCode();
    switch(responseCode) {
        case RESPONSE_CODE_ACCEPTED:
            final String receivedChecksum = EntityUtils.toString(response.getEntity());
            ((HttpInput) commSession.getInput()).setInputStream(new ByteArrayInputStream(receivedChecksum.getBytes()));
            ((HttpCommunicationsSession) commSession).setChecksum(receivedChecksum);
            logger.debug("receivedChecksum={}", receivedChecksum);
            break;
        default:
            try (InputStream content = response.getEntity().getContent()) {
                throw handleErrResponse(responseCode, content);
            }
    }
}

Example 5 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class TestHttpClientTransaction method testSendTwoFlowFiles.

@Test
public void testSendTwoFlowFiles() throws IOException {
    SiteToSiteRestApiClient apiClient = mock(SiteToSiteRestApiClient.class);
    final String transactionUrl = "http://www.example.com/data-transfer/input-ports/portId/transactions/transactionId";
    doNothing().when(apiClient).openConnectionForSend(eq("portId"), any(Peer.class));
    // Emulate that server returns correct checksum.
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            HttpCommunicationsSession commSession = (HttpCommunicationsSession) invocation.getArguments()[0];
            commSession.setChecksum("3359812065");
            return null;
        }
    }).when(apiClient).finishTransferFlowFiles(any(CommunicationsSession.class));
    TransactionResultEntity resultEntity = new TransactionResultEntity();
    resultEntity.setResponseCode(ResponseCode.TRANSACTION_FINISHED.getCode());
    doReturn(resultEntity).when(apiClient).commitTransferFlowFiles(eq(transactionUrl), eq(CONFIRM_TRANSACTION));
    ByteArrayOutputStream serverResponseBos = new ByteArrayOutputStream();
    ByteArrayInputStream serverResponse = new ByteArrayInputStream(serverResponseBos.toByteArray());
    ByteArrayOutputStream clientRequest = new ByteArrayOutputStream();
    HttpClientTransaction transaction = getClientTransaction(serverResponse, clientRequest, apiClient, TransferDirection.SEND, transactionUrl);
    execSendTwoFlowFiles(transaction);
    InputStream sentByClient = new ByteArrayInputStream(clientRequest.toByteArray());
    DataPacket packetByClient = codec.decode(sentByClient);
    assertEquals("contents on client 1", readContents(packetByClient));
    packetByClient = codec.decode(sentByClient);
    assertEquals("contents on client 2", readContents(packetByClient));
    assertEquals(-1, sentByClient.read());
    verify(apiClient).commitTransferFlowFiles(transactionUrl, CONFIRM_TRANSACTION);
}