Examples with HttpCommunicationsSession org.apache.nifi.remote.io.http.HttpCommunicationsSession used on opensource projects

Example 6 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class TestHttpClientTransaction method testSendWithInvalidChecksum.

@Test
public void testSendWithInvalidChecksum() throws IOException {
    SiteToSiteRestApiClient apiClient = mock(SiteToSiteRestApiClient.class);
    final String transactionUrl = "http://www.example.com/data-transfer/input-ports/portId/transactions/transactionId";
    doNothing().when(apiClient).openConnectionForSend(eq(transactionUrl), any(Peer.class));
    // Emulate that server returns incorrect checksum.
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            HttpCommunicationsSession commSession = (HttpCommunicationsSession) invocation.getArguments()[0];
            commSession.setChecksum("Different checksum");
            return null;
        }
    }).when(apiClient).finishTransferFlowFiles(any(CommunicationsSession.class));
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            TransactionResultEntity serverResult = new TransactionResultEntity();
            serverResult.setResponseCode(ResponseCode.CANCEL_TRANSACTION.getCode());
            return serverResult;
        }
    }).when(apiClient).commitTransferFlowFiles(eq(transactionUrl), eq(ResponseCode.BAD_CHECKSUM));
    ByteArrayOutputStream serverResponseBos = new ByteArrayOutputStream();
    ByteArrayInputStream serverResponse = new ByteArrayInputStream(serverResponseBos.toByteArray());
    ByteArrayOutputStream clientRequest = new ByteArrayOutputStream();
    HttpClientTransaction transaction = getClientTransaction(serverResponse, clientRequest, apiClient, TransferDirection.SEND, transactionUrl);
    execSendWithInvalidChecksum(transaction);
    InputStream sentByClient = new ByteArrayInputStream(clientRequest.toByteArray());
    DataPacket packetByClient = codec.decode(sentByClient);
    assertEquals("contents on client 1", readContents(packetByClient));
    packetByClient = codec.decode(sentByClient);
    assertEquals("contents on client 2", readContents(packetByClient));
    assertEquals(-1, sentByClient.read());
    verify(apiClient).commitTransferFlowFiles(transactionUrl, ResponseCode.BAD_CHECKSUM);
}

Example 7 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class TestHttpClientTransaction method testSendOneFlowFile.

@Test
public void testSendOneFlowFile() throws IOException {
    SiteToSiteRestApiClient apiClient = mock(SiteToSiteRestApiClient.class);
    final String transactionUrl = "http://www.example.com/data-transfer/input-ports/portId/transactions/transactionId";
    doNothing().when(apiClient).openConnectionForSend(eq(transactionUrl), any(Peer.class));
    // Emulate that server returns correct checksum.
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            HttpCommunicationsSession commSession = (HttpCommunicationsSession) invocation.getArguments()[0];
            commSession.setChecksum("2946083981");
            return null;
        }
    }).when(apiClient).finishTransferFlowFiles(any(CommunicationsSession.class));
    TransactionResultEntity resultEntity = new TransactionResultEntity();
    resultEntity.setResponseCode(ResponseCode.TRANSACTION_FINISHED.getCode());
    doReturn(resultEntity).when(apiClient).commitTransferFlowFiles(eq(transactionUrl), eq(CONFIRM_TRANSACTION));
    ByteArrayOutputStream serverResponseBos = new ByteArrayOutputStream();
    ByteArrayInputStream serverResponse = new ByteArrayInputStream(serverResponseBos.toByteArray());
    ByteArrayOutputStream clientRequest = new ByteArrayOutputStream();
    HttpClientTransaction transaction = getClientTransaction(serverResponse, clientRequest, apiClient, TransferDirection.SEND, transactionUrl);
    execSendOneFlowFile(transaction);
    InputStream sentByClient = new ByteArrayInputStream(clientRequest.toByteArray());
    DataPacket packetByClient = codec.decode(sentByClient);
    assertEquals("contents on client 1", readContents(packetByClient));
    assertEquals(-1, sentByClient.read());
    verify(apiClient).commitTransferFlowFiles(transactionUrl, CONFIRM_TRANSACTION);
}

Example 8 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class TestStandardRemoteGroupPort method testReceiveHttp.

@Test
public void testReceiveHttp() throws Exception {
    setupMock(SiteToSiteTransportProtocol.HTTP, TransferDirection.RECEIVE);
    setupMockProcessSession();
    final String peerUrl = "https://node1.example.com:8080/nifi";
    final PeerDescription peerDescription = new PeerDescription("node1.example.com", 8080, true);
    final HttpCommunicationsSession commsSession = new HttpCommunicationsSession();
    commsSession.setUserDn("nifi.node1.example.com");
    final Peer peer = new Peer(peerDescription, commsSession, peerUrl, REMOTE_CLUSTER_URL);
    final String flowFileEndpointUri = "https://node1.example.com:8080/nifi-api/output-ports/port-id/transactions/transaction-id/flow-files";
    doReturn(peer).when(transaction).getCommunicant();
    commsSession.setDataTransferUrl(flowFileEndpointUri);
    final Map<String, String> attributes = new HashMap<>();
    final byte[] dataPacketContents = "DataPacket Contents".getBytes();
    final ByteArrayInputStream dataPacketInputStream = new ByteArrayInputStream(dataPacketContents);
    final DataPacket dataPacket = new StandardDataPacket(attributes, dataPacketInputStream, dataPacketContents.length);
    // Return null when it gets called second time.
    doReturn(dataPacket).doReturn(null).when(transaction).receive();
    port.onTrigger(processContext, processSession);
    // Assert provenance.
    final List<ProvenanceEventRecord> provenanceEvents = sessionState.getProvenanceEvents();
    assertEquals(1, provenanceEvents.size());
    final ProvenanceEventRecord provenanceEvent = provenanceEvents.get(0);
    assertEquals(ProvenanceEventType.RECEIVE, provenanceEvent.getEventType());
    assertEquals(flowFileEndpointUri, provenanceEvent.getTransitUri());
    assertEquals("Remote DN=nifi.node1.example.com", provenanceEvent.getDetails());
    // Assert received flow files.
    processSession.assertAllFlowFilesTransferred(Relationship.ANONYMOUS);
    final List<MockFlowFile> flowFiles = processSession.getFlowFilesForRelationship(Relationship.ANONYMOUS);
    assertEquals(1, flowFiles.size());
    final MockFlowFile flowFile = flowFiles.get(0);
    flowFile.assertAttributeEquals(SiteToSiteAttributes.S2S_HOST.key(), peer.getHost());
    flowFile.assertAttributeEquals(SiteToSiteAttributes.S2S_ADDRESS.key(), peer.getHost() + ":" + peer.getPort());
}

Example 9 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class TestStandardRemoteGroupPort method testSendBatch.

/**
 * Generate flow files to be sent, and execute port's onTrigger method.
 * Finally, this method verifies whether packets are sent as expected.
 * @param expectedNumberOfPackets Specify how many packets should be sent by each transaction.
 *                                E.g. passing {2, 2, 1}, would generate 5 flow files in total.
 *                                Based on the siteToSiteClientConfig batch parameters,
 *                                it's expected to be sent via 3 transactions,
 *                                transaction 0 will send flow file 0 and 1,
 *                                transaction 1 will send flow file 2 and 3,
 *                                and transaction 2 will send flow file 4.
 *                                Each flow file has different content size generated automatically.
 *                                The content size starts with 10, and increases as more flow files are generated.
 *                                E.g. flow file 1 will have 10 bytes, flow file 2 has 11 bytes, f3 has 12 and so on.
 */
private void testSendBatch(final int[] expectedNumberOfPackets) throws Exception {
    setupMockProcessSession();
    final String peerUrl = "http://node1.example.com:8080/nifi";
    final PeerDescription peerDescription = new PeerDescription("node1.example.com", 8080, false);
    final HttpCommunicationsSession commsSession = new HttpCommunicationsSession();
    final Peer peer = new Peer(peerDescription, commsSession, peerUrl, REMOTE_CLUSTER_URL);
    final String flowFileEndpointUri = "http://node1.example.com:8080/nifi-api/output-ports/port-id/transactions/transaction-id/flow-files";
    doReturn(peer).when(transaction).getCommunicant();
    commsSession.setDataTransferUrl(flowFileEndpointUri);
    // Capture packets being sent to the remote peer
    final AtomicInteger totalPacketsSent = new AtomicInteger(0);
    final List<List<DataPacket>> sentPackets = new ArrayList<>(expectedNumberOfPackets.length);
    final List<DataPacket> sentPacketsPerTransaction = new ArrayList<>();
    doAnswer(invocation -> {
        sentPacketsPerTransaction.add((DataPacket) invocation.getArguments()[0]);
        totalPacketsSent.incrementAndGet();
        return null;
    }).when(transaction).send(any(DataPacket.class));
    doAnswer(invocation -> {
        sentPackets.add(new ArrayList<>(sentPacketsPerTransaction));
        sentPacketsPerTransaction.clear();
        return null;
    }).when(transaction).confirm();
    // Execute onTrigger while offering new flow files.
    final List<MockFlowFile> flowFiles = new ArrayList<>();
    for (int i = 0; i < expectedNumberOfPackets.length; i++) {
        int numOfPackets = expectedNumberOfPackets[i];
        int startF = flowFiles.size();
        int endF = startF + numOfPackets;
        IntStream.range(startF, endF).forEach(f -> {
            final StringBuilder flowFileContents = new StringBuilder("0123456789");
            for (int c = 0; c < f; c++) {
                flowFileContents.append(c);
            }
            final byte[] bytes = flowFileContents.toString().getBytes();
            final MockFlowFile flowFile = spy(processSession.createFlowFile(bytes));
            when(flowFile.getSize()).then(invocation -> {
                // For testSendBatchByDuration
                Thread.sleep(1);
                return bytes.length;
            });
            sessionState.getFlowFileQueue().offer(flowFile);
            flowFiles.add(flowFile);
        });
        port.onTrigger(processContext, processSession);
    }
    // Verify transactions, sent packets, and provenance events.
    assertEquals(flowFiles.size(), totalPacketsSent.get());
    assertEquals("The number of transactions should match as expected.", expectedNumberOfPackets.length, sentPackets.size());
    final List<ProvenanceEventRecord> provenanceEvents = sessionState.getProvenanceEvents();
    assertEquals(flowFiles.size(), provenanceEvents.size());
    int f = 0;
    for (int i = 0; i < expectedNumberOfPackets.length; i++) {
        final List<DataPacket> dataPackets = sentPackets.get(i);
        assertEquals(expectedNumberOfPackets[i], dataPackets.size());
        for (int p = 0; p < dataPackets.size(); p++) {
            final FlowFile flowFile = flowFiles.get(f);
            // Assert sent packet
            final DataPacket dataPacket = dataPackets.get(p);
            assertEquals(flowFile.getSize(), dataPacket.getSize());
            // Assert provenance event
            final ProvenanceEventRecord provenanceEvent = provenanceEvents.get(f);
            assertEquals(ProvenanceEventType.SEND, provenanceEvent.getEventType());
            assertEquals(flowFileEndpointUri, provenanceEvent.getTransitUri());
            f++;
        }
    }
}

Example 10 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class HttpClientTransaction method readTransactionResponse.

@Override
protected Response readTransactionResponse() throws IOException {
    HttpCommunicationsSession commSession = (HttpCommunicationsSession) peer.getCommunicationsSession();
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(bos);
    if (TransferDirection.RECEIVE.equals(direction)) {
        switch(state) {
            case TRANSACTION_STARTED:
            case DATA_EXCHANGED:
                logger.debug("{} {} readTransactionResponse. checksum={}", this, peer, commSession.getChecksum());
                if (StringUtils.isEmpty(commSession.getChecksum())) {
                    // We don't know if there's more data to receive, so just continue it.
                    ResponseCode.CONTINUE_TRANSACTION.writeResponse(dos);
                } else {
                    // We got a checksum to send to server.
                    if (TransactionState.TRANSACTION_STARTED.equals(state)) {
                        logger.debug("{} {} There's no transaction to confirm.", this, peer);
                        ResponseCode.CONFIRM_TRANSACTION.writeResponse(dos, "");
                    } else {
                        TransactionResultEntity transactionResult = apiClient.commitReceivingFlowFiles(transactionUrl, ResponseCode.CONFIRM_TRANSACTION, commSession.getChecksum());
                        ResponseCode responseCode = ResponseCode.fromCode(transactionResult.getResponseCode());
                        if (responseCode.containsMessage()) {
                            String message = transactionResult.getMessage();
                            responseCode.writeResponse(dos, message == null ? "" : message);
                        } else {
                            responseCode.writeResponse(dos);
                        }
                    }
                }
                break;
        }
    } else {
        switch(state) {
            case DATA_EXCHANGED:
                // Some flow files have been sent via stream, finish transferring.
                apiClient.finishTransferFlowFiles(commSession);
                ResponseCode.CONFIRM_TRANSACTION.writeResponse(dos, commSession.getChecksum());
                break;
            case TRANSACTION_CONFIRMED:
                TransactionResultEntity resultEntity = apiClient.commitTransferFlowFiles(transactionUrl, ResponseCode.CONFIRM_TRANSACTION);
                ResponseCode responseCode = ResponseCode.fromCode(resultEntity.getResponseCode());
                if (responseCode.containsMessage()) {
                    responseCode.writeResponse(dos, resultEntity.getMessage());
                } else {
                    responseCode.writeResponse(dos);
                }
                break;
        }
    }
    ByteArrayInputStream bis = new ByteArrayInputStream(bos.toByteArray());
    return Response.read(new DataInputStream(bis));
}