Examples with HttpCommunicationsSession org.apache.nifi.remote.io.http.HttpCommunicationsSession used on opensource projects

Example 11 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class SiteToSiteRestApiClient method openConnectionForSend.

public void openConnectionForSend(final String transactionUrl, final Peer peer) throws IOException {
    final CommunicationsSession commSession = peer.getCommunicationsSession();
    final String flowFilesPath = transactionUrl + "/flow-files";
    final HttpPost post = createPost(flowFilesPath);
    // Set uri so that it'll be used as transit uri.
    ((HttpCommunicationsSession) peer.getCommunicationsSession()).setDataTransferUrl(post.getURI().toString());
    post.setHeader("Content-Type", "application/octet-stream");
    post.setHeader("Accept", "text/plain");
    post.setHeader(HttpHeaders.PROTOCOL_VERSION, String.valueOf(transportProtocolVersionNegotiator.getVersion()));
    setHandshakeProperties(post);
    final CountDownLatch initConnectionLatch = new CountDownLatch(1);
    final URI requestUri = post.getURI();
    final PipedOutputStream outputStream = new PipedOutputStream();
    final PipedInputStream inputStream = new PipedInputStream(outputStream, DATA_PACKET_CHANNEL_READ_BUFFER_SIZE);
    final ReadableByteChannel dataPacketChannel = Channels.newChannel(inputStream);
    final HttpAsyncRequestProducer asyncRequestProducer = new HttpAsyncRequestProducer() {

        private final ByteBuffer buffer = ByteBuffer.allocate(DATA_PACKET_CHANNEL_READ_BUFFER_SIZE);

        private int totalRead = 0;

        private int totalProduced = 0;

        private boolean requestHasBeenReset = false;

        @Override
        public HttpHost getTarget() {
            return URIUtils.extractHost(requestUri);
        }

        @Override
        public HttpRequest generateRequest() throws IOException, HttpException {
            // Pass the output stream so that Site-to-Site client thread can send
            // data packet through this connection.
            logger.debug("sending data to {} has started...", flowFilesPath);
            ((HttpOutput) commSession.getOutput()).setOutputStream(outputStream);
            initConnectionLatch.countDown();
            final BasicHttpEntity entity = new BasicHttpEntity();
            entity.setChunked(true);
            entity.setContentType("application/octet-stream");
            post.setEntity(entity);
            return post;
        }

        private final AtomicBoolean bufferHasRemainingData = new AtomicBoolean(false);

        /**
         * If the proxy server requires authentication, the same POST request has to be sent again.
         * The first request will result 407, then the next one will be sent with auth headers and actual data.
         * This method produces a content only when it's need to be sent, to avoid producing the flow-file contents twice.
         * Whether we need to wait auth is determined heuristically by the previous POST request which creates transaction.
         * See {@link SiteToSiteRestApiClient#initiateTransactionForSend(HttpPost)} for further detail.
         */
        @Override
        public void produceContent(final ContentEncoder encoder, final IOControl ioControl) throws IOException {
            if (shouldCheckProxyAuth() && proxyAuthRequiresResend.get() && !requestHasBeenReset) {
                logger.debug("Need authentication with proxy server. Postpone producing content.");
                encoder.complete();
                return;
            }
            if (bufferHasRemainingData.get()) {
                // If there's remaining buffer last time, send it first.
                writeBuffer(encoder);
                if (bufferHasRemainingData.get()) {
                    return;
                }
            }
            int read;
            // or corresponding outputStream is closed.
            if ((read = dataPacketChannel.read(buffer)) > -1) {
                logger.trace("Read {} bytes from dataPacketChannel. {}", read, flowFilesPath);
                totalRead += read;
                buffer.flip();
                writeBuffer(encoder);
            } else {
                final long totalWritten = commSession.getOutput().getBytesWritten();
                logger.debug("sending data to {} has reached to its end. produced {} bytes by reading {} bytes from channel. {} bytes written in this transaction.", flowFilesPath, totalProduced, totalRead, totalWritten);
                if (totalRead != totalWritten || totalProduced != totalWritten) {
                    final String msg = "Sending data to %s has reached to its end, but produced : read : wrote byte sizes (%d : %d : %d) were not equal. Something went wrong.";
                    throw new RuntimeException(String.format(msg, flowFilesPath, totalProduced, totalRead, totalWritten));
                }
                transferDataLatch.countDown();
                encoder.complete();
                dataPacketChannel.close();
            }
        }

        private void writeBuffer(ContentEncoder encoder) throws IOException {
            while (buffer.hasRemaining()) {
                final int written = encoder.write(buffer);
                logger.trace("written {} bytes to encoder.", written);
                if (written == 0) {
                    logger.trace("Buffer still has remaining. {}", buffer);
                    bufferHasRemainingData.set(true);
                    return;
                }
                totalProduced += written;
            }
            bufferHasRemainingData.set(false);
            buffer.clear();
        }

        @Override
        public void requestCompleted(final HttpContext context) {
            logger.debug("Sending data to {} completed.", flowFilesPath);
            debugProxyAuthState(context);
        }

        @Override
        public void failed(final Exception ex) {
            final String msg = String.format("Failed to send data to %s due to %s", flowFilesPath, ex.toString());
            logger.error(msg, ex);
            eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, msg);
        }

        @Override
        public boolean isRepeatable() {
            // In order to pass authentication, request has to be repeatable.
            return true;
        }

        @Override
        public void resetRequest() throws IOException {
            logger.debug("Sending data request to {} has been reset...", flowFilesPath);
            requestHasBeenReset = true;
        }

        @Override
        public void close() throws IOException {
            logger.debug("Closing sending data request to {}", flowFilesPath);
            closeSilently(outputStream);
            closeSilently(dataPacketChannel);
            stopExtendingTtl();
        }
    };
    postResult = getHttpAsyncClient().execute(asyncRequestProducer, new BasicAsyncResponseConsumer(), null);
    try {
        // Need to wait the post request actually started so that we can write to its output stream.
        if (!initConnectionLatch.await(connectTimeoutMillis, TimeUnit.MILLISECONDS)) {
            throw new IOException("Awaiting initConnectionLatch has been timeout.");
        }
        // Started.
        transferDataLatch = new CountDownLatch(1);
        startExtendingTtl(transactionUrl, dataPacketChannel, null);
    } catch (final InterruptedException e) {
        throw new IOException("Awaiting initConnectionLatch has been interrupted.", e);
    }
}

Example 12 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class SiteToSiteRestApiClient method openConnectionForReceive.

public boolean openConnectionForReceive(final String transactionUrl, final Peer peer) throws IOException {
    final HttpGet get = createGet(transactionUrl + "/flow-files");
    // Set uri so that it'll be used as transit uri.
    ((HttpCommunicationsSession) peer.getCommunicationsSession()).setDataTransferUrl(get.getURI().toString());
    get.setHeader(HttpHeaders.PROTOCOL_VERSION, String.valueOf(transportProtocolVersionNegotiator.getVersion()));
    setHandshakeProperties(get);
    final CloseableHttpResponse response = getHttpClient().execute(get);
    final int responseCode = response.getStatusLine().getStatusCode();
    logger.debug("responseCode={}", responseCode);
    boolean keepItOpen = false;
    try {
        switch(responseCode) {
            case RESPONSE_CODE_OK:
                logger.debug("Server returned RESPONSE_CODE_OK, indicating there was no data.");
                EntityUtils.consume(response.getEntity());
                return false;
            case RESPONSE_CODE_ACCEPTED:
                final InputStream httpIn = response.getEntity().getContent();
                final InputStream streamCapture = new InputStream() {

                    boolean closed = false;

                    @Override
                    public int read() throws IOException {
                        if (closed) {
                            return -1;
                        }
                        final int r = httpIn.read();
                        if (r < 0) {
                            closed = true;
                            logger.debug("Reached to end of input stream. Closing resources...");
                            stopExtendingTtl();
                            closeSilently(httpIn);
                            closeSilently(response);
                        }
                        return r;
                    }
                };
                ((HttpInput) peer.getCommunicationsSession().getInput()).setInputStream(streamCapture);
                startExtendingTtl(transactionUrl, httpIn, response);
                keepItOpen = true;
                return true;
            default:
                try (InputStream content = response.getEntity().getContent()) {
                    throw handleErrResponse(responseCode, content);
                }
        }
    } finally {
        if (!keepItOpen) {
            response.close();
        }
    }
}

Example 13 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class TestHttpClientTransaction method getClientTransaction.

private HttpClientTransaction getClientTransaction(InputStream is, OutputStream os, SiteToSiteRestApiClient apiClient, TransferDirection direction, String transactionUrl) throws IOException {
    PeerDescription description = null;
    String peerUrl = "";
    HttpCommunicationsSession commsSession = new HttpCommunicationsSession();
    ((HttpInput) commsSession.getInput()).setInputStream(is);
    ((HttpOutput) commsSession.getOutput()).setOutputStream(os);
    String clusterUrl = "";
    Peer peer = new Peer(description, commsSession, peerUrl, clusterUrl);
    String portId = "portId";
    boolean useCompression = false;
    int penaltyMillis = 1000;
    EventReporter eventReporter = new EventReporter() {

        @Override
        public void reportEvent(Severity severity, String category, String message) {
            logger.info("Reporting event... severity={}, category={}, message={}", severity, category, message);
        }
    };
    int protocolVersion = 5;
    HttpClientTransaction transaction = new HttpClientTransaction(protocolVersion, peer, direction, useCompression, portId, penaltyMillis, eventReporter);
    transaction.initialize(apiClient, transactionUrl);
    return transaction;
}

Example 14 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class TestHttpClientTransaction method testSendButDestinationFull.

@Test
public void testSendButDestinationFull() throws IOException {
    SiteToSiteRestApiClient apiClient = mock(SiteToSiteRestApiClient.class);
    final String transactionUrl = "http://www.example.com/data-transfer/input-ports/portId/transactions/transactionId";
    doNothing().when(apiClient).openConnectionForSend(eq("portId"), any(Peer.class));
    // Emulate that server returns correct checksum.
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocation) throws Throwable {
            HttpCommunicationsSession commSession = (HttpCommunicationsSession) invocation.getArguments()[0];
            commSession.setChecksum("3359812065");
            return null;
        }
    }).when(apiClient).finishTransferFlowFiles(any(CommunicationsSession.class));
    TransactionResultEntity resultEntity = new TransactionResultEntity();
    resultEntity.setResponseCode(ResponseCode.TRANSACTION_FINISHED_BUT_DESTINATION_FULL.getCode());
    doReturn(resultEntity).when(apiClient).commitTransferFlowFiles(eq(transactionUrl), eq(CONFIRM_TRANSACTION));
    ByteArrayOutputStream serverResponseBos = new ByteArrayOutputStream();
    ByteArrayInputStream serverResponse = new ByteArrayInputStream(serverResponseBos.toByteArray());
    ByteArrayOutputStream clientRequest = new ByteArrayOutputStream();
    HttpClientTransaction transaction = getClientTransaction(serverResponse, clientRequest, apiClient, TransferDirection.SEND, transactionUrl);
    execSendButDestinationFull(transaction);
    InputStream sentByClient = new ByteArrayInputStream(clientRequest.toByteArray());
    DataPacket packetByClient = codec.decode(sentByClient);
    assertEquals("contents on client 1", readContents(packetByClient));
    packetByClient = codec.decode(sentByClient);
    assertEquals("contents on client 2", readContents(packetByClient));
    assertEquals(-1, sentByClient.read());
    verify(apiClient).commitTransferFlowFiles(transactionUrl, CONFIRM_TRANSACTION);
}

Example 15 with HttpCommunicationsSession

use of org.apache.nifi.remote.io.http.HttpCommunicationsSession in project nifi by apache.

the class DataTransferResource method initiateServerProtocol.

private HttpFlowFileServerProtocol initiateServerProtocol(final HttpServletRequest req, final Peer peer, final Integer transportProtocolVersion) throws IOException {
    // Switch transaction protocol version based on transport protocol version.
    TransportProtocolVersionNegotiator negotiatedTransportProtocolVersion = new TransportProtocolVersionNegotiator(transportProtocolVersion);
    VersionNegotiator versionNegotiator = new StandardVersionNegotiator(negotiatedTransportProtocolVersion.getTransactionProtocolVersion());
    final String dataTransferUrl = req.getRequestURL().toString();
    ((HttpCommunicationsSession) peer.getCommunicationsSession()).setDataTransferUrl(dataTransferUrl);
    HttpFlowFileServerProtocol serverProtocol = getHttpFlowFileServerProtocol(versionNegotiator);
    HttpRemoteSiteListener.getInstance(nifiProperties).setupServerProtocol(serverProtocol);
    serverProtocol.handshake(peer);
    return serverProtocol;
}