Examples with HttpOutput org.apache.nifi.remote.io.http.HttpOutput used on opensource projects

Example 1 with HttpOutput

use of org.apache.nifi.remote.io.http.HttpOutput in project nifi by apache.

the class DataTransferResource method transferFlowFiles.

@GET
@Consumes(MediaType.WILDCARD)
@Produces(MediaType.APPLICATION_OCTET_STREAM)
@Path("output-ports/{portId}/transactions/{transactionId}/flow-files")
@ApiOperation(value = "Transfer flow files from the output port", response = StreamingOutput.class, authorizations = { @Authorization(value = "Write - /data-transfer/output-ports/{uuid}") })
@ApiResponses(value = { @ApiResponse(code = 200, message = "There is no flow file to return."), @ApiResponse(code = 400, message = "NiFi was unable to complete the request because it was invalid. The request should not be retried without modification."), @ApiResponse(code = 401, message = "Client could not be authenticated."), @ApiResponse(code = 403, message = "Client is not authorized to make this request."), @ApiResponse(code = 404, message = "The specified resource could not be found."), @ApiResponse(code = 409, message = "The request was valid but NiFi was not in the appropriate state to process it. Retrying the same request later may be successful."), @ApiResponse(code = 503, message = "NiFi instance is not ready for serving request, or temporarily overloaded. Retrying the same request later may be successful") })
public Response transferFlowFiles(@ApiParam(value = "The output port id.", required = true) @PathParam("portId") String portId, @PathParam("transactionId") String transactionId, @Context HttpServletRequest req, @Context HttpServletResponse res, @Context ServletContext context, InputStream inputStream) {
    // authorize access
    serviceFacade.authorizeAccess(lookup -> {
        authorizeDataTransfer(lookup, ResourceType.OutputPort, portId);
    });
    final ValidateRequestResult validationResult = validateResult(req, portId, transactionId);
    if (validationResult.errResponse != null) {
        return validationResult.errResponse;
    }
    logger.debug("transferFlowFiles request: portId={}", portId);
    // Before opening the real output stream for HTTP response,
    // use this temporary output stream to buffer handshake result.
    final ByteArrayOutputStream tempBos = new ByteArrayOutputStream();
    final Peer peer = constructPeer(req, inputStream, tempBos, portId, transactionId);
    final int transportProtocolVersion = validationResult.transportProtocolVersion;
    try {
        final HttpFlowFileServerProtocol serverProtocol = initiateServerProtocol(req, peer, transportProtocolVersion);
        StreamingOutput flowFileContent = new StreamingOutput() {

            @Override
            public void write(OutputStream outputStream) throws IOException, WebApplicationException {
                HttpOutput output = (HttpOutput) peer.getCommunicationsSession().getOutput();
                output.setOutputStream(outputStream);
                try {
                    int numOfFlowFiles = serverProtocol.getPort().transferFlowFiles(peer, serverProtocol);
                    logger.debug("finished transferring flow files, numOfFlowFiles={}", numOfFlowFiles);
                    if (numOfFlowFiles < 1) {
                        // There was no flow file to transfer. Throw this exception to stop responding with SEE OTHER.
                        throw new WebApplicationException(Response.Status.OK);
                    }
                } catch (NotAuthorizedException | BadRequestException | RequestExpiredException e) {
                    // Handshake is done outside of write() method, so these exception wouldn't be thrown.
                    throw new IOException("Failed to process the request.", e);
                }
            }
        };
        return responseCreator.acceptedResponse(transactionManager, flowFileContent, transportProtocolVersion);
    } catch (HandshakeException e) {
        return responseCreator.handshakeExceptionResponse(e);
    } catch (Exception e) {
        return responseCreator.unexpectedErrorResponse(portId, e);
    }
}

Example 2 with HttpOutput

use of org.apache.nifi.remote.io.http.HttpOutput in project nifi by apache.

the class SiteToSiteRestApiClient method openConnectionForSend.

public void openConnectionForSend(final String transactionUrl, final Peer peer) throws IOException {
    final CommunicationsSession commSession = peer.getCommunicationsSession();
    final String flowFilesPath = transactionUrl + "/flow-files";
    final HttpPost post = createPost(flowFilesPath);
    // Set uri so that it'll be used as transit uri.
    ((HttpCommunicationsSession) peer.getCommunicationsSession()).setDataTransferUrl(post.getURI().toString());
    post.setHeader("Content-Type", "application/octet-stream");
    post.setHeader("Accept", "text/plain");
    post.setHeader(HttpHeaders.PROTOCOL_VERSION, String.valueOf(transportProtocolVersionNegotiator.getVersion()));
    setHandshakeProperties(post);
    final CountDownLatch initConnectionLatch = new CountDownLatch(1);
    final URI requestUri = post.getURI();
    final PipedOutputStream outputStream = new PipedOutputStream();
    final PipedInputStream inputStream = new PipedInputStream(outputStream, DATA_PACKET_CHANNEL_READ_BUFFER_SIZE);
    final ReadableByteChannel dataPacketChannel = Channels.newChannel(inputStream);
    final HttpAsyncRequestProducer asyncRequestProducer = new HttpAsyncRequestProducer() {

        private final ByteBuffer buffer = ByteBuffer.allocate(DATA_PACKET_CHANNEL_READ_BUFFER_SIZE);

        private int totalRead = 0;

        private int totalProduced = 0;

        private boolean requestHasBeenReset = false;

        @Override
        public HttpHost getTarget() {
            return URIUtils.extractHost(requestUri);
        }

        @Override
        public HttpRequest generateRequest() throws IOException, HttpException {
            // Pass the output stream so that Site-to-Site client thread can send
            // data packet through this connection.
            logger.debug("sending data to {} has started...", flowFilesPath);
            ((HttpOutput) commSession.getOutput()).setOutputStream(outputStream);
            initConnectionLatch.countDown();
            final BasicHttpEntity entity = new BasicHttpEntity();
            entity.setChunked(true);
            entity.setContentType("application/octet-stream");
            post.setEntity(entity);
            return post;
        }

        private final AtomicBoolean bufferHasRemainingData = new AtomicBoolean(false);

        /**
         * If the proxy server requires authentication, the same POST request has to be sent again.
         * The first request will result 407, then the next one will be sent with auth headers and actual data.
         * This method produces a content only when it's need to be sent, to avoid producing the flow-file contents twice.
         * Whether we need to wait auth is determined heuristically by the previous POST request which creates transaction.
         * See {@link SiteToSiteRestApiClient#initiateTransactionForSend(HttpPost)} for further detail.
         */
        @Override
        public void produceContent(final ContentEncoder encoder, final IOControl ioControl) throws IOException {
            if (shouldCheckProxyAuth() && proxyAuthRequiresResend.get() && !requestHasBeenReset) {
                logger.debug("Need authentication with proxy server. Postpone producing content.");
                encoder.complete();
                return;
            }
            if (bufferHasRemainingData.get()) {
                // If there's remaining buffer last time, send it first.
                writeBuffer(encoder);
                if (bufferHasRemainingData.get()) {
                    return;
                }
            }
            int read;
            // or corresponding outputStream is closed.
            if ((read = dataPacketChannel.read(buffer)) > -1) {
                logger.trace("Read {} bytes from dataPacketChannel. {}", read, flowFilesPath);
                totalRead += read;
                buffer.flip();
                writeBuffer(encoder);
            } else {
                final long totalWritten = commSession.getOutput().getBytesWritten();
                logger.debug("sending data to {} has reached to its end. produced {} bytes by reading {} bytes from channel. {} bytes written in this transaction.", flowFilesPath, totalProduced, totalRead, totalWritten);
                if (totalRead != totalWritten || totalProduced != totalWritten) {
                    final String msg = "Sending data to %s has reached to its end, but produced : read : wrote byte sizes (%d : %d : %d) were not equal. Something went wrong.";
                    throw new RuntimeException(String.format(msg, flowFilesPath, totalProduced, totalRead, totalWritten));
                }
                transferDataLatch.countDown();
                encoder.complete();
                dataPacketChannel.close();
            }
        }

        private void writeBuffer(ContentEncoder encoder) throws IOException {
            while (buffer.hasRemaining()) {
                final int written = encoder.write(buffer);
                logger.trace("written {} bytes to encoder.", written);
                if (written == 0) {
                    logger.trace("Buffer still has remaining. {}", buffer);
                    bufferHasRemainingData.set(true);
                    return;
                }
                totalProduced += written;
            }
            bufferHasRemainingData.set(false);
            buffer.clear();
        }

        @Override
        public void requestCompleted(final HttpContext context) {
            logger.debug("Sending data to {} completed.", flowFilesPath);
            debugProxyAuthState(context);
        }

        @Override
        public void failed(final Exception ex) {
            final String msg = String.format("Failed to send data to %s due to %s", flowFilesPath, ex.toString());
            logger.error(msg, ex);
            eventReporter.reportEvent(Severity.WARNING, EVENT_CATEGORY, msg);
        }

        @Override
        public boolean isRepeatable() {
            // In order to pass authentication, request has to be repeatable.
            return true;
        }

        @Override
        public void resetRequest() throws IOException {
            logger.debug("Sending data request to {} has been reset...", flowFilesPath);
            requestHasBeenReset = true;
        }

        @Override
        public void close() throws IOException {
            logger.debug("Closing sending data request to {}", flowFilesPath);
            closeSilently(outputStream);
            closeSilently(dataPacketChannel);
            stopExtendingTtl();
        }
    };
    postResult = getHttpAsyncClient().execute(asyncRequestProducer, new BasicAsyncResponseConsumer(), null);
    try {
        // Need to wait the post request actually started so that we can write to its output stream.
        if (!initConnectionLatch.await(connectTimeoutMillis, TimeUnit.MILLISECONDS)) {
            throw new IOException("Awaiting initConnectionLatch has been timeout.");
        }
        // Started.
        transferDataLatch = new CountDownLatch(1);
        startExtendingTtl(transactionUrl, dataPacketChannel, null);
    } catch (final InterruptedException e) {
        throw new IOException("Awaiting initConnectionLatch has been interrupted.", e);
    }
}

Example 3 with HttpOutput

use of org.apache.nifi.remote.io.http.HttpOutput in project nifi by apache.

the class TestHttpClientTransaction method getClientTransaction.

private HttpClientTransaction getClientTransaction(InputStream is, OutputStream os, SiteToSiteRestApiClient apiClient, TransferDirection direction, String transactionUrl) throws IOException {
    PeerDescription description = null;
    String peerUrl = "";
    HttpCommunicationsSession commsSession = new HttpCommunicationsSession();
    ((HttpInput) commsSession.getInput()).setInputStream(is);
    ((HttpOutput) commsSession.getOutput()).setOutputStream(os);
    String clusterUrl = "";
    Peer peer = new Peer(description, commsSession, peerUrl, clusterUrl);
    String portId = "portId";
    boolean useCompression = false;
    int penaltyMillis = 1000;
    EventReporter eventReporter = new EventReporter() {

        @Override
        public void reportEvent(Severity severity, String category, String message) {
            logger.info("Reporting event... severity={}, category={}, message={}", severity, category, message);
        }
    };
    int protocolVersion = 5;
    HttpClientTransaction transaction = new HttpClientTransaction(protocolVersion, peer, direction, useCompression, portId, penaltyMillis, eventReporter);
    transaction.initialize(apiClient, transactionUrl);
    return transaction;
}