Examples with StopWatch org.apache.nifi.util.StopWatch used on opensource projects

Example 51 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class PostHTTP method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile firstFlowFile = session.get();
    if (firstFlowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    final String url = context.getProperty(URL).evaluateAttributeExpressions(firstFlowFile).getValue();
    try {
        new java.net.URL(url);
    } catch (final MalformedURLException e) {
        logger.error("After substituting attribute values for {}, URL is {}; this is not a valid URL, so routing to failure", new Object[] { firstFlowFile, url });
        firstFlowFile = session.penalize(firstFlowFile);
        session.transfer(firstFlowFile, REL_FAILURE);
        return;
    }
    final List<FlowFile> toSend = new ArrayList<>();
    toSend.add(firstFlowFile);
    final boolean sendAsFlowFile = context.getProperty(SEND_AS_FLOWFILE).asBoolean();
    final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
    final String userAgent = context.getProperty(USER_AGENT).getValue();
    final RequestConfig.Builder requestConfigBuilder = RequestConfig.custom();
    requestConfigBuilder.setConnectionRequestTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setConnectTimeout(context.getProperty(CONNECTION_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    requestConfigBuilder.setRedirectsEnabled(false);
    requestConfigBuilder.setSocketTimeout(context.getProperty(DATA_TIMEOUT).asTimePeriod(TimeUnit.MILLISECONDS).intValue());
    final RequestConfig requestConfig = requestConfigBuilder.build();
    final StreamThrottler throttler = throttlerRef.get();
    final Double maxBatchBytes = context.getProperty(MAX_BATCH_SIZE).asDataSize(DataUnit.B);
    final AtomicLong bytesToSend = new AtomicLong(firstFlowFile.getSize());
    DestinationAccepts destinationAccepts = null;
    CloseableHttpClient client = null;
    final String transactionId = UUID.randomUUID().toString();
    final AtomicReference<String> dnHolder = new AtomicReference<>("none");
    final Config config = getConfig(url, context);
    final HttpClientConnectionManager conMan = config.getConnectionManager();
    final HttpClientBuilder clientBuilder = HttpClientBuilder.create();
    clientBuilder.setConnectionManager(conMan);
    clientBuilder.setUserAgent(userAgent);
    clientBuilder.addInterceptorFirst(new HttpResponseInterceptor() {

        @Override
        public void process(final HttpResponse response, final HttpContext httpContext) throws HttpException, IOException {
            final HttpCoreContext coreContext = HttpCoreContext.adapt(httpContext);
            final ManagedHttpClientConnection conn = coreContext.getConnection(ManagedHttpClientConnection.class);
            if (!conn.isOpen()) {
                return;
            }
            final SSLSession sslSession = conn.getSSLSession();
            if (sslSession != null) {
                final Certificate[] certChain = sslSession.getPeerCertificates();
                if (certChain == null || certChain.length == 0) {
                    throw new SSLPeerUnverifiedException("No certificates found");
                }
                try {
                    final X509Certificate cert = CertificateUtils.convertAbstractX509Certificate(certChain[0]);
                    dnHolder.set(cert.getSubjectDN().getName().trim());
                } catch (CertificateException e) {
                    final String msg = "Could not extract subject DN from SSL session peer certificate";
                    logger.warn(msg);
                    throw new SSLPeerUnverifiedException(msg);
                }
            }
        }
    });
    clientBuilder.disableAutomaticRetries();
    clientBuilder.disableContentCompression();
    final String username = context.getProperty(USERNAME).getValue();
    final String password = context.getProperty(PASSWORD).getValue();
    // set the credentials if appropriate
    if (username != null) {
        final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        if (password == null) {
            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username));
        } else {
            credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password));
        }
        clientBuilder.setDefaultCredentialsProvider(credentialsProvider);
    }
    // Set the proxy if specified
    if (context.getProperty(PROXY_HOST).isSet() && context.getProperty(PROXY_PORT).isSet()) {
        final String host = context.getProperty(PROXY_HOST).getValue();
        final int port = context.getProperty(PROXY_PORT).asInteger();
        clientBuilder.setProxy(new HttpHost(host, port));
    }
    client = clientBuilder.build();
    // determine whether or not destination accepts flowfile/gzip
    destinationAccepts = config.getDestinationAccepts();
    if (destinationAccepts == null) {
        try {
            destinationAccepts = getDestinationAcceptance(sendAsFlowFile, client, url, getLogger(), transactionId);
            config.setDestinationAccepts(destinationAccepts);
        } catch (final IOException e) {
            firstFlowFile = session.penalize(firstFlowFile);
            session.transfer(firstFlowFile, REL_FAILURE);
            logger.error("Unable to communicate with destination {} to determine whether or not it can accept " + "flowfiles/gzip; routing {} to failure due to {}", new Object[] { url, firstFlowFile, e });
            context.yield();
            return;
        }
    }
    // then we can get more flowfiles from the session up to MAX_BATCH_SIZE for the same URL
    if (sendAsFlowFile && (destinationAccepts.isFlowFileV3Accepted() || destinationAccepts.isFlowFileV2Accepted())) {
        toSend.addAll(session.get(new FlowFileFilter() {

            @Override
            public FlowFileFilterResult filter(FlowFile flowFile) {
                // if over MAX_BATCH_SIZE, then stop adding files
                if (bytesToSend.get() + flowFile.getSize() > maxBatchBytes) {
                    return FlowFileFilterResult.REJECT_AND_TERMINATE;
                }
                // check URL to see if this flowfile can be included in the batch
                final String urlToCheck = context.getProperty(URL).evaluateAttributeExpressions(flowFile).getValue();
                if (url.equals(urlToCheck)) {
                    bytesToSend.addAndGet(flowFile.getSize());
                    return FlowFileFilterResult.ACCEPT_AND_CONTINUE;
                } else {
                    return FlowFileFilterResult.REJECT_AND_CONTINUE;
                }
            }
        }));
    }
    final HttpPost post = new HttpPost(url);
    final DestinationAccepts accepts = destinationAccepts;
    final boolean isDestinationLegacyNiFi = accepts.getProtocolVersion() == null;
    final EntityTemplate entity = new EntityTemplate(new ContentProducer() {

        @Override
        public void writeTo(final OutputStream rawOut) throws IOException {
            final OutputStream throttled = throttler == null ? rawOut : throttler.newThrottledOutputStream(rawOut);
            OutputStream wrappedOut = new BufferedOutputStream(throttled);
            if (compressionLevel > 0 && accepts.isGzipAccepted()) {
                wrappedOut = new GZIPOutputStream(wrappedOut, compressionLevel);
            }
            try (final OutputStream out = wrappedOut) {
                for (final FlowFile flowFile : toSend) {
                    session.read(flowFile, new InputStreamCallback() {

                        @Override
                        public void process(final InputStream rawIn) throws IOException {
                            try (final InputStream in = new BufferedInputStream(rawIn)) {
                                FlowFilePackager packager = null;
                                if (!sendAsFlowFile) {
                                    packager = null;
                                } else if (accepts.isFlowFileV3Accepted()) {
                                    packager = new FlowFilePackagerV3();
                                } else if (accepts.isFlowFileV2Accepted()) {
                                    packager = new FlowFilePackagerV2();
                                } else if (accepts.isFlowFileV1Accepted()) {
                                    packager = new FlowFilePackagerV1();
                                }
                                // formats is acceptable if sending as FlowFile.
                                if (packager == null) {
                                    StreamUtils.copy(in, out);
                                } else {
                                    final Map<String, String> flowFileAttributes;
                                    if (isDestinationLegacyNiFi) {
                                        // Old versions of NiFi expect nf.file.name and nf.file.path to indicate filename & path;
                                        // in order to maintain backward compatibility, we copy the filename & path to those attribute keys.
                                        flowFileAttributes = new HashMap<>(flowFile.getAttributes());
                                        flowFileAttributes.put("nf.file.name", flowFile.getAttribute(CoreAttributes.FILENAME.key()));
                                        flowFileAttributes.put("nf.file.path", flowFile.getAttribute(CoreAttributes.PATH.key()));
                                    } else {
                                        flowFileAttributes = flowFile.getAttributes();
                                    }
                                    packager.packageFlowFile(in, out, flowFileAttributes, flowFile.getSize());
                                }
                            }
                        }
                    });
                }
                out.flush();
            }
        }
    }) {

        @Override
        public long getContentLength() {
            if (compressionLevel == 0 && !sendAsFlowFile && !context.getProperty(CHUNKED_ENCODING).asBoolean()) {
                return toSend.get(0).getSize();
            } else {
                return -1;
            }
        }
    };
    if (context.getProperty(CHUNKED_ENCODING).isSet()) {
        entity.setChunked(context.getProperty(CHUNKED_ENCODING).asBoolean());
    }
    post.setEntity(entity);
    post.setConfig(requestConfig);
    final String contentType;
    if (sendAsFlowFile) {
        if (accepts.isFlowFileV3Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V3;
        } else if (accepts.isFlowFileV2Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V2;
        } else if (accepts.isFlowFileV1Accepted()) {
            contentType = APPLICATION_FLOW_FILE_V1;
        } else {
            logger.error("Cannot send data to {} because the destination does not accept FlowFiles and this processor is " + "configured to deliver FlowFiles; rolling back session", new Object[] { url });
            session.rollback();
            context.yield();
            IOUtils.closeQuietly(client);
            return;
        }
    } else {
        final String contentTypeValue = context.getProperty(CONTENT_TYPE).evaluateAttributeExpressions(toSend.get(0)).getValue();
        contentType = StringUtils.isBlank(contentTypeValue) ? DEFAULT_CONTENT_TYPE : contentTypeValue;
    }
    final String attributeHeaderRegex = context.getProperty(ATTRIBUTES_AS_HEADERS_REGEX).getValue();
    if (attributeHeaderRegex != null && !sendAsFlowFile && toSend.size() == 1) {
        final Pattern pattern = Pattern.compile(attributeHeaderRegex);
        final Map<String, String> attributes = toSend.get(0).getAttributes();
        for (final Map.Entry<String, String> entry : attributes.entrySet()) {
            final String key = entry.getKey();
            if (pattern.matcher(key).matches()) {
                post.setHeader(entry.getKey(), entry.getValue());
            }
        }
    }
    post.setHeader(CONTENT_TYPE_HEADER, contentType);
    post.setHeader(FLOWFILE_CONFIRMATION_HEADER, "true");
    post.setHeader(PROTOCOL_VERSION_HEADER, PROTOCOL_VERSION);
    post.setHeader(TRANSACTION_ID_HEADER, transactionId);
    if (compressionLevel > 0 && accepts.isGzipAccepted()) {
        if (sendAsFlowFile) {
            post.setHeader(GZIPPED_HEADER, "true");
        } else {
            post.setHeader(CONTENT_ENCODING_HEADER, CONTENT_ENCODING_GZIP_VALUE);
        }
    }
    // Do the actual POST
    final String flowFileDescription = toSend.size() <= 10 ? toSend.toString() : toSend.size() + " FlowFiles";
    final String uploadDataRate;
    final long uploadMillis;
    CloseableHttpResponse response = null;
    try {
        final StopWatch stopWatch = new StopWatch(true);
        response = client.execute(post);
        // consume input stream entirely, ignoring its contents. If we
        // don't do this, the Connection will not be returned to the pool
        EntityUtils.consume(response.getEntity());
        stopWatch.stop();
        uploadDataRate = stopWatch.calculateDataRate(bytesToSend.get());
        uploadMillis = stopWatch.getDuration(TimeUnit.MILLISECONDS);
    } catch (final IOException e) {
        logger.error("Failed to Post {} due to {}; transferring to failure", new Object[] { flowFileDescription, e });
        context.yield();
        for (FlowFile flowFile : toSend) {
            flowFile = session.penalize(flowFile);
            session.transfer(flowFile, REL_FAILURE);
        }
        return;
    } finally {
        if (response != null) {
            try {
                response.close();
            } catch (final IOException e) {
                getLogger().warn("Failed to close HTTP Response due to {}", new Object[] { e });
            }
        }
    }
    // If we get a 'SEE OTHER' status code and an HTTP header that indicates that the intent
    // of the Location URI is a flowfile hold, we will store this holdUri. This prevents us
    // from posting to some other webservice and then attempting to delete some resource to which
    // we are redirected
    final int responseCode = response.getStatusLine().getStatusCode();
    final String responseReason = response.getStatusLine().getReasonPhrase();
    String holdUri = null;
    if (responseCode == HttpServletResponse.SC_SEE_OTHER) {
        final Header locationUriHeader = response.getFirstHeader(LOCATION_URI_INTENT_NAME);
        if (locationUriHeader != null) {
            if (LOCATION_URI_INTENT_VALUE.equals(locationUriHeader.getValue())) {
                final Header holdUriHeader = response.getFirstHeader(LOCATION_HEADER_NAME);
                if (holdUriHeader != null) {
                    holdUri = holdUriHeader.getValue();
                }
            }
        }
        if (holdUri == null) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error("Failed to Post {} to {}: sent content and received status code {}:{} but no Hold URI", new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
    }
    if (holdUri == null) {
        if (responseCode == HttpServletResponse.SC_SERVICE_UNAVAILABLE) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error("Failed to Post {} to {}: response code was {}:{}; will yield processing, " + "since the destination is temporarily unavailable", new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            context.yield();
            return;
        }
        if (responseCode >= 300) {
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                logger.error("Failed to Post {} to {}: response code was {}:{}", new Object[] { flowFile, url, responseCode, responseReason });
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
        logger.info("Successfully Posted {} to {} in {} at a rate of {}", new Object[] { flowFileDescription, url, FormatUtils.formatMinutesSeconds(uploadMillis, TimeUnit.MILLISECONDS), uploadDataRate });
        for (final FlowFile flowFile : toSend) {
            session.getProvenanceReporter().send(flowFile, url, "Remote DN=" + dnHolder.get(), uploadMillis, true);
            session.transfer(flowFile, REL_SUCCESS);
        }
        return;
    }
    // 
    // the response indicated a Hold URI; delete the Hold.
    // 
    // determine the full URI of the Flow File's Hold; Unfortunately, the responses that are returned have
    // changed over the past, so we have to take into account a few different possibilities.
    String fullHoldUri = holdUri;
    if (holdUri.startsWith("/contentListener")) {
        // If the Hold URI that we get starts with /contentListener, it may not really be /contentListener,
        // as this really indicates that it should be whatever we posted to -- if posting directly to the
        // ListenHTTP component, it will be /contentListener, but if posting to a proxy/load balancer, we may
        // be posting to some other URL.
        fullHoldUri = url + holdUri.substring(16);
    } else if (holdUri.startsWith("/")) {
        // URL indicates the full path but not hostname or port; use the same hostname & port that we posted
        // to but use the full path indicated by the response.
        int firstSlash = url.indexOf("/", 8);
        if (firstSlash < 0) {
            firstSlash = url.length();
        }
        final String beforeSlash = url.substring(0, firstSlash);
        fullHoldUri = beforeSlash + holdUri;
    } else if (!holdUri.startsWith("http")) {
        // Absolute URL
        fullHoldUri = url + (url.endsWith("/") ? "" : "/") + holdUri;
    }
    final HttpDelete delete = new HttpDelete(fullHoldUri);
    delete.setHeader(TRANSACTION_ID_HEADER, transactionId);
    while (true) {
        try {
            final HttpResponse holdResponse = client.execute(delete);
            EntityUtils.consume(holdResponse.getEntity());
            final int holdStatusCode = holdResponse.getStatusLine().getStatusCode();
            final String holdReason = holdResponse.getStatusLine().getReasonPhrase();
            if (holdStatusCode >= 300) {
                logger.error("Failed to delete Hold that destination placed on {}: got response code {}:{}; routing to failure", new Object[] { flowFileDescription, holdStatusCode, holdReason });
                for (FlowFile flowFile : toSend) {
                    flowFile = session.penalize(flowFile);
                    session.transfer(flowFile, REL_FAILURE);
                }
                return;
            }
            logger.info("Successfully Posted {} to {} in {} milliseconds at a rate of {}", new Object[] { flowFileDescription, url, uploadMillis, uploadDataRate });
            for (final FlowFile flowFile : toSend) {
                session.getProvenanceReporter().send(flowFile, url);
                session.transfer(flowFile, REL_SUCCESS);
            }
            return;
        } catch (final IOException e) {
            logger.warn("Failed to delete Hold that destination placed on {} due to {}", new Object[] { flowFileDescription, e });
        }
        if (!isScheduled()) {
            context.yield();
            logger.warn("Failed to delete Hold that destination placed on {}; Processor has been stopped so routing FlowFile(s) to failure", new Object[] { flowFileDescription });
            for (FlowFile flowFile : toSend) {
                flowFile = session.penalize(flowFile);
                session.transfer(flowFile, REL_FAILURE);
            }
            return;
        }
    }
}

Example 52 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class CompressContent method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    final long sizeBeforeCompression = flowFile.getSize();
    final String compressionMode = context.getProperty(MODE).getValue();
    String compressionFormatValue = context.getProperty(COMPRESSION_FORMAT).getValue();
    if (compressionFormatValue.equals(COMPRESSION_FORMAT_ATTRIBUTE)) {
        final String mimeType = flowFile.getAttribute(CoreAttributes.MIME_TYPE.key());
        if (mimeType == null) {
            logger.error("No {} attribute exists for {}; routing to failure", new Object[] { CoreAttributes.MIME_TYPE.key(), flowFile });
            session.transfer(flowFile, REL_FAILURE);
            return;
        }
        compressionFormatValue = compressionFormatMimeTypeMap.get(mimeType);
        if (compressionFormatValue == null) {
            logger.info("Mime Type of {} is '{}', which does not indicate a supported Compression Format; routing to success without decompressing", new Object[] { flowFile, mimeType });
            session.transfer(flowFile, REL_SUCCESS);
            return;
        }
    }
    final String compressionFormat = compressionFormatValue;
    final AtomicReference<String> mimeTypeRef = new AtomicReference<>(null);
    final StopWatch stopWatch = new StopWatch(true);
    final String fileExtension;
    switch(compressionFormat.toLowerCase()) {
        case COMPRESSION_FORMAT_GZIP:
            fileExtension = ".gz";
            break;
        case COMPRESSION_FORMAT_LZMA:
            fileExtension = ".lzma";
            break;
        case COMPRESSION_FORMAT_XZ_LZMA2:
            fileExtension = ".xz";
            break;
        case COMPRESSION_FORMAT_BZIP2:
            fileExtension = ".bz2";
            break;
        case COMPRESSION_FORMAT_SNAPPY:
            fileExtension = ".snappy";
            break;
        case COMPRESSION_FORMAT_SNAPPY_FRAMED:
            fileExtension = ".sz";
            break;
        default:
            fileExtension = "";
            break;
    }
    try {
        flowFile = session.write(flowFile, new StreamCallback() {

            @Override
            public void process(final InputStream rawIn, final OutputStream rawOut) throws IOException {
                final OutputStream compressionOut;
                final InputStream compressionIn;
                final OutputStream bufferedOut = new BufferedOutputStream(rawOut, 65536);
                final InputStream bufferedIn = new BufferedInputStream(rawIn, 65536);
                try {
                    if (MODE_COMPRESS.equalsIgnoreCase(compressionMode)) {
                        compressionIn = bufferedIn;
                        switch(compressionFormat.toLowerCase()) {
                            case COMPRESSION_FORMAT_GZIP:
                                final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
                                compressionOut = new GZIPOutputStream(bufferedOut, compressionLevel);
                                mimeTypeRef.set("application/gzip");
                                break;
                            case COMPRESSION_FORMAT_LZMA:
                                compressionOut = new LzmaOutputStream.Builder(bufferedOut).build();
                                mimeTypeRef.set("application/x-lzma");
                                break;
                            case COMPRESSION_FORMAT_XZ_LZMA2:
                                compressionOut = new XZOutputStream(bufferedOut, new LZMA2Options());
                                mimeTypeRef.set("application/x-xz");
                                break;
                            case COMPRESSION_FORMAT_SNAPPY:
                                compressionOut = new SnappyOutputStream(bufferedOut);
                                mimeTypeRef.set("application/x-snappy");
                                break;
                            case COMPRESSION_FORMAT_SNAPPY_FRAMED:
                                compressionOut = new SnappyFramedOutputStream(bufferedOut);
                                mimeTypeRef.set("application/x-snappy-framed");
                                break;
                            case COMPRESSION_FORMAT_BZIP2:
                            default:
                                mimeTypeRef.set("application/x-bzip2");
                                compressionOut = new CompressorStreamFactory().createCompressorOutputStream(compressionFormat.toLowerCase(), bufferedOut);
                                break;
                        }
                    } else {
                        compressionOut = bufferedOut;
                        switch(compressionFormat.toLowerCase()) {
                            case COMPRESSION_FORMAT_LZMA:
                                compressionIn = new LzmaInputStream(bufferedIn, new Decoder());
                                break;
                            case COMPRESSION_FORMAT_XZ_LZMA2:
                                compressionIn = new XZInputStream(bufferedIn);
                                break;
                            case COMPRESSION_FORMAT_BZIP2:
                                // need this two-arg constructor to support concatenated streams
                                compressionIn = new BZip2CompressorInputStream(bufferedIn, true);
                                break;
                            case COMPRESSION_FORMAT_GZIP:
                                compressionIn = new GzipCompressorInputStream(bufferedIn, true);
                                break;
                            case COMPRESSION_FORMAT_SNAPPY:
                                compressionIn = new SnappyInputStream(bufferedIn);
                                break;
                            case COMPRESSION_FORMAT_SNAPPY_FRAMED:
                                compressionIn = new SnappyFramedInputStream(bufferedIn);
                                break;
                            default:
                                compressionIn = new CompressorStreamFactory().createCompressorInputStream(compressionFormat.toLowerCase(), bufferedIn);
                        }
                    }
                } catch (final Exception e) {
                    closeQuietly(bufferedOut);
                    throw new IOException(e);
                }
                try (final InputStream in = compressionIn;
                    final OutputStream out = compressionOut) {
                    final byte[] buffer = new byte[8192];
                    int len;
                    while ((len = in.read(buffer)) > 0) {
                        out.write(buffer, 0, len);
                    }
                    out.flush();
                }
            }
        });
        stopWatch.stop();
        final long sizeAfterCompression = flowFile.getSize();
        if (MODE_DECOMPRESS.equalsIgnoreCase(compressionMode)) {
            flowFile = session.removeAttribute(flowFile, CoreAttributes.MIME_TYPE.key());
            if (context.getProperty(UPDATE_FILENAME).asBoolean()) {
                final String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key());
                if (filename.toLowerCase().endsWith(fileExtension)) {
                    flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), filename.substring(0, filename.length() - fileExtension.length()));
                }
            }
        } else {
            flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), mimeTypeRef.get());
            if (context.getProperty(UPDATE_FILENAME).asBoolean()) {
                final String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key());
                flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), filename + fileExtension);
            }
        }
        logger.info("Successfully {}ed {} using {} compression format; size changed from {} to {} bytes", new Object[] { compressionMode.toLowerCase(), flowFile, compressionFormat, sizeBeforeCompression, sizeAfterCompression });
        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getDuration(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
    } catch (final ProcessException e) {
        logger.error("Unable to {} {} using {} compression format due to {}; routing to failure", new Object[] { compressionMode.toLowerCase(), flowFile, compressionFormat, e });
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 53 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class ConvertCharacterSet method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    final Charset inputCharset = Charset.forName(context.getProperty(INPUT_CHARSET).evaluateAttributeExpressions(flowFile).getValue());
    final Charset outputCharset = Charset.forName(context.getProperty(OUTPUT_CHARSET).evaluateAttributeExpressions(flowFile).getValue());
    final CharBuffer charBuffer = CharBuffer.allocate(MAX_BUFFER_SIZE);
    final CharsetDecoder decoder = inputCharset.newDecoder();
    decoder.onMalformedInput(CodingErrorAction.REPLACE);
    decoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
    decoder.replaceWith("?");
    final CharsetEncoder encoder = outputCharset.newEncoder();
    encoder.onMalformedInput(CodingErrorAction.REPLACE);
    encoder.onUnmappableCharacter(CodingErrorAction.REPLACE);
    encoder.replaceWith("?".getBytes(outputCharset));
    try {
        final StopWatch stopWatch = new StopWatch(true);
        flowFile = session.write(flowFile, new StreamCallback() {

            @Override
            public void process(final InputStream rawIn, final OutputStream rawOut) throws IOException {
                try (final BufferedReader reader = new BufferedReader(new InputStreamReader(rawIn, decoder), MAX_BUFFER_SIZE);
                    final BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(rawOut, encoder), MAX_BUFFER_SIZE)) {
                    int charsRead;
                    while ((charsRead = reader.read(charBuffer)) != -1) {
                        charBuffer.flip();
                        writer.write(charBuffer.array(), 0, charsRead);
                    }
                    writer.flush();
                }
            }
        });
        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        logger.info("successfully converted characters from {} to {} for {}", new Object[] { inputCharset, outputCharset, flowFile });
        session.transfer(flowFile, REL_SUCCESS);
    } catch (final Exception e) {
        throw new ProcessException(e);
    }
}

Example 54 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class FetchFile method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final StopWatch stopWatch = new StopWatch(true);
    final String filename = context.getProperty(FILENAME).evaluateAttributeExpressions(flowFile).getValue();
    final LogLevel levelFileNotFound = LogLevel.valueOf(context.getProperty(FILE_NOT_FOUND_LOG_LEVEL).getValue());
    final LogLevel levelPermDenied = LogLevel.valueOf(context.getProperty(PERM_DENIED_LOG_LEVEL).getValue());
    final File file = new File(filename);
    // Verify that file system is reachable and file exists
    Path filePath = file.toPath();
    if (!Files.exists(filePath) && !Files.notExists(filePath)) {
        // see https://docs.oracle.com/javase/tutorial/essential/io/check.html for more details
        getLogger().log(levelFileNotFound, "Could not fetch file {} from file system for {} because the existence of the file cannot be verified; routing to failure", new Object[] { file, flowFile });
        session.transfer(session.penalize(flowFile), REL_FAILURE);
        return;
    } else if (!Files.exists(filePath)) {
        getLogger().log(levelFileNotFound, "Could not fetch file {} from file system for {} because the file does not exist; routing to not.found", new Object[] { file, flowFile });
        session.getProvenanceReporter().route(flowFile, REL_NOT_FOUND);
        session.transfer(session.penalize(flowFile), REL_NOT_FOUND);
        return;
    }
    // Verify read permission on file
    final String user = System.getProperty("user.name");
    if (!isReadable(file)) {
        getLogger().log(levelPermDenied, "Could not fetch file {} from file system for {} due to user {} not having sufficient permissions to read the file; routing to permission.denied", new Object[] { file, flowFile, user });
        session.getProvenanceReporter().route(flowFile, REL_PERMISSION_DENIED);
        session.transfer(session.penalize(flowFile), REL_PERMISSION_DENIED);
        return;
    }
    // If configured to move the file and fail if unable to do so, check that the existing file does not exist and that we have write permissions
    // for the parent file.
    final String completionStrategy = context.getProperty(COMPLETION_STRATEGY).getValue();
    final String targetDirectoryName = context.getProperty(MOVE_DESTINATION_DIR).evaluateAttributeExpressions(flowFile).getValue();
    if (targetDirectoryName != null) {
        final File targetDir = new File(targetDirectoryName);
        if (COMPLETION_MOVE.getValue().equalsIgnoreCase(completionStrategy)) {
            if (targetDir.exists() && (!isWritable(targetDir) || !isDirectory(targetDir))) {
                getLogger().error("Could not fetch file {} from file system for {} because Completion Strategy is configured to move the original file to {}, " + "but that is not a directory or user {} does not have permissions to write to that directory", new Object[] { file, flowFile, targetDir, user });
                session.transfer(flowFile, REL_FAILURE);
                return;
            }
            final String conflictStrategy = context.getProperty(CONFLICT_STRATEGY).getValue();
            if (CONFLICT_FAIL.getValue().equalsIgnoreCase(conflictStrategy)) {
                final File targetFile = new File(targetDir, file.getName());
                if (targetFile.exists()) {
                    getLogger().error("Could not fetch file {} from file system for {} because Completion Strategy is configured to move the original file to {}, " + "but a file with name {} already exists in that directory and the Move Conflict Strategy is configured for failure", new Object[] { file, flowFile, targetDir, file.getName() });
                    session.transfer(flowFile, REL_FAILURE);
                    return;
                }
            }
        }
    }
    // import content from file system
    try (final FileInputStream fis = new FileInputStream(file)) {
        flowFile = session.importFrom(fis, flowFile);
    } catch (final IOException ioe) {
        getLogger().error("Could not fetch file {} from file system for {} due to {}; routing to failure", new Object[] { file, flowFile, ioe.toString() }, ioe);
        session.transfer(session.penalize(flowFile), REL_FAILURE);
        return;
    }
    session.getProvenanceReporter().modifyContent(flowFile, "Replaced content of FlowFile with contents of " + file.toURI(), stopWatch.getElapsed(TimeUnit.MILLISECONDS));
    session.transfer(flowFile, REL_SUCCESS);
    // It is critical that we commit the session before we perform the Completion Strategy. Otherwise, we could have a case where we
    // ingest the file, delete/move the file, and then NiFi is restarted before the session is committed. That would result in data loss.
    // As long as we commit the session right here, before we perform the Completion Strategy, we are safe.
    session.commit();
    // Attempt to perform the Completion Strategy action
    Exception completionFailureException = null;
    if (COMPLETION_DELETE.getValue().equalsIgnoreCase(completionStrategy)) {
        // convert to path and use Files.delete instead of file.delete so that if we fail, we know why
        try {
            delete(file);
        } catch (final IOException ioe) {
            completionFailureException = ioe;
        }
    } else if (COMPLETION_MOVE.getValue().equalsIgnoreCase(completionStrategy)) {
        final File targetDirectory = new File(targetDirectoryName);
        final File targetFile = new File(targetDirectory, file.getName());
        try {
            if (targetFile.exists()) {
                final String conflictStrategy = context.getProperty(CONFLICT_STRATEGY).getValue();
                if (CONFLICT_KEEP_INTACT.getValue().equalsIgnoreCase(conflictStrategy)) {
                    // don't move, just delete the original
                    Files.delete(file.toPath());
                } else if (CONFLICT_RENAME.getValue().equalsIgnoreCase(conflictStrategy)) {
                    // rename to add a random UUID but keep the file extension if it has one.
                    final String simpleFilename = targetFile.getName();
                    final String newName;
                    if (simpleFilename.contains(".")) {
                        newName = StringUtils.substringBeforeLast(simpleFilename, ".") + "-" + UUID.randomUUID().toString() + "." + StringUtils.substringAfterLast(simpleFilename, ".");
                    } else {
                        newName = simpleFilename + "-" + UUID.randomUUID().toString();
                    }
                    move(file, new File(targetDirectory, newName), false);
                } else if (CONFLICT_REPLACE.getValue().equalsIgnoreCase(conflictStrategy)) {
                    move(file, targetFile, true);
                }
            } else {
                move(file, targetFile, false);
            }
        } catch (final IOException ioe) {
            completionFailureException = ioe;
        }
    }
    // Handle completion failures
    if (completionFailureException != null) {
        getLogger().warn("Successfully fetched the content from {} for {} but failed to perform Completion Action due to {}; routing to success", new Object[] { file, flowFile, completionFailureException }, completionFailureException);
    }
}

Example 55 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class FetchFileTransfer method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final StopWatch stopWatch = new StopWatch(true);
    final String host = context.getProperty(HOSTNAME).evaluateAttributeExpressions(flowFile).getValue();
    final int port = context.getProperty(UNDEFAULTED_PORT).evaluateAttributeExpressions(flowFile).asInteger();
    final String filename = context.getProperty(REMOTE_FILENAME).evaluateAttributeExpressions(flowFile).getValue();
    // Try to get a FileTransfer object from our cache.
    BlockingQueue<FileTransferIdleWrapper> transferQueue;
    synchronized (fileTransferMap) {
        final Tuple<String, Integer> tuple = new Tuple<>(host, port);
        transferQueue = fileTransferMap.get(tuple);
        if (transferQueue == null) {
            transferQueue = new LinkedBlockingQueue<>();
            fileTransferMap.put(tuple, transferQueue);
        }
        // periodically close idle connections
        if (System.currentTimeMillis() - lastClearTime > IDLE_CONNECTION_MILLIS) {
            closeConnections(false);
            lastClearTime = System.currentTimeMillis();
        }
    }
    // we have a queue of FileTransfer Objects. Get one from the queue or create a new one.
    FileTransfer transfer;
    FileTransferIdleWrapper transferWrapper = transferQueue.poll();
    if (transferWrapper == null) {
        transfer = createFileTransfer(context);
    } else {
        transfer = transferWrapper.getFileTransfer();
    }
    boolean closeConnection = false;
    try {
        // Pull data from remote system.
        final InputStream in;
        try {
            in = transfer.getInputStream(filename, flowFile);
            flowFile = session.write(flowFile, new OutputStreamCallback() {

                @Override
                public void process(final OutputStream out) throws IOException {
                    StreamUtils.copy(in, out);
                }
            });
            if (!transfer.flush(flowFile)) {
                throw new IOException("completePendingCommand returned false, file transfer failed");
            }
        } catch (final FileNotFoundException e) {
            closeConnection = false;
            getLogger().error("Failed to fetch content for {} from filename {} on remote host {} because the file could not be found on the remote system; routing to {}", new Object[] { flowFile, filename, host, REL_NOT_FOUND.getName() });
            session.transfer(session.penalize(flowFile), REL_NOT_FOUND);
            session.getProvenanceReporter().route(flowFile, REL_NOT_FOUND);
            return;
        } catch (final PermissionDeniedException e) {
            closeConnection = false;
            getLogger().error("Failed to fetch content for {} from filename {} on remote host {} due to insufficient permissions; routing to {}", new Object[] { flowFile, filename, host, REL_PERMISSION_DENIED.getName() });
            session.transfer(session.penalize(flowFile), REL_PERMISSION_DENIED);
            session.getProvenanceReporter().route(flowFile, REL_PERMISSION_DENIED);
            return;
        } catch (final ProcessException | IOException e) {
            closeConnection = true;
            getLogger().error("Failed to fetch content for {} from filename {} on remote host {}:{} due to {}; routing to comms.failure", new Object[] { flowFile, filename, host, port, e.toString() }, e);
            session.transfer(session.penalize(flowFile), REL_COMMS_FAILURE);
            return;
        }
        // Add FlowFile attributes
        final String protocolName = transfer.getProtocolName();
        final Map<String, String> attributes = new HashMap<>();
        attributes.put(protocolName + ".remote.host", host);
        attributes.put(protocolName + ".remote.port", String.valueOf(port));
        attributes.put(protocolName + ".remote.filename", filename);
        if (filename.contains("/")) {
            final String path = StringUtils.substringBeforeLast(filename, "/");
            final String filenameOnly = StringUtils.substringAfterLast(filename, "/");
            attributes.put(CoreAttributes.PATH.key(), path);
            attributes.put(CoreAttributes.FILENAME.key(), filenameOnly);
        } else {
            attributes.put(CoreAttributes.FILENAME.key(), filename);
        }
        flowFile = session.putAllAttributes(flowFile, attributes);
        // emit provenance event and transfer FlowFile
        session.getProvenanceReporter().fetch(flowFile, protocolName + "://" + host + ":" + port + "/" + filename, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
        // it is critical that we commit the session before moving/deleting the remote file. Otherwise, we could have a situation where
        // we ingest the data, delete/move the remote file, and then NiFi dies/is shut down before the session is committed. This would
        // result in data loss! If we commit the session first, we are safe.
        session.commit();
        final String completionStrategy = context.getProperty(COMPLETION_STRATEGY).getValue();
        if (COMPLETION_DELETE.getValue().equalsIgnoreCase(completionStrategy)) {
            try {
                transfer.deleteFile(flowFile, null, filename);
            } catch (final FileNotFoundException e) {
            // file doesn't exist -- effectively the same as removing it. Move on.
            } catch (final IOException ioe) {
                getLogger().warn("Successfully fetched the content for {} from {}:{}{} but failed to remove the remote file due to {}", new Object[] { flowFile, host, port, filename, ioe }, ioe);
            }
        } else if (COMPLETION_MOVE.getValue().equalsIgnoreCase(completionStrategy)) {
            String targetDir = context.getProperty(MOVE_DESTINATION_DIR).evaluateAttributeExpressions(flowFile).getValue();
            if (!targetDir.endsWith("/")) {
                targetDir = targetDir + "/";
            }
            final String simpleFilename = StringUtils.substringAfterLast(filename, "/");
            final String target = targetDir + simpleFilename;
            try {
                transfer.rename(flowFile, filename, target);
            } catch (final IOException ioe) {
                getLogger().warn("Successfully fetched the content for {} from {}:{}{} but failed to rename the remote file due to {}", new Object[] { flowFile, host, port, filename, ioe }, ioe);
            }
        }
    } finally {
        if (transfer != null) {
            if (closeConnection) {
                getLogger().debug("Closing FileTransfer...");
                try {
                    transfer.close();
                } catch (final IOException e) {
                    getLogger().warn("Failed to close connection to {}:{} due to {}", new Object[] { host, port, e.getMessage() }, e);
                }
            } else {
                getLogger().debug("Returning FileTransfer to pool...");
                transferQueue.offer(new FileTransferIdleWrapper(transfer, System.nanoTime()));
            }
        }
    }
}