Examples with StopWatch org.apache.nifi.util.StopWatch used on opensource projects

Example 21 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class ModifyBytes method onTrigger.

@Override
public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
    FlowFile ff = session.get();
    if (null == ff) {
        return;
    }
    final ComponentLog logger = getLogger();
    final long startOffset = context.getProperty(START_OFFSET).evaluateAttributeExpressions(ff).asDataSize(DataUnit.B).longValue();
    final long endOffset = context.getProperty(END_OFFSET).evaluateAttributeExpressions(ff).asDataSize(DataUnit.B).longValue();
    final boolean removeAll = context.getProperty(REMOVE_ALL).asBoolean();
    final long newFileSize = removeAll ? 0L : ff.getSize() - startOffset - endOffset;
    final StopWatch stopWatch = new StopWatch(true);
    if (newFileSize <= 0) {
        ff = session.write(ff, new OutputStreamCallback() {

            @Override
            public void process(final OutputStream out) throws IOException {
                out.write(new byte[0]);
            }
        });
    } else {
        ff = session.write(ff, new StreamCallback() {

            @Override
            public void process(final InputStream in, final OutputStream out) throws IOException {
                in.skip(startOffset);
                StreamUtils.copy(in, out, newFileSize);
            }
        });
    }
    logger.info("Transferred {} to 'success'", new Object[] { ff });
    session.getProvenanceReporter().modifyContent(ff, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
    session.transfer(ff, REL_SUCCESS);
}

Example 22 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class Base64EncodeContent method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    boolean encode = context.getProperty(MODE).getValue().equalsIgnoreCase(ENCODE_MODE);
    try {
        final StopWatch stopWatch = new StopWatch(true);
        if (encode) {
            flowFile = session.write(flowFile, new StreamCallback() {

                @Override
                public void process(InputStream in, OutputStream out) throws IOException {
                    try (Base64OutputStream bos = new Base64OutputStream(out)) {
                        int len = -1;
                        byte[] buf = new byte[8192];
                        while ((len = in.read(buf)) > 0) {
                            bos.write(buf, 0, len);
                        }
                        bos.flush();
                    }
                }
            });
        } else {
            flowFile = session.write(flowFile, new StreamCallback() {

                @Override
                public void process(InputStream in, OutputStream out) throws IOException {
                    try (Base64InputStream bis = new Base64InputStream(new ValidatingBase64InputStream(in))) {
                        int len = -1;
                        byte[] buf = new byte[8192];
                        while ((len = bis.read(buf)) > 0) {
                            out.write(buf, 0, len);
                        }
                        out.flush();
                    }
                }
            });
        }
        logger.info("Successfully {} {}", new Object[] { encode ? "encoded" : "decoded", flowFile });
        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
    } catch (ProcessException e) {
        logger.error("Failed to {} {} due to {}", new Object[] { encode ? "encode" : "decode", flowFile, e });
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 23 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class EncodeContent method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    boolean encode = context.getProperty(MODE).getValue().equalsIgnoreCase(ENCODE_MODE);
    String encoding = context.getProperty(ENCODING).getValue();
    StreamCallback encoder = null;
    // Select the encoder/decoder to use
    if (encode) {
        if (encoding.equalsIgnoreCase(BASE64_ENCODING)) {
            encoder = new EncodeBase64();
        } else if (encoding.equalsIgnoreCase(BASE32_ENCODING)) {
            encoder = new EncodeBase32();
        } else if (encoding.equalsIgnoreCase(HEX_ENCODING)) {
            encoder = new EncodeHex();
        }
    } else {
        if (encoding.equalsIgnoreCase(BASE64_ENCODING)) {
            encoder = new DecodeBase64();
        } else if (encoding.equalsIgnoreCase(BASE32_ENCODING)) {
            encoder = new DecodeBase32();
        } else if (encoding.equalsIgnoreCase(HEX_ENCODING)) {
            encoder = new DecodeHex();
        }
    }
    if (encoder == null) {
        logger.warn("Unknown operation: {} {}", new Object[] { encode ? "encode" : "decode", encoding });
        return;
    }
    try {
        final StopWatch stopWatch = new StopWatch(true);
        flowFile = session.write(flowFile, encoder);
        logger.info("Successfully {} {}", new Object[] { encode ? "encoded" : "decoded", flowFile });
        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
    } catch (Exception e) {
        logger.error("Failed to {} {} due to {}", new Object[] { encode ? "encode" : "decode", flowFile, e });
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 24 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class EncryptContent method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    final String method = context.getProperty(ENCRYPTION_ALGORITHM).getValue();
    final EncryptionMethod encryptionMethod = EncryptionMethod.valueOf(method);
    final String providerName = encryptionMethod.getProvider();
    final String algorithm = encryptionMethod.getAlgorithm();
    final String password = context.getProperty(PASSWORD).getValue();
    final KeyDerivationFunction kdf = KeyDerivationFunction.valueOf(context.getProperty(KEY_DERIVATION_FUNCTION).getValue());
    final boolean encrypt = context.getProperty(MODE).getValue().equalsIgnoreCase(ENCRYPT_MODE);
    Encryptor encryptor;
    StreamCallback callback;
    try {
        if (isPGPAlgorithm(algorithm)) {
            final String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key());
            final String publicKeyring = context.getProperty(PUBLIC_KEYRING).getValue();
            final String privateKeyring = context.getProperty(PRIVATE_KEYRING).getValue();
            if (encrypt && publicKeyring != null) {
                final String publicUserId = context.getProperty(PUBLIC_KEY_USERID).getValue();
                encryptor = new OpenPGPKeyBasedEncryptor(algorithm, providerName, publicKeyring, publicUserId, null, filename);
            } else if (!encrypt && privateKeyring != null) {
                final char[] keyringPassphrase = context.getProperty(PRIVATE_KEYRING_PASSPHRASE).evaluateAttributeExpressions().getValue().toCharArray();
                encryptor = new OpenPGPKeyBasedEncryptor(algorithm, providerName, privateKeyring, null, keyringPassphrase, filename);
            } else {
                final char[] passphrase = Normalizer.normalize(password, Normalizer.Form.NFC).toCharArray();
                encryptor = new OpenPGPPasswordBasedEncryptor(algorithm, providerName, passphrase, filename);
            }
        } else if (kdf.equals(KeyDerivationFunction.NONE)) {
            // Raw key
            final String keyHex = context.getProperty(RAW_KEY_HEX).getValue();
            encryptor = new KeyedEncryptor(encryptionMethod, Hex.decodeHex(keyHex.toCharArray()));
        } else {
            // PBE
            final char[] passphrase = Normalizer.normalize(password, Normalizer.Form.NFC).toCharArray();
            encryptor = new PasswordBasedEncryptor(encryptionMethod, passphrase, kdf);
        }
        if (encrypt) {
            callback = encryptor.getEncryptionCallback();
        } else {
            callback = encryptor.getDecryptionCallback();
        }
    } catch (final Exception e) {
        logger.error("Failed to initialize {}cryption algorithm because - ", new Object[] { encrypt ? "en" : "de", e });
        session.rollback();
        context.yield();
        return;
    }
    try {
        final StopWatch stopWatch = new StopWatch(true);
        flowFile = session.write(flowFile, callback);
        logger.info("successfully {}crypted {}", new Object[] { encrypt ? "en" : "de", flowFile });
        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
    } catch (final ProcessException e) {
        logger.error("Cannot {}crypt {} - ", new Object[] { encrypt ? "en" : "de", flowFile, e });
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 25 with StopWatch

use of org.apache.nifi.util.StopWatch in project nifi by apache.

the class ExecuteSQL method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile fileToProcess = null;
    if (context.hasIncomingConnection()) {
        fileToProcess = session.get();
        // we know that we should run only if we have a FlowFile.
        if (fileToProcess == null && context.hasNonLoopConnection()) {
            return;
        }
    }
    final ComponentLog logger = getLogger();
    final DBCPService dbcpService = context.getProperty(DBCP_SERVICE).asControllerService(DBCPService.class);
    final Integer queryTimeout = context.getProperty(QUERY_TIMEOUT).asTimePeriod(TimeUnit.SECONDS).intValue();
    final boolean convertNamesForAvro = context.getProperty(NORMALIZE_NAMES_FOR_AVRO).asBoolean();
    final Boolean useAvroLogicalTypes = context.getProperty(USE_AVRO_LOGICAL_TYPES).asBoolean();
    final Integer defaultPrecision = context.getProperty(DEFAULT_PRECISION).evaluateAttributeExpressions().asInteger();
    final Integer defaultScale = context.getProperty(DEFAULT_SCALE).evaluateAttributeExpressions().asInteger();
    final StopWatch stopWatch = new StopWatch(true);
    final String selectQuery;
    if (context.getProperty(SQL_SELECT_QUERY).isSet()) {
        selectQuery = context.getProperty(SQL_SELECT_QUERY).evaluateAttributeExpressions(fileToProcess).getValue();
    } else {
        // If the query is not set, then an incoming flow file is required, and expected to contain a valid SQL select query.
        // If there is no incoming connection, onTrigger will not be called as the processor will fail when scheduled.
        final StringBuilder queryContents = new StringBuilder();
        session.read(fileToProcess, in -> queryContents.append(IOUtils.toString(in, Charset.defaultCharset())));
        selectQuery = queryContents.toString();
    }
    int resultCount = 0;
    try (final Connection con = dbcpService.getConnection();
        final PreparedStatement st = con.prepareStatement(selectQuery)) {
        // timeout in seconds
        st.setQueryTimeout(queryTimeout);
        if (fileToProcess != null) {
            JdbcCommon.setParameters(st, fileToProcess.getAttributes());
        }
        logger.debug("Executing query {}", new Object[] { selectQuery });
        boolean results = st.execute();
        while (results) {
            FlowFile resultSetFF;
            if (fileToProcess == null) {
                resultSetFF = session.create();
            } else {
                resultSetFF = session.create(fileToProcess);
                resultSetFF = session.putAllAttributes(resultSetFF, fileToProcess.getAttributes());
            }
            final AtomicLong nrOfRows = new AtomicLong(0L);
            resultSetFF = session.write(resultSetFF, out -> {
                try {
                    final ResultSet resultSet = st.getResultSet();
                    final JdbcCommon.AvroConversionOptions options = JdbcCommon.AvroConversionOptions.builder().convertNames(convertNamesForAvro).useLogicalTypes(useAvroLogicalTypes).defaultPrecision(defaultPrecision).defaultScale(defaultScale).build();
                    nrOfRows.set(JdbcCommon.convertToAvroStream(resultSet, out, options, null));
                } catch (final SQLException e) {
                    throw new ProcessException(e);
                }
            });
            long duration = stopWatch.getElapsed(TimeUnit.MILLISECONDS);
            // set attribute how many rows were selected
            resultSetFF = session.putAttribute(resultSetFF, RESULT_ROW_COUNT, String.valueOf(nrOfRows.get()));
            resultSetFF = session.putAttribute(resultSetFF, RESULT_QUERY_DURATION, String.valueOf(duration));
            resultSetFF = session.putAttribute(resultSetFF, CoreAttributes.MIME_TYPE.key(), JdbcCommon.MIME_TYPE_AVRO_BINARY);
            logger.info("{} contains {} Avro records; transferring to 'success'", new Object[] { resultSetFF, nrOfRows.get() });
            session.getProvenanceReporter().modifyContent(resultSetFF, "Retrieved " + nrOfRows.get() + " rows", duration);
            session.transfer(resultSetFF, REL_SUCCESS);
            resultCount++;
            // are there anymore result sets?
            try {
                results = st.getMoreResults();
            } catch (SQLException ex) {
                results = false;
            }
        }
        // pass the original flow file down the line to trigger downstream processors
        if (fileToProcess != null) {
            if (resultCount > 0) {
                session.remove(fileToProcess);
            } else {
                fileToProcess = session.write(fileToProcess, JdbcCommon::createEmptyAvroStream);
                session.transfer(fileToProcess, REL_SUCCESS);
            }
        }
    } catch (final ProcessException | SQLException e) {
        // pass the original flow file down the line to trigger downstream processors
        if (fileToProcess == null) {
            // This can happen if any exceptions occur while setting up the connection, statement, etc.
            logger.error("Unable to execute SQL select query {} due to {}. No FlowFile to route to failure", new Object[] { selectQuery, e });
            context.yield();
        } else {
            if (context.hasIncomingConnection()) {
                logger.error("Unable to execute SQL select query {} for {} due to {}; routing to failure", new Object[] { selectQuery, fileToProcess, e });
                fileToProcess = session.penalize(fileToProcess);
            } else {
                logger.error("Unable to execute SQL select query {} due to {}; routing to failure", new Object[] { selectQuery, e });
                context.yield();
            }
            session.transfer(fileToProcess, REL_FAILURE);
        }
    }
}