Examples with StreamCallback org.apache.nifi.processor.io.StreamCallback used on opensource projects

Example 6 with StreamCallback

use of org.apache.nifi.processor.io.StreamCallback in project nifi by apache.

the class Base64EncodeContent method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    boolean encode = context.getProperty(MODE).getValue().equalsIgnoreCase(ENCODE_MODE);
    try {
        final StopWatch stopWatch = new StopWatch(true);
        if (encode) {
            flowFile = session.write(flowFile, new StreamCallback() {

                @Override
                public void process(InputStream in, OutputStream out) throws IOException {
                    try (Base64OutputStream bos = new Base64OutputStream(out)) {
                        int len = -1;
                        byte[] buf = new byte[8192];
                        while ((len = in.read(buf)) > 0) {
                            bos.write(buf, 0, len);
                        }
                        bos.flush();
                    }
                }
            });
        } else {
            flowFile = session.write(flowFile, new StreamCallback() {

                @Override
                public void process(InputStream in, OutputStream out) throws IOException {
                    try (Base64InputStream bis = new Base64InputStream(new ValidatingBase64InputStream(in))) {
                        int len = -1;
                        byte[] buf = new byte[8192];
                        while ((len = bis.read(buf)) > 0) {
                            out.write(buf, 0, len);
                        }
                        out.flush();
                    }
                }
            });
        }
        logger.info("Successfully {} {}", new Object[] { encode ? "encoded" : "decoded", flowFile });
        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
    } catch (ProcessException e) {
        logger.error("Failed to {} {} due to {}", new Object[] { encode ? "encode" : "decode", flowFile, e });
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 7 with StreamCallback

use of org.apache.nifi.processor.io.StreamCallback in project nifi by apache.

the class EncodeContent method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    boolean encode = context.getProperty(MODE).getValue().equalsIgnoreCase(ENCODE_MODE);
    String encoding = context.getProperty(ENCODING).getValue();
    StreamCallback encoder = null;
    // Select the encoder/decoder to use
    if (encode) {
        if (encoding.equalsIgnoreCase(BASE64_ENCODING)) {
            encoder = new EncodeBase64();
        } else if (encoding.equalsIgnoreCase(BASE32_ENCODING)) {
            encoder = new EncodeBase32();
        } else if (encoding.equalsIgnoreCase(HEX_ENCODING)) {
            encoder = new EncodeHex();
        }
    } else {
        if (encoding.equalsIgnoreCase(BASE64_ENCODING)) {
            encoder = new DecodeBase64();
        } else if (encoding.equalsIgnoreCase(BASE32_ENCODING)) {
            encoder = new DecodeBase32();
        } else if (encoding.equalsIgnoreCase(HEX_ENCODING)) {
            encoder = new DecodeHex();
        }
    }
    if (encoder == null) {
        logger.warn("Unknown operation: {} {}", new Object[] { encode ? "encode" : "decode", encoding });
        return;
    }
    try {
        final StopWatch stopWatch = new StopWatch(true);
        flowFile = session.write(flowFile, encoder);
        logger.info("Successfully {} {}", new Object[] { encode ? "encoded" : "decoded", flowFile });
        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
    } catch (Exception e) {
        logger.error("Failed to {} {} due to {}", new Object[] { encode ? "encode" : "decode", flowFile, e });
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 8 with StreamCallback

use of org.apache.nifi.processor.io.StreamCallback in project nifi by apache.

the class EncryptContent method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    final String method = context.getProperty(ENCRYPTION_ALGORITHM).getValue();
    final EncryptionMethod encryptionMethod = EncryptionMethod.valueOf(method);
    final String providerName = encryptionMethod.getProvider();
    final String algorithm = encryptionMethod.getAlgorithm();
    final String password = context.getProperty(PASSWORD).getValue();
    final KeyDerivationFunction kdf = KeyDerivationFunction.valueOf(context.getProperty(KEY_DERIVATION_FUNCTION).getValue());
    final boolean encrypt = context.getProperty(MODE).getValue().equalsIgnoreCase(ENCRYPT_MODE);
    Encryptor encryptor;
    StreamCallback callback;
    try {
        if (isPGPAlgorithm(algorithm)) {
            final String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key());
            final String publicKeyring = context.getProperty(PUBLIC_KEYRING).getValue();
            final String privateKeyring = context.getProperty(PRIVATE_KEYRING).getValue();
            if (encrypt && publicKeyring != null) {
                final String publicUserId = context.getProperty(PUBLIC_KEY_USERID).getValue();
                encryptor = new OpenPGPKeyBasedEncryptor(algorithm, providerName, publicKeyring, publicUserId, null, filename);
            } else if (!encrypt && privateKeyring != null) {
                final char[] keyringPassphrase = context.getProperty(PRIVATE_KEYRING_PASSPHRASE).evaluateAttributeExpressions().getValue().toCharArray();
                encryptor = new OpenPGPKeyBasedEncryptor(algorithm, providerName, privateKeyring, null, keyringPassphrase, filename);
            } else {
                final char[] passphrase = Normalizer.normalize(password, Normalizer.Form.NFC).toCharArray();
                encryptor = new OpenPGPPasswordBasedEncryptor(algorithm, providerName, passphrase, filename);
            }
        } else if (kdf.equals(KeyDerivationFunction.NONE)) {
            // Raw key
            final String keyHex = context.getProperty(RAW_KEY_HEX).getValue();
            encryptor = new KeyedEncryptor(encryptionMethod, Hex.decodeHex(keyHex.toCharArray()));
        } else {
            // PBE
            final char[] passphrase = Normalizer.normalize(password, Normalizer.Form.NFC).toCharArray();
            encryptor = new PasswordBasedEncryptor(encryptionMethod, passphrase, kdf);
        }
        if (encrypt) {
            callback = encryptor.getEncryptionCallback();
        } else {
            callback = encryptor.getDecryptionCallback();
        }
    } catch (final Exception e) {
        logger.error("Failed to initialize {}cryption algorithm because - ", new Object[] { encrypt ? "en" : "de", e });
        session.rollback();
        context.yield();
        return;
    }
    try {
        final StopWatch stopWatch = new StopWatch(true);
        flowFile = session.write(flowFile, callback);
        logger.info("successfully {}crypted {}", new Object[] { encrypt ? "en" : "de", flowFile });
        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getElapsed(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
    } catch (final ProcessException e) {
        logger.error("Cannot {}crypt {} - ", new Object[] { encrypt ? "en" : "de", flowFile, e });
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 9 with StreamCallback

use of org.apache.nifi.processor.io.StreamCallback in project nifi by apache.

the class ExtractGrok method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final StopWatch stopWatch = new StopWatch(true);
    final Charset charset = Charset.forName(context.getProperty(CHARACTER_SET).getValue());
    final String contentString;
    byte[] buffer = bufferQueue.poll();
    if (buffer == null) {
        final int maxBufferSize = context.getProperty(MAX_BUFFER_SIZE).asDataSize(DataUnit.B).intValue();
        buffer = new byte[maxBufferSize];
    }
    try {
        final byte[] byteBuffer = buffer;
        session.read(flowFile, new InputStreamCallback() {

            @Override
            public void process(InputStream in) throws IOException {
                StreamUtils.fillBuffer(in, byteBuffer, false);
            }
        });
        final long len = Math.min(byteBuffer.length, flowFile.getSize());
        contentString = new String(byteBuffer, 0, (int) len, charset);
    } finally {
        bufferQueue.offer(buffer);
    }
    final Match gm = grok.match(contentString);
    gm.captures();
    if (gm.toMap().isEmpty()) {
        session.transfer(flowFile, REL_NO_MATCH);
        getLogger().info("Did not match any Grok Expressions for FlowFile {}", new Object[] { flowFile });
        return;
    }
    final ObjectMapper objectMapper = new ObjectMapper();
    switch(context.getProperty(DESTINATION).getValue()) {
        case FLOWFILE_ATTRIBUTE:
            Map<String, String> grokResults = new HashMap<>();
            for (Map.Entry<String, Object> entry : gm.toMap().entrySet()) {
                if (null != entry.getValue()) {
                    grokResults.put("grok." + entry.getKey(), entry.getValue().toString());
                }
            }
            flowFile = session.putAllAttributes(flowFile, grokResults);
            session.getProvenanceReporter().modifyAttributes(flowFile);
            session.transfer(flowFile, REL_MATCH);
            getLogger().info("Matched {} Grok Expressions and added attributes to FlowFile {}", new Object[] { grokResults.size(), flowFile });
            break;
        case FLOWFILE_CONTENT:
            FlowFile conFlowfile = session.write(flowFile, new StreamCallback() {

                @Override
                public void process(InputStream in, OutputStream out) throws IOException {
                    out.write(objectMapper.writeValueAsBytes(gm.toMap()));
                }
            });
            conFlowfile = session.putAttribute(conFlowfile, CoreAttributes.MIME_TYPE.key(), APPLICATION_JSON);
            session.getProvenanceReporter().modifyContent(conFlowfile, "Replaced content with parsed Grok fields and values", stopWatch.getElapsed(TimeUnit.MILLISECONDS));
            session.transfer(conFlowfile, REL_MATCH);
            break;
    }
}

Example 10 with StreamCallback

use of org.apache.nifi.processor.io.StreamCallback in project nifi by apache.

the class PutHTMLElement method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    final FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final Document doc;
    final Elements eles;
    try {
        doc = parseHTMLDocumentFromFlowfile(flowFile, context, session);
        eles = doc.select(context.getProperty(CSS_SELECTOR).evaluateAttributeExpressions().getValue());
    } catch (Exception ex) {
        getLogger().error("Failed to extract HTML from {} due to {}; routing to {}", new Object[] { flowFile, ex.toString(), REL_INVALID_HTML.getName() }, ex);
        session.transfer(flowFile, REL_INVALID_HTML);
        return;
    }
    if (eles == null || eles.isEmpty()) {
        // No element found
        session.transfer(flowFile, REL_NOT_FOUND);
    } else {
        final String putValue = context.getProperty(PUT_VALUE).evaluateAttributeExpressions(flowFile).getValue();
        for (final Element ele : eles) {
            switch(context.getProperty(PUT_LOCATION_TYPE).getValue()) {
                case APPEND_ELEMENT:
                    ele.append(putValue);
                    break;
                case PREPEND_ELEMENT:
                    ele.prepend(putValue);
                    break;
            }
        }
        FlowFile ff = session.write(session.create(flowFile), new StreamCallback() {

            @Override
            public void process(final InputStream in, final OutputStream out) throws IOException {
                out.write(doc.html().getBytes(StandardCharsets.UTF_8));
            }
        });
        session.transfer(ff, REL_SUCCESS);
        // Transfer the original HTML
        session.transfer(flowFile, REL_ORIGINAL);
    }
}