Examples with Relationship org.apache.nifi.processor.Relationship used on opensource projects

Example 81 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class EvaluateXPath method onTrigger.

@Override
@SuppressWarnings("unchecked")
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final List<FlowFile> flowFiles = session.get(50);
    if (flowFiles.isEmpty()) {
        return;
    }
    final ComponentLog logger = getLogger();
    final XMLReader xmlReader;
    try {
        xmlReader = XMLReaderFactory.createXMLReader();
    } catch (SAXException e) {
        logger.error("Error while constructing XMLReader {}", new Object[] { e });
        throw new ProcessException(e.getMessage());
    }
    if (!context.getProperty(VALIDATE_DTD).asBoolean()) {
        xmlReader.setEntityResolver(new EntityResolver() {

            @Override
            public InputSource resolveEntity(String publicId, String systemId) throws SAXException, IOException {
                return new InputSource(new StringReader(""));
            }
        });
    }
    final XPathFactory factory = factoryRef.get();
    final XPathEvaluator xpathEvaluator = (XPathEvaluator) factory.newXPath();
    final Map<String, XPathExpression> attributeToXPathMap = new HashMap<>();
    for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
        if (!entry.getKey().isDynamic()) {
            continue;
        }
        final XPathExpression xpathExpression;
        try {
            xpathExpression = xpathEvaluator.compile(entry.getValue());
            attributeToXPathMap.put(entry.getKey().getName(), xpathExpression);
        } catch (XPathExpressionException e) {
            // should not happen because we've already validated the XPath (in XPathValidator)
            throw new ProcessException(e);
        }
    }
    final XPathExpression slashExpression;
    try {
        slashExpression = xpathEvaluator.compile("/");
    } catch (XPathExpressionException e) {
        logger.error("unable to compile XPath expression due to {}", new Object[] { e });
        session.transfer(flowFiles, REL_FAILURE);
        return;
    }
    final String destination = context.getProperty(DESTINATION).getValue();
    final QName returnType;
    switch(context.getProperty(RETURN_TYPE).getValue()) {
        case RETURN_TYPE_AUTO:
            if (DESTINATION_ATTRIBUTE.equals(destination)) {
                returnType = STRING;
            } else if (DESTINATION_CONTENT.equals(destination)) {
                returnType = NODESET;
            } else {
                throw new IllegalStateException("The only possible destinations should be CONTENT or ATTRIBUTE...");
            }
            break;
        case RETURN_TYPE_NODESET:
            returnType = NODESET;
            break;
        case RETURN_TYPE_STRING:
            returnType = STRING;
            break;
        default:
            throw new IllegalStateException("There are no other return types...");
    }
    flowFileLoop: for (FlowFile flowFile : flowFiles) {
        final AtomicReference<Throwable> error = new AtomicReference<>(null);
        final AtomicReference<Source> sourceRef = new AtomicReference<>(null);
        session.read(flowFile, new InputStreamCallback() {

            @Override
            public void process(final InputStream rawIn) throws IOException {
                try (final InputStream in = new BufferedInputStream(rawIn)) {
                    final List<Source> rootList = (List<Source>) slashExpression.evaluate(new SAXSource(xmlReader, new InputSource(in)), NODESET);
                    sourceRef.set(rootList.get(0));
                } catch (final Exception e) {
                    error.set(e);
                }
            }
        });
        if (error.get() != null) {
            logger.error("unable to evaluate XPath against {} due to {}; routing to 'failure'", new Object[] { flowFile, error.get() });
            session.transfer(flowFile, REL_FAILURE);
            continue;
        }
        final Map<String, String> xpathResults = new HashMap<>();
        for (final Map.Entry<String, XPathExpression> entry : attributeToXPathMap.entrySet()) {
            Object result = null;
            try {
                result = entry.getValue().evaluate(sourceRef.get(), returnType);
                if (result == null) {
                    continue;
                }
            } catch (final XPathExpressionException e) {
                logger.error("failed to evaluate XPath for {} for Property {} due to {}; routing to failure", new Object[] { flowFile, entry.getKey(), e });
                session.transfer(flowFile, REL_FAILURE);
                continue flowFileLoop;
            }
            if (returnType == NODESET) {
                List<Source> nodeList = (List<Source>) result;
                if (nodeList.isEmpty()) {
                    logger.info("Routing {} to 'unmatched'", new Object[] { flowFile });
                    session.transfer(flowFile, REL_NO_MATCH);
                    continue flowFileLoop;
                } else if (nodeList.size() > 1) {
                    logger.error("Routing {} to 'failure' because the XPath evaluated to {} XML nodes", new Object[] { flowFile, nodeList.size() });
                    session.transfer(flowFile, REL_FAILURE);
                    continue flowFileLoop;
                }
                final Source sourceNode = nodeList.get(0);
                if (DESTINATION_ATTRIBUTE.equals(destination)) {
                    try {
                        ByteArrayOutputStream baos = new ByteArrayOutputStream();
                        doTransform(sourceNode, baos);
                        xpathResults.put(entry.getKey(), baos.toString("UTF-8"));
                    } catch (UnsupportedEncodingException e) {
                        throw new ProcessException(e);
                    } catch (TransformerException e) {
                        error.set(e);
                    }
                } else if (DESTINATION_CONTENT.equals(destination)) {
                    flowFile = session.write(flowFile, new OutputStreamCallback() {

                        @Override
                        public void process(final OutputStream rawOut) throws IOException {
                            try (final OutputStream out = new BufferedOutputStream(rawOut)) {
                                doTransform(sourceNode, out);
                            } catch (TransformerException e) {
                                error.set(e);
                            }
                        }
                    });
                }
            } else if (returnType == STRING) {
                final String resultString = (String) result;
                if (DESTINATION_ATTRIBUTE.equals(destination)) {
                    xpathResults.put(entry.getKey(), resultString);
                } else if (DESTINATION_CONTENT.equals(destination)) {
                    flowFile = session.write(flowFile, new OutputStreamCallback() {

                        @Override
                        public void process(final OutputStream rawOut) throws IOException {
                            try (final OutputStream out = new BufferedOutputStream(rawOut)) {
                                out.write(resultString.getBytes("UTF-8"));
                            }
                        }
                    });
                }
            }
        }
        if (error.get() == null) {
            if (DESTINATION_ATTRIBUTE.equals(destination)) {
                flowFile = session.putAllAttributes(flowFile, xpathResults);
                final Relationship destRel = xpathResults.isEmpty() ? REL_NO_MATCH : REL_MATCH;
                logger.info("Successfully evaluated XPaths against {} and found {} matches; routing to {}", new Object[] { flowFile, xpathResults.size(), destRel.getName() });
                session.transfer(flowFile, destRel);
                session.getProvenanceReporter().modifyAttributes(flowFile);
            } else if (DESTINATION_CONTENT.equals(destination)) {
                logger.info("Successfully updated content for {}; routing to 'matched'", new Object[] { flowFile });
                session.transfer(flowFile, REL_MATCH);
                session.getProvenanceReporter().modifyContent(flowFile);
            }
        } else {
            logger.error("Failed to write XPath result for {} due to {}; routing original to 'failure'", new Object[] { flowFile, error.get() });
            session.transfer(flowFile, REL_FAILURE);
        }
    }
}

Example 82 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class EvaluateXQuery method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    final List<FlowFile> flowFileBatch = session.get(50);
    if (flowFileBatch.isEmpty()) {
        return;
    }
    final ComponentLog logger = getLogger();
    final Map<String, XQueryExecutable> attributeToXQueryMap = new HashMap<>();
    final Processor proc = new Processor(false);
    final XMLReader xmlReader;
    try {
        xmlReader = XMLReaderFactory.createXMLReader();
    } catch (SAXException e) {
        logger.error("Error while constructing XMLReader {}", new Object[] { e });
        throw new ProcessException(e.getMessage());
    }
    if (!context.getProperty(VALIDATE_DTD).asBoolean()) {
        xmlReader.setEntityResolver(new EntityResolver() {

            @Override
            public InputSource resolveEntity(String publicId, String systemId) throws SAXException, IOException {
                return new InputSource(new StringReader(""));
            }
        });
    }
    final XQueryCompiler comp = proc.newXQueryCompiler();
    for (final Map.Entry<PropertyDescriptor, String> entry : context.getProperties().entrySet()) {
        if (!entry.getKey().isDynamic()) {
            continue;
        }
        final XQueryExecutable exp;
        try {
            exp = comp.compile(entry.getValue());
            attributeToXQueryMap.put(entry.getKey().getName(), exp);
        } catch (SaxonApiException e) {
            // should not happen because we've already validated the XQuery (in XQueryValidator)
            throw new ProcessException(e);
        }
    }
    final XQueryExecutable slashExpression;
    try {
        slashExpression = comp.compile("/");
    } catch (SaxonApiException e) {
        logger.error("unable to compile XQuery expression due to {}", new Object[] { e });
        session.transfer(flowFileBatch, REL_FAILURE);
        return;
    }
    final String destination = context.getProperty(DESTINATION).getValue();
    flowFileLoop: for (FlowFile flowFile : flowFileBatch) {
        if (!isScheduled()) {
            session.rollback();
            return;
        }
        final AtomicReference<Throwable> error = new AtomicReference<>(null);
        final AtomicReference<XdmNode> sourceRef = new AtomicReference<>(null);
        session.read(flowFile, new InputStreamCallback() {

            @Override
            public void process(final InputStream rawIn) throws IOException {
                try (final InputStream in = new BufferedInputStream(rawIn)) {
                    XQueryEvaluator qe = slashExpression.load();
                    qe.setSource(new SAXSource(xmlReader, new InputSource(in)));
                    DocumentBuilderFactory dfactory = DocumentBuilderFactory.newInstance();
                    dfactory.setNamespaceAware(true);
                    Document dom = dfactory.newDocumentBuilder().newDocument();
                    qe.run(new DOMDestination(dom));
                    XdmNode rootNode = proc.newDocumentBuilder().wrap(dom);
                    sourceRef.set(rootNode);
                } catch (final Exception e) {
                    error.set(e);
                }
            }
        });
        if (error.get() != null) {
            logger.error("unable to evaluate XQuery against {} due to {}; routing to 'failure'", new Object[] { flowFile, error.get() });
            session.transfer(flowFile, REL_FAILURE);
            continue;
        }
        final Map<String, String> xQueryResults = new HashMap<>();
        List<FlowFile> childrenFlowFiles = new ArrayList<>();
        for (final Map.Entry<String, XQueryExecutable> entry : attributeToXQueryMap.entrySet()) {
            try {
                XQueryEvaluator qe = entry.getValue().load();
                qe.setContextItem(sourceRef.get());
                XdmValue result = qe.evaluate();
                if (DESTINATION_ATTRIBUTE.equals(destination)) {
                    int index = 1;
                    for (XdmItem item : result) {
                        String value = formatItem(item, context);
                        String attributeName = entry.getKey();
                        if (result.size() > 1) {
                            attributeName += "." + index++;
                        }
                        xQueryResults.put(attributeName, value);
                    }
                } else {
                    // if (DESTINATION_CONTENT.equals(destination)){
                    if (result.size() == 0) {
                        logger.info("Routing {} to 'unmatched'", new Object[] { flowFile });
                        session.transfer(flowFile, REL_NO_MATCH);
                        continue flowFileLoop;
                    } else if (result.size() == 1) {
                        final XdmItem item = result.itemAt(0);
                        flowFile = session.write(flowFile, new OutputStreamCallback() {

                            @Override
                            public void process(final OutputStream rawOut) throws IOException {
                                try (final OutputStream out = new BufferedOutputStream(rawOut)) {
                                    writeformattedItem(item, context, out);
                                } catch (TransformerFactoryConfigurationError | TransformerException e) {
                                    throw new IOException(e);
                                }
                            }
                        });
                    } else {
                        for (final XdmItem item : result) {
                            FlowFile ff = session.clone(flowFile);
                            ff = session.write(ff, new OutputStreamCallback() {

                                @Override
                                public void process(final OutputStream rawOut) throws IOException {
                                    try (final OutputStream out = new BufferedOutputStream(rawOut)) {
                                        try {
                                            writeformattedItem(item, context, out);
                                        } catch (TransformerFactoryConfigurationError | TransformerException e) {
                                            throw new IOException(e);
                                        }
                                    }
                                }
                            });
                            childrenFlowFiles.add(ff);
                        }
                    }
                }
            } catch (final SaxonApiException e) {
                logger.error("failed to evaluate XQuery for {} for Property {} due to {}; routing to failure", new Object[] { flowFile, entry.getKey(), e });
                session.transfer(flowFile, REL_FAILURE);
                session.remove(childrenFlowFiles);
                continue flowFileLoop;
            } catch (TransformerFactoryConfigurationError | TransformerException | IOException e) {
                logger.error("Failed to write XQuery result for {} due to {}; routing original to 'failure'", new Object[] { flowFile, error.get() });
                session.transfer(flowFile, REL_FAILURE);
                session.remove(childrenFlowFiles);
                continue flowFileLoop;
            }
        }
        if (DESTINATION_ATTRIBUTE.equals(destination)) {
            flowFile = session.putAllAttributes(flowFile, xQueryResults);
            final Relationship destRel = xQueryResults.isEmpty() ? REL_NO_MATCH : REL_MATCH;
            logger.info("Successfully evaluated XQueries against {} and found {} matches; routing to {}", new Object[] { flowFile, xQueryResults.size(), destRel.getName() });
            session.transfer(flowFile, destRel);
            session.getProvenanceReporter().modifyAttributes(flowFile);
        } else {
            // if (DESTINATION_CONTENT.equals(destination)) {
            if (!childrenFlowFiles.isEmpty()) {
                logger.info("Successfully created {} new FlowFiles from {}; routing all to 'matched'", new Object[] { childrenFlowFiles.size(), flowFile });
                session.transfer(childrenFlowFiles, REL_MATCH);
                session.remove(flowFile);
            } else {
                logger.info("Successfully updated content for {}; routing to 'matched'", new Object[] { flowFile });
                session.transfer(flowFile, REL_MATCH);
                session.getProvenanceReporter().modifyContent(flowFile);
            }
        }
    }
// end flowFileLoop
}

Example 83 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class InvokeScriptedProcessor method getRelationships.

/**
 * Returns the valid relationships for this processor as supplied by the
 * script itself.
 *
 * @return a Set of Relationships supported by this processor
 */
@Override
public Set<Relationship> getRelationships() {
    final Set<Relationship> relationships = new HashSet<>();
    final Processor instance = processor.get();
    if (instance != null) {
        try {
            final Set<Relationship> rels = instance.getRelationships();
            if (rels != null && !rels.isEmpty()) {
                relationships.addAll(rels);
            }
        } catch (final Throwable t) {
            final ComponentLog logger = getLogger();
            final String message = "Unable to get relationships from scripted Processor: " + t;
            logger.error(message);
            if (logger.isDebugEnabled()) {
                logger.error(message, t);
            }
        }
    }
    return Collections.unmodifiableSet(relationships);
}

Example 84 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class TestInvokeJavascript method testScriptDefinedRelationship.

/**
 * Tests a scripted processor written in Javascript reads the first line of text from the flowfiles content and
 * stores the value in an attribute of the outgoing flowfile.
 * Confirms that the scripted processor can return relationships defined in it.
 *
 * @throws Exception Any error encountered while testing
 */
@Test
public void testScriptDefinedRelationship() throws Exception {
    InvokeScriptedProcessor processor = new InvokeScriptedProcessor();
    MockProcessContext context = new MockProcessContext(processor);
    MockProcessorInitializationContext initContext = new MockProcessorInitializationContext(processor, context);
    processor.initialize(initContext);
    context.setProperty(scriptingComponent.getScriptingComponentHelper().SCRIPT_ENGINE, "ECMAScript");
    context.setProperty(ScriptingComponentUtils.SCRIPT_FILE, "target/test/resources/javascript/test_reader.js");
    context.setProperty(ScriptingComponentUtils.MODULES, "target/test/resources/javascript");
    // State Manger is unused, and a null reference is specified
    processor.customValidate(new MockValidationContext(context));
    processor.setup(context);
    Set<Relationship> relationships = processor.getRelationships();
    assertNotNull(relationships);
    assertTrue(relationships.size() > 0);
    boolean found = false;
    for (Relationship relationship : relationships) {
        if (relationship.getName().equals("test")) {
            found = true;
            break;
        }
    }
    assertTrue(found);
}

Example 85 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class PutHiveStreaming method init.

@Override
protected void init(ProcessorInitializationContext context) {
    List<PropertyDescriptor> props = new ArrayList<>();
    props.add(METASTORE_URI);
    props.add(HIVE_CONFIGURATION_RESOURCES);
    props.add(DB_NAME);
    props.add(TABLE_NAME);
    props.add(PARTITION_COLUMNS);
    props.add(AUTOCREATE_PARTITIONS);
    props.add(MAX_OPEN_CONNECTIONS);
    props.add(HEARTBEAT_INTERVAL);
    props.add(TXNS_PER_BATCH);
    props.add(RECORDS_PER_TXN);
    props.add(CALL_TIMEOUT);
    props.add(ROLLBACK_ON_FAILURE);
    props.add(KERBEROS_CREDENTIALS_SERVICE);
    kerberosConfigFile = context.getKerberosConfigurationFile();
    kerberosProperties = new KerberosProperties(kerberosConfigFile);
    props.add(kerberosProperties.getKerberosPrincipal());
    props.add(kerberosProperties.getKerberosKeytab());
    propertyDescriptors = Collections.unmodifiableList(props);
    Set<Relationship> _relationships = new HashSet<>();
    _relationships.add(REL_SUCCESS);
    _relationships.add(REL_FAILURE);
    _relationships.add(REL_RETRY);
    relationships = Collections.unmodifiableSet(_relationships);
}