Examples with Relationship org.apache.nifi.processor.Relationship used on opensource projects

Example 76 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class InsertRowsWriter method writeEvent.

/**
 * Creates and transfers a new flow file whose contents are the JSON-serialized value of the specified event, and the sequence ID attribute set
 *
 * @param session   A reference to a ProcessSession from which the flow file(s) will be created and transferred
 * @param eventInfo An event whose value will become the contents of the flow file
 * @return The next available CDC sequence ID for use by the CDC processor
 */
@Override
public long writeEvent(final ProcessSession session, String transitUri, final InsertRowsEventInfo eventInfo, final long currentSequenceId, Relationship relationship) {
    final AtomicLong seqId = new AtomicLong(currentSequenceId);
    for (Serializable[] row : eventInfo.getRows()) {
        FlowFile flowFile = session.create();
        flowFile = session.write(flowFile, outputStream -> {
            super.startJson(outputStream, eventInfo);
            super.writeJson(eventInfo);
            final BitSet bitSet = eventInfo.getIncludedColumns();
            writeRow(eventInfo, row, bitSet);
            super.endJson();
        });
        flowFile = session.putAllAttributes(flowFile, getCommonAttributes(seqId.get(), eventInfo));
        session.transfer(flowFile, relationship);
        session.getProvenanceReporter().receive(flowFile, transitUri);
        seqId.getAndIncrement();
    }
    return seqId.get();
}

Example 77 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class AbstractEmailProcessor method transfer.

/**
 * Disposes the message by converting it to a {@link FlowFile} transferring
 * it to the REL_SUCCESS relationship.
 */
private void transfer(Message emailMessage, ProcessContext context, ProcessSession processSession) {
    long start = System.nanoTime();
    FlowFile flowFile = processSession.create();
    flowFile = processSession.append(flowFile, out -> {
        try {
            emailMessage.writeTo(out);
        } catch (MessagingException e) {
            throw new IOException(e);
        }
    });
    long executionDuration = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
    String fromAddressesString = "";
    try {
        Address[] fromAddresses = emailMessage.getFrom();
        if (fromAddresses != null) {
            fromAddressesString = Arrays.asList(fromAddresses).toString();
        }
    } catch (MessagingException e) {
        this.logger.warn("Failed to retrieve 'From' attribute from Message.");
    }
    processSession.getProvenanceReporter().receive(flowFile, this.displayUrl, "Received message from " + fromAddressesString, executionDuration);
    this.getLogger().info("Successfully received {} from {} in {} millis", new Object[] { flowFile, fromAddressesString, executionDuration });
    processSession.transfer(flowFile, REL_SUCCESS);
}

Example 78 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class FlowController method getGroupStatus.

/**
 * Returns the status for the components in the specified group with the
 * specified report. The results will be filtered by executing the specified
 * predicate.
 *
 * @param group group id
 * @param statusReport report
 * @param isAuthorized is authorized check
 * @return the component status
 */
public ProcessGroupStatus getGroupStatus(final ProcessGroup group, final RepositoryStatusReport statusReport, final Predicate<Authorizable> isAuthorized) {
    if (group == null) {
        return null;
    }
    final ProcessGroupStatus status = new ProcessGroupStatus();
    status.setId(group.getIdentifier());
    status.setName(isAuthorized.evaluate(group) ? group.getName() : group.getIdentifier());
    int activeGroupThreads = 0;
    long bytesRead = 0L;
    long bytesWritten = 0L;
    int queuedCount = 0;
    long queuedContentSize = 0L;
    int flowFilesIn = 0;
    long bytesIn = 0L;
    int flowFilesOut = 0;
    long bytesOut = 0L;
    int flowFilesReceived = 0;
    long bytesReceived = 0L;
    int flowFilesSent = 0;
    long bytesSent = 0L;
    int flowFilesTransferred = 0;
    long bytesTransferred = 0;
    // set status for processors
    final Collection<ProcessorStatus> processorStatusCollection = new ArrayList<>();
    status.setProcessorStatus(processorStatusCollection);
    for (final ProcessorNode procNode : group.getProcessors()) {
        final ProcessorStatus procStat = getProcessorStatus(statusReport, procNode, isAuthorized);
        processorStatusCollection.add(procStat);
        activeGroupThreads += procStat.getActiveThreadCount();
        bytesRead += procStat.getBytesRead();
        bytesWritten += procStat.getBytesWritten();
        flowFilesReceived += procStat.getFlowFilesReceived();
        bytesReceived += procStat.getBytesReceived();
        flowFilesSent += procStat.getFlowFilesSent();
        bytesSent += procStat.getBytesSent();
    }
    // set status for local child groups
    final Collection<ProcessGroupStatus> localChildGroupStatusCollection = new ArrayList<>();
    status.setProcessGroupStatus(localChildGroupStatusCollection);
    for (final ProcessGroup childGroup : group.getProcessGroups()) {
        final ProcessGroupStatus childGroupStatus = getGroupStatus(childGroup, statusReport, isAuthorized);
        localChildGroupStatusCollection.add(childGroupStatus);
        activeGroupThreads += childGroupStatus.getActiveThreadCount();
        bytesRead += childGroupStatus.getBytesRead();
        bytesWritten += childGroupStatus.getBytesWritten();
        queuedCount += childGroupStatus.getQueuedCount();
        queuedContentSize += childGroupStatus.getQueuedContentSize();
        flowFilesReceived += childGroupStatus.getFlowFilesReceived();
        bytesReceived += childGroupStatus.getBytesReceived();
        flowFilesSent += childGroupStatus.getFlowFilesSent();
        bytesSent += childGroupStatus.getBytesSent();
        flowFilesTransferred += childGroupStatus.getFlowFilesTransferred();
        bytesTransferred += childGroupStatus.getBytesTransferred();
    }
    // set status for remote child groups
    final Collection<RemoteProcessGroupStatus> remoteProcessGroupStatusCollection = new ArrayList<>();
    status.setRemoteProcessGroupStatus(remoteProcessGroupStatusCollection);
    for (final RemoteProcessGroup remoteGroup : group.getRemoteProcessGroups()) {
        final RemoteProcessGroupStatus remoteStatus = createRemoteGroupStatus(remoteGroup, statusReport, isAuthorized);
        if (remoteStatus != null) {
            remoteProcessGroupStatusCollection.add(remoteStatus);
            flowFilesReceived += remoteStatus.getReceivedCount();
            bytesReceived += remoteStatus.getReceivedContentSize();
            flowFilesSent += remoteStatus.getSentCount();
            bytesSent += remoteStatus.getSentContentSize();
        }
    }
    // connection status
    final Collection<ConnectionStatus> connectionStatusCollection = new ArrayList<>();
    status.setConnectionStatus(connectionStatusCollection);
    // get the connection and remote port status
    for (final Connection conn : group.getConnections()) {
        final boolean isConnectionAuthorized = isAuthorized.evaluate(conn);
        final boolean isSourceAuthorized = isAuthorized.evaluate(conn.getSource());
        final boolean isDestinationAuthorized = isAuthorized.evaluate(conn.getDestination());
        final ConnectionStatus connStatus = new ConnectionStatus();
        connStatus.setId(conn.getIdentifier());
        connStatus.setGroupId(conn.getProcessGroup().getIdentifier());
        connStatus.setSourceId(conn.getSource().getIdentifier());
        connStatus.setSourceName(isSourceAuthorized ? conn.getSource().getName() : conn.getSource().getIdentifier());
        connStatus.setDestinationId(conn.getDestination().getIdentifier());
        connStatus.setDestinationName(isDestinationAuthorized ? conn.getDestination().getName() : conn.getDestination().getIdentifier());
        connStatus.setBackPressureDataSizeThreshold(conn.getFlowFileQueue().getBackPressureDataSizeThreshold());
        connStatus.setBackPressureObjectThreshold(conn.getFlowFileQueue().getBackPressureObjectThreshold());
        final FlowFileEvent connectionStatusReport = statusReport.getReportEntry(conn.getIdentifier());
        if (connectionStatusReport != null) {
            connStatus.setInputBytes(connectionStatusReport.getContentSizeIn());
            connStatus.setInputCount(connectionStatusReport.getFlowFilesIn());
            connStatus.setOutputBytes(connectionStatusReport.getContentSizeOut());
            connStatus.setOutputCount(connectionStatusReport.getFlowFilesOut());
            flowFilesTransferred += connectionStatusReport.getFlowFilesIn() + connectionStatusReport.getFlowFilesOut();
            bytesTransferred += connectionStatusReport.getContentSizeIn() + connectionStatusReport.getContentSizeOut();
        }
        if (isConnectionAuthorized) {
            if (StringUtils.isNotBlank(conn.getName())) {
                connStatus.setName(conn.getName());
            } else if (conn.getRelationships() != null && !conn.getRelationships().isEmpty()) {
                final Collection<String> relationships = new ArrayList<>(conn.getRelationships().size());
                for (final Relationship relationship : conn.getRelationships()) {
                    relationships.add(relationship.getName());
                }
                connStatus.setName(StringUtils.join(relationships, ", "));
            }
        } else {
            connStatus.setName(conn.getIdentifier());
        }
        final QueueSize queueSize = conn.getFlowFileQueue().size();
        final int connectionQueuedCount = queueSize.getObjectCount();
        final long connectionQueuedBytes = queueSize.getByteCount();
        if (connectionQueuedCount > 0) {
            connStatus.setQueuedBytes(connectionQueuedBytes);
            connStatus.setQueuedCount(connectionQueuedCount);
        }
        connectionStatusCollection.add(connStatus);
        queuedCount += connectionQueuedCount;
        queuedContentSize += connectionQueuedBytes;
        final Connectable source = conn.getSource();
        if (ConnectableType.REMOTE_OUTPUT_PORT.equals(source.getConnectableType())) {
            final RemoteGroupPort remoteOutputPort = (RemoteGroupPort) source;
            activeGroupThreads += processScheduler.getActiveThreadCount(remoteOutputPort);
        }
        final Connectable destination = conn.getDestination();
        if (ConnectableType.REMOTE_INPUT_PORT.equals(destination.getConnectableType())) {
            final RemoteGroupPort remoteInputPort = (RemoteGroupPort) destination;
            activeGroupThreads += processScheduler.getActiveThreadCount(remoteInputPort);
        }
    }
    // status for input ports
    final Collection<PortStatus> inputPortStatusCollection = new ArrayList<>();
    status.setInputPortStatus(inputPortStatusCollection);
    final Set<Port> inputPorts = group.getInputPorts();
    for (final Port port : inputPorts) {
        final boolean isInputPortAuthorized = isAuthorized.evaluate(port);
        final PortStatus portStatus = new PortStatus();
        portStatus.setId(port.getIdentifier());
        portStatus.setGroupId(port.getProcessGroup().getIdentifier());
        portStatus.setName(isInputPortAuthorized ? port.getName() : port.getIdentifier());
        portStatus.setActiveThreadCount(processScheduler.getActiveThreadCount(port));
        // determine the run status
        if (ScheduledState.RUNNING.equals(port.getScheduledState())) {
            portStatus.setRunStatus(RunStatus.Running);
        } else if (ScheduledState.DISABLED.equals(port.getScheduledState())) {
            portStatus.setRunStatus(RunStatus.Disabled);
        } else if (!port.isValid()) {
            portStatus.setRunStatus(RunStatus.Invalid);
        } else {
            portStatus.setRunStatus(RunStatus.Stopped);
        }
        // special handling for root group ports
        if (port instanceof RootGroupPort) {
            final RootGroupPort rootGroupPort = (RootGroupPort) port;
            portStatus.setTransmitting(rootGroupPort.isTransmitting());
        }
        final FlowFileEvent entry = statusReport.getReportEntries().get(port.getIdentifier());
        if (entry == null) {
            portStatus.setInputBytes(0L);
            portStatus.setInputCount(0);
            portStatus.setOutputBytes(0L);
            portStatus.setOutputCount(0);
        } else {
            final int processedCount = entry.getFlowFilesOut();
            final long numProcessedBytes = entry.getContentSizeOut();
            portStatus.setOutputBytes(numProcessedBytes);
            portStatus.setOutputCount(processedCount);
            final int inputCount = entry.getFlowFilesIn();
            final long inputBytes = entry.getContentSizeIn();
            portStatus.setInputBytes(inputBytes);
            portStatus.setInputCount(inputCount);
            flowFilesIn += inputCount;
            bytesIn += inputBytes;
            bytesWritten += entry.getBytesWritten();
            flowFilesReceived += entry.getFlowFilesReceived();
            bytesReceived += entry.getBytesReceived();
        }
        inputPortStatusCollection.add(portStatus);
        activeGroupThreads += portStatus.getActiveThreadCount();
    }
    // status for output ports
    final Collection<PortStatus> outputPortStatusCollection = new ArrayList<>();
    status.setOutputPortStatus(outputPortStatusCollection);
    final Set<Port> outputPorts = group.getOutputPorts();
    for (final Port port : outputPorts) {
        final boolean isOutputPortAuthorized = isAuthorized.evaluate(port);
        final PortStatus portStatus = new PortStatus();
        portStatus.setId(port.getIdentifier());
        portStatus.setGroupId(port.getProcessGroup().getIdentifier());
        portStatus.setName(isOutputPortAuthorized ? port.getName() : port.getIdentifier());
        portStatus.setActiveThreadCount(processScheduler.getActiveThreadCount(port));
        // determine the run status
        if (ScheduledState.RUNNING.equals(port.getScheduledState())) {
            portStatus.setRunStatus(RunStatus.Running);
        } else if (ScheduledState.DISABLED.equals(port.getScheduledState())) {
            portStatus.setRunStatus(RunStatus.Disabled);
        } else if (!port.isValid()) {
            portStatus.setRunStatus(RunStatus.Invalid);
        } else {
            portStatus.setRunStatus(RunStatus.Stopped);
        }
        // special handling for root group ports
        if (port instanceof RootGroupPort) {
            final RootGroupPort rootGroupPort = (RootGroupPort) port;
            portStatus.setTransmitting(rootGroupPort.isTransmitting());
        }
        final FlowFileEvent entry = statusReport.getReportEntries().get(port.getIdentifier());
        if (entry == null) {
            portStatus.setInputBytes(0L);
            portStatus.setInputCount(0);
            portStatus.setOutputBytes(0L);
            portStatus.setOutputCount(0);
        } else {
            final int processedCount = entry.getFlowFilesOut();
            final long numProcessedBytes = entry.getContentSizeOut();
            portStatus.setOutputBytes(numProcessedBytes);
            portStatus.setOutputCount(processedCount);
            final int inputCount = entry.getFlowFilesIn();
            final long inputBytes = entry.getContentSizeIn();
            portStatus.setInputBytes(inputBytes);
            portStatus.setInputCount(inputCount);
            bytesRead += entry.getBytesRead();
            flowFilesOut += entry.getFlowFilesOut();
            bytesOut += entry.getContentSizeOut();
            flowFilesSent = entry.getFlowFilesSent();
            bytesSent += entry.getBytesSent();
        }
        outputPortStatusCollection.add(portStatus);
        activeGroupThreads += portStatus.getActiveThreadCount();
    }
    for (final Funnel funnel : group.getFunnels()) {
        activeGroupThreads += processScheduler.getActiveThreadCount(funnel);
    }
    status.setActiveThreadCount(activeGroupThreads);
    status.setBytesRead(bytesRead);
    status.setBytesWritten(bytesWritten);
    status.setQueuedCount(queuedCount);
    status.setQueuedContentSize(queuedContentSize);
    status.setInputContentSize(bytesIn);
    status.setInputCount(flowFilesIn);
    status.setOutputContentSize(bytesOut);
    status.setOutputCount(flowFilesOut);
    status.setFlowFilesReceived(flowFilesReceived);
    status.setBytesReceived(bytesReceived);
    status.setFlowFilesSent(flowFilesSent);
    status.setBytesSent(bytesSent);
    status.setFlowFilesTransferred(flowFilesTransferred);
    status.setBytesTransferred(bytesTransferred);
    final VersionControlInformation vci = group.getVersionControlInformation();
    if (vci != null && vci.getStatus() != null && vci.getStatus().getState() != null) {
        status.setVersionedFlowState(vci.getStatus().getState());
    }
    return status;
}

Example 79 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class HtmlProcessorDocumentationWriter method writeRelationships.

/**
 * Writes a table describing the relations a processor has.
 *
 * @param processor the processor to describe
 * @param xmlStreamWriter the stream writer to use
 * @throws XMLStreamException thrown if there was a problem writing the xml
 */
private void writeRelationships(final Processor processor, final XMLStreamWriter xmlStreamWriter) throws XMLStreamException {
    writeSimpleElement(xmlStreamWriter, "h3", "Relationships: ");
    if (processor.getRelationships().size() > 0) {
        xmlStreamWriter.writeStartElement("table");
        xmlStreamWriter.writeAttribute("id", "relationships");
        xmlStreamWriter.writeStartElement("tr");
        writeSimpleElement(xmlStreamWriter, "th", "Name");
        writeSimpleElement(xmlStreamWriter, "th", "Description");
        xmlStreamWriter.writeEndElement();
        for (Relationship relationship : processor.getRelationships()) {
            xmlStreamWriter.writeStartElement("tr");
            writeSimpleElement(xmlStreamWriter, "td", relationship.getName());
            writeSimpleElement(xmlStreamWriter, "td", relationship.getDescription());
            xmlStreamWriter.writeEndElement();
        }
        xmlStreamWriter.writeEndElement();
    } else {
        xmlStreamWriter.writeCharacters("This processor has no relationships.");
    }
}

Example 80 with Relationship

use of org.apache.nifi.processor.Relationship in project nifi by apache.

the class LookupRecord method route.

@Override
protected Set<Relationship> route(final Record record, final RecordSchema writeSchema, final FlowFile flowFile, final ProcessContext context, final Tuple<Map<String, RecordPath>, RecordPath> flowFileContext) {
    final Map<String, RecordPath> recordPaths = flowFileContext.getKey();
    final Map<String, Object> lookupCoordinates = new HashMap<>(recordPaths.size());
    for (final Map.Entry<String, RecordPath> entry : recordPaths.entrySet()) {
        final String coordinateKey = entry.getKey();
        final RecordPath recordPath = entry.getValue();
        final RecordPathResult pathResult = recordPath.evaluate(record);
        final List<FieldValue> lookupFieldValues = pathResult.getSelectedFields().filter(fieldVal -> fieldVal.getValue() != null).collect(Collectors.toList());
        if (lookupFieldValues.isEmpty()) {
            final Set<Relationship> rels = routeToMatchedUnmatched ? UNMATCHED_COLLECTION : SUCCESS_COLLECTION;
            getLogger().debug("RecordPath for property '{}' did not match any fields in a record for {}; routing record to {}", new Object[] { coordinateKey, flowFile, rels });
            return rels;
        }
        if (lookupFieldValues.size() > 1) {
            final Set<Relationship> rels = routeToMatchedUnmatched ? UNMATCHED_COLLECTION : SUCCESS_COLLECTION;
            getLogger().debug("RecordPath for property '{}' matched {} fields in a record for {}; routing record to {}", new Object[] { coordinateKey, lookupFieldValues.size(), flowFile, rels });
            return rels;
        }
        final FieldValue fieldValue = lookupFieldValues.get(0);
        final Object coordinateValue = (fieldValue.getValue() instanceof Number || fieldValue.getValue() instanceof Boolean) ? fieldValue.getValue() : DataTypeUtils.toString(fieldValue.getValue(), (String) null);
        lookupCoordinates.put(coordinateKey, coordinateValue);
    }
    final Optional<?> lookupValueOption;
    try {
        lookupValueOption = lookupService.lookup(lookupCoordinates);
    } catch (final Exception e) {
        throw new ProcessException("Failed to lookup coordinates " + lookupCoordinates + " in Lookup Service", e);
    }
    if (!lookupValueOption.isPresent()) {
        final Set<Relationship> rels = routeToMatchedUnmatched ? UNMATCHED_COLLECTION : SUCCESS_COLLECTION;
        return rels;
    }
    // Ensure that the Record has the appropriate schema to account for the newly added values
    final RecordPath resultPath = flowFileContext.getValue();
    if (resultPath != null) {
        record.incorporateSchema(writeSchema);
        final Object lookupValue = lookupValueOption.get();
        final RecordPathResult resultPathResult = flowFileContext.getValue().evaluate(record);
        final String resultContentsValue = context.getProperty(RESULT_CONTENTS).getValue();
        if (RESULT_RECORD_FIELDS.getValue().equals(resultContentsValue) && lookupValue instanceof Record) {
            final Record lookupRecord = (Record) lookupValue;
            // Use wants to add all fields of the resultant Record to the specified Record Path.
            // If the destination Record Path returns to us a Record, then we will add all field values of
            // the Lookup Record to the destination Record. However, if the destination Record Path returns
            // something other than a Record, then we can't add the fields to it. We can only replace it,
            // because it doesn't make sense to add fields to anything but a Record.
            resultPathResult.getSelectedFields().forEach(fieldVal -> {
                final Object destinationValue = fieldVal.getValue();
                if (destinationValue instanceof Record) {
                    final Record destinationRecord = (Record) destinationValue;
                    for (final String fieldName : lookupRecord.getRawFieldNames()) {
                        final Object value = lookupRecord.getValue(fieldName);
                        destinationRecord.setValue(fieldName, value);
                    }
                } else {
                    final Optional<Record> parentOption = fieldVal.getParentRecord();
                    if (parentOption.isPresent()) {
                        parentOption.get().setValue(fieldVal.getField().getFieldName(), lookupRecord);
                    }
                }
            });
        } else {
            resultPathResult.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(lookupValue));
        }
    }
    final Set<Relationship> rels = routeToMatchedUnmatched ? MATCHED_COLLECTION : SUCCESS_COLLECTION;
    return rels;
}