Examples with Funnel org.apache.nifi.connectable.Funnel used on opensource projects

Example 21 with Funnel

use of org.apache.nifi.connectable.Funnel in project nifi by apache.

the class StandardFlowSerializer method addProcessGroup.

private void addProcessGroup(final Element parentElement, final ProcessGroup group, final String elementName, final ScheduledStateLookup scheduledStateLookup) {
    final Document doc = parentElement.getOwnerDocument();
    final Element element = doc.createElement(elementName);
    parentElement.appendChild(element);
    addTextElement(element, "id", group.getIdentifier());
    addTextElement(element, "versionedComponentId", group.getVersionedComponentId());
    addTextElement(element, "name", group.getName());
    addPosition(element, group.getPosition());
    addTextElement(element, "comment", group.getComments());
    final VersionControlInformation versionControlInfo = group.getVersionControlInformation();
    if (versionControlInfo != null) {
        final Element versionControlInfoElement = doc.createElement("versionControlInformation");
        addTextElement(versionControlInfoElement, "registryId", versionControlInfo.getRegistryIdentifier());
        addTextElement(versionControlInfoElement, "bucketId", versionControlInfo.getBucketIdentifier());
        addTextElement(versionControlInfoElement, "bucketName", versionControlInfo.getBucketName());
        addTextElement(versionControlInfoElement, "flowId", versionControlInfo.getFlowIdentifier());
        addTextElement(versionControlInfoElement, "flowName", versionControlInfo.getFlowName());
        addTextElement(versionControlInfoElement, "flowDescription", versionControlInfo.getFlowDescription());
        addTextElement(versionControlInfoElement, "version", versionControlInfo.getVersion());
        element.appendChild(versionControlInfoElement);
    }
    for (final ProcessorNode processor : group.getProcessors()) {
        addProcessor(element, processor, scheduledStateLookup);
    }
    if (group.isRootGroup()) {
        for (final Port port : group.getInputPorts()) {
            addRootGroupPort(element, (RootGroupPort) port, "inputPort", scheduledStateLookup);
        }
        for (final Port port : group.getOutputPorts()) {
            addRootGroupPort(element, (RootGroupPort) port, "outputPort", scheduledStateLookup);
        }
    } else {
        for (final Port port : group.getInputPorts()) {
            addPort(element, port, "inputPort", scheduledStateLookup);
        }
        for (final Port port : group.getOutputPorts()) {
            addPort(element, port, "outputPort", scheduledStateLookup);
        }
    }
    for (final Label label : group.getLabels()) {
        addLabel(element, label);
    }
    for (final Funnel funnel : group.getFunnels()) {
        addFunnel(element, funnel);
    }
    for (final ProcessGroup childGroup : group.getProcessGroups()) {
        addProcessGroup(element, childGroup, "processGroup", scheduledStateLookup);
    }
    for (final RemoteProcessGroup remoteRef : group.getRemoteProcessGroups()) {
        addRemoteProcessGroup(element, remoteRef, scheduledStateLookup);
    }
    for (final Connection connection : group.getConnections()) {
        addConnection(element, connection);
    }
    for (final ControllerServiceNode service : group.getControllerServices(false)) {
        addControllerService(element, service);
    }
    for (final Template template : group.getTemplates()) {
        addTemplate(element, template);
    }
    final VariableRegistry variableRegistry = group.getVariableRegistry();
    for (final Map.Entry<VariableDescriptor, String> entry : variableRegistry.getVariableMap().entrySet()) {
        addVariable(element, entry.getKey().getName(), entry.getValue());
    }
}

Example 22 with Funnel

use of org.apache.nifi.connectable.Funnel in project nifi by apache.

the class FunnelAuditor method createFunnelAdvice.

/**
 * Audits the creation of a funnel.
 *
 * @param proceedingJoinPoint join point
 * @return funnel
 * @throws Throwable ex
 */
@Around("within(org.apache.nifi.web.dao.FunnelDAO+) && " + "execution(org.apache.nifi.connectable.Funnel createFunnel(java.lang.String, org.apache.nifi.web.api.dto.FunnelDTO))")
public Funnel createFunnelAdvice(ProceedingJoinPoint proceedingJoinPoint) throws Throwable {
    // perform the underlying operation
    Funnel funnel = (Funnel) proceedingJoinPoint.proceed();
    // perform the audit
    final Action action = generateAuditRecord(funnel, Operation.Add);
    // save the actions
    if (action != null) {
        saveAction(action, logger);
    }
    return funnel;
}

Example 23 with Funnel

use of org.apache.nifi.connectable.Funnel in project nifi by apache.

the class SnippetAuditor method removeSnippetAdvice.

/**
 * Audits bulk delete.
 *
 * @param proceedingJoinPoint join point
 * @param snippetId snippet id
 * @param snippetDAO dao
 * @throws Throwable ex
 */
@Around("within(org.apache.nifi.web.dao.SnippetDAO+) && " + "execution(void deleteSnippetComponents(java.lang.String)) && " + "args(snippetId) && " + "target(snippetDAO)")
public void removeSnippetAdvice(ProceedingJoinPoint proceedingJoinPoint, String snippetId, SnippetDAO snippetDAO) throws Throwable {
    // get the snippet before removing it
    final Snippet snippet = snippetDAO.getSnippet(snippetId);
    // locate all the components being removed
    final Set<Funnel> funnels = new HashSet<>();
    for (String id : snippet.getFunnels().keySet()) {
        funnels.add(funnelDAO.getFunnel(id));
    }
    final Set<Port> inputPorts = new HashSet<>();
    for (String id : snippet.getInputPorts().keySet()) {
        inputPorts.add(inputPortDAO.getPort(id));
    }
    final Set<Port> outputPorts = new HashSet<>();
    for (String id : snippet.getOutputPorts().keySet()) {
        outputPorts.add(outputPortDAO.getPort(id));
    }
    final Set<RemoteProcessGroup> remoteProcessGroups = new HashSet<>();
    for (String id : snippet.getRemoteProcessGroups().keySet()) {
        remoteProcessGroups.add(remoteProcessGroupDAO.getRemoteProcessGroup(id));
    }
    final Set<ProcessGroup> processGroups = new HashSet<>();
    final ProcessGroupDAO processGroupDAO = getProcessGroupDAO();
    for (String id : snippet.getProcessGroups().keySet()) {
        processGroups.add(processGroupDAO.getProcessGroup(id));
    }
    final Set<ProcessorNode> processors = new HashSet<>();
    for (String id : snippet.getProcessors().keySet()) {
        processors.add(processorDAO.getProcessor(id));
    }
    final Set<Connection> connections = new HashSet<>();
    for (String id : snippet.getConnections().keySet()) {
        connections.add(connectionDAO.getConnection(id));
    }
    // remove the snippet and components
    proceedingJoinPoint.proceed();
    final Collection<Action> actions = new ArrayList<>();
    // audit funnel removal
    for (Funnel funnel : funnels) {
        final Action action = funnelAuditor.generateAuditRecord(funnel, Operation.Remove);
        if (action != null) {
            actions.add(action);
        }
    }
    for (Port inputPort : inputPorts) {
        final Action action = portAuditor.generateAuditRecord(inputPort, Operation.Remove);
        if (action != null) {
            actions.add(action);
        }
    }
    for (Port outputPort : outputPorts) {
        final Action action = portAuditor.generateAuditRecord(outputPort, Operation.Remove);
        if (action != null) {
            actions.add(action);
        }
    }
    for (RemoteProcessGroup remoteProcessGroup : remoteProcessGroups) {
        final Action action = remoteProcessGroupAuditor.generateAuditRecord(remoteProcessGroup, Operation.Remove);
        if (action != null) {
            actions.add(action);
        }
    }
    for (ProcessGroup processGroup : processGroups) {
        final Action action = processGroupAuditor.generateAuditRecord(processGroup, Operation.Remove);
        if (action != null) {
            actions.add(action);
        }
    }
    for (ProcessorNode processor : processors) {
        final Action action = processorAuditor.generateAuditRecord(processor, Operation.Remove);
        if (action != null) {
            actions.add(action);
        }
    }
    for (Connection connection : connections) {
        final ConnectDetails connectDetails = relationshipAuditor.createConnectDetails(connection, connection.getRelationships());
        final Action action = relationshipAuditor.generateAuditRecordForConnection(connection, Operation.Disconnect, connectDetails);
        if (action != null) {
            actions.add(action);
        }
    }
    // save the actions
    if (CollectionUtils.isNotEmpty(actions)) {
        saveActions(actions, logger);
    }
}

Example 24 with Funnel

use of org.apache.nifi.connectable.Funnel in project nifi by apache.

the class FlowController method onFlowInitialized.

/**
 * <p>
 * Causes any processors that were added to the flow with a 'delayStart'
 * flag of true to now start
 * </p>
 *
 * @param startDelayedComponents true if start
 */
public void onFlowInitialized(final boolean startDelayedComponents) {
    writeLock.lock();
    try {
        if (startDelayedComponents) {
            LOG.info("Starting {} processors/ports/funnels", startConnectablesAfterInitialization.size() + startRemoteGroupPortsAfterInitialization.size());
            for (final Connectable connectable : startConnectablesAfterInitialization) {
                if (connectable.getScheduledState() == ScheduledState.DISABLED) {
                    continue;
                }
                try {
                    if (connectable instanceof ProcessorNode) {
                        connectable.getProcessGroup().startProcessor((ProcessorNode) connectable, true);
                    } else {
                        startConnectable(connectable);
                    }
                } catch (final Throwable t) {
                    LOG.error("Unable to start {} due to {}", new Object[] { connectable, t.toString() });
                    if (LOG.isDebugEnabled()) {
                        LOG.error("", t);
                    }
                }
            }
            startConnectablesAfterInitialization.clear();
            int startedTransmitting = 0;
            for (final RemoteGroupPort remoteGroupPort : startRemoteGroupPortsAfterInitialization) {
                try {
                    remoteGroupPort.getRemoteProcessGroup().startTransmitting(remoteGroupPort);
                    startedTransmitting++;
                } catch (final Throwable t) {
                    LOG.error("Unable to start transmitting with {} due to {}", new Object[] { remoteGroupPort, t });
                }
            }
            LOG.info("Started {} Remote Group Ports transmitting", startedTransmitting);
            startRemoteGroupPortsAfterInitialization.clear();
        } else {
            // because we don't provide users the ability to start or stop them - they are just notional.
            for (final Connectable connectable : startConnectablesAfterInitialization) {
                try {
                    if (connectable instanceof Funnel) {
                        startConnectable(connectable);
                    }
                } catch (final Throwable t) {
                    LOG.error("Unable to start {} due to {}", new Object[] { connectable, t });
                }
            }
            startConnectablesAfterInitialization.clear();
            startRemoteGroupPortsAfterInitialization.clear();
        }
    } finally {
        writeLock.unlock();
    }
}

Example 25 with Funnel

use of org.apache.nifi.connectable.Funnel in project nifi by apache.

the class FlowController method getGroupStatus.

/**
 * Returns the status for the components in the specified group with the
 * specified report. The results will be filtered by executing the specified
 * predicate.
 *
 * @param group group id
 * @param statusReport report
 * @param isAuthorized is authorized check
 * @return the component status
 */
public ProcessGroupStatus getGroupStatus(final ProcessGroup group, final RepositoryStatusReport statusReport, final Predicate<Authorizable> isAuthorized) {
    if (group == null) {
        return null;
    }
    final ProcessGroupStatus status = new ProcessGroupStatus();
    status.setId(group.getIdentifier());
    status.setName(isAuthorized.evaluate(group) ? group.getName() : group.getIdentifier());
    int activeGroupThreads = 0;
    long bytesRead = 0L;
    long bytesWritten = 0L;
    int queuedCount = 0;
    long queuedContentSize = 0L;
    int flowFilesIn = 0;
    long bytesIn = 0L;
    int flowFilesOut = 0;
    long bytesOut = 0L;
    int flowFilesReceived = 0;
    long bytesReceived = 0L;
    int flowFilesSent = 0;
    long bytesSent = 0L;
    int flowFilesTransferred = 0;
    long bytesTransferred = 0;
    // set status for processors
    final Collection<ProcessorStatus> processorStatusCollection = new ArrayList<>();
    status.setProcessorStatus(processorStatusCollection);
    for (final ProcessorNode procNode : group.getProcessors()) {
        final ProcessorStatus procStat = getProcessorStatus(statusReport, procNode, isAuthorized);
        processorStatusCollection.add(procStat);
        activeGroupThreads += procStat.getActiveThreadCount();
        bytesRead += procStat.getBytesRead();
        bytesWritten += procStat.getBytesWritten();
        flowFilesReceived += procStat.getFlowFilesReceived();
        bytesReceived += procStat.getBytesReceived();
        flowFilesSent += procStat.getFlowFilesSent();
        bytesSent += procStat.getBytesSent();
    }
    // set status for local child groups
    final Collection<ProcessGroupStatus> localChildGroupStatusCollection = new ArrayList<>();
    status.setProcessGroupStatus(localChildGroupStatusCollection);
    for (final ProcessGroup childGroup : group.getProcessGroups()) {
        final ProcessGroupStatus childGroupStatus = getGroupStatus(childGroup, statusReport, isAuthorized);
        localChildGroupStatusCollection.add(childGroupStatus);
        activeGroupThreads += childGroupStatus.getActiveThreadCount();
        bytesRead += childGroupStatus.getBytesRead();
        bytesWritten += childGroupStatus.getBytesWritten();
        queuedCount += childGroupStatus.getQueuedCount();
        queuedContentSize += childGroupStatus.getQueuedContentSize();
        flowFilesReceived += childGroupStatus.getFlowFilesReceived();
        bytesReceived += childGroupStatus.getBytesReceived();
        flowFilesSent += childGroupStatus.getFlowFilesSent();
        bytesSent += childGroupStatus.getBytesSent();
        flowFilesTransferred += childGroupStatus.getFlowFilesTransferred();
        bytesTransferred += childGroupStatus.getBytesTransferred();
    }
    // set status for remote child groups
    final Collection<RemoteProcessGroupStatus> remoteProcessGroupStatusCollection = new ArrayList<>();
    status.setRemoteProcessGroupStatus(remoteProcessGroupStatusCollection);
    for (final RemoteProcessGroup remoteGroup : group.getRemoteProcessGroups()) {
        final RemoteProcessGroupStatus remoteStatus = createRemoteGroupStatus(remoteGroup, statusReport, isAuthorized);
        if (remoteStatus != null) {
            remoteProcessGroupStatusCollection.add(remoteStatus);
            flowFilesReceived += remoteStatus.getReceivedCount();
            bytesReceived += remoteStatus.getReceivedContentSize();
            flowFilesSent += remoteStatus.getSentCount();
            bytesSent += remoteStatus.getSentContentSize();
        }
    }
    // connection status
    final Collection<ConnectionStatus> connectionStatusCollection = new ArrayList<>();
    status.setConnectionStatus(connectionStatusCollection);
    // get the connection and remote port status
    for (final Connection conn : group.getConnections()) {
        final boolean isConnectionAuthorized = isAuthorized.evaluate(conn);
        final boolean isSourceAuthorized = isAuthorized.evaluate(conn.getSource());
        final boolean isDestinationAuthorized = isAuthorized.evaluate(conn.getDestination());
        final ConnectionStatus connStatus = new ConnectionStatus();
        connStatus.setId(conn.getIdentifier());
        connStatus.setGroupId(conn.getProcessGroup().getIdentifier());
        connStatus.setSourceId(conn.getSource().getIdentifier());
        connStatus.setSourceName(isSourceAuthorized ? conn.getSource().getName() : conn.getSource().getIdentifier());
        connStatus.setDestinationId(conn.getDestination().getIdentifier());
        connStatus.setDestinationName(isDestinationAuthorized ? conn.getDestination().getName() : conn.getDestination().getIdentifier());
        connStatus.setBackPressureDataSizeThreshold(conn.getFlowFileQueue().getBackPressureDataSizeThreshold());
        connStatus.setBackPressureObjectThreshold(conn.getFlowFileQueue().getBackPressureObjectThreshold());
        final FlowFileEvent connectionStatusReport = statusReport.getReportEntry(conn.getIdentifier());
        if (connectionStatusReport != null) {
            connStatus.setInputBytes(connectionStatusReport.getContentSizeIn());
            connStatus.setInputCount(connectionStatusReport.getFlowFilesIn());
            connStatus.setOutputBytes(connectionStatusReport.getContentSizeOut());
            connStatus.setOutputCount(connectionStatusReport.getFlowFilesOut());
            flowFilesTransferred += connectionStatusReport.getFlowFilesIn() + connectionStatusReport.getFlowFilesOut();
            bytesTransferred += connectionStatusReport.getContentSizeIn() + connectionStatusReport.getContentSizeOut();
        }
        if (isConnectionAuthorized) {
            if (StringUtils.isNotBlank(conn.getName())) {
                connStatus.setName(conn.getName());
            } else if (conn.getRelationships() != null && !conn.getRelationships().isEmpty()) {
                final Collection<String> relationships = new ArrayList<>(conn.getRelationships().size());
                for (final Relationship relationship : conn.getRelationships()) {
                    relationships.add(relationship.getName());
                }
                connStatus.setName(StringUtils.join(relationships, ", "));
            }
        } else {
            connStatus.setName(conn.getIdentifier());
        }
        final QueueSize queueSize = conn.getFlowFileQueue().size();
        final int connectionQueuedCount = queueSize.getObjectCount();
        final long connectionQueuedBytes = queueSize.getByteCount();
        if (connectionQueuedCount > 0) {
            connStatus.setQueuedBytes(connectionQueuedBytes);
            connStatus.setQueuedCount(connectionQueuedCount);
        }
        connectionStatusCollection.add(connStatus);
        queuedCount += connectionQueuedCount;
        queuedContentSize += connectionQueuedBytes;
        final Connectable source = conn.getSource();
        if (ConnectableType.REMOTE_OUTPUT_PORT.equals(source.getConnectableType())) {
            final RemoteGroupPort remoteOutputPort = (RemoteGroupPort) source;
            activeGroupThreads += processScheduler.getActiveThreadCount(remoteOutputPort);
        }
        final Connectable destination = conn.getDestination();
        if (ConnectableType.REMOTE_INPUT_PORT.equals(destination.getConnectableType())) {
            final RemoteGroupPort remoteInputPort = (RemoteGroupPort) destination;
            activeGroupThreads += processScheduler.getActiveThreadCount(remoteInputPort);
        }
    }
    // status for input ports
    final Collection<PortStatus> inputPortStatusCollection = new ArrayList<>();
    status.setInputPortStatus(inputPortStatusCollection);
    final Set<Port> inputPorts = group.getInputPorts();
    for (final Port port : inputPorts) {
        final boolean isInputPortAuthorized = isAuthorized.evaluate(port);
        final PortStatus portStatus = new PortStatus();
        portStatus.setId(port.getIdentifier());
        portStatus.setGroupId(port.getProcessGroup().getIdentifier());
        portStatus.setName(isInputPortAuthorized ? port.getName() : port.getIdentifier());
        portStatus.setActiveThreadCount(processScheduler.getActiveThreadCount(port));
        // determine the run status
        if (ScheduledState.RUNNING.equals(port.getScheduledState())) {
            portStatus.setRunStatus(RunStatus.Running);
        } else if (ScheduledState.DISABLED.equals(port.getScheduledState())) {
            portStatus.setRunStatus(RunStatus.Disabled);
        } else if (!port.isValid()) {
            portStatus.setRunStatus(RunStatus.Invalid);
        } else {
            portStatus.setRunStatus(RunStatus.Stopped);
        }
        // special handling for root group ports
        if (port instanceof RootGroupPort) {
            final RootGroupPort rootGroupPort = (RootGroupPort) port;
            portStatus.setTransmitting(rootGroupPort.isTransmitting());
        }
        final FlowFileEvent entry = statusReport.getReportEntries().get(port.getIdentifier());
        if (entry == null) {
            portStatus.setInputBytes(0L);
            portStatus.setInputCount(0);
            portStatus.setOutputBytes(0L);
            portStatus.setOutputCount(0);
        } else {
            final int processedCount = entry.getFlowFilesOut();
            final long numProcessedBytes = entry.getContentSizeOut();
            portStatus.setOutputBytes(numProcessedBytes);
            portStatus.setOutputCount(processedCount);
            final int inputCount = entry.getFlowFilesIn();
            final long inputBytes = entry.getContentSizeIn();
            portStatus.setInputBytes(inputBytes);
            portStatus.setInputCount(inputCount);
            flowFilesIn += inputCount;
            bytesIn += inputBytes;
            bytesWritten += entry.getBytesWritten();
            flowFilesReceived += entry.getFlowFilesReceived();
            bytesReceived += entry.getBytesReceived();
        }
        inputPortStatusCollection.add(portStatus);
        activeGroupThreads += portStatus.getActiveThreadCount();
    }
    // status for output ports
    final Collection<PortStatus> outputPortStatusCollection = new ArrayList<>();
    status.setOutputPortStatus(outputPortStatusCollection);
    final Set<Port> outputPorts = group.getOutputPorts();
    for (final Port port : outputPorts) {
        final boolean isOutputPortAuthorized = isAuthorized.evaluate(port);
        final PortStatus portStatus = new PortStatus();
        portStatus.setId(port.getIdentifier());
        portStatus.setGroupId(port.getProcessGroup().getIdentifier());
        portStatus.setName(isOutputPortAuthorized ? port.getName() : port.getIdentifier());
        portStatus.setActiveThreadCount(processScheduler.getActiveThreadCount(port));
        // determine the run status
        if (ScheduledState.RUNNING.equals(port.getScheduledState())) {
            portStatus.setRunStatus(RunStatus.Running);
        } else if (ScheduledState.DISABLED.equals(port.getScheduledState())) {
            portStatus.setRunStatus(RunStatus.Disabled);
        } else if (!port.isValid()) {
            portStatus.setRunStatus(RunStatus.Invalid);
        } else {
            portStatus.setRunStatus(RunStatus.Stopped);
        }
        // special handling for root group ports
        if (port instanceof RootGroupPort) {
            final RootGroupPort rootGroupPort = (RootGroupPort) port;
            portStatus.setTransmitting(rootGroupPort.isTransmitting());
        }
        final FlowFileEvent entry = statusReport.getReportEntries().get(port.getIdentifier());
        if (entry == null) {
            portStatus.setInputBytes(0L);
            portStatus.setInputCount(0);
            portStatus.setOutputBytes(0L);
            portStatus.setOutputCount(0);
        } else {
            final int processedCount = entry.getFlowFilesOut();
            final long numProcessedBytes = entry.getContentSizeOut();
            portStatus.setOutputBytes(numProcessedBytes);
            portStatus.setOutputCount(processedCount);
            final int inputCount = entry.getFlowFilesIn();
            final long inputBytes = entry.getContentSizeIn();
            portStatus.setInputBytes(inputBytes);
            portStatus.setInputCount(inputCount);
            bytesRead += entry.getBytesRead();
            flowFilesOut += entry.getFlowFilesOut();
            bytesOut += entry.getContentSizeOut();
            flowFilesSent = entry.getFlowFilesSent();
            bytesSent += entry.getBytesSent();
        }
        outputPortStatusCollection.add(portStatus);
        activeGroupThreads += portStatus.getActiveThreadCount();
    }
    for (final Funnel funnel : group.getFunnels()) {
        activeGroupThreads += processScheduler.getActiveThreadCount(funnel);
    }
    status.setActiveThreadCount(activeGroupThreads);
    status.setBytesRead(bytesRead);
    status.setBytesWritten(bytesWritten);
    status.setQueuedCount(queuedCount);
    status.setQueuedContentSize(queuedContentSize);
    status.setInputContentSize(bytesIn);
    status.setInputCount(flowFilesIn);
    status.setOutputContentSize(bytesOut);
    status.setOutputCount(flowFilesOut);
    status.setFlowFilesReceived(flowFilesReceived);
    status.setBytesReceived(bytesReceived);
    status.setFlowFilesSent(flowFilesSent);
    status.setBytesSent(bytesSent);
    status.setFlowFilesTransferred(flowFilesTransferred);
    status.setBytesTransferred(bytesTransferred);
    final VersionControlInformation vci = group.getVersionControlInformation();
    if (vci != null && vci.getStatus() != null && vci.getStatus().getState() != null) {
        status.setVersionedFlowState(vci.getStatus().getState());
    }
    return status;
}