Examples with VersionControlInformation org.apache.nifi.registry.flow.VersionControlInformation used on opensource projects

Example 16 with VersionControlInformation

use of org.apache.nifi.registry.flow.VersionControlInformation in project nifi by apache.

the class StandardProcessGroup method getCounts.

@Override
public ProcessGroupCounts getCounts() {
    int inputPortCount = 0;
    int outputPortCount = 0;
    int running = 0;
    int stopped = 0;
    int invalid = 0;
    int disabled = 0;
    int activeRemotePorts = 0;
    int inactiveRemotePorts = 0;
    int upToDate = 0;
    int locallyModified = 0;
    int stale = 0;
    int locallyModifiedAndStale = 0;
    int syncFailure = 0;
    readLock.lock();
    try {
        for (final ProcessorNode procNode : processors.values()) {
            if (ScheduledState.DISABLED.equals(procNode.getScheduledState())) {
                disabled++;
            } else if (procNode.isRunning()) {
                running++;
            } else if (!procNode.isValid()) {
                invalid++;
            } else {
                stopped++;
            }
        }
        inputPortCount = inputPorts.size();
        for (final Port port : inputPorts.values()) {
            if (ScheduledState.DISABLED.equals(port.getScheduledState())) {
                disabled++;
            } else if (port.isRunning()) {
                running++;
            } else if (!port.isValid()) {
                invalid++;
            } else {
                stopped++;
            }
        }
        outputPortCount = outputPorts.size();
        for (final Port port : outputPorts.values()) {
            if (ScheduledState.DISABLED.equals(port.getScheduledState())) {
                disabled++;
            } else if (port.isRunning()) {
                running++;
            } else if (!port.isValid()) {
                invalid++;
            } else {
                stopped++;
            }
        }
        for (final ProcessGroup childGroup : processGroups.values()) {
            final ProcessGroupCounts childCounts = childGroup.getCounts();
            running += childCounts.getRunningCount();
            stopped += childCounts.getStoppedCount();
            invalid += childCounts.getInvalidCount();
            disabled += childCounts.getDisabledCount();
            // update the vci counts for this child group
            final VersionControlInformation vci = childGroup.getVersionControlInformation();
            if (vci != null) {
                switch(vci.getStatus().getState()) {
                    case LOCALLY_MODIFIED:
                        locallyModified++;
                        break;
                    case LOCALLY_MODIFIED_AND_STALE:
                        locallyModifiedAndStale++;
                        break;
                    case STALE:
                        stale++;
                        break;
                    case SYNC_FAILURE:
                        syncFailure++;
                        break;
                    case UP_TO_DATE:
                        upToDate++;
                        break;
                }
            }
            // update the vci counts for all nested groups within the child
            upToDate += childCounts.getUpToDateCount();
            locallyModified += childCounts.getLocallyModifiedCount();
            stale += childCounts.getStaleCount();
            locallyModifiedAndStale += childCounts.getLocallyModifiedAndStaleCount();
            syncFailure += childCounts.getSyncFailureCount();
        }
        for (final RemoteProcessGroup remoteGroup : findAllRemoteProcessGroups()) {
            // Count only input ports that have incoming connections
            for (final Port port : remoteGroup.getInputPorts()) {
                if (port.hasIncomingConnection()) {
                    if (port.isRunning()) {
                        activeRemotePorts++;
                    } else {
                        inactiveRemotePorts++;
                    }
                }
            }
            // Count only output ports that have outgoing connections
            for (final Port port : remoteGroup.getOutputPorts()) {
                if (!port.getConnections().isEmpty()) {
                    if (port.isRunning()) {
                        activeRemotePorts++;
                    } else {
                        inactiveRemotePorts++;
                    }
                }
            }
            final String authIssue = remoteGroup.getAuthorizationIssue();
            if (authIssue != null) {
                invalid++;
            }
        }
    } finally {
        readLock.unlock();
    }
    return new ProcessGroupCounts(inputPortCount, outputPortCount, running, stopped, invalid, disabled, activeRemotePorts, inactiveRemotePorts, upToDate, locallyModified, stale, locallyModifiedAndStale, syncFailure);
}

Example 17 with VersionControlInformation

use of org.apache.nifi.registry.flow.VersionControlInformation in project nifi by apache.

the class SnippetUtils method verifyNoDuplicateVersionControlInfo.

private static void verifyNoDuplicateVersionControlInfo(final ProcessGroup group, final Collection<VersionControlInformation> snippetVcis) {
    final VersionControlInformation vci = group.getVersionControlInformation();
    if (vci != null) {
        for (final VersionControlInformation snippetVci : snippetVcis) {
            if (vci.getBucketIdentifier().equals(snippetVci.getBucketIdentifier()) && vci.getFlowIdentifier().equals(snippetVci.getFlowIdentifier())) {
                throw new IllegalArgumentException("Cannot place the given Process Group into the desired destination because the destination group or one of its ancestor groups is " + "under Version Control and one of the selected Process Groups is also under Version Control with the same Flow. A Process Group that is under Version Control " + "cannot contain a child Process Group that points to the same Versioned Flow.");
            }
        }
    }
    final ProcessGroup parent = group.getParent();
    if (parent != null) {
        verifyNoDuplicateVersionControlInfo(parent, snippetVcis);
    }
}

Example 18 with VersionControlInformation

use of org.apache.nifi.registry.flow.VersionControlInformation in project nifi by apache.

the class StandardFlowSerializer method addProcessGroup.

private void addProcessGroup(final Element parentElement, final ProcessGroup group, final String elementName, final ScheduledStateLookup scheduledStateLookup) {
    final Document doc = parentElement.getOwnerDocument();
    final Element element = doc.createElement(elementName);
    parentElement.appendChild(element);
    addTextElement(element, "id", group.getIdentifier());
    addTextElement(element, "versionedComponentId", group.getVersionedComponentId());
    addTextElement(element, "name", group.getName());
    addPosition(element, group.getPosition());
    addTextElement(element, "comment", group.getComments());
    final VersionControlInformation versionControlInfo = group.getVersionControlInformation();
    if (versionControlInfo != null) {
        final Element versionControlInfoElement = doc.createElement("versionControlInformation");
        addTextElement(versionControlInfoElement, "registryId", versionControlInfo.getRegistryIdentifier());
        addTextElement(versionControlInfoElement, "bucketId", versionControlInfo.getBucketIdentifier());
        addTextElement(versionControlInfoElement, "bucketName", versionControlInfo.getBucketName());
        addTextElement(versionControlInfoElement, "flowId", versionControlInfo.getFlowIdentifier());
        addTextElement(versionControlInfoElement, "flowName", versionControlInfo.getFlowName());
        addTextElement(versionControlInfoElement, "flowDescription", versionControlInfo.getFlowDescription());
        addTextElement(versionControlInfoElement, "version", versionControlInfo.getVersion());
        element.appendChild(versionControlInfoElement);
    }
    for (final ProcessorNode processor : group.getProcessors()) {
        addProcessor(element, processor, scheduledStateLookup);
    }
    if (group.isRootGroup()) {
        for (final Port port : group.getInputPorts()) {
            addRootGroupPort(element, (RootGroupPort) port, "inputPort", scheduledStateLookup);
        }
        for (final Port port : group.getOutputPorts()) {
            addRootGroupPort(element, (RootGroupPort) port, "outputPort", scheduledStateLookup);
        }
    } else {
        for (final Port port : group.getInputPorts()) {
            addPort(element, port, "inputPort", scheduledStateLookup);
        }
        for (final Port port : group.getOutputPorts()) {
            addPort(element, port, "outputPort", scheduledStateLookup);
        }
    }
    for (final Label label : group.getLabels()) {
        addLabel(element, label);
    }
    for (final Funnel funnel : group.getFunnels()) {
        addFunnel(element, funnel);
    }
    for (final ProcessGroup childGroup : group.getProcessGroups()) {
        addProcessGroup(element, childGroup, "processGroup", scheduledStateLookup);
    }
    for (final RemoteProcessGroup remoteRef : group.getRemoteProcessGroups()) {
        addRemoteProcessGroup(element, remoteRef, scheduledStateLookup);
    }
    for (final Connection connection : group.getConnections()) {
        addConnection(element, connection);
    }
    for (final ControllerServiceNode service : group.getControllerServices(false)) {
        addControllerService(element, service);
    }
    for (final Template template : group.getTemplates()) {
        addTemplate(element, template);
    }
    final VariableRegistry variableRegistry = group.getVariableRegistry();
    for (final Map.Entry<VariableDescriptor, String> entry : variableRegistry.getVariableMap().entrySet()) {
        addVariable(element, entry.getKey().getName(), entry.getValue());
    }
}

Example 19 with VersionControlInformation

use of org.apache.nifi.registry.flow.VersionControlInformation in project nifi by apache.

the class ProcessGroupAuditor method updateProcessGroupFlowAdvice.

@Around("within(org.apache.nifi.web.dao.ProcessGroupDAO+) && " + "execution(org.apache.nifi.groups.ProcessGroup updateProcessGroupFlow(..))")
public ProcessGroup updateProcessGroupFlowAdvice(final ProceedingJoinPoint proceedingJoinPoint) throws Throwable {
    final Object[] args = proceedingJoinPoint.getArgs();
    final String groupId = (String) args[0];
    final NiFiUser user = (NiFiUser) args[1];
    final ProcessGroupDAO processGroupDAO = getProcessGroupDAO();
    final ProcessGroup processGroup = processGroupDAO.getProcessGroup(groupId);
    final VersionControlInformation vci = processGroup.getVersionControlInformation();
    final ProcessGroup updatedProcessGroup = (ProcessGroup) proceedingJoinPoint.proceed();
    final VersionControlInformation updatedVci = updatedProcessGroup.getVersionControlInformation();
    final Operation operation;
    if (vci == null) {
        operation = Operation.StartVersionControl;
    } else {
        if (updatedVci == null) {
            operation = Operation.StopVersionControl;
        } else if (vci.getVersion() == updatedVci.getVersion()) {
            operation = Operation.RevertLocalChanges;
        } else {
            operation = Operation.ChangeVersion;
        }
    }
    saveUpdateAction(user, groupId, operation);
    return updatedProcessGroup;
}

Example 20 with VersionControlInformation

use of org.apache.nifi.registry.flow.VersionControlInformation in project nifi by apache.

the class FlowController method getGroupStatus.

/**
 * Returns the status for the components in the specified group with the
 * specified report. The results will be filtered by executing the specified
 * predicate.
 *
 * @param group group id
 * @param statusReport report
 * @param isAuthorized is authorized check
 * @return the component status
 */
public ProcessGroupStatus getGroupStatus(final ProcessGroup group, final RepositoryStatusReport statusReport, final Predicate<Authorizable> isAuthorized) {
    if (group == null) {
        return null;
    }
    final ProcessGroupStatus status = new ProcessGroupStatus();
    status.setId(group.getIdentifier());
    status.setName(isAuthorized.evaluate(group) ? group.getName() : group.getIdentifier());
    int activeGroupThreads = 0;
    long bytesRead = 0L;
    long bytesWritten = 0L;
    int queuedCount = 0;
    long queuedContentSize = 0L;
    int flowFilesIn = 0;
    long bytesIn = 0L;
    int flowFilesOut = 0;
    long bytesOut = 0L;
    int flowFilesReceived = 0;
    long bytesReceived = 0L;
    int flowFilesSent = 0;
    long bytesSent = 0L;
    int flowFilesTransferred = 0;
    long bytesTransferred = 0;
    // set status for processors
    final Collection<ProcessorStatus> processorStatusCollection = new ArrayList<>();
    status.setProcessorStatus(processorStatusCollection);
    for (final ProcessorNode procNode : group.getProcessors()) {
        final ProcessorStatus procStat = getProcessorStatus(statusReport, procNode, isAuthorized);
        processorStatusCollection.add(procStat);
        activeGroupThreads += procStat.getActiveThreadCount();
        bytesRead += procStat.getBytesRead();
        bytesWritten += procStat.getBytesWritten();
        flowFilesReceived += procStat.getFlowFilesReceived();
        bytesReceived += procStat.getBytesReceived();
        flowFilesSent += procStat.getFlowFilesSent();
        bytesSent += procStat.getBytesSent();
    }
    // set status for local child groups
    final Collection<ProcessGroupStatus> localChildGroupStatusCollection = new ArrayList<>();
    status.setProcessGroupStatus(localChildGroupStatusCollection);
    for (final ProcessGroup childGroup : group.getProcessGroups()) {
        final ProcessGroupStatus childGroupStatus = getGroupStatus(childGroup, statusReport, isAuthorized);
        localChildGroupStatusCollection.add(childGroupStatus);
        activeGroupThreads += childGroupStatus.getActiveThreadCount();
        bytesRead += childGroupStatus.getBytesRead();
        bytesWritten += childGroupStatus.getBytesWritten();
        queuedCount += childGroupStatus.getQueuedCount();
        queuedContentSize += childGroupStatus.getQueuedContentSize();
        flowFilesReceived += childGroupStatus.getFlowFilesReceived();
        bytesReceived += childGroupStatus.getBytesReceived();
        flowFilesSent += childGroupStatus.getFlowFilesSent();
        bytesSent += childGroupStatus.getBytesSent();
        flowFilesTransferred += childGroupStatus.getFlowFilesTransferred();
        bytesTransferred += childGroupStatus.getBytesTransferred();
    }
    // set status for remote child groups
    final Collection<RemoteProcessGroupStatus> remoteProcessGroupStatusCollection = new ArrayList<>();
    status.setRemoteProcessGroupStatus(remoteProcessGroupStatusCollection);
    for (final RemoteProcessGroup remoteGroup : group.getRemoteProcessGroups()) {
        final RemoteProcessGroupStatus remoteStatus = createRemoteGroupStatus(remoteGroup, statusReport, isAuthorized);
        if (remoteStatus != null) {
            remoteProcessGroupStatusCollection.add(remoteStatus);
            flowFilesReceived += remoteStatus.getReceivedCount();
            bytesReceived += remoteStatus.getReceivedContentSize();
            flowFilesSent += remoteStatus.getSentCount();
            bytesSent += remoteStatus.getSentContentSize();
        }
    }
    // connection status
    final Collection<ConnectionStatus> connectionStatusCollection = new ArrayList<>();
    status.setConnectionStatus(connectionStatusCollection);
    // get the connection and remote port status
    for (final Connection conn : group.getConnections()) {
        final boolean isConnectionAuthorized = isAuthorized.evaluate(conn);
        final boolean isSourceAuthorized = isAuthorized.evaluate(conn.getSource());
        final boolean isDestinationAuthorized = isAuthorized.evaluate(conn.getDestination());
        final ConnectionStatus connStatus = new ConnectionStatus();
        connStatus.setId(conn.getIdentifier());
        connStatus.setGroupId(conn.getProcessGroup().getIdentifier());
        connStatus.setSourceId(conn.getSource().getIdentifier());
        connStatus.setSourceName(isSourceAuthorized ? conn.getSource().getName() : conn.getSource().getIdentifier());
        connStatus.setDestinationId(conn.getDestination().getIdentifier());
        connStatus.setDestinationName(isDestinationAuthorized ? conn.getDestination().getName() : conn.getDestination().getIdentifier());
        connStatus.setBackPressureDataSizeThreshold(conn.getFlowFileQueue().getBackPressureDataSizeThreshold());
        connStatus.setBackPressureObjectThreshold(conn.getFlowFileQueue().getBackPressureObjectThreshold());
        final FlowFileEvent connectionStatusReport = statusReport.getReportEntry(conn.getIdentifier());
        if (connectionStatusReport != null) {
            connStatus.setInputBytes(connectionStatusReport.getContentSizeIn());
            connStatus.setInputCount(connectionStatusReport.getFlowFilesIn());
            connStatus.setOutputBytes(connectionStatusReport.getContentSizeOut());
            connStatus.setOutputCount(connectionStatusReport.getFlowFilesOut());
            flowFilesTransferred += connectionStatusReport.getFlowFilesIn() + connectionStatusReport.getFlowFilesOut();
            bytesTransferred += connectionStatusReport.getContentSizeIn() + connectionStatusReport.getContentSizeOut();
        }
        if (isConnectionAuthorized) {
            if (StringUtils.isNotBlank(conn.getName())) {
                connStatus.setName(conn.getName());
            } else if (conn.getRelationships() != null && !conn.getRelationships().isEmpty()) {
                final Collection<String> relationships = new ArrayList<>(conn.getRelationships().size());
                for (final Relationship relationship : conn.getRelationships()) {
                    relationships.add(relationship.getName());
                }
                connStatus.setName(StringUtils.join(relationships, ", "));
            }
        } else {
            connStatus.setName(conn.getIdentifier());
        }
        final QueueSize queueSize = conn.getFlowFileQueue().size();
        final int connectionQueuedCount = queueSize.getObjectCount();
        final long connectionQueuedBytes = queueSize.getByteCount();
        if (connectionQueuedCount > 0) {
            connStatus.setQueuedBytes(connectionQueuedBytes);
            connStatus.setQueuedCount(connectionQueuedCount);
        }
        connectionStatusCollection.add(connStatus);
        queuedCount += connectionQueuedCount;
        queuedContentSize += connectionQueuedBytes;
        final Connectable source = conn.getSource();
        if (ConnectableType.REMOTE_OUTPUT_PORT.equals(source.getConnectableType())) {
            final RemoteGroupPort remoteOutputPort = (RemoteGroupPort) source;
            activeGroupThreads += processScheduler.getActiveThreadCount(remoteOutputPort);
        }
        final Connectable destination = conn.getDestination();
        if (ConnectableType.REMOTE_INPUT_PORT.equals(destination.getConnectableType())) {
            final RemoteGroupPort remoteInputPort = (RemoteGroupPort) destination;
            activeGroupThreads += processScheduler.getActiveThreadCount(remoteInputPort);
        }
    }
    // status for input ports
    final Collection<PortStatus> inputPortStatusCollection = new ArrayList<>();
    status.setInputPortStatus(inputPortStatusCollection);
    final Set<Port> inputPorts = group.getInputPorts();
    for (final Port port : inputPorts) {
        final boolean isInputPortAuthorized = isAuthorized.evaluate(port);
        final PortStatus portStatus = new PortStatus();
        portStatus.setId(port.getIdentifier());
        portStatus.setGroupId(port.getProcessGroup().getIdentifier());
        portStatus.setName(isInputPortAuthorized ? port.getName() : port.getIdentifier());
        portStatus.setActiveThreadCount(processScheduler.getActiveThreadCount(port));
        // determine the run status
        if (ScheduledState.RUNNING.equals(port.getScheduledState())) {
            portStatus.setRunStatus(RunStatus.Running);
        } else if (ScheduledState.DISABLED.equals(port.getScheduledState())) {
            portStatus.setRunStatus(RunStatus.Disabled);
        } else if (!port.isValid()) {
            portStatus.setRunStatus(RunStatus.Invalid);
        } else {
            portStatus.setRunStatus(RunStatus.Stopped);
        }
        // special handling for root group ports
        if (port instanceof RootGroupPort) {
            final RootGroupPort rootGroupPort = (RootGroupPort) port;
            portStatus.setTransmitting(rootGroupPort.isTransmitting());
        }
        final FlowFileEvent entry = statusReport.getReportEntries().get(port.getIdentifier());
        if (entry == null) {
            portStatus.setInputBytes(0L);
            portStatus.setInputCount(0);
            portStatus.setOutputBytes(0L);
            portStatus.setOutputCount(0);
        } else {
            final int processedCount = entry.getFlowFilesOut();
            final long numProcessedBytes = entry.getContentSizeOut();
            portStatus.setOutputBytes(numProcessedBytes);
            portStatus.setOutputCount(processedCount);
            final int inputCount = entry.getFlowFilesIn();
            final long inputBytes = entry.getContentSizeIn();
            portStatus.setInputBytes(inputBytes);
            portStatus.setInputCount(inputCount);
            flowFilesIn += inputCount;
            bytesIn += inputBytes;
            bytesWritten += entry.getBytesWritten();
            flowFilesReceived += entry.getFlowFilesReceived();
            bytesReceived += entry.getBytesReceived();
        }
        inputPortStatusCollection.add(portStatus);
        activeGroupThreads += portStatus.getActiveThreadCount();
    }
    // status for output ports
    final Collection<PortStatus> outputPortStatusCollection = new ArrayList<>();
    status.setOutputPortStatus(outputPortStatusCollection);
    final Set<Port> outputPorts = group.getOutputPorts();
    for (final Port port : outputPorts) {
        final boolean isOutputPortAuthorized = isAuthorized.evaluate(port);
        final PortStatus portStatus = new PortStatus();
        portStatus.setId(port.getIdentifier());
        portStatus.setGroupId(port.getProcessGroup().getIdentifier());
        portStatus.setName(isOutputPortAuthorized ? port.getName() : port.getIdentifier());
        portStatus.setActiveThreadCount(processScheduler.getActiveThreadCount(port));
        // determine the run status
        if (ScheduledState.RUNNING.equals(port.getScheduledState())) {
            portStatus.setRunStatus(RunStatus.Running);
        } else if (ScheduledState.DISABLED.equals(port.getScheduledState())) {
            portStatus.setRunStatus(RunStatus.Disabled);
        } else if (!port.isValid()) {
            portStatus.setRunStatus(RunStatus.Invalid);
        } else {
            portStatus.setRunStatus(RunStatus.Stopped);
        }
        // special handling for root group ports
        if (port instanceof RootGroupPort) {
            final RootGroupPort rootGroupPort = (RootGroupPort) port;
            portStatus.setTransmitting(rootGroupPort.isTransmitting());
        }
        final FlowFileEvent entry = statusReport.getReportEntries().get(port.getIdentifier());
        if (entry == null) {
            portStatus.setInputBytes(0L);
            portStatus.setInputCount(0);
            portStatus.setOutputBytes(0L);
            portStatus.setOutputCount(0);
        } else {
            final int processedCount = entry.getFlowFilesOut();
            final long numProcessedBytes = entry.getContentSizeOut();
            portStatus.setOutputBytes(numProcessedBytes);
            portStatus.setOutputCount(processedCount);
            final int inputCount = entry.getFlowFilesIn();
            final long inputBytes = entry.getContentSizeIn();
            portStatus.setInputBytes(inputBytes);
            portStatus.setInputCount(inputCount);
            bytesRead += entry.getBytesRead();
            flowFilesOut += entry.getFlowFilesOut();
            bytesOut += entry.getContentSizeOut();
            flowFilesSent = entry.getFlowFilesSent();
            bytesSent += entry.getBytesSent();
        }
        outputPortStatusCollection.add(portStatus);
        activeGroupThreads += portStatus.getActiveThreadCount();
    }
    for (final Funnel funnel : group.getFunnels()) {
        activeGroupThreads += processScheduler.getActiveThreadCount(funnel);
    }
    status.setActiveThreadCount(activeGroupThreads);
    status.setBytesRead(bytesRead);
    status.setBytesWritten(bytesWritten);
    status.setQueuedCount(queuedCount);
    status.setQueuedContentSize(queuedContentSize);
    status.setInputContentSize(bytesIn);
    status.setInputCount(flowFilesIn);
    status.setOutputContentSize(bytesOut);
    status.setOutputCount(flowFilesOut);
    status.setFlowFilesReceived(flowFilesReceived);
    status.setBytesReceived(bytesReceived);
    status.setFlowFilesSent(flowFilesSent);
    status.setBytesSent(bytesSent);
    status.setFlowFilesTransferred(flowFilesTransferred);
    status.setBytesTransferred(bytesTransferred);
    final VersionControlInformation vci = group.getVersionControlInformation();
    if (vci != null && vci.getStatus() != null && vci.getStatus().getState() != null) {
        status.setVersionedFlowState(vci.getStatus().getState());
    }
    return status;
}