use of org.apache.nifi.controller.status.ProcessorStatus in project nifi by apache.
the class FlowController method getGroupStatus.
/**
* Returns the status for the components in the specified group with the
* specified report. The results will be filtered by executing the specified
* predicate.
*
* @param group group id
* @param statusReport report
* @param isAuthorized is authorized check
* @return the component status
*/
public ProcessGroupStatus getGroupStatus(final ProcessGroup group, final RepositoryStatusReport statusReport, final Predicate<Authorizable> isAuthorized) {
if (group == null) {
return null;
}
final ProcessGroupStatus status = new ProcessGroupStatus();
status.setId(group.getIdentifier());
status.setName(isAuthorized.evaluate(group) ? group.getName() : group.getIdentifier());
int activeGroupThreads = 0;
long bytesRead = 0L;
long bytesWritten = 0L;
int queuedCount = 0;
long queuedContentSize = 0L;
int flowFilesIn = 0;
long bytesIn = 0L;
int flowFilesOut = 0;
long bytesOut = 0L;
int flowFilesReceived = 0;
long bytesReceived = 0L;
int flowFilesSent = 0;
long bytesSent = 0L;
int flowFilesTransferred = 0;
long bytesTransferred = 0;
// set status for processors
final Collection<ProcessorStatus> processorStatusCollection = new ArrayList<>();
status.setProcessorStatus(processorStatusCollection);
for (final ProcessorNode procNode : group.getProcessors()) {
final ProcessorStatus procStat = getProcessorStatus(statusReport, procNode, isAuthorized);
processorStatusCollection.add(procStat);
activeGroupThreads += procStat.getActiveThreadCount();
bytesRead += procStat.getBytesRead();
bytesWritten += procStat.getBytesWritten();
flowFilesReceived += procStat.getFlowFilesReceived();
bytesReceived += procStat.getBytesReceived();
flowFilesSent += procStat.getFlowFilesSent();
bytesSent += procStat.getBytesSent();
}
// set status for local child groups
final Collection<ProcessGroupStatus> localChildGroupStatusCollection = new ArrayList<>();
status.setProcessGroupStatus(localChildGroupStatusCollection);
for (final ProcessGroup childGroup : group.getProcessGroups()) {
final ProcessGroupStatus childGroupStatus = getGroupStatus(childGroup, statusReport, isAuthorized);
localChildGroupStatusCollection.add(childGroupStatus);
activeGroupThreads += childGroupStatus.getActiveThreadCount();
bytesRead += childGroupStatus.getBytesRead();
bytesWritten += childGroupStatus.getBytesWritten();
queuedCount += childGroupStatus.getQueuedCount();
queuedContentSize += childGroupStatus.getQueuedContentSize();
flowFilesReceived += childGroupStatus.getFlowFilesReceived();
bytesReceived += childGroupStatus.getBytesReceived();
flowFilesSent += childGroupStatus.getFlowFilesSent();
bytesSent += childGroupStatus.getBytesSent();
flowFilesTransferred += childGroupStatus.getFlowFilesTransferred();
bytesTransferred += childGroupStatus.getBytesTransferred();
}
// set status for remote child groups
final Collection<RemoteProcessGroupStatus> remoteProcessGroupStatusCollection = new ArrayList<>();
status.setRemoteProcessGroupStatus(remoteProcessGroupStatusCollection);
for (final RemoteProcessGroup remoteGroup : group.getRemoteProcessGroups()) {
final RemoteProcessGroupStatus remoteStatus = createRemoteGroupStatus(remoteGroup, statusReport, isAuthorized);
if (remoteStatus != null) {
remoteProcessGroupStatusCollection.add(remoteStatus);
flowFilesReceived += remoteStatus.getReceivedCount();
bytesReceived += remoteStatus.getReceivedContentSize();
flowFilesSent += remoteStatus.getSentCount();
bytesSent += remoteStatus.getSentContentSize();
}
}
// connection status
final Collection<ConnectionStatus> connectionStatusCollection = new ArrayList<>();
status.setConnectionStatus(connectionStatusCollection);
// get the connection and remote port status
for (final Connection conn : group.getConnections()) {
final boolean isConnectionAuthorized = isAuthorized.evaluate(conn);
final boolean isSourceAuthorized = isAuthorized.evaluate(conn.getSource());
final boolean isDestinationAuthorized = isAuthorized.evaluate(conn.getDestination());
final ConnectionStatus connStatus = new ConnectionStatus();
connStatus.setId(conn.getIdentifier());
connStatus.setGroupId(conn.getProcessGroup().getIdentifier());
connStatus.setSourceId(conn.getSource().getIdentifier());
connStatus.setSourceName(isSourceAuthorized ? conn.getSource().getName() : conn.getSource().getIdentifier());
connStatus.setDestinationId(conn.getDestination().getIdentifier());
connStatus.setDestinationName(isDestinationAuthorized ? conn.getDestination().getName() : conn.getDestination().getIdentifier());
connStatus.setBackPressureDataSizeThreshold(conn.getFlowFileQueue().getBackPressureDataSizeThreshold());
connStatus.setBackPressureObjectThreshold(conn.getFlowFileQueue().getBackPressureObjectThreshold());
final FlowFileEvent connectionStatusReport = statusReport.getReportEntry(conn.getIdentifier());
if (connectionStatusReport != null) {
connStatus.setInputBytes(connectionStatusReport.getContentSizeIn());
connStatus.setInputCount(connectionStatusReport.getFlowFilesIn());
connStatus.setOutputBytes(connectionStatusReport.getContentSizeOut());
connStatus.setOutputCount(connectionStatusReport.getFlowFilesOut());
flowFilesTransferred += connectionStatusReport.getFlowFilesIn() + connectionStatusReport.getFlowFilesOut();
bytesTransferred += connectionStatusReport.getContentSizeIn() + connectionStatusReport.getContentSizeOut();
}
if (isConnectionAuthorized) {
if (StringUtils.isNotBlank(conn.getName())) {
connStatus.setName(conn.getName());
} else if (conn.getRelationships() != null && !conn.getRelationships().isEmpty()) {
final Collection<String> relationships = new ArrayList<>(conn.getRelationships().size());
for (final Relationship relationship : conn.getRelationships()) {
relationships.add(relationship.getName());
}
connStatus.setName(StringUtils.join(relationships, ", "));
}
} else {
connStatus.setName(conn.getIdentifier());
}
final QueueSize queueSize = conn.getFlowFileQueue().size();
final int connectionQueuedCount = queueSize.getObjectCount();
final long connectionQueuedBytes = queueSize.getByteCount();
if (connectionQueuedCount > 0) {
connStatus.setQueuedBytes(connectionQueuedBytes);
connStatus.setQueuedCount(connectionQueuedCount);
}
connectionStatusCollection.add(connStatus);
queuedCount += connectionQueuedCount;
queuedContentSize += connectionQueuedBytes;
final Connectable source = conn.getSource();
if (ConnectableType.REMOTE_OUTPUT_PORT.equals(source.getConnectableType())) {
final RemoteGroupPort remoteOutputPort = (RemoteGroupPort) source;
activeGroupThreads += processScheduler.getActiveThreadCount(remoteOutputPort);
}
final Connectable destination = conn.getDestination();
if (ConnectableType.REMOTE_INPUT_PORT.equals(destination.getConnectableType())) {
final RemoteGroupPort remoteInputPort = (RemoteGroupPort) destination;
activeGroupThreads += processScheduler.getActiveThreadCount(remoteInputPort);
}
}
// status for input ports
final Collection<PortStatus> inputPortStatusCollection = new ArrayList<>();
status.setInputPortStatus(inputPortStatusCollection);
final Set<Port> inputPorts = group.getInputPorts();
for (final Port port : inputPorts) {
final boolean isInputPortAuthorized = isAuthorized.evaluate(port);
final PortStatus portStatus = new PortStatus();
portStatus.setId(port.getIdentifier());
portStatus.setGroupId(port.getProcessGroup().getIdentifier());
portStatus.setName(isInputPortAuthorized ? port.getName() : port.getIdentifier());
portStatus.setActiveThreadCount(processScheduler.getActiveThreadCount(port));
// determine the run status
if (ScheduledState.RUNNING.equals(port.getScheduledState())) {
portStatus.setRunStatus(RunStatus.Running);
} else if (ScheduledState.DISABLED.equals(port.getScheduledState())) {
portStatus.setRunStatus(RunStatus.Disabled);
} else if (!port.isValid()) {
portStatus.setRunStatus(RunStatus.Invalid);
} else {
portStatus.setRunStatus(RunStatus.Stopped);
}
// special handling for root group ports
if (port instanceof RootGroupPort) {
final RootGroupPort rootGroupPort = (RootGroupPort) port;
portStatus.setTransmitting(rootGroupPort.isTransmitting());
}
final FlowFileEvent entry = statusReport.getReportEntries().get(port.getIdentifier());
if (entry == null) {
portStatus.setInputBytes(0L);
portStatus.setInputCount(0);
portStatus.setOutputBytes(0L);
portStatus.setOutputCount(0);
} else {
final int processedCount = entry.getFlowFilesOut();
final long numProcessedBytes = entry.getContentSizeOut();
portStatus.setOutputBytes(numProcessedBytes);
portStatus.setOutputCount(processedCount);
final int inputCount = entry.getFlowFilesIn();
final long inputBytes = entry.getContentSizeIn();
portStatus.setInputBytes(inputBytes);
portStatus.setInputCount(inputCount);
flowFilesIn += inputCount;
bytesIn += inputBytes;
bytesWritten += entry.getBytesWritten();
flowFilesReceived += entry.getFlowFilesReceived();
bytesReceived += entry.getBytesReceived();
}
inputPortStatusCollection.add(portStatus);
activeGroupThreads += portStatus.getActiveThreadCount();
}
// status for output ports
final Collection<PortStatus> outputPortStatusCollection = new ArrayList<>();
status.setOutputPortStatus(outputPortStatusCollection);
final Set<Port> outputPorts = group.getOutputPorts();
for (final Port port : outputPorts) {
final boolean isOutputPortAuthorized = isAuthorized.evaluate(port);
final PortStatus portStatus = new PortStatus();
portStatus.setId(port.getIdentifier());
portStatus.setGroupId(port.getProcessGroup().getIdentifier());
portStatus.setName(isOutputPortAuthorized ? port.getName() : port.getIdentifier());
portStatus.setActiveThreadCount(processScheduler.getActiveThreadCount(port));
// determine the run status
if (ScheduledState.RUNNING.equals(port.getScheduledState())) {
portStatus.setRunStatus(RunStatus.Running);
} else if (ScheduledState.DISABLED.equals(port.getScheduledState())) {
portStatus.setRunStatus(RunStatus.Disabled);
} else if (!port.isValid()) {
portStatus.setRunStatus(RunStatus.Invalid);
} else {
portStatus.setRunStatus(RunStatus.Stopped);
}
// special handling for root group ports
if (port instanceof RootGroupPort) {
final RootGroupPort rootGroupPort = (RootGroupPort) port;
portStatus.setTransmitting(rootGroupPort.isTransmitting());
}
final FlowFileEvent entry = statusReport.getReportEntries().get(port.getIdentifier());
if (entry == null) {
portStatus.setInputBytes(0L);
portStatus.setInputCount(0);
portStatus.setOutputBytes(0L);
portStatus.setOutputCount(0);
} else {
final int processedCount = entry.getFlowFilesOut();
final long numProcessedBytes = entry.getContentSizeOut();
portStatus.setOutputBytes(numProcessedBytes);
portStatus.setOutputCount(processedCount);
final int inputCount = entry.getFlowFilesIn();
final long inputBytes = entry.getContentSizeIn();
portStatus.setInputBytes(inputBytes);
portStatus.setInputCount(inputCount);
bytesRead += entry.getBytesRead();
flowFilesOut += entry.getFlowFilesOut();
bytesOut += entry.getContentSizeOut();
flowFilesSent = entry.getFlowFilesSent();
bytesSent += entry.getBytesSent();
}
outputPortStatusCollection.add(portStatus);
activeGroupThreads += portStatus.getActiveThreadCount();
}
for (final Funnel funnel : group.getFunnels()) {
activeGroupThreads += processScheduler.getActiveThreadCount(funnel);
}
status.setActiveThreadCount(activeGroupThreads);
status.setBytesRead(bytesRead);
status.setBytesWritten(bytesWritten);
status.setQueuedCount(queuedCount);
status.setQueuedContentSize(queuedContentSize);
status.setInputContentSize(bytesIn);
status.setInputCount(flowFilesIn);
status.setOutputContentSize(bytesOut);
status.setOutputCount(flowFilesOut);
status.setFlowFilesReceived(flowFilesReceived);
status.setBytesReceived(bytesReceived);
status.setFlowFilesSent(flowFilesSent);
status.setBytesSent(bytesSent);
status.setFlowFilesTransferred(flowFilesTransferred);
status.setBytesTransferred(bytesTransferred);
final VersionControlInformation vci = group.getVersionControlInformation();
if (vci != null && vci.getStatus() != null && vci.getStatus().getState() != null) {
status.setVersionedFlowState(vci.getStatus().getState());
}
return status;
}
use of org.apache.nifi.controller.status.ProcessorStatus in project nifi by apache.
the class ComponentMapHolder method createComponentMap.
private static ComponentMapHolder createComponentMap(final ProcessGroupStatus status, final ParentProcessGroupSearchNode thisProcessGroupNode) {
final ComponentMapHolder holder = new ComponentMapHolder();
final Map<String, String> componentNameMap = holder.componentNameMap;
final Map<String, ParentProcessGroupSearchNode> componentToParentGroupMap = holder.componentToParentGroupMap;
final Map<String, String> sourceToConnectionParentGroupMap = holder.sourceToConnectionParentGroupMap;
final Map<String, String> destinationToConnectionParentGroupMap = holder.destinationToConnectionParentGroupMap;
if (status != null) {
componentNameMap.put(status.getId(), status.getName());
for (final ProcessorStatus procStatus : status.getProcessorStatus()) {
componentNameMap.put(procStatus.getId(), procStatus.getName());
componentToParentGroupMap.put(procStatus.getId(), thisProcessGroupNode);
}
for (final PortStatus portStatus : status.getInputPortStatus()) {
componentNameMap.put(portStatus.getId(), portStatus.getName());
componentToParentGroupMap.put(portStatus.getId(), thisProcessGroupNode);
}
for (final PortStatus portStatus : status.getOutputPortStatus()) {
componentNameMap.put(portStatus.getId(), portStatus.getName());
componentToParentGroupMap.put(portStatus.getId(), thisProcessGroupNode);
}
for (final RemoteProcessGroupStatus rpgStatus : status.getRemoteProcessGroupStatus()) {
componentNameMap.put(rpgStatus.getId(), rpgStatus.getName());
componentToParentGroupMap.put(rpgStatus.getId(), thisProcessGroupNode);
}
for (final ConnectionStatus connectionStatus : status.getConnectionStatus()) {
componentNameMap.put(connectionStatus.getId(), connectionStatus.getName());
componentToParentGroupMap.put(connectionStatus.getId(), thisProcessGroupNode);
// Add source and destination for Remote Input/Output Ports because metadata for those are only available at ConnectionStatus.
componentNameMap.computeIfAbsent(connectionStatus.getSourceId(), k -> connectionStatus.getSourceName());
componentNameMap.computeIfAbsent(connectionStatus.getDestinationId(), k -> connectionStatus.getDestinationName());
sourceToConnectionParentGroupMap.put(connectionStatus.getSourceId(), connectionStatus.getGroupId());
destinationToConnectionParentGroupMap.put(connectionStatus.getDestinationId(), connectionStatus.getGroupId());
}
for (final ProcessGroupStatus childGroup : status.getProcessGroupStatus()) {
componentNameMap.put(childGroup.getId(), childGroup.getName());
ParentProcessGroupSearchNode childProcessGroupNode = new ParentProcessGroupSearchNode(childGroup.getId(), thisProcessGroupNode);
componentToParentGroupMap.put(childGroup.getId(), thisProcessGroupNode);
holder.putAll(createComponentMap(childGroup, childProcessGroupNode));
}
}
return holder;
}
use of org.apache.nifi.controller.status.ProcessorStatus in project nifi by apache.
the class TestSiteToSiteProvenanceReportingTask method setup.
private MockSiteToSiteProvenanceReportingTask setup(ProvenanceEventRecord event, Map<PropertyDescriptor, String> properties, long maxEventId) throws IOException {
final MockSiteToSiteProvenanceReportingTask task = new MockSiteToSiteProvenanceReportingTask();
when(context.getStateManager()).thenReturn(new MockStateManager(task));
Mockito.doAnswer(new Answer<PropertyValue>() {
@Override
public PropertyValue answer(final InvocationOnMock invocation) throws Throwable {
final PropertyDescriptor descriptor = invocation.getArgumentAt(0, PropertyDescriptor.class);
return new MockPropertyValue(properties.get(descriptor));
}
}).when(context).getProperty(Mockito.any(PropertyDescriptor.class));
Mockito.doAnswer(new Answer<PropertyValue>() {
@Override
public PropertyValue answer(final InvocationOnMock invocation) throws Throwable {
final PropertyDescriptor descriptor = invocation.getArgumentAt(0, PropertyDescriptor.class);
return new MockPropertyValue(properties.get(descriptor));
}
}).when(confContext).getProperty(Mockito.any(PropertyDescriptor.class));
final AtomicInteger totalEvents = new AtomicInteger(0);
final EventAccess eventAccess = Mockito.mock(EventAccess.class);
Mockito.doAnswer(new Answer<List<ProvenanceEventRecord>>() {
@Override
public List<ProvenanceEventRecord> answer(final InvocationOnMock invocation) throws Throwable {
final long startId = invocation.getArgumentAt(0, long.class);
final int maxRecords = invocation.getArgumentAt(1, int.class);
final List<ProvenanceEventRecord> eventsToReturn = new ArrayList<>();
for (int i = (int) Math.max(0, startId); i < (int) (startId + maxRecords) && totalEvents.get() < maxEventId; i++) {
if (event != null) {
eventsToReturn.add(event);
}
totalEvents.getAndIncrement();
}
return eventsToReturn;
}
}).when(eventAccess).getProvenanceEvents(Mockito.anyLong(), Mockito.anyInt());
ProcessGroupStatus pgRoot = new ProcessGroupStatus();
pgRoot.setId("root");
when(eventAccess.getControllerStatus()).thenReturn(pgRoot);
// Add child Process Groups.
// Root -> (A, B -> (B2 -> (B3)))
final ProcessGroupStatus pgA = new ProcessGroupStatus();
pgA.setId("pgA");
final ProcessGroupStatus pgB = new ProcessGroupStatus();
pgB.setId("pgB");
final ProcessGroupStatus pgB2 = new ProcessGroupStatus();
pgB2.setId("pgB2");
final ProcessGroupStatus pgB3 = new ProcessGroupStatus();
pgB3.setId("pgB3");
final Collection<ProcessGroupStatus> childPGs = pgRoot.getProcessGroupStatus();
childPGs.add(pgA);
childPGs.add(pgB);
pgB.getProcessGroupStatus().add(pgB2);
pgB2.getProcessGroupStatus().add(pgB3);
// Add Processors.
final ProcessorStatus prcRoot = new ProcessorStatus();
prcRoot.setId("1234");
pgRoot.getProcessorStatus().add(prcRoot);
final ProcessorStatus prcA = new ProcessorStatus();
prcA.setId("A001");
prcA.setName("Processor in PGA");
pgA.getProcessorStatus().add(prcA);
final ProcessorStatus prcB = new ProcessorStatus();
prcB.setId("B001");
prcB.setName("Processor in PGB");
pgB.getProcessorStatus().add(prcB);
final ProcessorStatus prcB2 = new ProcessorStatus();
prcB2.setId("B201");
prcB2.setName("Processor in PGB2");
pgB2.getProcessorStatus().add(prcB2);
final ProcessorStatus prcB3 = new ProcessorStatus();
prcB3.setId("B301");
prcB3.setName("Processor in PGB3");
pgB3.getProcessorStatus().add(prcB3);
// Add connection status to test Remote Input/Output Ports
final ConnectionStatus b2RemoteInputPort = new ConnectionStatus();
b2RemoteInputPort.setGroupId("pgB2");
b2RemoteInputPort.setSourceId("B201");
b2RemoteInputPort.setDestinationId("riB2");
b2RemoteInputPort.setDestinationName("Remote Input Port name");
pgB2.getConnectionStatus().add(b2RemoteInputPort);
final ConnectionStatus b3RemoteOutputPort = new ConnectionStatus();
b3RemoteOutputPort.setGroupId("pgB3");
b3RemoteOutputPort.setSourceId("roB3");
b3RemoteOutputPort.setSourceName("Remote Output Port name");
b3RemoteOutputPort.setDestinationId("B301");
pgB3.getConnectionStatus().add(b3RemoteOutputPort);
final ProvenanceEventRepository provenanceRepository = Mockito.mock(ProvenanceEventRepository.class);
Mockito.doAnswer(new Answer<Long>() {
@Override
public Long answer(final InvocationOnMock invocation) throws Throwable {
return maxEventId;
}
}).when(provenanceRepository).getMaxEventId();
when(context.getEventAccess()).thenReturn(eventAccess);
when(eventAccess.getProvenanceRepository()).thenReturn(provenanceRepository);
final ComponentLog logger = Mockito.mock(ComponentLog.class);
when(initContext.getIdentifier()).thenReturn(UUID.randomUUID().toString());
when(initContext.getLogger()).thenReturn(logger);
return task;
}
use of org.apache.nifi.controller.status.ProcessorStatus in project nifi by apache.
the class TestSiteToSiteStatusReportingTask method generateProcessorStatus.
public static ProcessorStatus generateProcessorStatus(String id, String namePrefix) {
ProcessorStatus pStatus = new ProcessorStatus();
pStatus.setId(id);
pStatus.setName(namePrefix + "-" + UUID.randomUUID().toString());
pStatus.setActiveThreadCount(0);
pStatus.setAverageLineageDuration(1l);
pStatus.setBytesRead(2l);
pStatus.setBytesReceived(3l);
pStatus.setBytesSent(4l);
pStatus.setBytesWritten(5l);
pStatus.setFlowFilesReceived(6);
pStatus.setFlowFilesRemoved(7);
pStatus.setFlowFilesSent(8);
pStatus.setInputBytes(9l);
pStatus.setInputCount(10);
pStatus.setInvocations(11);
pStatus.setOutputBytes(12l);
pStatus.setOutputCount(13);
pStatus.setProcessingNanos(14l);
pStatus.setType("type");
return pStatus;
}
use of org.apache.nifi.controller.status.ProcessorStatus in project nifi by apache.
the class TestAmbariReportingTask method setup.
@Before
public void setup() {
status = new ProcessGroupStatus();
status.setId("1234");
status.setFlowFilesReceived(5);
status.setBytesReceived(10000);
status.setFlowFilesSent(10);
status.setBytesSent(20000);
status.setQueuedCount(100);
status.setQueuedContentSize(1024L);
status.setBytesRead(60000L);
status.setBytesWritten(80000L);
status.setActiveThreadCount(5);
// create a processor status with processing time
ProcessorStatus procStatus = new ProcessorStatus();
procStatus.setProcessingNanos(123456789);
Collection<ProcessorStatus> processorStatuses = new ArrayList<>();
processorStatuses.add(procStatus);
status.setProcessorStatus(processorStatuses);
// create a group status with processing time
ProcessGroupStatus groupStatus = new ProcessGroupStatus();
groupStatus.setProcessorStatus(processorStatuses);
Collection<ProcessGroupStatus> groupStatuses = new ArrayList<>();
groupStatuses.add(groupStatus);
status.setProcessGroupStatus(groupStatuses);
}
Aggregations