use of org.apache.nifi.controller.status.RemoteProcessGroupStatus in project nifi by apache.
the class FlowController method createRemoteGroupStatus.
private RemoteProcessGroupStatus createRemoteGroupStatus(final RemoteProcessGroup remoteGroup, final RepositoryStatusReport statusReport, final Predicate<Authorizable> isAuthorized) {
final boolean isRemoteProcessGroupAuthorized = isAuthorized.evaluate(remoteGroup);
int receivedCount = 0;
long receivedContentSize = 0L;
int sentCount = 0;
long sentContentSize = 0L;
int activeThreadCount = 0;
int activePortCount = 0;
int inactivePortCount = 0;
final RemoteProcessGroupStatus status = new RemoteProcessGroupStatus();
status.setGroupId(remoteGroup.getProcessGroup().getIdentifier());
status.setName(isRemoteProcessGroupAuthorized ? remoteGroup.getName() : remoteGroup.getIdentifier());
status.setTargetUri(isRemoteProcessGroupAuthorized ? remoteGroup.getTargetUri() : null);
long lineageMillis = 0L;
int flowFilesRemoved = 0;
int flowFilesTransferred = 0;
for (final Port port : remoteGroup.getInputPorts()) {
// determine if this input port is connected
final boolean isConnected = port.hasIncomingConnection();
// we only want to consider remote ports that we are connected to
if (isConnected) {
if (port.isRunning()) {
activePortCount++;
} else {
inactivePortCount++;
}
activeThreadCount += processScheduler.getActiveThreadCount(port);
final FlowFileEvent portEvent = statusReport.getReportEntry(port.getIdentifier());
if (portEvent != null) {
lineageMillis += portEvent.getAggregateLineageMillis();
flowFilesRemoved += portEvent.getFlowFilesRemoved();
flowFilesTransferred += portEvent.getFlowFilesOut();
sentCount += portEvent.getFlowFilesSent();
sentContentSize += portEvent.getBytesSent();
}
}
}
for (final Port port : remoteGroup.getOutputPorts()) {
// determine if this output port is connected
final boolean isConnected = !port.getConnections().isEmpty();
// we only want to consider remote ports that we are connected from
if (isConnected) {
if (port.isRunning()) {
activePortCount++;
} else {
inactivePortCount++;
}
activeThreadCount += processScheduler.getActiveThreadCount(port);
final FlowFileEvent portEvent = statusReport.getReportEntry(port.getIdentifier());
if (portEvent != null) {
receivedCount += portEvent.getFlowFilesReceived();
receivedContentSize += portEvent.getBytesReceived();
}
}
}
status.setId(remoteGroup.getIdentifier());
status.setTransmissionStatus(remoteGroup.isTransmitting() ? TransmissionStatus.Transmitting : TransmissionStatus.NotTransmitting);
status.setActiveThreadCount(activeThreadCount);
status.setReceivedContentSize(receivedContentSize);
status.setReceivedCount(receivedCount);
status.setSentContentSize(sentContentSize);
status.setSentCount(sentCount);
status.setActiveRemotePortCount(activePortCount);
status.setInactiveRemotePortCount(inactivePortCount);
final int flowFilesOutOrRemoved = flowFilesTransferred + flowFilesRemoved;
status.setAverageLineageDuration(flowFilesOutOrRemoved == 0 ? 0 : lineageMillis / flowFilesOutOrRemoved, TimeUnit.MILLISECONDS);
return status;
}
use of org.apache.nifi.controller.status.RemoteProcessGroupStatus in project nifi-minifi by apache.
the class StatusConfigReporter method handleRemoteProcessGroupRequest.
private static void handleRemoteProcessGroupRequest(String[] sections, ProcessGroupStatus rootGroupStatus, FlowController flowController, List<RemoteProcessGroupStatusBean> remoteProcessGroupStatusList, Map<String, RemoteProcessGroupStatus> remoteProcessGroupStatusMap, Logger logger) throws StatusRequestException {
if (remoteProcessGroupStatusMap == null) {
remoteProcessGroupStatusMap = transformStatusCollection(rootGroupStatus.getRemoteProcessGroupStatus());
}
if (sections[1].equalsIgnoreCase("all")) {
if (!remoteProcessGroupStatusMap.isEmpty()) {
for (RemoteProcessGroupStatus remoteProcessGroupStatus : remoteProcessGroupStatusMap.values()) {
remoteProcessGroupStatusList.add(parseRemoteProcessGroupStatusRequest(remoteProcessGroupStatus, sections[2], flowController));
}
}
} else {
if (remoteProcessGroupStatusMap.containsKey(sections[1])) {
RemoteProcessGroupStatus remoteProcessGroupStatus = remoteProcessGroupStatusMap.get(sections[1]);
remoteProcessGroupStatusList.add(parseRemoteProcessGroupStatusRequest(remoteProcessGroupStatus, sections[2], flowController));
} else {
logger.warn("Status for Remote Process Group with key " + sections[1] + " was requested but one does not exist");
throw new StatusRequestException("No Remote Process Group with key " + sections[1] + " to report status on");
}
}
}
use of org.apache.nifi.controller.status.RemoteProcessGroupStatus in project nifi-minifi by apache.
the class StatusConfigReporter method getStatus.
public static FlowStatusReport getStatus(FlowController flowController, String statusRequest, Logger logger) throws StatusRequestException {
if (statusRequest == null) {
logger.error("Received a status request which was null");
throw new StatusRequestException("Cannot complete status request because the statusRequest is null");
}
if (flowController == null) {
logger.error("Received a status but the Flow Controller is null");
throw new StatusRequestException("Cannot complete status request because the Flow Controller is null");
}
FlowStatusReport flowStatusReport = new FlowStatusReport();
List<String> errorsGeneratingReport = new LinkedList<>();
flowStatusReport.setErrorsGeneratingReport(errorsGeneratingReport);
String[] itemsToReport = statusRequest.split(";");
ProcessGroupStatus rootGroupStatus = flowController.getControllerStatus();
Map<String, ProcessorStatus> processorStatusMap = null;
Map<String, ConnectionStatus> connectionStatusMap = null;
Map<String, RemoteProcessGroupStatus> remoteProcessGroupStatusMap = null;
for (String item : itemsToReport) {
String[] sections = item.split(":");
try {
switch(sections[0].toLowerCase().trim()) {
case "systemdiagnostics":
SystemDiagnosticsStatus systemDiagnosticsStatus = parseSystemDiagnosticsRequest(flowController.getSystemDiagnostics(), sections[1]);
flowStatusReport.setSystemDiagnosticsStatus(systemDiagnosticsStatus);
break;
case "instance":
InstanceStatus instanceStatus = parseInstanceRequest(sections[1], flowController, rootGroupStatus);
flowStatusReport.setInstanceStatus(instanceStatus);
break;
case "remoteprocessgroup":
if (flowStatusReport.getRemoteProcessGroupStatusList() == null) {
List<RemoteProcessGroupStatusBean> remoteProcessGroupStatusList = new LinkedList<>();
flowStatusReport.setRemoteProcessGroupStatusList(remoteProcessGroupStatusList);
}
handleRemoteProcessGroupRequest(sections, rootGroupStatus, flowController, flowStatusReport.getRemoteProcessGroupStatusList(), remoteProcessGroupStatusMap, logger);
break;
case "processor":
if (flowStatusReport.getProcessorStatusList() == null) {
List<ProcessorStatusBean> processorStatusList = new LinkedList<>();
flowStatusReport.setProcessorStatusList(processorStatusList);
}
handleProcessorRequest(sections, rootGroupStatus, flowController, flowStatusReport.getProcessorStatusList(), processorStatusMap, logger);
break;
case "connection":
if (flowStatusReport.getConnectionStatusList() == null) {
List<ConnectionStatusBean> connectionStatusList = new LinkedList<>();
flowStatusReport.setConnectionStatusList(connectionStatusList);
}
handleConnectionRequest(sections, rootGroupStatus, flowStatusReport.getConnectionStatusList(), connectionStatusMap, logger);
break;
case "provenancereporting":
if (flowStatusReport.getRemoteProcessGroupStatusList() == null) {
List<ReportingTaskStatus> reportingTaskStatusList = new LinkedList<>();
flowStatusReport.setReportingTaskStatusList(reportingTaskStatusList);
}
handleReportingTaskRequest(sections, flowController, flowStatusReport.getReportingTaskStatusList(), logger);
break;
case "controllerservices":
if (flowStatusReport.getControllerServiceStatusList() == null) {
List<ControllerServiceStatus> controllerServiceStatusList = new LinkedList<>();
flowStatusReport.setControllerServiceStatusList(controllerServiceStatusList);
}
handleControllerServices(sections, flowController, flowStatusReport.getControllerServiceStatusList(), logger);
break;
}
} catch (Exception e) {
logger.error("Hit exception while requesting status for item '" + item + "'", e);
errorsGeneratingReport.add("Unable to get status for request '" + item + "' due to:" + e);
}
}
return flowStatusReport;
}
use of org.apache.nifi.controller.status.RemoteProcessGroupStatus in project nifi-minifi by apache.
the class StatusConfigReporterTest method populateRemoteProcessGroup.
private void populateRemoteProcessGroup(boolean addBulletins, boolean addAuthIssues) {
when(mockFlowController.getGroup(mockFlowController.getRootGroupId())).thenReturn(processGroup);
RemoteProcessGroup remoteProcessGroup = mock(RemoteProcessGroup.class);
when(processGroup.getRemoteProcessGroup(any())).thenReturn(remoteProcessGroup);
RemoteGroupPort remoteGroupPort = mock(RemoteGroupPort.class);
when(remoteGroupPort.getName()).thenReturn("inputPort");
when(remoteGroupPort.getTargetExists()).thenReturn(true);
when(remoteGroupPort.isTargetRunning()).thenReturn(false);
when(remoteProcessGroup.getInputPorts()).thenReturn(Collections.singleton(remoteGroupPort));
remoteGroupPort = mock(RemoteGroupPort.class);
when(remoteGroupPort.getName()).thenReturn("outputPort");
when(remoteGroupPort.getTargetExists()).thenReturn(true);
when(remoteGroupPort.isTargetRunning()).thenReturn(false);
when(remoteProcessGroup.getOutputPorts()).thenReturn(Collections.singleton(remoteGroupPort));
RemoteProcessGroupStatus remoteProcessGroupStatus = new RemoteProcessGroupStatus();
addRemoteProcessGroupStatus(remoteProcessGroupStatus);
if (addBulletins) {
addBulletins("Bulletin message", remoteProcessGroupStatus.getId());
}
when(rootGroupStatus.getRemoteProcessGroupStatus()).thenReturn(Collections.singletonList(remoteProcessGroupStatus));
}
use of org.apache.nifi.controller.status.RemoteProcessGroupStatus in project nifi by apache.
the class FlowController method getGroupStatus.
/**
* Returns the status for the components in the specified group with the
* specified report. The results will be filtered by executing the specified
* predicate.
*
* @param group group id
* @param statusReport report
* @param isAuthorized is authorized check
* @return the component status
*/
public ProcessGroupStatus getGroupStatus(final ProcessGroup group, final RepositoryStatusReport statusReport, final Predicate<Authorizable> isAuthorized) {
if (group == null) {
return null;
}
final ProcessGroupStatus status = new ProcessGroupStatus();
status.setId(group.getIdentifier());
status.setName(isAuthorized.evaluate(group) ? group.getName() : group.getIdentifier());
int activeGroupThreads = 0;
long bytesRead = 0L;
long bytesWritten = 0L;
int queuedCount = 0;
long queuedContentSize = 0L;
int flowFilesIn = 0;
long bytesIn = 0L;
int flowFilesOut = 0;
long bytesOut = 0L;
int flowFilesReceived = 0;
long bytesReceived = 0L;
int flowFilesSent = 0;
long bytesSent = 0L;
int flowFilesTransferred = 0;
long bytesTransferred = 0;
// set status for processors
final Collection<ProcessorStatus> processorStatusCollection = new ArrayList<>();
status.setProcessorStatus(processorStatusCollection);
for (final ProcessorNode procNode : group.getProcessors()) {
final ProcessorStatus procStat = getProcessorStatus(statusReport, procNode, isAuthorized);
processorStatusCollection.add(procStat);
activeGroupThreads += procStat.getActiveThreadCount();
bytesRead += procStat.getBytesRead();
bytesWritten += procStat.getBytesWritten();
flowFilesReceived += procStat.getFlowFilesReceived();
bytesReceived += procStat.getBytesReceived();
flowFilesSent += procStat.getFlowFilesSent();
bytesSent += procStat.getBytesSent();
}
// set status for local child groups
final Collection<ProcessGroupStatus> localChildGroupStatusCollection = new ArrayList<>();
status.setProcessGroupStatus(localChildGroupStatusCollection);
for (final ProcessGroup childGroup : group.getProcessGroups()) {
final ProcessGroupStatus childGroupStatus = getGroupStatus(childGroup, statusReport, isAuthorized);
localChildGroupStatusCollection.add(childGroupStatus);
activeGroupThreads += childGroupStatus.getActiveThreadCount();
bytesRead += childGroupStatus.getBytesRead();
bytesWritten += childGroupStatus.getBytesWritten();
queuedCount += childGroupStatus.getQueuedCount();
queuedContentSize += childGroupStatus.getQueuedContentSize();
flowFilesReceived += childGroupStatus.getFlowFilesReceived();
bytesReceived += childGroupStatus.getBytesReceived();
flowFilesSent += childGroupStatus.getFlowFilesSent();
bytesSent += childGroupStatus.getBytesSent();
flowFilesTransferred += childGroupStatus.getFlowFilesTransferred();
bytesTransferred += childGroupStatus.getBytesTransferred();
}
// set status for remote child groups
final Collection<RemoteProcessGroupStatus> remoteProcessGroupStatusCollection = new ArrayList<>();
status.setRemoteProcessGroupStatus(remoteProcessGroupStatusCollection);
for (final RemoteProcessGroup remoteGroup : group.getRemoteProcessGroups()) {
final RemoteProcessGroupStatus remoteStatus = createRemoteGroupStatus(remoteGroup, statusReport, isAuthorized);
if (remoteStatus != null) {
remoteProcessGroupStatusCollection.add(remoteStatus);
flowFilesReceived += remoteStatus.getReceivedCount();
bytesReceived += remoteStatus.getReceivedContentSize();
flowFilesSent += remoteStatus.getSentCount();
bytesSent += remoteStatus.getSentContentSize();
}
}
// connection status
final Collection<ConnectionStatus> connectionStatusCollection = new ArrayList<>();
status.setConnectionStatus(connectionStatusCollection);
// get the connection and remote port status
for (final Connection conn : group.getConnections()) {
final boolean isConnectionAuthorized = isAuthorized.evaluate(conn);
final boolean isSourceAuthorized = isAuthorized.evaluate(conn.getSource());
final boolean isDestinationAuthorized = isAuthorized.evaluate(conn.getDestination());
final ConnectionStatus connStatus = new ConnectionStatus();
connStatus.setId(conn.getIdentifier());
connStatus.setGroupId(conn.getProcessGroup().getIdentifier());
connStatus.setSourceId(conn.getSource().getIdentifier());
connStatus.setSourceName(isSourceAuthorized ? conn.getSource().getName() : conn.getSource().getIdentifier());
connStatus.setDestinationId(conn.getDestination().getIdentifier());
connStatus.setDestinationName(isDestinationAuthorized ? conn.getDestination().getName() : conn.getDestination().getIdentifier());
connStatus.setBackPressureDataSizeThreshold(conn.getFlowFileQueue().getBackPressureDataSizeThreshold());
connStatus.setBackPressureObjectThreshold(conn.getFlowFileQueue().getBackPressureObjectThreshold());
final FlowFileEvent connectionStatusReport = statusReport.getReportEntry(conn.getIdentifier());
if (connectionStatusReport != null) {
connStatus.setInputBytes(connectionStatusReport.getContentSizeIn());
connStatus.setInputCount(connectionStatusReport.getFlowFilesIn());
connStatus.setOutputBytes(connectionStatusReport.getContentSizeOut());
connStatus.setOutputCount(connectionStatusReport.getFlowFilesOut());
flowFilesTransferred += connectionStatusReport.getFlowFilesIn() + connectionStatusReport.getFlowFilesOut();
bytesTransferred += connectionStatusReport.getContentSizeIn() + connectionStatusReport.getContentSizeOut();
}
if (isConnectionAuthorized) {
if (StringUtils.isNotBlank(conn.getName())) {
connStatus.setName(conn.getName());
} else if (conn.getRelationships() != null && !conn.getRelationships().isEmpty()) {
final Collection<String> relationships = new ArrayList<>(conn.getRelationships().size());
for (final Relationship relationship : conn.getRelationships()) {
relationships.add(relationship.getName());
}
connStatus.setName(StringUtils.join(relationships, ", "));
}
} else {
connStatus.setName(conn.getIdentifier());
}
final QueueSize queueSize = conn.getFlowFileQueue().size();
final int connectionQueuedCount = queueSize.getObjectCount();
final long connectionQueuedBytes = queueSize.getByteCount();
if (connectionQueuedCount > 0) {
connStatus.setQueuedBytes(connectionQueuedBytes);
connStatus.setQueuedCount(connectionQueuedCount);
}
connectionStatusCollection.add(connStatus);
queuedCount += connectionQueuedCount;
queuedContentSize += connectionQueuedBytes;
final Connectable source = conn.getSource();
if (ConnectableType.REMOTE_OUTPUT_PORT.equals(source.getConnectableType())) {
final RemoteGroupPort remoteOutputPort = (RemoteGroupPort) source;
activeGroupThreads += processScheduler.getActiveThreadCount(remoteOutputPort);
}
final Connectable destination = conn.getDestination();
if (ConnectableType.REMOTE_INPUT_PORT.equals(destination.getConnectableType())) {
final RemoteGroupPort remoteInputPort = (RemoteGroupPort) destination;
activeGroupThreads += processScheduler.getActiveThreadCount(remoteInputPort);
}
}
// status for input ports
final Collection<PortStatus> inputPortStatusCollection = new ArrayList<>();
status.setInputPortStatus(inputPortStatusCollection);
final Set<Port> inputPorts = group.getInputPorts();
for (final Port port : inputPorts) {
final boolean isInputPortAuthorized = isAuthorized.evaluate(port);
final PortStatus portStatus = new PortStatus();
portStatus.setId(port.getIdentifier());
portStatus.setGroupId(port.getProcessGroup().getIdentifier());
portStatus.setName(isInputPortAuthorized ? port.getName() : port.getIdentifier());
portStatus.setActiveThreadCount(processScheduler.getActiveThreadCount(port));
// determine the run status
if (ScheduledState.RUNNING.equals(port.getScheduledState())) {
portStatus.setRunStatus(RunStatus.Running);
} else if (ScheduledState.DISABLED.equals(port.getScheduledState())) {
portStatus.setRunStatus(RunStatus.Disabled);
} else if (!port.isValid()) {
portStatus.setRunStatus(RunStatus.Invalid);
} else {
portStatus.setRunStatus(RunStatus.Stopped);
}
// special handling for root group ports
if (port instanceof RootGroupPort) {
final RootGroupPort rootGroupPort = (RootGroupPort) port;
portStatus.setTransmitting(rootGroupPort.isTransmitting());
}
final FlowFileEvent entry = statusReport.getReportEntries().get(port.getIdentifier());
if (entry == null) {
portStatus.setInputBytes(0L);
portStatus.setInputCount(0);
portStatus.setOutputBytes(0L);
portStatus.setOutputCount(0);
} else {
final int processedCount = entry.getFlowFilesOut();
final long numProcessedBytes = entry.getContentSizeOut();
portStatus.setOutputBytes(numProcessedBytes);
portStatus.setOutputCount(processedCount);
final int inputCount = entry.getFlowFilesIn();
final long inputBytes = entry.getContentSizeIn();
portStatus.setInputBytes(inputBytes);
portStatus.setInputCount(inputCount);
flowFilesIn += inputCount;
bytesIn += inputBytes;
bytesWritten += entry.getBytesWritten();
flowFilesReceived += entry.getFlowFilesReceived();
bytesReceived += entry.getBytesReceived();
}
inputPortStatusCollection.add(portStatus);
activeGroupThreads += portStatus.getActiveThreadCount();
}
// status for output ports
final Collection<PortStatus> outputPortStatusCollection = new ArrayList<>();
status.setOutputPortStatus(outputPortStatusCollection);
final Set<Port> outputPorts = group.getOutputPorts();
for (final Port port : outputPorts) {
final boolean isOutputPortAuthorized = isAuthorized.evaluate(port);
final PortStatus portStatus = new PortStatus();
portStatus.setId(port.getIdentifier());
portStatus.setGroupId(port.getProcessGroup().getIdentifier());
portStatus.setName(isOutputPortAuthorized ? port.getName() : port.getIdentifier());
portStatus.setActiveThreadCount(processScheduler.getActiveThreadCount(port));
// determine the run status
if (ScheduledState.RUNNING.equals(port.getScheduledState())) {
portStatus.setRunStatus(RunStatus.Running);
} else if (ScheduledState.DISABLED.equals(port.getScheduledState())) {
portStatus.setRunStatus(RunStatus.Disabled);
} else if (!port.isValid()) {
portStatus.setRunStatus(RunStatus.Invalid);
} else {
portStatus.setRunStatus(RunStatus.Stopped);
}
// special handling for root group ports
if (port instanceof RootGroupPort) {
final RootGroupPort rootGroupPort = (RootGroupPort) port;
portStatus.setTransmitting(rootGroupPort.isTransmitting());
}
final FlowFileEvent entry = statusReport.getReportEntries().get(port.getIdentifier());
if (entry == null) {
portStatus.setInputBytes(0L);
portStatus.setInputCount(0);
portStatus.setOutputBytes(0L);
portStatus.setOutputCount(0);
} else {
final int processedCount = entry.getFlowFilesOut();
final long numProcessedBytes = entry.getContentSizeOut();
portStatus.setOutputBytes(numProcessedBytes);
portStatus.setOutputCount(processedCount);
final int inputCount = entry.getFlowFilesIn();
final long inputBytes = entry.getContentSizeIn();
portStatus.setInputBytes(inputBytes);
portStatus.setInputCount(inputCount);
bytesRead += entry.getBytesRead();
flowFilesOut += entry.getFlowFilesOut();
bytesOut += entry.getContentSizeOut();
flowFilesSent = entry.getFlowFilesSent();
bytesSent += entry.getBytesSent();
}
outputPortStatusCollection.add(portStatus);
activeGroupThreads += portStatus.getActiveThreadCount();
}
for (final Funnel funnel : group.getFunnels()) {
activeGroupThreads += processScheduler.getActiveThreadCount(funnel);
}
status.setActiveThreadCount(activeGroupThreads);
status.setBytesRead(bytesRead);
status.setBytesWritten(bytesWritten);
status.setQueuedCount(queuedCount);
status.setQueuedContentSize(queuedContentSize);
status.setInputContentSize(bytesIn);
status.setInputCount(flowFilesIn);
status.setOutputContentSize(bytesOut);
status.setOutputCount(flowFilesOut);
status.setFlowFilesReceived(flowFilesReceived);
status.setBytesReceived(bytesReceived);
status.setFlowFilesSent(flowFilesSent);
status.setBytesSent(bytesSent);
status.setFlowFilesTransferred(flowFilesTransferred);
status.setBytesTransferred(bytesTransferred);
final VersionControlInformation vci = group.getVersionControlInformation();
if (vci != null && vci.getStatus() != null && vci.getStatus().getState() != null) {
status.setVersionedFlowState(vci.getStatus().getState());
}
return status;
}
Aggregations