use of org.apache.nifi.controller.status.ProcessGroupStatus in project nifi by apache.
the class StandardGangliaReporter method onConfigure.
@OnScheduled
public void onConfigure(final ConfigurationContext config) throws InitializationException {
metricsRegistry = new MetricsRegistry();
metricsRegistry.newGauge(new MetricName(METRICS_GROUP, "int32", "FlowFiles Received Last 5 mins"), new Gauge<Integer>() {
@Override
public Integer value() {
final ProcessGroupStatus status = latestStatus.get();
if (status == null) {
return 0;
}
final Integer value = status.getFlowFilesReceived();
return (value == null) ? 0 : value;
}
});
metricsRegistry.newGauge(new MetricName(METRICS_GROUP, "int64", "Bytes Received Last 5 mins"), new Gauge<Long>() {
@Override
public Long value() {
final ProcessGroupStatus status = latestStatus.get();
if (status == null) {
return 0L;
}
return status.getBytesReceived();
}
});
metricsRegistry.newGauge(new MetricName(METRICS_GROUP, "int32", "FlowFiles Sent Last 5 mins"), new Gauge<Integer>() {
@Override
public Integer value() {
final ProcessGroupStatus status = latestStatus.get();
if (status == null) {
return 0;
}
return status.getFlowFilesSent();
}
});
metricsRegistry.newGauge(new MetricName(METRICS_GROUP, "int64", "Bytes Sent Last 5 mins"), new Gauge<Long>() {
@Override
public Long value() {
final ProcessGroupStatus status = latestStatus.get();
if (status == null) {
return 0L;
}
return status.getBytesSent();
}
});
metricsRegistry.newGauge(new MetricName(METRICS_GROUP, "int32", "FlowFiles Queued"), new Gauge<Integer>() {
@Override
public Integer value() {
final ProcessGroupStatus status = latestStatus.get();
if (status == null) {
return 0;
}
final Integer value = status.getQueuedCount();
return (value == null) ? 0 : value;
}
});
metricsRegistry.newGauge(new MetricName(METRICS_GROUP, "int64", "Bytes Queued"), new Gauge<Long>() {
@Override
public Long value() {
final ProcessGroupStatus status = latestStatus.get();
if (status == null) {
return 0L;
}
final Long value = status.getQueuedContentSize();
return (value == null) ? 0L : value;
}
});
metricsRegistry.newGauge(new MetricName(METRICS_GROUP, "int64", "Bytes Read (5 mins)"), new Gauge<Long>() {
@Override
public Long value() {
final ProcessGroupStatus status = latestStatus.get();
if (status == null) {
return 0L;
}
final Long value = status.getBytesRead();
return (value == null) ? 0L : value;
}
});
metricsRegistry.newGauge(new MetricName(METRICS_GROUP, "int64", "Bytes Written (5 mins)"), new Gauge<Long>() {
@Override
public Long value() {
final ProcessGroupStatus status = latestStatus.get();
if (status == null) {
return 0L;
}
final Long value = status.getBytesWritten();
return (value == null) ? 0L : value;
}
});
metricsRegistry.newGauge(new MetricName(METRICS_GROUP, "int32", "Active Threads"), new Gauge<Integer>() {
@Override
public Integer value() {
final ProcessGroupStatus status = latestStatus.get();
if (status == null) {
return 0;
}
final Integer value = status.getActiveThreadCount();
return (value == null) ? 0 : value;
}
});
metricsRegistry.newGauge(new MetricName(METRICS_GROUP, "int32", "Total Task Duration Seconds"), new Gauge<Integer>() {
@Override
public Integer value() {
final ProcessGroupStatus status = latestStatus.get();
if (status == null) {
return 0;
}
final long nanos = calculateProcessingNanos(status);
return (int) TimeUnit.NANOSECONDS.toSeconds(nanos);
}
});
final String gangliaHost = config.getProperty(HOSTNAME).getValue();
final int port = config.getProperty(PORT).asInteger();
try {
gangliaReporter = new GangliaReporter(metricsRegistry, gangliaHost, port, METRICS_GROUP) {
@Override
protected String sanitizeName(MetricName name) {
return name.getName();
}
};
gangliaReporter.printVMMetrics = config.getProperty(SEND_JVM_METRICS).asBoolean();
} catch (final IOException e) {
throw new InitializationException(e);
}
}
use of org.apache.nifi.controller.status.ProcessGroupStatus in project nifi-minifi by apache.
the class StatusConfigReporter method getStatus.
public static FlowStatusReport getStatus(FlowController flowController, String statusRequest, Logger logger) throws StatusRequestException {
if (statusRequest == null) {
logger.error("Received a status request which was null");
throw new StatusRequestException("Cannot complete status request because the statusRequest is null");
}
if (flowController == null) {
logger.error("Received a status but the Flow Controller is null");
throw new StatusRequestException("Cannot complete status request because the Flow Controller is null");
}
FlowStatusReport flowStatusReport = new FlowStatusReport();
List<String> errorsGeneratingReport = new LinkedList<>();
flowStatusReport.setErrorsGeneratingReport(errorsGeneratingReport);
String[] itemsToReport = statusRequest.split(";");
ProcessGroupStatus rootGroupStatus = flowController.getControllerStatus();
Map<String, ProcessorStatus> processorStatusMap = null;
Map<String, ConnectionStatus> connectionStatusMap = null;
Map<String, RemoteProcessGroupStatus> remoteProcessGroupStatusMap = null;
for (String item : itemsToReport) {
String[] sections = item.split(":");
try {
switch(sections[0].toLowerCase().trim()) {
case "systemdiagnostics":
SystemDiagnosticsStatus systemDiagnosticsStatus = parseSystemDiagnosticsRequest(flowController.getSystemDiagnostics(), sections[1]);
flowStatusReport.setSystemDiagnosticsStatus(systemDiagnosticsStatus);
break;
case "instance":
InstanceStatus instanceStatus = parseInstanceRequest(sections[1], flowController, rootGroupStatus);
flowStatusReport.setInstanceStatus(instanceStatus);
break;
case "remoteprocessgroup":
if (flowStatusReport.getRemoteProcessGroupStatusList() == null) {
List<RemoteProcessGroupStatusBean> remoteProcessGroupStatusList = new LinkedList<>();
flowStatusReport.setRemoteProcessGroupStatusList(remoteProcessGroupStatusList);
}
handleRemoteProcessGroupRequest(sections, rootGroupStatus, flowController, flowStatusReport.getRemoteProcessGroupStatusList(), remoteProcessGroupStatusMap, logger);
break;
case "processor":
if (flowStatusReport.getProcessorStatusList() == null) {
List<ProcessorStatusBean> processorStatusList = new LinkedList<>();
flowStatusReport.setProcessorStatusList(processorStatusList);
}
handleProcessorRequest(sections, rootGroupStatus, flowController, flowStatusReport.getProcessorStatusList(), processorStatusMap, logger);
break;
case "connection":
if (flowStatusReport.getConnectionStatusList() == null) {
List<ConnectionStatusBean> connectionStatusList = new LinkedList<>();
flowStatusReport.setConnectionStatusList(connectionStatusList);
}
handleConnectionRequest(sections, rootGroupStatus, flowStatusReport.getConnectionStatusList(), connectionStatusMap, logger);
break;
case "provenancereporting":
if (flowStatusReport.getRemoteProcessGroupStatusList() == null) {
List<ReportingTaskStatus> reportingTaskStatusList = new LinkedList<>();
flowStatusReport.setReportingTaskStatusList(reportingTaskStatusList);
}
handleReportingTaskRequest(sections, flowController, flowStatusReport.getReportingTaskStatusList(), logger);
break;
case "controllerservices":
if (flowStatusReport.getControllerServiceStatusList() == null) {
List<ControllerServiceStatus> controllerServiceStatusList = new LinkedList<>();
flowStatusReport.setControllerServiceStatusList(controllerServiceStatusList);
}
handleControllerServices(sections, flowController, flowStatusReport.getControllerServiceStatusList(), logger);
break;
}
} catch (Exception e) {
logger.error("Hit exception while requesting status for item '" + item + "'", e);
errorsGeneratingReport.add("Unable to get status for request '" + item + "' due to:" + e);
}
}
return flowStatusReport;
}
use of org.apache.nifi.controller.status.ProcessGroupStatus in project nifi by apache.
the class DataDogReportingTask method onTrigger.
@Override
public void onTrigger(ReportingContext context) {
final ProcessGroupStatus status = context.getEventAccess().getControllerStatus();
metricsPrefix = context.getProperty(METRICS_PREFIX).evaluateAttributeExpressions().getValue();
environment = context.getProperty(ENVIRONMENT).evaluateAttributeExpressions().getValue();
statusId = status.getId();
defaultTags = ImmutableMap.of("env", environment, "dataflow_id", statusId);
try {
updateDataDogTransport(context);
} catch (IOException e) {
e.printStackTrace();
}
updateAllMetricGroups(status);
ddMetricRegistryBuilder.getDatadogReporter().report();
}
use of org.apache.nifi.controller.status.ProcessGroupStatus in project nifi by apache.
the class TestMetricsService method init.
@Before
public void init() {
status = new ProcessGroupStatus();
metricsService = new MetricsService();
status.setId("1234");
status.setFlowFilesReceived(5);
status.setBytesReceived(10000);
status.setFlowFilesSent(10);
status.setBytesSent(20000);
status.setQueuedCount(100);
status.setQueuedContentSize(1024L);
status.setBytesRead(60000L);
status.setBytesWritten(80000L);
status.setActiveThreadCount(5);
}
use of org.apache.nifi.controller.status.ProcessGroupStatus in project nifi by apache.
the class FlowController method getGroupStatus.
/**
* Returns the status for the components in the specified group with the
* specified report. The results will be filtered by executing the specified
* predicate.
*
* @param group group id
* @param statusReport report
* @param isAuthorized is authorized check
* @return the component status
*/
public ProcessGroupStatus getGroupStatus(final ProcessGroup group, final RepositoryStatusReport statusReport, final Predicate<Authorizable> isAuthorized) {
if (group == null) {
return null;
}
final ProcessGroupStatus status = new ProcessGroupStatus();
status.setId(group.getIdentifier());
status.setName(isAuthorized.evaluate(group) ? group.getName() : group.getIdentifier());
int activeGroupThreads = 0;
long bytesRead = 0L;
long bytesWritten = 0L;
int queuedCount = 0;
long queuedContentSize = 0L;
int flowFilesIn = 0;
long bytesIn = 0L;
int flowFilesOut = 0;
long bytesOut = 0L;
int flowFilesReceived = 0;
long bytesReceived = 0L;
int flowFilesSent = 0;
long bytesSent = 0L;
int flowFilesTransferred = 0;
long bytesTransferred = 0;
// set status for processors
final Collection<ProcessorStatus> processorStatusCollection = new ArrayList<>();
status.setProcessorStatus(processorStatusCollection);
for (final ProcessorNode procNode : group.getProcessors()) {
final ProcessorStatus procStat = getProcessorStatus(statusReport, procNode, isAuthorized);
processorStatusCollection.add(procStat);
activeGroupThreads += procStat.getActiveThreadCount();
bytesRead += procStat.getBytesRead();
bytesWritten += procStat.getBytesWritten();
flowFilesReceived += procStat.getFlowFilesReceived();
bytesReceived += procStat.getBytesReceived();
flowFilesSent += procStat.getFlowFilesSent();
bytesSent += procStat.getBytesSent();
}
// set status for local child groups
final Collection<ProcessGroupStatus> localChildGroupStatusCollection = new ArrayList<>();
status.setProcessGroupStatus(localChildGroupStatusCollection);
for (final ProcessGroup childGroup : group.getProcessGroups()) {
final ProcessGroupStatus childGroupStatus = getGroupStatus(childGroup, statusReport, isAuthorized);
localChildGroupStatusCollection.add(childGroupStatus);
activeGroupThreads += childGroupStatus.getActiveThreadCount();
bytesRead += childGroupStatus.getBytesRead();
bytesWritten += childGroupStatus.getBytesWritten();
queuedCount += childGroupStatus.getQueuedCount();
queuedContentSize += childGroupStatus.getQueuedContentSize();
flowFilesReceived += childGroupStatus.getFlowFilesReceived();
bytesReceived += childGroupStatus.getBytesReceived();
flowFilesSent += childGroupStatus.getFlowFilesSent();
bytesSent += childGroupStatus.getBytesSent();
flowFilesTransferred += childGroupStatus.getFlowFilesTransferred();
bytesTransferred += childGroupStatus.getBytesTransferred();
}
// set status for remote child groups
final Collection<RemoteProcessGroupStatus> remoteProcessGroupStatusCollection = new ArrayList<>();
status.setRemoteProcessGroupStatus(remoteProcessGroupStatusCollection);
for (final RemoteProcessGroup remoteGroup : group.getRemoteProcessGroups()) {
final RemoteProcessGroupStatus remoteStatus = createRemoteGroupStatus(remoteGroup, statusReport, isAuthorized);
if (remoteStatus != null) {
remoteProcessGroupStatusCollection.add(remoteStatus);
flowFilesReceived += remoteStatus.getReceivedCount();
bytesReceived += remoteStatus.getReceivedContentSize();
flowFilesSent += remoteStatus.getSentCount();
bytesSent += remoteStatus.getSentContentSize();
}
}
// connection status
final Collection<ConnectionStatus> connectionStatusCollection = new ArrayList<>();
status.setConnectionStatus(connectionStatusCollection);
// get the connection and remote port status
for (final Connection conn : group.getConnections()) {
final boolean isConnectionAuthorized = isAuthorized.evaluate(conn);
final boolean isSourceAuthorized = isAuthorized.evaluate(conn.getSource());
final boolean isDestinationAuthorized = isAuthorized.evaluate(conn.getDestination());
final ConnectionStatus connStatus = new ConnectionStatus();
connStatus.setId(conn.getIdentifier());
connStatus.setGroupId(conn.getProcessGroup().getIdentifier());
connStatus.setSourceId(conn.getSource().getIdentifier());
connStatus.setSourceName(isSourceAuthorized ? conn.getSource().getName() : conn.getSource().getIdentifier());
connStatus.setDestinationId(conn.getDestination().getIdentifier());
connStatus.setDestinationName(isDestinationAuthorized ? conn.getDestination().getName() : conn.getDestination().getIdentifier());
connStatus.setBackPressureDataSizeThreshold(conn.getFlowFileQueue().getBackPressureDataSizeThreshold());
connStatus.setBackPressureObjectThreshold(conn.getFlowFileQueue().getBackPressureObjectThreshold());
final FlowFileEvent connectionStatusReport = statusReport.getReportEntry(conn.getIdentifier());
if (connectionStatusReport != null) {
connStatus.setInputBytes(connectionStatusReport.getContentSizeIn());
connStatus.setInputCount(connectionStatusReport.getFlowFilesIn());
connStatus.setOutputBytes(connectionStatusReport.getContentSizeOut());
connStatus.setOutputCount(connectionStatusReport.getFlowFilesOut());
flowFilesTransferred += connectionStatusReport.getFlowFilesIn() + connectionStatusReport.getFlowFilesOut();
bytesTransferred += connectionStatusReport.getContentSizeIn() + connectionStatusReport.getContentSizeOut();
}
if (isConnectionAuthorized) {
if (StringUtils.isNotBlank(conn.getName())) {
connStatus.setName(conn.getName());
} else if (conn.getRelationships() != null && !conn.getRelationships().isEmpty()) {
final Collection<String> relationships = new ArrayList<>(conn.getRelationships().size());
for (final Relationship relationship : conn.getRelationships()) {
relationships.add(relationship.getName());
}
connStatus.setName(StringUtils.join(relationships, ", "));
}
} else {
connStatus.setName(conn.getIdentifier());
}
final QueueSize queueSize = conn.getFlowFileQueue().size();
final int connectionQueuedCount = queueSize.getObjectCount();
final long connectionQueuedBytes = queueSize.getByteCount();
if (connectionQueuedCount > 0) {
connStatus.setQueuedBytes(connectionQueuedBytes);
connStatus.setQueuedCount(connectionQueuedCount);
}
connectionStatusCollection.add(connStatus);
queuedCount += connectionQueuedCount;
queuedContentSize += connectionQueuedBytes;
final Connectable source = conn.getSource();
if (ConnectableType.REMOTE_OUTPUT_PORT.equals(source.getConnectableType())) {
final RemoteGroupPort remoteOutputPort = (RemoteGroupPort) source;
activeGroupThreads += processScheduler.getActiveThreadCount(remoteOutputPort);
}
final Connectable destination = conn.getDestination();
if (ConnectableType.REMOTE_INPUT_PORT.equals(destination.getConnectableType())) {
final RemoteGroupPort remoteInputPort = (RemoteGroupPort) destination;
activeGroupThreads += processScheduler.getActiveThreadCount(remoteInputPort);
}
}
// status for input ports
final Collection<PortStatus> inputPortStatusCollection = new ArrayList<>();
status.setInputPortStatus(inputPortStatusCollection);
final Set<Port> inputPorts = group.getInputPorts();
for (final Port port : inputPorts) {
final boolean isInputPortAuthorized = isAuthorized.evaluate(port);
final PortStatus portStatus = new PortStatus();
portStatus.setId(port.getIdentifier());
portStatus.setGroupId(port.getProcessGroup().getIdentifier());
portStatus.setName(isInputPortAuthorized ? port.getName() : port.getIdentifier());
portStatus.setActiveThreadCount(processScheduler.getActiveThreadCount(port));
// determine the run status
if (ScheduledState.RUNNING.equals(port.getScheduledState())) {
portStatus.setRunStatus(RunStatus.Running);
} else if (ScheduledState.DISABLED.equals(port.getScheduledState())) {
portStatus.setRunStatus(RunStatus.Disabled);
} else if (!port.isValid()) {
portStatus.setRunStatus(RunStatus.Invalid);
} else {
portStatus.setRunStatus(RunStatus.Stopped);
}
// special handling for root group ports
if (port instanceof RootGroupPort) {
final RootGroupPort rootGroupPort = (RootGroupPort) port;
portStatus.setTransmitting(rootGroupPort.isTransmitting());
}
final FlowFileEvent entry = statusReport.getReportEntries().get(port.getIdentifier());
if (entry == null) {
portStatus.setInputBytes(0L);
portStatus.setInputCount(0);
portStatus.setOutputBytes(0L);
portStatus.setOutputCount(0);
} else {
final int processedCount = entry.getFlowFilesOut();
final long numProcessedBytes = entry.getContentSizeOut();
portStatus.setOutputBytes(numProcessedBytes);
portStatus.setOutputCount(processedCount);
final int inputCount = entry.getFlowFilesIn();
final long inputBytes = entry.getContentSizeIn();
portStatus.setInputBytes(inputBytes);
portStatus.setInputCount(inputCount);
flowFilesIn += inputCount;
bytesIn += inputBytes;
bytesWritten += entry.getBytesWritten();
flowFilesReceived += entry.getFlowFilesReceived();
bytesReceived += entry.getBytesReceived();
}
inputPortStatusCollection.add(portStatus);
activeGroupThreads += portStatus.getActiveThreadCount();
}
// status for output ports
final Collection<PortStatus> outputPortStatusCollection = new ArrayList<>();
status.setOutputPortStatus(outputPortStatusCollection);
final Set<Port> outputPorts = group.getOutputPorts();
for (final Port port : outputPorts) {
final boolean isOutputPortAuthorized = isAuthorized.evaluate(port);
final PortStatus portStatus = new PortStatus();
portStatus.setId(port.getIdentifier());
portStatus.setGroupId(port.getProcessGroup().getIdentifier());
portStatus.setName(isOutputPortAuthorized ? port.getName() : port.getIdentifier());
portStatus.setActiveThreadCount(processScheduler.getActiveThreadCount(port));
// determine the run status
if (ScheduledState.RUNNING.equals(port.getScheduledState())) {
portStatus.setRunStatus(RunStatus.Running);
} else if (ScheduledState.DISABLED.equals(port.getScheduledState())) {
portStatus.setRunStatus(RunStatus.Disabled);
} else if (!port.isValid()) {
portStatus.setRunStatus(RunStatus.Invalid);
} else {
portStatus.setRunStatus(RunStatus.Stopped);
}
// special handling for root group ports
if (port instanceof RootGroupPort) {
final RootGroupPort rootGroupPort = (RootGroupPort) port;
portStatus.setTransmitting(rootGroupPort.isTransmitting());
}
final FlowFileEvent entry = statusReport.getReportEntries().get(port.getIdentifier());
if (entry == null) {
portStatus.setInputBytes(0L);
portStatus.setInputCount(0);
portStatus.setOutputBytes(0L);
portStatus.setOutputCount(0);
} else {
final int processedCount = entry.getFlowFilesOut();
final long numProcessedBytes = entry.getContentSizeOut();
portStatus.setOutputBytes(numProcessedBytes);
portStatus.setOutputCount(processedCount);
final int inputCount = entry.getFlowFilesIn();
final long inputBytes = entry.getContentSizeIn();
portStatus.setInputBytes(inputBytes);
portStatus.setInputCount(inputCount);
bytesRead += entry.getBytesRead();
flowFilesOut += entry.getFlowFilesOut();
bytesOut += entry.getContentSizeOut();
flowFilesSent = entry.getFlowFilesSent();
bytesSent += entry.getBytesSent();
}
outputPortStatusCollection.add(portStatus);
activeGroupThreads += portStatus.getActiveThreadCount();
}
for (final Funnel funnel : group.getFunnels()) {
activeGroupThreads += processScheduler.getActiveThreadCount(funnel);
}
status.setActiveThreadCount(activeGroupThreads);
status.setBytesRead(bytesRead);
status.setBytesWritten(bytesWritten);
status.setQueuedCount(queuedCount);
status.setQueuedContentSize(queuedContentSize);
status.setInputContentSize(bytesIn);
status.setInputCount(flowFilesIn);
status.setOutputContentSize(bytesOut);
status.setOutputCount(flowFilesOut);
status.setFlowFilesReceived(flowFilesReceived);
status.setBytesReceived(bytesReceived);
status.setFlowFilesSent(flowFilesSent);
status.setBytesSent(bytesSent);
status.setFlowFilesTransferred(flowFilesTransferred);
status.setBytesTransferred(bytesTransferred);
final VersionControlInformation vci = group.getVersionControlInformation();
if (vci != null && vci.getStatus() != null && vci.getStatus().getState() != null) {
status.setVersionedFlowState(vci.getStatus().getState());
}
return status;
}
Aggregations