Examples with ProcessGroupStatus org.apache.nifi.controller.status.ProcessGroupStatus used on opensource projects

Example 26 with ProcessGroupStatus

use of org.apache.nifi.controller.status.ProcessGroupStatus in project nifi by apache.

the class ComponentMapHolder method createComponentMap.

private static ComponentMapHolder createComponentMap(final ProcessGroupStatus status, final ParentProcessGroupSearchNode thisProcessGroupNode) {
    final ComponentMapHolder holder = new ComponentMapHolder();
    final Map<String, String> componentNameMap = holder.componentNameMap;
    final Map<String, ParentProcessGroupSearchNode> componentToParentGroupMap = holder.componentToParentGroupMap;
    final Map<String, String> sourceToConnectionParentGroupMap = holder.sourceToConnectionParentGroupMap;
    final Map<String, String> destinationToConnectionParentGroupMap = holder.destinationToConnectionParentGroupMap;
    if (status != null) {
        componentNameMap.put(status.getId(), status.getName());
        for (final ProcessorStatus procStatus : status.getProcessorStatus()) {
            componentNameMap.put(procStatus.getId(), procStatus.getName());
            componentToParentGroupMap.put(procStatus.getId(), thisProcessGroupNode);
        }
        for (final PortStatus portStatus : status.getInputPortStatus()) {
            componentNameMap.put(portStatus.getId(), portStatus.getName());
            componentToParentGroupMap.put(portStatus.getId(), thisProcessGroupNode);
        }
        for (final PortStatus portStatus : status.getOutputPortStatus()) {
            componentNameMap.put(portStatus.getId(), portStatus.getName());
            componentToParentGroupMap.put(portStatus.getId(), thisProcessGroupNode);
        }
        for (final RemoteProcessGroupStatus rpgStatus : status.getRemoteProcessGroupStatus()) {
            componentNameMap.put(rpgStatus.getId(), rpgStatus.getName());
            componentToParentGroupMap.put(rpgStatus.getId(), thisProcessGroupNode);
        }
        for (final ConnectionStatus connectionStatus : status.getConnectionStatus()) {
            componentNameMap.put(connectionStatus.getId(), connectionStatus.getName());
            componentToParentGroupMap.put(connectionStatus.getId(), thisProcessGroupNode);
            // Add source and destination for Remote Input/Output Ports because metadata for those are only available at ConnectionStatus.
            componentNameMap.computeIfAbsent(connectionStatus.getSourceId(), k -> connectionStatus.getSourceName());
            componentNameMap.computeIfAbsent(connectionStatus.getDestinationId(), k -> connectionStatus.getDestinationName());
            sourceToConnectionParentGroupMap.put(connectionStatus.getSourceId(), connectionStatus.getGroupId());
            destinationToConnectionParentGroupMap.put(connectionStatus.getDestinationId(), connectionStatus.getGroupId());
        }
        for (final ProcessGroupStatus childGroup : status.getProcessGroupStatus()) {
            componentNameMap.put(childGroup.getId(), childGroup.getName());
            ParentProcessGroupSearchNode childProcessGroupNode = new ParentProcessGroupSearchNode(childGroup.getId(), thisProcessGroupNode);
            componentToParentGroupMap.put(childGroup.getId(), thisProcessGroupNode);
            holder.putAll(createComponentMap(childGroup, childProcessGroupNode));
        }
    }
    return holder;
}

Example 27 with ProcessGroupStatus

use of org.apache.nifi.controller.status.ProcessGroupStatus in project nifi by apache.

the class ProvenanceEventConsumer method consumeEvents.

public void consumeEvents(final ReportingContext context, final BiConsumer<ComponentMapHolder, List<ProvenanceEventRecord>> consumer) throws ProcessException {
    if (context == null) {
        logger.debug("No ReportingContext available.");
        return;
    }
    final EventAccess eventAccess = context.getEventAccess();
    final ProcessGroupStatus procGroupStatus = eventAccess.getControllerStatus();
    final ComponentMapHolder componentMapHolder = ComponentMapHolder.createComponentMap(procGroupStatus);
    final StateManager stateManager = context.getStateManager();
    Long currMaxId = eventAccess.getProvenanceRepository().getMaxEventId();
    if (currMaxId == null) {
        logger.debug("No events to send because no events have been created yet.");
        return;
    }
    if (firstEventId < 0) {
        Map<String, String> state;
        try {
            state = stateManager.getState(Scope.LOCAL).toMap();
        } catch (IOException e) {
            logger.error("Failed to get state at start up due to:" + e.getMessage(), e);
            return;
        }
        if (state.containsKey(LAST_EVENT_ID_KEY)) {
            firstEventId = Long.parseLong(state.get(LAST_EVENT_ID_KEY)) + 1;
        } else {
            if (END_OF_STREAM.getValue().equals(startPositionValue)) {
                firstEventId = currMaxId;
            }
        }
        if (currMaxId < (firstEventId - 1)) {
            if (BEGINNING_OF_STREAM.getValue().equals(startPositionValue)) {
                logger.warn("Current provenance max id is {} which is less than what was stored in state as the last queried event, which was {}. This means the provenance restarted its " + "ids. Restarting querying from the beginning.", new Object[] { currMaxId, firstEventId });
                firstEventId = -1;
            } else {
                logger.warn("Current provenance max id is {} which is less than what was stored in state as the last queried event, which was {}. This means the provenance restarted its " + "ids. Restarting querying from the latest event in the Provenance Repository.", new Object[] { currMaxId, firstEventId });
                firstEventId = currMaxId;
            }
        }
    }
    if (currMaxId == (firstEventId - 1)) {
        logger.debug("No events to send due to the current max id being equal to the last id that was queried.");
        return;
    }
    List<ProvenanceEventRecord> rawEvents;
    List<ProvenanceEventRecord> filteredEvents;
    try {
        rawEvents = eventAccess.getProvenanceEvents(firstEventId, batchSize);
        filteredEvents = filterEvents(componentMapHolder, rawEvents);
    } catch (final IOException ioe) {
        logger.error("Failed to retrieve Provenance Events from repository due to: " + ioe.getMessage(), ioe);
        return;
    }
    if (rawEvents == null || rawEvents.isEmpty()) {
        logger.debug("No events to send due to 'events' being null or empty.");
        return;
    }
    // Consume while there are more events and not stopped.
    while (rawEvents != null && !rawEvents.isEmpty() && isScheduled()) {
        if (!filteredEvents.isEmpty()) {
            // Executes callback.
            consumer.accept(componentMapHolder, filteredEvents);
        }
        firstEventId = updateLastEventId(rawEvents, stateManager);
        // Retrieve the next batch
        try {
            rawEvents = eventAccess.getProvenanceEvents(firstEventId, batchSize);
            filteredEvents = filterEvents(componentMapHolder, rawEvents);
        } catch (final IOException ioe) {
            logger.error("Failed to retrieve Provenance Events from repository due to: " + ioe.getMessage(), ioe);
            return;
        }
    }
}

Example 28 with ProcessGroupStatus

use of org.apache.nifi.controller.status.ProcessGroupStatus in project nifi by apache.

the class SiteToSiteProvenanceReportingTask method onTrigger.

@Override
public void onTrigger(final ReportingContext context) {
    final boolean isClustered = context.isClustered();
    final String nodeId = context.getClusterNodeIdentifier();
    if (nodeId == null && isClustered) {
        getLogger().debug("This instance of NiFi is configured for clustering, but the Cluster Node Identifier is not yet available. " + "Will wait for Node Identifier to be established.");
        return;
    }
    final ProcessGroupStatus procGroupStatus = context.getEventAccess().getControllerStatus();
    final String rootGroupName = procGroupStatus == null ? null : procGroupStatus.getName();
    final String nifiUrl = context.getProperty(INSTANCE_URL).evaluateAttributeExpressions().getValue();
    URL url;
    try {
        url = new URL(nifiUrl);
    } catch (final MalformedURLException e1) {
        // already validated
        throw new AssertionError();
    }
    final String hostname = url.getHost();
    final String platform = context.getProperty(PLATFORM).evaluateAttributeExpressions().getValue();
    final Map<String, ?> config = Collections.emptyMap();
    final JsonBuilderFactory factory = Json.createBuilderFactory(config);
    final JsonObjectBuilder builder = factory.createObjectBuilder();
    final DateFormat df = new SimpleDateFormat(TIMESTAMP_FORMAT);
    df.setTimeZone(TimeZone.getTimeZone("Z"));
    consumer.consumeEvents(context, (mapHolder, events) -> {
        final long start = System.nanoTime();
        // Create a JSON array of all the events in the current batch
        final JsonArrayBuilder arrayBuilder = factory.createArrayBuilder();
        for (final ProvenanceEventRecord event : events) {
            final String componentName = mapHolder.getComponentName(event.getComponentId());
            final String processGroupId = mapHolder.getProcessGroupId(event.getComponentId(), event.getComponentType());
            final String processGroupName = mapHolder.getComponentName(processGroupId);
            arrayBuilder.add(serialize(factory, builder, event, df, componentName, processGroupId, processGroupName, hostname, url, rootGroupName, platform, nodeId));
        }
        final JsonArray jsonArray = arrayBuilder.build();
        // Send the JSON document for the current batch
        try {
            final Transaction transaction = getClient().createTransaction(TransferDirection.SEND);
            if (transaction == null) {
                // Throw an exception to avoid provenance event id will not proceed so that those can be consumed again.
                throw new ProcessException("All destination nodes are penalized; will attempt to send data later");
            }
            final Map<String, String> attributes = new HashMap<>();
            final String transactionId = UUID.randomUUID().toString();
            attributes.put("reporting.task.transaction.id", transactionId);
            attributes.put("reporting.task.name", getName());
            attributes.put("reporting.task.uuid", getIdentifier());
            attributes.put("reporting.task.type", this.getClass().getSimpleName());
            attributes.put("mime.type", "application/json");
            final byte[] data = jsonArray.toString().getBytes(StandardCharsets.UTF_8);
            transaction.send(data, attributes);
            transaction.confirm();
            transaction.complete();
            final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
            getLogger().info("Successfully sent {} Provenance Events to destination in {} ms; Transaction ID = {}; First Event ID = {}", new Object[] { events.size(), transferMillis, transactionId, events.get(0).getEventId() });
        } catch (final IOException e) {
            throw new ProcessException("Failed to send Provenance Events to destination due to IOException:" + e.getMessage(), e);
        }
    });
}

Example 29 with ProcessGroupStatus

use of org.apache.nifi.controller.status.ProcessGroupStatus in project nifi by apache.

the class SiteToSiteStatusReportingTask method onTrigger.

@Override
public void onTrigger(final ReportingContext context) {
    final boolean isClustered = context.isClustered();
    final String nodeId = context.getClusterNodeIdentifier();
    if (nodeId == null && isClustered) {
        getLogger().debug("This instance of NiFi is configured for clustering, but the Cluster Node Identifier is not yet available. " + "Will wait for Node Identifier to be established.");
        return;
    }
    componentTypeFilter = Pattern.compile(context.getProperty(COMPONENT_TYPE_FILTER_REGEX).evaluateAttributeExpressions().getValue());
    componentNameFilter = Pattern.compile(context.getProperty(COMPONENT_NAME_FILTER_REGEX).evaluateAttributeExpressions().getValue());
    final ProcessGroupStatus procGroupStatus = context.getEventAccess().getControllerStatus();
    final String rootGroupName = procGroupStatus == null ? null : procGroupStatus.getName();
    final String nifiUrl = context.getProperty(INSTANCE_URL).evaluateAttributeExpressions().getValue();
    URL url;
    try {
        url = new URL(nifiUrl);
    } catch (final MalformedURLException e1) {
        // already validated
        throw new AssertionError();
    }
    final String hostname = url.getHost();
    final String platform = context.getProperty(PLATFORM).evaluateAttributeExpressions().getValue();
    final Map<String, ?> config = Collections.emptyMap();
    final JsonBuilderFactory factory = Json.createBuilderFactory(config);
    final DateFormat df = new SimpleDateFormat(TIMESTAMP_FORMAT);
    df.setTimeZone(TimeZone.getTimeZone("Z"));
    final JsonArrayBuilder arrayBuilder = factory.createArrayBuilder();
    serializeProcessGroupStatus(arrayBuilder, factory, procGroupStatus, df, hostname, rootGroupName, platform, null, new Date());
    final JsonArray jsonArray = arrayBuilder.build();
    final int batchSize = context.getProperty(BATCH_SIZE).asInteger();
    int fromIndex = 0;
    int toIndex = Math.min(batchSize, jsonArray.size());
    List<JsonValue> jsonBatch = jsonArray.subList(fromIndex, toIndex);
    while (!jsonBatch.isEmpty()) {
        // Send the JSON document for the current batch
        try {
            long start = System.nanoTime();
            final Transaction transaction = getClient().createTransaction(TransferDirection.SEND);
            if (transaction == null) {
                getLogger().debug("All destination nodes are penalized; will attempt to send data later");
                return;
            }
            final Map<String, String> attributes = new HashMap<>();
            final String transactionId = UUID.randomUUID().toString();
            attributes.put("reporting.task.transaction.id", transactionId);
            attributes.put("reporting.task.name", getName());
            attributes.put("reporting.task.uuid", getIdentifier());
            attributes.put("reporting.task.type", this.getClass().getSimpleName());
            attributes.put("mime.type", "application/json");
            JsonArrayBuilder jsonBatchArrayBuilder = factory.createArrayBuilder();
            for (JsonValue jsonValue : jsonBatch) {
                jsonBatchArrayBuilder.add(jsonValue);
            }
            final JsonArray jsonBatchArray = jsonBatchArrayBuilder.build();
            final byte[] data = jsonBatchArray.toString().getBytes(StandardCharsets.UTF_8);
            transaction.send(data, attributes);
            transaction.confirm();
            transaction.complete();
            final long transferMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
            getLogger().info("Successfully sent {} Status Records to destination in {} ms; Transaction ID = {}", new Object[] { jsonArray.size(), transferMillis, transactionId });
            fromIndex = toIndex;
            toIndex = Math.min(fromIndex + batchSize, jsonArray.size());
            jsonBatch = jsonArray.subList(fromIndex, toIndex);
        } catch (final IOException e) {
            throw new ProcessException("Failed to send Status Records to destination due to IOException:" + e.getMessage(), e);
        }
    }
}

Example 30 with ProcessGroupStatus

use of org.apache.nifi.controller.status.ProcessGroupStatus in project nifi by apache.

the class TestSiteToSiteProvenanceReportingTask method setup.

private MockSiteToSiteProvenanceReportingTask setup(ProvenanceEventRecord event, Map<PropertyDescriptor, String> properties, long maxEventId) throws IOException {
    final MockSiteToSiteProvenanceReportingTask task = new MockSiteToSiteProvenanceReportingTask();
    when(context.getStateManager()).thenReturn(new MockStateManager(task));
    Mockito.doAnswer(new Answer<PropertyValue>() {

        @Override
        public PropertyValue answer(final InvocationOnMock invocation) throws Throwable {
            final PropertyDescriptor descriptor = invocation.getArgumentAt(0, PropertyDescriptor.class);
            return new MockPropertyValue(properties.get(descriptor));
        }
    }).when(context).getProperty(Mockito.any(PropertyDescriptor.class));
    Mockito.doAnswer(new Answer<PropertyValue>() {

        @Override
        public PropertyValue answer(final InvocationOnMock invocation) throws Throwable {
            final PropertyDescriptor descriptor = invocation.getArgumentAt(0, PropertyDescriptor.class);
            return new MockPropertyValue(properties.get(descriptor));
        }
    }).when(confContext).getProperty(Mockito.any(PropertyDescriptor.class));
    final AtomicInteger totalEvents = new AtomicInteger(0);
    final EventAccess eventAccess = Mockito.mock(EventAccess.class);
    Mockito.doAnswer(new Answer<List<ProvenanceEventRecord>>() {

        @Override
        public List<ProvenanceEventRecord> answer(final InvocationOnMock invocation) throws Throwable {
            final long startId = invocation.getArgumentAt(0, long.class);
            final int maxRecords = invocation.getArgumentAt(1, int.class);
            final List<ProvenanceEventRecord> eventsToReturn = new ArrayList<>();
            for (int i = (int) Math.max(0, startId); i < (int) (startId + maxRecords) && totalEvents.get() < maxEventId; i++) {
                if (event != null) {
                    eventsToReturn.add(event);
                }
                totalEvents.getAndIncrement();
            }
            return eventsToReturn;
        }
    }).when(eventAccess).getProvenanceEvents(Mockito.anyLong(), Mockito.anyInt());
    ProcessGroupStatus pgRoot = new ProcessGroupStatus();
    pgRoot.setId("root");
    when(eventAccess.getControllerStatus()).thenReturn(pgRoot);
    // Add child Process Groups.
    // Root -> (A, B -> (B2 -> (B3)))
    final ProcessGroupStatus pgA = new ProcessGroupStatus();
    pgA.setId("pgA");
    final ProcessGroupStatus pgB = new ProcessGroupStatus();
    pgB.setId("pgB");
    final ProcessGroupStatus pgB2 = new ProcessGroupStatus();
    pgB2.setId("pgB2");
    final ProcessGroupStatus pgB3 = new ProcessGroupStatus();
    pgB3.setId("pgB3");
    final Collection<ProcessGroupStatus> childPGs = pgRoot.getProcessGroupStatus();
    childPGs.add(pgA);
    childPGs.add(pgB);
    pgB.getProcessGroupStatus().add(pgB2);
    pgB2.getProcessGroupStatus().add(pgB3);
    // Add Processors.
    final ProcessorStatus prcRoot = new ProcessorStatus();
    prcRoot.setId("1234");
    pgRoot.getProcessorStatus().add(prcRoot);
    final ProcessorStatus prcA = new ProcessorStatus();
    prcA.setId("A001");
    prcA.setName("Processor in PGA");
    pgA.getProcessorStatus().add(prcA);
    final ProcessorStatus prcB = new ProcessorStatus();
    prcB.setId("B001");
    prcB.setName("Processor in PGB");
    pgB.getProcessorStatus().add(prcB);
    final ProcessorStatus prcB2 = new ProcessorStatus();
    prcB2.setId("B201");
    prcB2.setName("Processor in PGB2");
    pgB2.getProcessorStatus().add(prcB2);
    final ProcessorStatus prcB3 = new ProcessorStatus();
    prcB3.setId("B301");
    prcB3.setName("Processor in PGB3");
    pgB3.getProcessorStatus().add(prcB3);
    // Add connection status to test Remote Input/Output Ports
    final ConnectionStatus b2RemoteInputPort = new ConnectionStatus();
    b2RemoteInputPort.setGroupId("pgB2");
    b2RemoteInputPort.setSourceId("B201");
    b2RemoteInputPort.setDestinationId("riB2");
    b2RemoteInputPort.setDestinationName("Remote Input Port name");
    pgB2.getConnectionStatus().add(b2RemoteInputPort);
    final ConnectionStatus b3RemoteOutputPort = new ConnectionStatus();
    b3RemoteOutputPort.setGroupId("pgB3");
    b3RemoteOutputPort.setSourceId("roB3");
    b3RemoteOutputPort.setSourceName("Remote Output Port name");
    b3RemoteOutputPort.setDestinationId("B301");
    pgB3.getConnectionStatus().add(b3RemoteOutputPort);
    final ProvenanceEventRepository provenanceRepository = Mockito.mock(ProvenanceEventRepository.class);
    Mockito.doAnswer(new Answer<Long>() {

        @Override
        public Long answer(final InvocationOnMock invocation) throws Throwable {
            return maxEventId;
        }
    }).when(provenanceRepository).getMaxEventId();
    when(context.getEventAccess()).thenReturn(eventAccess);
    when(eventAccess.getProvenanceRepository()).thenReturn(provenanceRepository);
    final ComponentLog logger = Mockito.mock(ComponentLog.class);
    when(initContext.getIdentifier()).thenReturn(UUID.randomUUID().toString());
    when(initContext.getLogger()).thenReturn(logger);
    return task;
}