Examples with EventAccess org.apache.nifi.reporting.EventAccess used on opensource projects

Example 1 with EventAccess

use of org.apache.nifi.reporting.EventAccess in project nifi by apache.

the class TestAmbariReportingTask method testOnTrigger.

@Test
public void testOnTrigger() throws InitializationException, IOException {
    final String metricsUrl = "http://myambari:6188/ws/v1/timeline/metrics";
    final String applicationId = "NIFI";
    final String hostName = "localhost";
    // create the jersey client mocks for handling the post
    final Client client = Mockito.mock(Client.class);
    final WebTarget target = Mockito.mock(WebTarget.class);
    final Invocation.Builder builder = Mockito.mock(Invocation.Builder.class);
    final Response response = Mockito.mock(Response.class);
    Mockito.when(response.getStatus()).thenReturn(200);
    Mockito.when(client.target(metricsUrl)).thenReturn(target);
    Mockito.when(target.request()).thenReturn(builder);
    Mockito.when(builder.post(Matchers.any(Entity.class))).thenReturn(response);
    // mock the ReportingInitializationContext for initialize(...)
    final ComponentLog logger = Mockito.mock(ComponentLog.class);
    final ReportingInitializationContext initContext = Mockito.mock(ReportingInitializationContext.class);
    Mockito.when(initContext.getIdentifier()).thenReturn(UUID.randomUUID().toString());
    Mockito.when(initContext.getLogger()).thenReturn(logger);
    // mock the ConfigurationContext for setup(...)
    final ConfigurationContext configurationContext = Mockito.mock(ConfigurationContext.class);
    // mock the ReportingContext for onTrigger(...)
    final ReportingContext context = Mockito.mock(ReportingContext.class);
    Mockito.when(context.getProperty(AmbariReportingTask.METRICS_COLLECTOR_URL)).thenReturn(new MockPropertyValue(metricsUrl));
    Mockito.when(context.getProperty(AmbariReportingTask.APPLICATION_ID)).thenReturn(new MockPropertyValue(applicationId));
    Mockito.when(context.getProperty(AmbariReportingTask.HOSTNAME)).thenReturn(new MockPropertyValue(hostName));
    Mockito.when(context.getProperty(AmbariReportingTask.PROCESS_GROUP_ID)).thenReturn(new MockPropertyValue("1234"));
    final EventAccess eventAccess = Mockito.mock(EventAccess.class);
    Mockito.when(context.getEventAccess()).thenReturn(eventAccess);
    Mockito.when(eventAccess.getControllerStatus()).thenReturn(status);
    // create a testable instance of the reporting task
    final AmbariReportingTask task = new TestableAmbariReportingTask(client);
    task.initialize(initContext);
    task.setup(configurationContext);
    task.onTrigger(context);
}

Example 2 with EventAccess

use of org.apache.nifi.reporting.EventAccess in project nifi by apache.

the class ReportLineageToAtlas method consumeNiFiProvenanceEvents.

private void consumeNiFiProvenanceEvents(ReportingContext context, NiFiFlow nifiFlow) {
    final EventAccess eventAccess = context.getEventAccess();
    final AnalysisContext analysisContext = new StandardAnalysisContext(nifiFlow, clusterResolvers, // FIXME: This class cast shouldn't be necessary to query lineage. Possible refactor target in next major update.
    (ProvenanceRepository) eventAccess.getProvenanceRepository());
    consumer.consumeEvents(context, (componentMapHolder, events) -> {
        for (ProvenanceEventRecord event : events) {
            try {
                lineageStrategy.processEvent(analysisContext, nifiFlow, event);
            } catch (Exception e) {
                // If something went wrong, log it and continue with other records.
                getLogger().error("Skipping failed analyzing event {} due to {}.", new Object[] { event, e, e });
            }
        }
        nifiAtlasHook.commitMessages();
    });
}

Example 3 with EventAccess

use of org.apache.nifi.reporting.EventAccess in project nifi by apache.

the class TestDataDogReportingTask method initContexts.

// init all contexts
private void initContexts() {
    configurationContext = Mockito.mock(ConfigurationContext.class);
    context = Mockito.mock(ReportingContext.class);
    Mockito.when(context.getProperty(DataDogReportingTask.ENVIRONMENT)).thenReturn(new MockPropertyValue(env, null));
    Mockito.when(context.getProperty(DataDogReportingTask.METRICS_PREFIX)).thenReturn(new MockPropertyValue(prefix, null));
    Mockito.when(context.getProperty(DataDogReportingTask.API_KEY)).thenReturn(new MockPropertyValue("agent", null));
    Mockito.when(context.getProperty(DataDogReportingTask.DATADOG_TRANSPORT)).thenReturn(new MockPropertyValue("DataDog Agent", null));
    EventAccess eventAccess = Mockito.mock(EventAccess.class);
    Mockito.when(eventAccess.getControllerStatus()).thenReturn(status);
    Mockito.when(context.getEventAccess()).thenReturn(eventAccess);
    logger = Mockito.mock(Logger.class);
    initContext = Mockito.mock(ReportingInitializationContext.class);
    Mockito.when(initContext.getIdentifier()).thenReturn(UUID.randomUUID().toString());
    // Mockito.when(initContext.getLogger()).thenReturn(logger);
    metricsMap = new ConcurrentHashMap<>();
    metricRegistry = Mockito.mock(MetricRegistry.class);
    virtualMachineMetrics = VirtualMachineMetrics.getInstance();
    metricsService = Mockito.mock(MetricsService.class);
}

Example 4 with EventAccess

use of org.apache.nifi.reporting.EventAccess in project nifi by apache.

the class ProvenanceEventConsumer method consumeEvents.

public void consumeEvents(final ReportingContext context, final BiConsumer<ComponentMapHolder, List<ProvenanceEventRecord>> consumer) throws ProcessException {
    if (context == null) {
        logger.debug("No ReportingContext available.");
        return;
    }
    final EventAccess eventAccess = context.getEventAccess();
    final ProcessGroupStatus procGroupStatus = eventAccess.getControllerStatus();
    final ComponentMapHolder componentMapHolder = ComponentMapHolder.createComponentMap(procGroupStatus);
    final StateManager stateManager = context.getStateManager();
    Long currMaxId = eventAccess.getProvenanceRepository().getMaxEventId();
    if (currMaxId == null) {
        logger.debug("No events to send because no events have been created yet.");
        return;
    }
    if (firstEventId < 0) {
        Map<String, String> state;
        try {
            state = stateManager.getState(Scope.LOCAL).toMap();
        } catch (IOException e) {
            logger.error("Failed to get state at start up due to:" + e.getMessage(), e);
            return;
        }
        if (state.containsKey(LAST_EVENT_ID_KEY)) {
            firstEventId = Long.parseLong(state.get(LAST_EVENT_ID_KEY)) + 1;
        } else {
            if (END_OF_STREAM.getValue().equals(startPositionValue)) {
                firstEventId = currMaxId;
            }
        }
        if (currMaxId < (firstEventId - 1)) {
            if (BEGINNING_OF_STREAM.getValue().equals(startPositionValue)) {
                logger.warn("Current provenance max id is {} which is less than what was stored in state as the last queried event, which was {}. This means the provenance restarted its " + "ids. Restarting querying from the beginning.", new Object[] { currMaxId, firstEventId });
                firstEventId = -1;
            } else {
                logger.warn("Current provenance max id is {} which is less than what was stored in state as the last queried event, which was {}. This means the provenance restarted its " + "ids. Restarting querying from the latest event in the Provenance Repository.", new Object[] { currMaxId, firstEventId });
                firstEventId = currMaxId;
            }
        }
    }
    if (currMaxId == (firstEventId - 1)) {
        logger.debug("No events to send due to the current max id being equal to the last id that was queried.");
        return;
    }
    List<ProvenanceEventRecord> rawEvents;
    List<ProvenanceEventRecord> filteredEvents;
    try {
        rawEvents = eventAccess.getProvenanceEvents(firstEventId, batchSize);
        filteredEvents = filterEvents(componentMapHolder, rawEvents);
    } catch (final IOException ioe) {
        logger.error("Failed to retrieve Provenance Events from repository due to: " + ioe.getMessage(), ioe);
        return;
    }
    if (rawEvents == null || rawEvents.isEmpty()) {
        logger.debug("No events to send due to 'events' being null or empty.");
        return;
    }
    // Consume while there are more events and not stopped.
    while (rawEvents != null && !rawEvents.isEmpty() && isScheduled()) {
        if (!filteredEvents.isEmpty()) {
            // Executes callback.
            consumer.accept(componentMapHolder, filteredEvents);
        }
        firstEventId = updateLastEventId(rawEvents, stateManager);
        // Retrieve the next batch
        try {
            rawEvents = eventAccess.getProvenanceEvents(firstEventId, batchSize);
            filteredEvents = filterEvents(componentMapHolder, rawEvents);
        } catch (final IOException ioe) {
            logger.error("Failed to retrieve Provenance Events from repository due to: " + ioe.getMessage(), ioe);
            return;
        }
    }
}

Example 5 with EventAccess

use of org.apache.nifi.reporting.EventAccess in project nifi by apache.

the class ITReportLineageToAtlas method test.

private void test(TestConfiguration tc) throws InitializationException, IOException {
    final ReportLineageToAtlas reportingTask = new ReportLineageToAtlas();
    final MockComponentLog logger = new MockComponentLog("reporting-task-id", reportingTask);
    final ReportingInitializationContext initializationContext = mock(ReportingInitializationContext.class);
    when(initializationContext.getLogger()).thenReturn(logger);
    final ConfigurationContext configurationContext = new MockConfigurationContext(tc.properties, null);
    final ValidationContext validationContext = mock(ValidationContext.class);
    when(validationContext.getProperty(any())).then(invocation -> new MockPropertyValue(tc.properties.get(invocation.getArguments()[0])));
    final ReportingContext reportingContext = mock(ReportingContext.class);
    final MockStateManager stateManager = new MockStateManager(reportingTask);
    final EventAccess eventAccess = mock(EventAccess.class);
    when(reportingContext.getProperties()).thenReturn(tc.properties);
    when(reportingContext.getProperty(any())).then(invocation -> new MockPropertyValue(tc.properties.get(invocation.getArguments()[0])));
    when(reportingContext.getStateManager()).thenReturn(stateManager);
    when(reportingContext.getEventAccess()).thenReturn(eventAccess);
    when(eventAccess.getGroupStatus(eq("root"))).thenReturn(tc.rootPgStatus);
    final ProvenanceRepository provenanceRepository = mock(ProvenanceRepository.class);
    when(eventAccess.getControllerStatus()).thenReturn(tc.rootPgStatus);
    when(eventAccess.getProvenanceRepository()).thenReturn(provenanceRepository);
    when(eventAccess.getProvenanceEvents(eq(-1L), anyInt())).thenReturn(tc.provenanceRecords);
    when(provenanceRepository.getMaxEventId()).thenReturn((long) tc.provenanceRecords.size() - 1);
    when(provenanceRepository.getEvent(anyLong())).then(invocation -> tc.provenanceRecords.get(((Long) invocation.getArguments()[0]).intValue()));
    // To mock this async method invocations, keep the requested event ids in a stack.
    final ComputeLineageSubmission lineageComputationSubmission = mock(ComputeLineageSubmission.class);
    when(provenanceRepository.submitLineageComputation(anyLong(), any())).thenAnswer(invocation -> {
        requestedLineageComputationIds.push((Long) invocation.getArguments()[0]);
        return lineageComputationSubmission;
    });
    when(lineageComputationSubmission.getResult()).then(invocation -> tc.lineageResults.get(requestedLineageComputationIds.pop()));
    final ComputeLineageSubmission expandParentsSubmission = mock(ComputeLineageSubmission.class);
    when(provenanceRepository.submitExpandParents(anyLong(), any())).thenAnswer(invocation -> {
        requestedExpandParentsIds.push(((Long) invocation.getArguments()[0]));
        return expandParentsSubmission;
    });
    when(expandParentsSubmission.getResult()).then(invocation -> tc.parentLineageResults.get(requestedExpandParentsIds.pop()));
    tc.properties.put(ATLAS_NIFI_URL, "http://localhost:8080/nifi");
    tc.properties.put(ATLAS_URLS, TARGET_ATLAS_URL);
    tc.properties.put(ATLAS_USER, "admin");
    tc.properties.put(ATLAS_PASSWORD, "admin");
    tc.properties.put(new PropertyDescriptor.Builder().name("hostnamePattern.example").dynamic(true).build(), ".*");
    reportingTask.initialize(initializationContext);
    reportingTask.validate(validationContext);
    reportingTask.setup(configurationContext);
    reportingTask.onTrigger(reportingContext);
    reportingTask.onUnscheduled();
    reportingTask.onStopped();
}