use of org.apache.nifi.provenance.store.ArrayListEventStore in project nifi by apache.
the class TestLuceneEventIndex method testUnauthorizedEventsGetFilteredForQuery.
@Test(timeout = 60000)
public void testUnauthorizedEventsGetFilteredForQuery() throws InterruptedException {
assumeFalse(isWindowsEnvironment());
final RepositoryConfiguration repoConfig = createConfig(1);
repoConfig.setDesiredIndexSize(1L);
final IndexManager indexManager = new SimpleIndexManager(repoConfig);
final ArrayListEventStore eventStore = new ArrayListEventStore();
final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
index.initialize(eventStore);
for (int i = 0; i < 3; i++) {
final ProvenanceEventRecord event = createEvent("1234");
final StorageResult storageResult = eventStore.addEvent(event);
index.addEvents(storageResult.getStorageLocations());
}
final Query query = new Query(UUID.randomUUID().toString());
final EventAuthorizer authorizer = new EventAuthorizer() {
@Override
public boolean isAuthorized(ProvenanceEventRecord event) {
return event.getEventId() % 2 == 0;
}
@Override
public void authorize(ProvenanceEventRecord event) throws AccessDeniedException {
throw new AccessDeniedException();
}
};
List<ProvenanceEventRecord> events = Collections.emptyList();
while (events.size() < 2) {
final QuerySubmission submission = index.submitQuery(query, authorizer, "unit test");
assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));
events = submission.getResult().getMatchingEvents();
Thread.sleep(25L);
}
assertEquals(2, events.size());
}
use of org.apache.nifi.provenance.store.ArrayListEventStore in project nifi by apache.
the class TestLuceneEventIndex method testUnauthorizedEventsGetPlaceholdersForFindParents.
@Test(timeout = 60000)
public void testUnauthorizedEventsGetPlaceholdersForFindParents() throws InterruptedException {
assumeFalse(isWindowsEnvironment());
final RepositoryConfiguration repoConfig = createConfig(1);
repoConfig.setDesiredIndexSize(1L);
final IndexManager indexManager = new SimpleIndexManager(repoConfig);
final ArrayListEventStore eventStore = new ArrayListEventStore();
final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
index.initialize(eventStore);
final ProvenanceEventRecord firstEvent = createEvent("4444");
final Map<String, String> previousAttributes = new HashMap<>();
previousAttributes.put("uuid", "4444");
final Map<String, String> updatedAttributes = new HashMap<>();
updatedAttributes.put("updated", "true");
final ProvenanceEventRecord join = new StandardProvenanceEventRecord.Builder().setEventType(ProvenanceEventType.JOIN).setAttributes(previousAttributes, updatedAttributes).addParentUuid("4444").addChildFlowFile("1234").setComponentId("component-1").setComponentType("unit test").setEventId(idGenerator.getAndIncrement()).setEventTime(System.currentTimeMillis()).setFlowFileEntryDate(System.currentTimeMillis()).setFlowFileUUID("1234").setLineageStartDate(System.currentTimeMillis()).setCurrentContentClaim("container", "section", "unit-test-id", 0L, 1024L).build();
index.addEvents(eventStore.addEvent(firstEvent).getStorageLocations());
index.addEvents(eventStore.addEvent(join).getStorageLocations());
for (int i = 0; i < 3; i++) {
final ProvenanceEventRecord event = createEvent("1234");
final StorageResult storageResult = eventStore.addEvent(event);
index.addEvents(storageResult.getStorageLocations());
}
final NiFiUser user = createUser();
final EventAuthorizer allowJoinEvents = new EventAuthorizer() {
@Override
public boolean isAuthorized(ProvenanceEventRecord event) {
return event.getEventType() == ProvenanceEventType.JOIN;
}
@Override
public void authorize(ProvenanceEventRecord event) throws AccessDeniedException {
}
};
List<LineageNode> nodes = Collections.emptyList();
while (nodes.size() < 2) {
final ComputeLineageSubmission submission = index.submitExpandParents(1L, user, allowJoinEvents);
assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));
nodes = submission.getResult().getNodes();
Thread.sleep(25L);
}
assertEquals(2, nodes.size());
final Map<ProvenanceEventType, List<LineageNode>> eventMap = nodes.stream().filter(n -> n.getNodeType() == LineageNodeType.PROVENANCE_EVENT_NODE).collect(Collectors.groupingBy(n -> ((ProvenanceEventLineageNode) n).getEventType()));
assertEquals(2, eventMap.size());
assertEquals(1, eventMap.get(ProvenanceEventType.JOIN).size());
assertEquals(1, eventMap.get(ProvenanceEventType.UNKNOWN).size());
assertEquals("4444", eventMap.get(ProvenanceEventType.UNKNOWN).get(0).getFlowFileUuid());
}
use of org.apache.nifi.provenance.store.ArrayListEventStore in project nifi by apache.
the class TestLuceneEventIndex method testUnauthorizedEventsGetPlaceholdersForLineage.
@Test(timeout = 60000)
public void testUnauthorizedEventsGetPlaceholdersForLineage() throws InterruptedException {
assumeFalse(isWindowsEnvironment());
final RepositoryConfiguration repoConfig = createConfig(1);
repoConfig.setDesiredIndexSize(1L);
final IndexManager indexManager = new SimpleIndexManager(repoConfig);
final ArrayListEventStore eventStore = new ArrayListEventStore();
final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
index.initialize(eventStore);
for (int i = 0; i < 3; i++) {
final ProvenanceEventRecord event = createEvent("1234");
final StorageResult storageResult = eventStore.addEvent(event);
index.addEvents(storageResult.getStorageLocations());
}
final NiFiUser user = createUser();
List<LineageNode> nodes = Collections.emptyList();
while (nodes.size() < 3) {
final ComputeLineageSubmission submission = index.submitLineageComputation(1L, user, EventAuthorizer.DENY_ALL);
assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));
nodes = submission.getResult().getNodes();
Thread.sleep(25L);
}
assertEquals(3, nodes.size());
for (final LineageNode node : nodes) {
assertEquals(LineageNodeType.PROVENANCE_EVENT_NODE, node.getNodeType());
final ProvenanceEventLineageNode eventNode = (ProvenanceEventLineageNode) node;
assertEquals(ProvenanceEventType.UNKNOWN, eventNode.getEventType());
}
}
use of org.apache.nifi.provenance.store.ArrayListEventStore in project nifi by apache.
the class TestLuceneEventIndex method testGetMinimumIdToReindex.
@Test(timeout = 60000)
public void testGetMinimumIdToReindex() throws InterruptedException {
assumeFalse(isWindowsEnvironment());
final RepositoryConfiguration repoConfig = createConfig(1);
repoConfig.setDesiredIndexSize(1L);
final IndexManager indexManager = new SimpleIndexManager(repoConfig);
final ArrayListEventStore eventStore = new ArrayListEventStore();
final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 20_000, EventReporter.NO_OP);
index.initialize(eventStore);
for (int i = 0; i < 50_000; i++) {
final ProvenanceEventRecord event = createEvent("1234");
final StorageResult storageResult = eventStore.addEvent(event);
index.addEvents(storageResult.getStorageLocations());
}
while (index.getMaxEventId("1") < 40_000L) {
Thread.sleep(25);
}
final long id = index.getMinimumEventIdToReindex("1");
assertTrue(id >= 30000L);
}
use of org.apache.nifi.provenance.store.ArrayListEventStore in project nifi by apache.
the class TestLuceneEventIndex method addThenQueryWithEmptyQuery.
@Test(timeout = 60000)
public void addThenQueryWithEmptyQuery() throws InterruptedException {
assumeFalse(isWindowsEnvironment());
final RepositoryConfiguration repoConfig = createConfig();
final IndexManager indexManager = new SimpleIndexManager(repoConfig);
final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP);
final ProvenanceEventRecord event = createEvent();
index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));
final Query query = new Query(UUID.randomUUID().toString());
final ArrayListEventStore eventStore = new ArrayListEventStore();
eventStore.addEvent(event);
index.initialize(eventStore);
// We don't know how long it will take for the event to be indexed, so keep querying until
// we get a result. The test will timeout after 5 seconds if we've still not succeeded.
List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
while (matchingEvents.isEmpty()) {
final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
assertNotNull(submission);
final QueryResult result = submission.getResult();
assertNotNull(result);
result.awaitCompletion(100, TimeUnit.MILLISECONDS);
assertTrue(result.isFinished());
assertNull(result.getError());
matchingEvents = result.getMatchingEvents();
assertNotNull(matchingEvents);
// avoid crushing the CPU
Thread.sleep(100L);
}
assertEquals(1, matchingEvents.size());
assertEquals(event, matchingEvents.get(0));
}
Aggregations