Examples with RepositoryConfiguration org.apache.nifi.provenance.RepositoryConfiguration used on opensource projects

Example 6 with RepositoryConfiguration

use of org.apache.nifi.provenance.RepositoryConfiguration in project nifi by apache.

the class TestLuceneEventIndex method testUnauthorizedEventsGetPlaceholdersForLineage.

@Test(timeout = 60000)
public void testUnauthorizedEventsGetPlaceholdersForLineage() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);
    final ArrayListEventStore eventStore = new ArrayListEventStore();
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 3, EventReporter.NO_OP);
    index.initialize(eventStore);
    for (int i = 0; i < 3; i++) {
        final ProvenanceEventRecord event = createEvent("1234");
        final StorageResult storageResult = eventStore.addEvent(event);
        index.addEvents(storageResult.getStorageLocations());
    }
    final NiFiUser user = createUser();
    List<LineageNode> nodes = Collections.emptyList();
    while (nodes.size() < 3) {
        final ComputeLineageSubmission submission = index.submitLineageComputation(1L, user, EventAuthorizer.DENY_ALL);
        assertTrue(submission.getResult().awaitCompletion(5, TimeUnit.SECONDS));
        nodes = submission.getResult().getNodes();
        Thread.sleep(25L);
    }
    assertEquals(3, nodes.size());
    for (final LineageNode node : nodes) {
        assertEquals(LineageNodeType.PROVENANCE_EVENT_NODE, node.getNodeType());
        final ProvenanceEventLineageNode eventNode = (ProvenanceEventLineageNode) node;
        assertEquals(ProvenanceEventType.UNKNOWN, eventNode.getEventType());
    }
}

Example 7 with RepositoryConfiguration

use of org.apache.nifi.provenance.RepositoryConfiguration in project nifi by apache.

the class TestSimpleIndexManager method testWriterCloseIfPreviouslyMarkedCloseable.

@Test
public void testWriterCloseIfPreviouslyMarkedCloseable() throws IOException {
    final AtomicInteger closeCount = new AtomicInteger(0);
    final SimpleIndexManager mgr = new SimpleIndexManager(new RepositoryConfiguration()) {

        @Override
        protected void close(IndexWriterCount count) throws IOException {
            closeCount.incrementAndGet();
        }
    };
    final File dir = new File("target/" + UUID.randomUUID().toString());
    final EventIndexWriter writer1 = mgr.borrowIndexWriter(dir);
    final EventIndexWriter writer2 = mgr.borrowIndexWriter(dir);
    assertTrue(writer1 == writer2);
    mgr.returnIndexWriter(writer1, true, true);
    assertEquals(0, closeCount.get());
    final EventIndexWriter[] writers = new EventIndexWriter[10];
    for (int i = 0; i < writers.length; i++) {
        writers[i] = mgr.borrowIndexWriter(dir);
        assertTrue(writers[i] == writer1);
    }
    for (int i = 0; i < writers.length; i++) {
        mgr.returnIndexWriter(writers[i], true, false);
        assertEquals(0, closeCount.get());
        assertEquals(1, mgr.getWriterCount());
    }
    // this should close the index writer even though 'false' is passed in
    // because the previous call marked the writer as closeable and this is
    // the last reference to the writer.
    mgr.returnIndexWriter(writer2, false, false);
    assertEquals(1, closeCount.get());
    assertEquals(0, mgr.getWriterCount());
}

Example 8 with RepositoryConfiguration

use of org.apache.nifi.provenance.RepositoryConfiguration in project nifi by apache.

the class TestSimpleIndexManager method testWriterLeftOpenIfNotCloseable.

@Test
public void testWriterLeftOpenIfNotCloseable() throws IOException {
    final AtomicInteger closeCount = new AtomicInteger(0);
    final SimpleIndexManager mgr = new SimpleIndexManager(new RepositoryConfiguration()) {

        @Override
        protected void close(IndexWriterCount count) throws IOException {
            closeCount.incrementAndGet();
        }
    };
    final File dir = new File("target/" + UUID.randomUUID().toString());
    final EventIndexWriter writer = mgr.borrowIndexWriter(dir);
    mgr.returnIndexWriter(writer, true, false);
    assertEquals(0, closeCount.get());
}

Example 9 with RepositoryConfiguration

use of org.apache.nifi.provenance.RepositoryConfiguration in project nifi by apache.

the class TestPartitionedWriteAheadEventStore method testGetEventsWithMinIdAndCount.

@Test
public void testGetEventsWithMinIdAndCount() throws IOException {
    final RepositoryConfiguration config = createConfig();
    config.setMaxEventFileCount(100);
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();
    final int numEvents = 50_000;
    final List<ProvenanceEventRecord> events = new ArrayList<>(numEvents);
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        store.addEvents(Collections.singleton(event));
        if (i < 1000) {
            events.add(event);
        }
    }
    assertTrue(store.getEvents(-1000L, 1000).isEmpty());
    assertEquals(events, store.getEvents(0, events.size()));
    assertEquals(events, store.getEvents(-30, events.size()));
    assertEquals(events.subList(10, events.size()), store.getEvents(10L, events.size() - 10));
    assertTrue(store.getEvents(numEvents, 100).isEmpty());
}

Example 10 with RepositoryConfiguration

use of org.apache.nifi.provenance.RepositoryConfiguration in project nifi by apache.

the class TestPartitionedWriteAheadEventStore method testGetEventsWithStartOffsetAndCountWithNothingAuthorized.

@Test
public void testGetEventsWithStartOffsetAndCountWithNothingAuthorized() throws IOException {
    final RepositoryConfiguration config = createConfig();
    final PartitionedWriteAheadEventStore store = new PartitionedWriteAheadEventStore(config, writerFactory, readerFactory, EventReporter.NO_OP, new EventFileManager());
    store.initialize();
    final int numEvents = 20;
    final List<ProvenanceEventRecord> events = new ArrayList<>(numEvents);
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        store.addEvents(Collections.singleton(event));
        events.add(event);
    }
    final EventAuthorizer allowEventNumberedEventIds = EventAuthorizer.DENY_ALL;
    final List<ProvenanceEventRecord> storedEvents = store.getEvents(0, 20, allowEventNumberedEventIds, EventTransformer.EMPTY_TRANSFORMER);
    assertTrue(storedEvents.isEmpty());
}