Examples with RepositoryConfiguration org.apache.nifi.provenance.RepositoryConfiguration used on opensource projects

Example 11 with RepositoryConfiguration

use of org.apache.nifi.provenance.RepositoryConfiguration in project nifi by apache.

the class TestWriteAheadStorePartition method createConfig.

private RepositoryConfiguration createConfig(final int numStorageDirs, final String testName) {
    final RepositoryConfiguration config = new RepositoryConfiguration();
    final File storageDir = new File("target/storage/" + testName + "/" + UUID.randomUUID().toString());
    for (int i = 1; i <= numStorageDirs; i++) {
        config.addStorageDirectory(String.valueOf(1), new File(storageDir, String.valueOf(i)));
    }
    config.setJournalCount(4);
    return config;
}

Example 12 with RepositoryConfiguration

use of org.apache.nifi.provenance.RepositoryConfiguration in project nifi by apache.

the class TestWriteAheadStorePartition method testReindex.

@Test
@SuppressWarnings("unchecked")
public void testReindex() throws IOException {
    final RepositoryConfiguration repoConfig = createConfig(1, "testReindex");
    repoConfig.setMaxEventFileCount(5);
    final String partitionName = repoConfig.getStorageDirectories().keySet().iterator().next();
    final File storageDirectory = repoConfig.getStorageDirectories().values().iterator().next();
    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
        return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, 32 * 1024, IdentifierLookup.EMPTY);
    };
    final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> RecordReaders.newRecordReader(file, logs, maxChars);
    final WriteAheadStorePartition partition = new WriteAheadStorePartition(storageDirectory, partitionName, repoConfig, recordWriterFactory, recordReaderFactory, new LinkedBlockingQueue<>(), new AtomicLong(0L), EventReporter.NO_OP);
    for (int i = 0; i < 100; i++) {
        partition.addEvents(Collections.singleton(TestUtil.createEvent()));
    }
    final Map<ProvenanceEventRecord, StorageSummary> reindexedEvents = new ConcurrentHashMap<>();
    final EventIndex eventIndex = Mockito.mock(EventIndex.class);
    Mockito.doAnswer(new Answer<Object>() {

        @Override
        public Object answer(final InvocationOnMock invocation) throws Throwable {
            final Map<ProvenanceEventRecord, StorageSummary> events = invocation.getArgumentAt(0, Map.class);
            reindexedEvents.putAll(events);
            return null;
        }
    }).when(eventIndex).reindexEvents(Mockito.anyMap());
    Mockito.doReturn(18L).when(eventIndex).getMinimumEventIdToReindex("1");
    partition.reindexLatestEvents(eventIndex);
    final List<Long> eventIdsReindexed = reindexedEvents.values().stream().map(StorageSummary::getEventId).sorted().collect(Collectors.toList());
    assertEquals(82, eventIdsReindexed.size());
    for (int i = 0; i < eventIdsReindexed.size(); i++) {
        assertEquals(18 + i, eventIdsReindexed.get(i).intValue());
    }
}

Example 13 with RepositoryConfiguration

use of org.apache.nifi.provenance.RepositoryConfiguration in project nifi by apache.

the class TestIndexDirectoryManager method testGetDirectoriesOnlyObtainsDirectoriesForDesiredPartition.

@Test
public void testGetDirectoriesOnlyObtainsDirectoriesForDesiredPartition() {
    final RepositoryConfiguration config = createConfig(2);
    final File storageDir1 = config.getStorageDirectories().get("1");
    final File storageDir2 = config.getStorageDirectories().get("2");
    final File index1 = new File(storageDir1, "index-1");
    final File index2 = new File(storageDir1, "index-2");
    final File index3 = new File(storageDir2, "index-3");
    final File index4 = new File(storageDir2, "index-4");
    final File[] allIndices = new File[] { index1, index2, index3, index4 };
    for (final File file : allIndices) {
        assertTrue(file.mkdirs() || file.exists());
    }
    try {
        final IndexDirectoryManager mgr = new IndexDirectoryManager(config);
        mgr.initialize();
        final List<File> indexes1 = mgr.getDirectories(0L, Long.MAX_VALUE, "1");
        final List<File> indexes2 = mgr.getDirectories(0L, Long.MAX_VALUE, "2");
        assertEquals(2, indexes1.size());
        assertTrue(indexes1.contains(index1));
        assertTrue(indexes1.contains(index2));
        assertEquals(2, indexes2.size());
        assertTrue(indexes2.contains(index3));
        assertTrue(indexes2.contains(index4));
    } finally {
        for (final File file : allIndices) {
            file.delete();
        }
    }
}

Example 14 with RepositoryConfiguration

use of org.apache.nifi.provenance.RepositoryConfiguration in project nifi by apache.

the class TestLuceneEventIndex method testGetMinimumIdToReindex.

@Test(timeout = 60000)
public void testGetMinimumIdToReindex() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig(1);
    repoConfig.setDesiredIndexSize(1L);
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);
    final ArrayListEventStore eventStore = new ArrayListEventStore();
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 20_000, EventReporter.NO_OP);
    index.initialize(eventStore);
    for (int i = 0; i < 50_000; i++) {
        final ProvenanceEventRecord event = createEvent("1234");
        final StorageResult storageResult = eventStore.addEvent(event);
        index.addEvents(storageResult.getStorageLocations());
    }
    while (index.getMaxEventId("1") < 40_000L) {
        Thread.sleep(25);
    }
    final long id = index.getMinimumEventIdToReindex("1");
    assertTrue(id >= 30000L);
}

Example 15 with RepositoryConfiguration

use of org.apache.nifi.provenance.RepositoryConfiguration in project nifi by apache.

the class TestLuceneEventIndex method addThenQueryWithEmptyQuery.

@Test(timeout = 60000)
public void addThenQueryWithEmptyQuery() throws InterruptedException {
    assumeFalse(isWindowsEnvironment());
    final RepositoryConfiguration repoConfig = createConfig();
    final IndexManager indexManager = new SimpleIndexManager(repoConfig);
    final LuceneEventIndex index = new LuceneEventIndex(repoConfig, indexManager, 1, EventReporter.NO_OP);
    final ProvenanceEventRecord event = createEvent();
    index.addEvent(event, new StorageSummary(event.getEventId(), "1.prov", "1", 1, 2L, 2L));
    final Query query = new Query(UUID.randomUUID().toString());
    final ArrayListEventStore eventStore = new ArrayListEventStore();
    eventStore.addEvent(event);
    index.initialize(eventStore);
    // We don't know how long it will take for the event to be indexed, so keep querying until
    // we get a result. The test will timeout after 5 seconds if we've still not succeeded.
    List<ProvenanceEventRecord> matchingEvents = Collections.emptyList();
    while (matchingEvents.isEmpty()) {
        final QuerySubmission submission = index.submitQuery(query, EventAuthorizer.GRANT_ALL, "unit test user");
        assertNotNull(submission);
        final QueryResult result = submission.getResult();
        assertNotNull(result);
        result.awaitCompletion(100, TimeUnit.MILLISECONDS);
        assertTrue(result.isFinished());
        assertNull(result.getError());
        matchingEvents = result.getMatchingEvents();
        assertNotNull(matchingEvents);
        // avoid crushing the CPU
        Thread.sleep(100L);
    }
    assertEquals(1, matchingEvents.size());
    assertEquals(event, matchingEvents.get(0));
}