Examples with RecordReaderFactory org.apache.nifi.provenance.store.RecordReaderFactory used on opensource projects

Example 1 with RecordReaderFactory

use of org.apache.nifi.provenance.store.RecordReaderFactory in project nifi by apache.

the class TestSelectiveRecordReaderEventIterator method testPerformanceOfRandomAccessReads.

@Test
@Ignore("For local testing only. Runs indefinitely")
public void testPerformanceOfRandomAccessReads() throws Exception {
    final File dir = new File("target/storage/" + UUID.randomUUID().toString());
    final File journalFile = new File(dir, "/4.prov.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final int blockSize = 1024 * 32;
    try (final RecordWriter writer = createWriter(journalFile, new StandardTocWriter(tocFile, true, false), true, blockSize)) {
        writer.writeHeader(0L);
        for (int i = 0; i < 100_000; i++) {
            writer.writeRecord(TestUtil.createEvent());
        }
    }
    final Long[] eventIds = new Long[] { 4L, 80L, 1024L, 1025L, 1026L, 1027L, 1028L, 1029L, 1030L, 40_000L, 80_000L, 99_000L };
    final RecordReaderFactory readerFactory = (file, logs, maxChars) -> RecordReaders.newRecordReader(file, logs, maxChars);
    final List<File> files = new ArrayList<>();
    files.add(new File(dir, "0.prov"));
    files.add(new File(dir, "0.prov"));
    files.add(new File(dir, "1.prov"));
    files.add(new File(dir, "2.prov"));
    files.add(new File(dir, "3.prov"));
    files.add(journalFile);
    files.add(new File(dir, "100000000.prov"));
    boolean loopForever = true;
    while (loopForever) {
        final long start = System.nanoTime();
        for (int i = 0; i < 1000; i++) {
            final SelectiveRecordReaderEventIterator iterator = new SelectiveRecordReaderEventIterator(Collections.singletonList(journalFile), readerFactory, Arrays.asList(eventIds), 32 * 1024);
            for (final long id : eventIds) {
                time(() -> {
                    return iterator.nextEvent().orElse(null);
                }, id);
            }
        }
        final long ms = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
        System.out.println(ms + " ms total");
    }
}

Example 2 with RecordReaderFactory

use of org.apache.nifi.provenance.store.RecordReaderFactory in project nifi by apache.

the class TestSelectiveRecordReaderEventIterator method testFileNotFound.

@Test
public void testFileNotFound() throws IOException {
    final File file1 = new File("1.prov");
    // Filter out the first file.
    final List<File> files = new ArrayList<>();
    files.add(file1);
    List<Long> eventIds = new ArrayList<>();
    eventIds.add(1L);
    eventIds.add(5L);
    final RecordReaderFactory readerFactory = (file, logs, maxChars) -> {
        return RecordReaders.newRecordReader(file, logs, maxChars);
    };
    final SelectiveRecordReaderEventIterator itr = new SelectiveRecordReaderEventIterator(files, readerFactory, eventIds, 65536);
    final Optional<ProvenanceEventRecord> firstRecordOption = itr.nextEvent();
    assertFalse(firstRecordOption.isPresent());
}

Example 3 with RecordReaderFactory

use of org.apache.nifi.provenance.store.RecordReaderFactory in project nifi by apache.

the class WriteAheadProvenanceRepository method initialize.

@Override
public synchronized void initialize(final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory, final IdentifierLookup idLookup) throws IOException {
    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
        return new EventIdFirstSchemaRecordWriter(file, idGenerator, tocWriter, compressed, BLOCK_SIZE, idLookup);
    };
    final EventFileManager fileManager = new EventFileManager();
    final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> {
        fileManager.obtainReadLock(file);
        try {
            return RecordReaders.newRecordReader(file, logs, maxChars);
        } finally {
            fileManager.releaseReadLock(file);
        }
    };
    init(recordWriterFactory, recordReaderFactory, eventReporter, authorizer, resourceFactory);
}

Example 4 with RecordReaderFactory

use of org.apache.nifi.provenance.store.RecordReaderFactory in project nifi by apache.

the class EncryptedWriteAheadProvenanceRepository method initialize.

/**
 * This method initializes the repository. It first builds the key provider and event encryptor
 * from the config values, then creates the encrypted record writer and reader, then delegates
 * back to the superclass for the common implementation.
 *
 * @param eventReporter   the event reporter
 * @param authorizer      the authorizer
 * @param resourceFactory the authorizable factory
 * @param idLookup        the lookup provider
 * @throws IOException if there is an error initializing this repository
 */
@Override
public synchronized void initialize(final EventReporter eventReporter, final Authorizer authorizer, final ProvenanceAuthorizableFactory resourceFactory, final IdentifierLookup idLookup) throws IOException {
    // Initialize the encryption-specific fields
    ProvenanceEventEncryptor provenanceEventEncryptor;
    if (getConfig().supportsEncryption()) {
        try {
            KeyProvider keyProvider;
            if (KeyProviderFactory.requiresMasterKey(getConfig().getKeyProviderImplementation())) {
                SecretKey masterKey = getMasterKey();
                keyProvider = buildKeyProvider(masterKey);
            } else {
                keyProvider = buildKeyProvider();
            }
            provenanceEventEncryptor = new AESProvenanceEventEncryptor();
            provenanceEventEncryptor.initialize(keyProvider);
        } catch (KeyManagementException e) {
            String msg = "Encountered an error building the key provider";
            logger.error(msg, e);
            throw new IOException(msg, e);
        }
    } else {
        throw new IOException("The provided configuration does not support a encrypted repository");
    }
    // Build a factory using lambda which injects the encryptor
    final RecordWriterFactory recordWriterFactory = (file, idGenerator, compressed, createToc) -> {
        try {
            final TocWriter tocWriter = createToc ? new StandardTocWriter(TocUtil.getTocFile(file), false, false) : null;
            return new EncryptedSchemaRecordWriter(file, idGenerator, tocWriter, compressed, BLOCK_SIZE, idLookup, provenanceEventEncryptor, getConfig().getDebugFrequency());
        } catch (EncryptionException e) {
            logger.error("Encountered an error building the schema record writer factory: ", e);
            throw new IOException(e);
        }
    };
    // Build a factory using lambda which injects the encryptor
    final EventFileManager fileManager = new EventFileManager();
    final RecordReaderFactory recordReaderFactory = (file, logs, maxChars) -> {
        fileManager.obtainReadLock(file);
        try {
            EncryptedSchemaRecordReader tempReader = (EncryptedSchemaRecordReader) RecordReaders.newRecordReader(file, logs, maxChars);
            tempReader.setProvenanceEventEncryptor(provenanceEventEncryptor);
            return tempReader;
        } finally {
            fileManager.releaseReadLock(file);
        }
    };
    // Delegate the init to the parent impl
    super.init(recordWriterFactory, recordReaderFactory, eventReporter, authorizer, resourceFactory);
}