Examples with StandardTocWriter org.apache.nifi.provenance.toc.StandardTocWriter used on opensource projects

Example 11 with StandardTocWriter

use of org.apache.nifi.provenance.toc.StandardTocWriter in project nifi by apache.

the class TestSelectiveRecordReaderEventIterator method testPerformanceOfRandomAccessReads.

@Test
@Ignore("For local testing only. Runs indefinitely")
public void testPerformanceOfRandomAccessReads() throws Exception {
    final File dir = new File("target/storage/" + UUID.randomUUID().toString());
    final File journalFile = new File(dir, "/4.prov.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final int blockSize = 1024 * 32;
    try (final RecordWriter writer = createWriter(journalFile, new StandardTocWriter(tocFile, true, false), true, blockSize)) {
        writer.writeHeader(0L);
        for (int i = 0; i < 100_000; i++) {
            writer.writeRecord(TestUtil.createEvent());
        }
    }
    final Long[] eventIds = new Long[] { 4L, 80L, 1024L, 1025L, 1026L, 1027L, 1028L, 1029L, 1030L, 40_000L, 80_000L, 99_000L };
    final RecordReaderFactory readerFactory = (file, logs, maxChars) -> RecordReaders.newRecordReader(file, logs, maxChars);
    final List<File> files = new ArrayList<>();
    files.add(new File(dir, "0.prov"));
    files.add(new File(dir, "0.prov"));
    files.add(new File(dir, "1.prov"));
    files.add(new File(dir, "2.prov"));
    files.add(new File(dir, "3.prov"));
    files.add(journalFile);
    files.add(new File(dir, "100000000.prov"));
    boolean loopForever = true;
    while (loopForever) {
        final long start = System.nanoTime();
        for (int i = 0; i < 1000; i++) {
            final SelectiveRecordReaderEventIterator iterator = new SelectiveRecordReaderEventIterator(Collections.singletonList(journalFile), readerFactory, Arrays.asList(eventIds), 32 * 1024);
            for (final long id : eventIds) {
                time(() -> {
                    return iterator.nextEvent().orElse(null);
                }, id);
            }
        }
        final long ms = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
        System.out.println(ms + " ms total");
    }
}

Example 12 with StandardTocWriter

use of org.apache.nifi.provenance.toc.StandardTocWriter in project nifi by apache.

the class AbstractTestRecordReaderWriter method testMultipleRecordsSameBlockCompressed.

@Test
public void testMultipleRecordsSameBlockCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new record each 1 MB of uncompressed data
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 1024 * 1024);
    writer.writeHeader(1L);
    for (int i = 0; i < 10; i++) {
        writer.writeRecord(createEvent());
    }
    writer.close();
    final TocReader tocReader = new StandardTocReader(tocFile);
    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 10; i++) {
            assertEquals(0, reader.getBlockIndex());
            // the other half of the time to ensure that it's okay.
            if (i <= 5) {
                reader.skipToBlock(0);
            }
            final StandardProvenanceEventRecord recovered = reader.nextRecord();
            assertNotNull(recovered);
            assertEquals("nifi://unit-test", recovered.getTransitUri());
        }
        assertNull(reader.nextRecord());
    }
    FileUtils.deleteFile(journalFile.getParentFile(), true);
}

Example 13 with StandardTocWriter

use of org.apache.nifi.provenance.toc.StandardTocWriter in project nifi by apache.

the class AbstractTestRecordReaderWriter method testSingleRecordCompressed.

@Test
public void testSingleRecordCompressed() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);
    writer.writeHeader(1L);
    writer.writeRecord(createEvent());
    writer.close();
    final TocReader tocReader = new StandardTocReader(tocFile);
    assertRecoveredRecord(journalFile, tocReader, "nifi://unit-test", 0);
    FileUtils.deleteFile(journalFile.getParentFile(), true);
}

Example 14 with StandardTocWriter

use of org.apache.nifi.provenance.toc.StandardTocWriter in project nifi by apache.

the class AbstractTestRecordReaderWriter method testSkipToEvent.

@Test
public void testSkipToEvent() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // new block each 10 bytes
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 100);
    writer.writeHeader(0L);
    final int numEvents = 10;
    final List<ProvenanceEventRecord> events = new ArrayList<>();
    for (int i = 0; i < numEvents; i++) {
        final ProvenanceEventRecord event = createEvent();
        events.add(event);
        writer.writeRecord(event);
    }
    writer.close();
    final TocReader tocReader = new StandardTocReader(tocFile);
    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < numEvents; i++) {
            final Optional<ProvenanceEventRecord> eventOption = reader.skipToEvent(i);
            assertTrue(eventOption.isPresent());
            assertEquals(i, eventOption.get().getEventId());
            assertEquals(events.get(i), eventOption.get());
            final StandardProvenanceEventRecord consumedEvent = reader.nextRecord();
            assertEquals(eventOption.get(), consumedEvent);
        }
        assertFalse(reader.skipToEvent(numEvents + 1).isPresent());
    }
    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        for (int i = 0; i < 3; i++) {
            final Optional<ProvenanceEventRecord> eventOption = reader.skipToEvent(8);
            assertTrue(eventOption.isPresent());
            assertEquals(events.get(8), eventOption.get());
        }
        final StandardProvenanceEventRecord consumedEvent = reader.nextRecord();
        assertEquals(events.get(8), consumedEvent);
    }
}

Example 15 with StandardTocWriter

use of org.apache.nifi.provenance.toc.StandardTocWriter in project nifi by apache.

the class TestEventIdFirstSchemaRecordReaderWriter method testContentClaimAdded.

@Test
public void testContentClaimAdded() throws IOException {
    final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
    final File tocFile = TocUtil.getTocFile(journalFile);
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);
    final Map<String, String> attributes = new HashMap<>();
    attributes.put("filename", "1.txt");
    attributes.put("uuid", UUID.randomUUID().toString());
    final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
    builder.setEventTime(System.currentTimeMillis());
    builder.setEventType(ProvenanceEventType.RECEIVE);
    builder.setTransitUri("nifi://unit-test");
    builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
    builder.setComponentId("1234");
    builder.setComponentType("dummy processor");
    builder.setCurrentContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
    final ProvenanceEventRecord record = builder.build();
    writer.writeHeader(1L);
    writer.writeRecord(record);
    writer.close();
    final TocReader tocReader = new StandardTocReader(tocFile);
    try (final FileInputStream fis = new FileInputStream(journalFile);
        final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
        assertEquals(0, reader.getBlockIndex());
        reader.skipToBlock(0);
        final StandardProvenanceEventRecord recovered = reader.nextRecord();
        assertNotNull(recovered);
        assertEquals("nifi://unit-test", recovered.getTransitUri());
        assertEquals("container-1", recovered.getContentClaimContainer());
        assertNull(recovered.getPreviousContentClaimContainer());
        assertEquals("section-1", recovered.getContentClaimSection());
        assertNull(recovered.getPreviousContentClaimSection());
        assertEquals("identifier-1", recovered.getContentClaimIdentifier());
        assertNull(recovered.getPreviousContentClaimIdentifier());
        assertEquals(1L, recovered.getContentClaimOffset().longValue());
        assertNull(recovered.getPreviousContentClaimOffset());
        assertEquals(1L, recovered.getFileSize());
        assertNull(recovered.getPreviousContentClaimOffset());
        assertNull(reader.nextRecord());
    }
    FileUtils.deleteFile(journalFile.getParentFile(), true);
}