use of org.apache.nifi.provenance.serialization.RecordWriter in project nifi by apache.
the class AbstractTestRecordReaderWriter method testSkipToEvent.
@Test
public void testSkipToEvent() throws IOException {
final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
final File tocFile = TocUtil.getTocFile(journalFile);
final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
// new block each 10 bytes
final RecordWriter writer = createWriter(journalFile, tocWriter, true, 100);
writer.writeHeader(0L);
final int numEvents = 10;
final List<ProvenanceEventRecord> events = new ArrayList<>();
for (int i = 0; i < numEvents; i++) {
final ProvenanceEventRecord event = createEvent();
events.add(event);
writer.writeRecord(event);
}
writer.close();
final TocReader tocReader = new StandardTocReader(tocFile);
try (final FileInputStream fis = new FileInputStream(journalFile);
final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
for (int i = 0; i < numEvents; i++) {
final Optional<ProvenanceEventRecord> eventOption = reader.skipToEvent(i);
assertTrue(eventOption.isPresent());
assertEquals(i, eventOption.get().getEventId());
assertEquals(events.get(i), eventOption.get());
final StandardProvenanceEventRecord consumedEvent = reader.nextRecord();
assertEquals(eventOption.get(), consumedEvent);
}
assertFalse(reader.skipToEvent(numEvents + 1).isPresent());
}
try (final FileInputStream fis = new FileInputStream(journalFile);
final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
for (int i = 0; i < 3; i++) {
final Optional<ProvenanceEventRecord> eventOption = reader.skipToEvent(8);
assertTrue(eventOption.isPresent());
assertEquals(events.get(8), eventOption.get());
}
final StandardProvenanceEventRecord consumedEvent = reader.nextRecord();
assertEquals(events.get(8), consumedEvent);
}
}
use of org.apache.nifi.provenance.serialization.RecordWriter in project nifi by apache.
the class TestEventIdFirstSchemaRecordReaderWriter method testContentClaimAdded.
@Test
public void testContentClaimAdded() throws IOException {
final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
final File tocFile = TocUtil.getTocFile(journalFile);
final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);
final Map<String, String> attributes = new HashMap<>();
attributes.put("filename", "1.txt");
attributes.put("uuid", UUID.randomUUID().toString());
final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
builder.setEventTime(System.currentTimeMillis());
builder.setEventType(ProvenanceEventType.RECEIVE);
builder.setTransitUri("nifi://unit-test");
builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
builder.setComponentId("1234");
builder.setComponentType("dummy processor");
builder.setCurrentContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
final ProvenanceEventRecord record = builder.build();
writer.writeHeader(1L);
writer.writeRecord(record);
writer.close();
final TocReader tocReader = new StandardTocReader(tocFile);
try (final FileInputStream fis = new FileInputStream(journalFile);
final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
assertEquals(0, reader.getBlockIndex());
reader.skipToBlock(0);
final StandardProvenanceEventRecord recovered = reader.nextRecord();
assertNotNull(recovered);
assertEquals("nifi://unit-test", recovered.getTransitUri());
assertEquals("container-1", recovered.getContentClaimContainer());
assertNull(recovered.getPreviousContentClaimContainer());
assertEquals("section-1", recovered.getContentClaimSection());
assertNull(recovered.getPreviousContentClaimSection());
assertEquals("identifier-1", recovered.getContentClaimIdentifier());
assertNull(recovered.getPreviousContentClaimIdentifier());
assertEquals(1L, recovered.getContentClaimOffset().longValue());
assertNull(recovered.getPreviousContentClaimOffset());
assertEquals(1L, recovered.getFileSize());
assertNull(recovered.getPreviousContentClaimOffset());
assertNull(reader.nextRecord());
}
FileUtils.deleteFile(journalFile.getParentFile(), true);
}
use of org.apache.nifi.provenance.serialization.RecordWriter in project nifi by apache.
the class TestEventIdFirstSchemaRecordReaderWriter method testContentClaimChanged.
@Test
public void testContentClaimChanged() throws IOException {
final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
final File tocFile = TocUtil.getTocFile(journalFile);
final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);
final Map<String, String> attributes = new HashMap<>();
attributes.put("filename", "1.txt");
attributes.put("uuid", UUID.randomUUID().toString());
final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
builder.setEventTime(System.currentTimeMillis());
builder.setEventType(ProvenanceEventType.RECEIVE);
builder.setTransitUri("nifi://unit-test");
builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
builder.setComponentId("1234");
builder.setComponentType("dummy processor");
builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
builder.setCurrentContentClaim("container-2", "section-2", "identifier-2", 2L, 2L);
final ProvenanceEventRecord record = builder.build();
writer.writeHeader(1L);
writer.writeRecord(record);
writer.close();
final TocReader tocReader = new StandardTocReader(tocFile);
try (final FileInputStream fis = new FileInputStream(journalFile);
final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
assertEquals(0, reader.getBlockIndex());
reader.skipToBlock(0);
final StandardProvenanceEventRecord recovered = reader.nextRecord();
assertNotNull(recovered);
assertEquals("nifi://unit-test", recovered.getTransitUri());
assertEquals("container-1", recovered.getPreviousContentClaimContainer());
assertEquals("container-2", recovered.getContentClaimContainer());
assertEquals("section-1", recovered.getPreviousContentClaimSection());
assertEquals("section-2", recovered.getContentClaimSection());
assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
assertEquals("identifier-2", recovered.getContentClaimIdentifier());
assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
assertEquals(2L, recovered.getContentClaimOffset().longValue());
assertEquals(1L, recovered.getPreviousFileSize().longValue());
assertEquals(2L, recovered.getContentClaimOffset().longValue());
assertNull(reader.nextRecord());
}
FileUtils.deleteFile(journalFile.getParentFile(), true);
}
use of org.apache.nifi.provenance.serialization.RecordWriter in project nifi by apache.
the class TestEventIdFirstSchemaRecordReaderWriter method testContentClaimUnchanged.
@Test
public void testContentClaimUnchanged() throws IOException {
final File journalFile = new File("target/storage/" + UUID.randomUUID().toString() + "/testSimpleWrite.gz");
final File tocFile = TocUtil.getTocFile(journalFile);
final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
final RecordWriter writer = createWriter(journalFile, tocWriter, true, 8192);
final Map<String, String> attributes = new HashMap<>();
attributes.put("filename", "1.txt");
attributes.put("uuid", UUID.randomUUID().toString());
final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
builder.setEventTime(System.currentTimeMillis());
builder.setEventType(ProvenanceEventType.RECEIVE);
builder.setTransitUri("nifi://unit-test");
builder.fromFlowFile(TestUtil.createFlowFile(3L, 3000L, attributes));
builder.setComponentId("1234");
builder.setComponentType("dummy processor");
builder.setPreviousContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
builder.setCurrentContentClaim("container-1", "section-1", "identifier-1", 1L, 1L);
final ProvenanceEventRecord record = builder.build();
writer.writeHeader(1L);
writer.writeRecord(record);
writer.close();
final TocReader tocReader = new StandardTocReader(tocFile);
try (final FileInputStream fis = new FileInputStream(journalFile);
final RecordReader reader = createReader(fis, journalFile.getName(), tocReader, 2048)) {
assertEquals(0, reader.getBlockIndex());
reader.skipToBlock(0);
final StandardProvenanceEventRecord recovered = reader.nextRecord();
assertNotNull(recovered);
assertEquals("nifi://unit-test", recovered.getTransitUri());
assertEquals("container-1", recovered.getPreviousContentClaimContainer());
assertEquals("container-1", recovered.getContentClaimContainer());
assertEquals("section-1", recovered.getPreviousContentClaimSection());
assertEquals("section-1", recovered.getContentClaimSection());
assertEquals("identifier-1", recovered.getPreviousContentClaimIdentifier());
assertEquals("identifier-1", recovered.getContentClaimIdentifier());
assertEquals(1L, recovered.getPreviousContentClaimOffset().longValue());
assertEquals(1L, recovered.getContentClaimOffset().longValue());
assertEquals(1L, recovered.getPreviousFileSize().longValue());
assertEquals(1L, recovered.getContentClaimOffset().longValue());
assertNull(reader.nextRecord());
}
FileUtils.deleteFile(journalFile.getParentFile(), true);
}
use of org.apache.nifi.provenance.serialization.RecordWriter in project nifi by apache.
the class TestPersistentProvenanceRepository method findJournalSizes.
@BeforeClass
public static void findJournalSizes() throws IOException {
// determine header and record size
final Map<String, String> attributes = new HashMap<>();
final ProvenanceEventBuilder builder = new StandardProvenanceEventRecord.Builder();
builder.setEventTime(System.currentTimeMillis());
builder.setEventType(ProvenanceEventType.RECEIVE);
builder.setTransitUri("nifi://unit-test");
attributes.put("uuid", "12345678-0000-0000-0000-012345678912");
builder.fromFlowFile(createFlowFile(3L, 3000L, attributes));
builder.setComponentId("1234");
builder.setComponentType("dummy processor");
final ProvenanceEventRecord record = builder.build();
builder.setComponentId("2345");
final ProvenanceEventRecord record2 = builder.build();
final File tempRecordFile = tempFolder.newFile("record.tmp");
System.out.println("findJournalSizes position 0 = " + tempRecordFile.length());
final AtomicLong idGenerator = new AtomicLong(0L);
final RecordWriter writer = RecordWriters.newSchemaRecordWriter(tempRecordFile, idGenerator, false, false);
writer.writeHeader(12345L);
writer.flush();
headerSize = Long.valueOf(tempRecordFile.length()).intValue();
writer.writeRecord(record);
writer.flush();
recordSize = Long.valueOf(tempRecordFile.length()).intValue() - headerSize;
writer.writeRecord(record2);
writer.flush();
recordSize2 = Long.valueOf(tempRecordFile.length()).intValue() - headerSize - recordSize;
writer.close();
System.out.println("headerSize =" + headerSize);
System.out.println("recordSize =" + recordSize);
System.out.println("recordSize2=" + recordSize2);
}
Aggregations