Examples with EventRecord org.apache.nifi.provenance.schema.EventRecord used on opensource projects

Example 1 with EventRecord

use of org.apache.nifi.provenance.schema.EventRecord in project nifi by apache.

the class TestSchemaRecordReaderWriter method testFieldRemovedFromSchema.

@Test
public void testFieldRemovedFromSchema() throws IOException {
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    try {
        // Create a schema that has the fields modified
        final RecordSchema schemaV1 = ProvenanceEventSchema.PROVENANCE_EVENT_SCHEMA_V1;
        final List<RecordField> fields = new ArrayList<>(schemaV1.getFields());
        fields.remove(new SimpleRecordField(EventFieldNames.UPDATED_ATTRIBUTES, FieldType.STRING, Repetition.EXACTLY_ONE));
        fields.remove(new SimpleRecordField(EventFieldNames.PREVIOUS_ATTRIBUTES, FieldType.STRING, Repetition.EXACTLY_ONE));
        final RecordSchema recordSchema = new RecordSchema(fields);
        // Create a record writer whose schema does not contain updated attributes or previous attributes.
        // This means that we must also override the method that writes out attributes so that we are able
        // to avoid actually writing them out.
        final ByteArraySchemaRecordWriter writer = new ByteArraySchemaRecordWriter(journalFile, idGenerator, tocWriter, false, 0) {

            @Override
            public void writeHeader(long firstEventId, DataOutputStream out) throws IOException {
                final ByteArrayOutputStream baos = new ByteArrayOutputStream();
                recordSchema.writeTo(baos);
                out.writeInt(baos.size());
                baos.writeTo(out);
            }

            @Override
            protected Record createRecord(final ProvenanceEventRecord event, final long eventId) {
                final RecordSchema contentClaimSchema = new RecordSchema(recordSchema.getField(EventFieldNames.CONTENT_CLAIM).getSubFields());
                return new EventRecord(event, eventId, recordSchema, contentClaimSchema);
            }
        };
        try {
            writer.writeHeader(1L);
            writer.writeRecord(createEvent());
            writer.writeRecord(createEvent());
        } finally {
            writer.close();
        }
    } finally {
        tocWriter.close();
    }
    // Read the records in and make sure that they have the info that we expect.
    try (final InputStream in = new FileInputStream(journalFile);
        final TocReader tocReader = new StandardTocReader(tocFile);
        final RecordReader reader = createReader(in, journalFile.getName(), tocReader, 10000)) {
        for (int i = 0; i < 2; i++) {
            final StandardProvenanceEventRecord event = reader.nextRecord();
            assertNotNull(event);
            assertEquals(ProvenanceEventType.RECEIVE, event.getEventType());
            // We will still have a Map<String, String> for updated attributes because the
            // Provenance Event Builder will create an empty map.
            assertNotNull(event.getUpdatedAttributes());
            assertTrue(event.getUpdatedAttributes().isEmpty());
        }
    }
}

Example 2 with EventRecord

use of org.apache.nifi.provenance.schema.EventRecord in project nifi by apache.

the class ByteArraySchemaRecordWriter method writeRecord.

@Override
protected void writeRecord(final ProvenanceEventRecord event, final long eventId, final DataOutputStream out) throws IOException {
    try (final ByteArrayOutputStream baos = new ByteArrayOutputStream(256)) {
        final Record eventRecord = createRecord(event, eventId);
        recordWriter.writeRecord(eventRecord, baos);
        out.writeInt(baos.size());
        baos.writeTo(out);
    }
}

Example 3 with EventRecord

use of org.apache.nifi.provenance.schema.EventRecord in project nifi by apache.

the class TestSchemaRecordReaderWriter method createSchemaWriter.

/**
 * Creates a SchemaRecordWriter that uses a modified schema
 *
 * @param fieldModifier the callback for modifying the schema
 * @return a SchemaRecordWriter that uses the modified schema
 * @throws IOException if unable to create the writer
 */
private ByteArraySchemaRecordWriter createSchemaWriter(final Consumer<List<RecordField>> fieldModifier, final Map<RecordField, Object> fieldsToAdd) throws IOException {
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // Create a schema that has the fields modified
    final RecordSchema schemaV1 = ProvenanceEventSchema.PROVENANCE_EVENT_SCHEMA_V1;
    final List<RecordField> fields = new ArrayList<>(schemaV1.getFields());
    fieldModifier.accept(fields);
    final RecordSchema recordSchema = new RecordSchema(fields);
    final RecordSchema contentClaimSchema = new RecordSchema(recordSchema.getField(EventFieldNames.CONTENT_CLAIM).getSubFields());
    final ByteArraySchemaRecordWriter writer = new ByteArraySchemaRecordWriter(journalFile, idGenerator, tocWriter, false, 0) {

        @Override
        public void writeHeader(long firstEventId, DataOutputStream out) throws IOException {
            final ByteArrayOutputStream baos = new ByteArrayOutputStream();
            recordSchema.writeTo(baos);
            out.writeInt(baos.size());
            baos.writeTo(out);
        }

        @Override
        protected Record createRecord(final ProvenanceEventRecord event, final long eventId) {
            final Map<RecordField, Object> values = new HashMap<>();
            final EventRecord eventRecord = new EventRecord(event, eventId, recordSchema, contentClaimSchema);
            for (final RecordField field : recordSchema.getFields()) {
                final Object value = eventRecord.getFieldValue(field);
                values.put(field, value);
            }
            values.putAll(fieldsToAdd);
            return new FieldMapRecord(values, recordSchema);
        }
    };
    return writer;
}

Example 4 with EventRecord

use of org.apache.nifi.provenance.schema.EventRecord in project nifi by apache.

the class ByteArraySchemaRecordReader method nextRecord.

@Override
protected StandardProvenanceEventRecord nextRecord(final DataInputStream in, final int serializationVersion) throws IOException {
    verifySerializationVersion(serializationVersion);
    final long byteOffset = getBytesConsumed();
    final int recordLength = in.readInt();
    final InputStream limitedIn = new LimitingInputStream(in, recordLength);
    final Record eventRecord = recordReader.readRecord(limitedIn);
    if (eventRecord == null) {
        return null;
    }
    return EventRecord.getEvent(eventRecord, getFilename(), byteOffset, getMaxAttributeLength());
}