use of org.apache.nifi.repository.schema.RecordSchema in project nifi by apache.
the class TestSchemaRecordReaderWriter method testFieldRemovedFromSchema.
@Test
public void testFieldRemovedFromSchema() throws IOException {
final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
try {
// Create a schema that has the fields modified
final RecordSchema schemaV1 = ProvenanceEventSchema.PROVENANCE_EVENT_SCHEMA_V1;
final List<RecordField> fields = new ArrayList<>(schemaV1.getFields());
fields.remove(new SimpleRecordField(EventFieldNames.UPDATED_ATTRIBUTES, FieldType.STRING, Repetition.EXACTLY_ONE));
fields.remove(new SimpleRecordField(EventFieldNames.PREVIOUS_ATTRIBUTES, FieldType.STRING, Repetition.EXACTLY_ONE));
final RecordSchema recordSchema = new RecordSchema(fields);
// Create a record writer whose schema does not contain updated attributes or previous attributes.
// This means that we must also override the method that writes out attributes so that we are able
// to avoid actually writing them out.
final ByteArraySchemaRecordWriter writer = new ByteArraySchemaRecordWriter(journalFile, idGenerator, tocWriter, false, 0) {
@Override
public void writeHeader(long firstEventId, DataOutputStream out) throws IOException {
final ByteArrayOutputStream baos = new ByteArrayOutputStream();
recordSchema.writeTo(baos);
out.writeInt(baos.size());
baos.writeTo(out);
}
@Override
protected Record createRecord(final ProvenanceEventRecord event, final long eventId) {
final RecordSchema contentClaimSchema = new RecordSchema(recordSchema.getField(EventFieldNames.CONTENT_CLAIM).getSubFields());
return new EventRecord(event, eventId, recordSchema, contentClaimSchema);
}
};
try {
writer.writeHeader(1L);
writer.writeRecord(createEvent());
writer.writeRecord(createEvent());
} finally {
writer.close();
}
} finally {
tocWriter.close();
}
// Read the records in and make sure that they have the info that we expect.
try (final InputStream in = new FileInputStream(journalFile);
final TocReader tocReader = new StandardTocReader(tocFile);
final RecordReader reader = createReader(in, journalFile.getName(), tocReader, 10000)) {
for (int i = 0; i < 2; i++) {
final StandardProvenanceEventRecord event = reader.nextRecord();
assertNotNull(event);
assertEquals(ProvenanceEventType.RECEIVE, event.getEventType());
// We will still have a Map<String, String> for updated attributes because the
// Provenance Event Builder will create an empty map.
assertNotNull(event.getUpdatedAttributes());
assertTrue(event.getUpdatedAttributes().isEmpty());
}
}
}
use of org.apache.nifi.repository.schema.RecordSchema in project nifi by apache.
the class EventIdFirstSchemaRecordReader method readHeader.
@Override
@SuppressWarnings("unchecked")
protected synchronized void readHeader(final DataInputStream in, final int serializationVersion) throws IOException {
verifySerializationVersion(serializationVersion);
final int eventSchemaLength = in.readInt();
final byte[] buffer = new byte[eventSchemaLength];
StreamUtils.fillBuffer(in, buffer);
try (final ByteArrayInputStream bais = new ByteArrayInputStream(buffer)) {
schema = RecordSchema.readFrom(bais);
}
recordReader = SchemaRecordReader.fromSchema(schema);
final int headerSchemaLength = in.readInt();
final byte[] headerSchemaBuffer = new byte[headerSchemaLength];
StreamUtils.fillBuffer(in, headerSchemaBuffer);
final RecordSchema headerSchema;
try (final ByteArrayInputStream bais = new ByteArrayInputStream(headerSchemaBuffer)) {
headerSchema = RecordSchema.readFrom(bais);
}
final SchemaRecordReader headerReader = SchemaRecordReader.fromSchema(headerSchema);
final Record headerRecord = headerReader.readRecord(in);
componentIds = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.COMPONENT_IDS);
componentTypes = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.COMPONENT_TYPES);
queueIds = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.QUEUE_IDS);
eventTypes = (List<String>) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.EVENT_TYPES);
firstEventId = (Long) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.FIRST_EVENT_ID);
systemTimeOffset = (Long) headerRecord.getFieldValue(EventIdFirstHeaderSchema.FieldNames.TIMESTAMP_OFFSET);
}
use of org.apache.nifi.repository.schema.RecordSchema in project nifi by apache.
the class EventIdFirstHeaderSchema method buildSchema.
private static RecordSchema buildSchema() {
final List<RecordField> fields = new ArrayList<>();
fields.add(new SimpleRecordField(FieldNames.FIRST_EVENT_ID, FieldType.LONG, Repetition.EXACTLY_ONE));
fields.add(new SimpleRecordField(FieldNames.TIMESTAMP_OFFSET, FieldType.LONG, Repetition.EXACTLY_ONE));
fields.add(new SimpleRecordField(FieldNames.COMPONENT_IDS, FieldType.STRING, Repetition.ZERO_OR_MORE));
fields.add(new SimpleRecordField(FieldNames.COMPONENT_TYPES, FieldType.STRING, Repetition.ZERO_OR_MORE));
fields.add(new SimpleRecordField(FieldNames.QUEUE_IDS, FieldType.STRING, Repetition.ZERO_OR_MORE));
fields.add(new SimpleRecordField(FieldNames.EVENT_TYPES, FieldType.STRING, Repetition.ZERO_OR_MORE));
return new RecordSchema(fields);
}
use of org.apache.nifi.repository.schema.RecordSchema in project nifi by apache.
the class LookupTableEventSchema method buildSchemaV1.
private static RecordSchema buildSchemaV1(final boolean includeEventId) {
final List<RecordField> fields = new ArrayList<>();
if (includeEventId) {
fields.add(RECORD_IDENTIFIER_OFFSET);
}
fields.add(EVENT_TYPE_ORDINAL);
fields.add(EVENT_TIME_OFFSET);
fields.add(FLOWFILE_ENTRY_DATE_OFFSET);
fields.add(EVENT_DURATION);
fields.add(LINEAGE_START_DATE_OFFSET);
fields.add(COMPONENT_ID);
fields.add(COMPONENT_TYPE);
fields.add(EVENT_DETAILS);
fields.add(PREVIOUS_ATTRIBUTES);
fields.add(UPDATED_ATTRIBUTES);
fields.add(CURRENT_CONTENT_CLAIM);
fields.add(PREVIOUS_CONTENT_CLAIM);
fields.add(SOURCE_QUEUE_ID);
// EventType-Specific fields
// for FORK, JOIN, CLONE, REPLAY events
fields.add(PARENT_UUIDS);
// for FORK, JOIN, CLONE, REPLAY events
fields.add(CHILD_UUIDS);
// for SEND/RECEIVE/FETCH events
fields.add(TRANSIT_URI);
// for SEND/RECEIVE events
fields.add(SOURCE_SYSTEM_FLOWFILE_IDENTIFIER);
// for ADD_INFO events
fields.add(ALTERNATE_IDENTIFIER);
// for ROUTE events
fields.add(RELATIONSHIP);
final RecordSchema schema = new RecordSchema(fields);
return schema;
}
use of org.apache.nifi.repository.schema.RecordSchema in project nifi by apache.
the class ProvenanceEventSchema method buildSchemaV1.
private static RecordSchema buildSchemaV1(final boolean includeEventId) {
final List<RecordField> fields = new ArrayList<>();
if (includeEventId) {
fields.add(RECORD_IDENTIFIER);
}
fields.add(EVENT_TYPE);
fields.add(EVENT_TIME);
fields.add(FLOWFILE_ENTRY_DATE);
fields.add(EVENT_DURATION);
fields.add(LINEAGE_START_DATE);
fields.add(COMPONENT_ID);
fields.add(COMPONENT_TYPE);
fields.add(FLOWFILE_UUID);
fields.add(EVENT_DETAILS);
fields.add(PREVIOUS_ATTRIBUTES);
fields.add(UPDATED_ATTRIBUTES);
fields.add(CURRENT_CONTENT_CLAIM);
fields.add(PREVIOUS_CONTENT_CLAIM);
fields.add(SOURCE_QUEUE_IDENTIFIER);
// EventType-Specific fields
// for FORK, JOIN, CLONE, REPLAY events
fields.add(PARENT_UUIDS);
// for FORK, JOIN, CLONE, REPLAY events
fields.add(CHILD_UUIDS);
// for SEND/RECEIVE/FETCH events
fields.add(TRANSIT_URI);
// for SEND/RECEIVE events
fields.add(SOURCE_SYSTEM_FLOWFILE_IDENTIFIER);
// for ADD_INFO events
fields.add(ALTERNATE_IDENTIFIER);
// for ROUTE events
fields.add(RELATIONSHIP);
final RecordSchema schema = new RecordSchema(fields);
return schema;
}
Aggregations