Examples with RecordField org.apache.nifi.repository.schema.RecordField used on opensource projects

Example 1 with RecordField

use of org.apache.nifi.repository.schema.RecordField in project nifi by apache.

the class TestSchemaRecordReaderWriter method testFieldRemovedFromSchema.

@Test
public void testFieldRemovedFromSchema() throws IOException {
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    try {
        // Create a schema that has the fields modified
        final RecordSchema schemaV1 = ProvenanceEventSchema.PROVENANCE_EVENT_SCHEMA_V1;
        final List<RecordField> fields = new ArrayList<>(schemaV1.getFields());
        fields.remove(new SimpleRecordField(EventFieldNames.UPDATED_ATTRIBUTES, FieldType.STRING, Repetition.EXACTLY_ONE));
        fields.remove(new SimpleRecordField(EventFieldNames.PREVIOUS_ATTRIBUTES, FieldType.STRING, Repetition.EXACTLY_ONE));
        final RecordSchema recordSchema = new RecordSchema(fields);
        // Create a record writer whose schema does not contain updated attributes or previous attributes.
        // This means that we must also override the method that writes out attributes so that we are able
        // to avoid actually writing them out.
        final ByteArraySchemaRecordWriter writer = new ByteArraySchemaRecordWriter(journalFile, idGenerator, tocWriter, false, 0) {

            @Override
            public void writeHeader(long firstEventId, DataOutputStream out) throws IOException {
                final ByteArrayOutputStream baos = new ByteArrayOutputStream();
                recordSchema.writeTo(baos);
                out.writeInt(baos.size());
                baos.writeTo(out);
            }

            @Override
            protected Record createRecord(final ProvenanceEventRecord event, final long eventId) {
                final RecordSchema contentClaimSchema = new RecordSchema(recordSchema.getField(EventFieldNames.CONTENT_CLAIM).getSubFields());
                return new EventRecord(event, eventId, recordSchema, contentClaimSchema);
            }
        };
        try {
            writer.writeHeader(1L);
            writer.writeRecord(createEvent());
            writer.writeRecord(createEvent());
        } finally {
            writer.close();
        }
    } finally {
        tocWriter.close();
    }
    // Read the records in and make sure that they have the info that we expect.
    try (final InputStream in = new FileInputStream(journalFile);
        final TocReader tocReader = new StandardTocReader(tocFile);
        final RecordReader reader = createReader(in, journalFile.getName(), tocReader, 10000)) {
        for (int i = 0; i < 2; i++) {
            final StandardProvenanceEventRecord event = reader.nextRecord();
            assertNotNull(event);
            assertEquals(ProvenanceEventType.RECEIVE, event.getEventType());
            // We will still have a Map<String, String> for updated attributes because the
            // Provenance Event Builder will create an empty map.
            assertNotNull(event.getUpdatedAttributes());
            assertTrue(event.getUpdatedAttributes().isEmpty());
        }
    }
}

Example 2 with RecordField

use of org.apache.nifi.repository.schema.RecordField in project nifi by apache.

the class TestSchemaRecordReaderWriter method testAddOneRecordReadTwice.

@Test
public void testAddOneRecordReadTwice() throws IOException {
    final RecordField unitTestField = new SimpleRecordField("Unit Test Field", FieldType.STRING, Repetition.EXACTLY_ONE);
    final Consumer<List<RecordField>> schemaModifier = fields -> fields.add(unitTestField);
    final Map<RecordField, Object> toAdd = new HashMap<>();
    toAdd.put(unitTestField, "hello");
    try (final ByteArraySchemaRecordWriter writer = createSchemaWriter(schemaModifier, toAdd)) {
        writer.writeHeader(1L);
        writer.writeRecord(createEvent());
    }
    try (final InputStream in = new FileInputStream(journalFile);
        final TocReader tocReader = new StandardTocReader(tocFile);
        final RecordReader reader = createReader(in, journalFile.getName(), tocReader, 10000)) {
        final ProvenanceEventRecord firstEvent = reader.nextRecord();
        assertNotNull(firstEvent);
        final ProvenanceEventRecord secondEvent = reader.nextRecord();
        assertNull(secondEvent);
    }
}

Example 3 with RecordField

use of org.apache.nifi.repository.schema.RecordField in project nifi by apache.

the class EventIdFirstHeaderSchema method buildSchema.

private static RecordSchema buildSchema() {
    final List<RecordField> fields = new ArrayList<>();
    fields.add(new SimpleRecordField(FieldNames.FIRST_EVENT_ID, FieldType.LONG, Repetition.EXACTLY_ONE));
    fields.add(new SimpleRecordField(FieldNames.TIMESTAMP_OFFSET, FieldType.LONG, Repetition.EXACTLY_ONE));
    fields.add(new SimpleRecordField(FieldNames.COMPONENT_IDS, FieldType.STRING, Repetition.ZERO_OR_MORE));
    fields.add(new SimpleRecordField(FieldNames.COMPONENT_TYPES, FieldType.STRING, Repetition.ZERO_OR_MORE));
    fields.add(new SimpleRecordField(FieldNames.QUEUE_IDS, FieldType.STRING, Repetition.ZERO_OR_MORE));
    fields.add(new SimpleRecordField(FieldNames.EVENT_TYPES, FieldType.STRING, Repetition.ZERO_OR_MORE));
    return new RecordSchema(fields);
}

Example 4 with RecordField

use of org.apache.nifi.repository.schema.RecordField in project nifi by apache.

the class LookupTableEventRecord method createLookupValue.

private NamedValue createLookupValue(final String literalValue, final Map<String, Integer> lookup) {
    if (literalValue == null) {
        final Map<RecordField, Object> lookupValues = Collections.singletonMap(LookupTableEventRecordFields.NO_VALUE, EventFieldNames.NO_VALUE);
        final Record record = new FieldMapRecord(lookupValues, LookupTableEventSchema.NO_VALUE_SCHEMA);
        final NamedValue namedValue = new NamedValue(EventFieldNames.NO_VALUE, record);
        return namedValue;
    }
    final Integer index = lookup.get(literalValue);
    if (index == null) {
        final Map<RecordField, Object> lookupValues = Collections.singletonMap(LookupTableEventRecordFields.EXPLICIT_STRING, literalValue);
        final Record record = new FieldMapRecord(lookupValues, LookupTableEventSchema.EXPLICIT_STRING_SCHEMA);
        final NamedValue namedValue = new NamedValue(EventFieldNames.EXPLICIT_VALUE, record);
        return namedValue;
    } else {
        final Map<RecordField, Object> lookupValues = Collections.singletonMap(LookupTableEventRecordFields.LOOKUP_VALUE, index);
        final Record record = new FieldMapRecord(lookupValues, LookupTableEventSchema.LOOKUP_VALUE_SCHEMA);
        final NamedValue namedValue = new NamedValue(EventFieldNames.LOOKUP_VALUE, record);
        return namedValue;
    }
}

Example 5 with RecordField

use of org.apache.nifi.repository.schema.RecordField in project nifi by apache.

the class LookupTableEventSchema method buildSchemaV1.

private static RecordSchema buildSchemaV1(final boolean includeEventId) {
    final List<RecordField> fields = new ArrayList<>();
    if (includeEventId) {
        fields.add(RECORD_IDENTIFIER_OFFSET);
    }
    fields.add(EVENT_TYPE_ORDINAL);
    fields.add(EVENT_TIME_OFFSET);
    fields.add(FLOWFILE_ENTRY_DATE_OFFSET);
    fields.add(EVENT_DURATION);
    fields.add(LINEAGE_START_DATE_OFFSET);
    fields.add(COMPONENT_ID);
    fields.add(COMPONENT_TYPE);
    fields.add(EVENT_DETAILS);
    fields.add(PREVIOUS_ATTRIBUTES);
    fields.add(UPDATED_ATTRIBUTES);
    fields.add(CURRENT_CONTENT_CLAIM);
    fields.add(PREVIOUS_CONTENT_CLAIM);
    fields.add(SOURCE_QUEUE_ID);
    // EventType-Specific fields
    // for FORK, JOIN, CLONE, REPLAY events
    fields.add(PARENT_UUIDS);
    // for FORK, JOIN, CLONE, REPLAY events
    fields.add(CHILD_UUIDS);
    // for SEND/RECEIVE/FETCH events
    fields.add(TRANSIT_URI);
    // for SEND/RECEIVE events
    fields.add(SOURCE_SYSTEM_FLOWFILE_IDENTIFIER);
    // for ADD_INFO events
    fields.add(ALTERNATE_IDENTIFIER);
    // for ROUTE events
    fields.add(RELATIONSHIP);
    final RecordSchema schema = new RecordSchema(fields);
    return schema;
}