Search in sources :

Example 6 with RecordSchema

use of org.apache.nifi.repository.schema.RecordSchema in project nifi by apache.

the class SchemaSwapDeserializer method getSwapSummary.

@Override
public SwapSummary getSwapSummary(final DataInputStream in, final String swapLocation, final ResourceClaimManager claimManager) throws IOException {
    final RecordSchema schema = RecordSchema.readFrom(in);
    final List<RecordField> summaryFields = schema.getField(SwapSchema.SWAP_SUMMARY).getSubFields();
    final RecordField summaryRecordField = new ComplexRecordField(SwapSchema.SWAP_SUMMARY, Repetition.EXACTLY_ONE, summaryFields);
    final RecordSchema summarySchema = new RecordSchema(Collections.singletonList(summaryRecordField));
    final Record summaryRecordParent = SchemaRecordReader.fromSchema(summarySchema).readRecord(in);
    final Record summaryRecord = (Record) summaryRecordParent.getFieldValue(SwapSchema.SWAP_SUMMARY);
    final SwapSummary swapSummary = SwapSummaryFieldMap.getSwapSummary(summaryRecord, claimManager);
    return swapSummary;
}
Also used : RecordField(org.apache.nifi.repository.schema.RecordField) ComplexRecordField(org.apache.nifi.repository.schema.ComplexRecordField) SwapSummary(org.apache.nifi.controller.repository.SwapSummary) Record(org.apache.nifi.repository.schema.Record) FlowFileRecord(org.apache.nifi.controller.repository.FlowFileRecord) ComplexRecordField(org.apache.nifi.repository.schema.ComplexRecordField) RecordSchema(org.apache.nifi.repository.schema.RecordSchema)

Example 7 with RecordSchema

use of org.apache.nifi.repository.schema.RecordSchema in project nifi by apache.

the class SchemaSwapSerializer method serializeFlowFiles.

@Override
public void serializeFlowFiles(final List<FlowFileRecord> toSwap, final FlowFileQueue queue, final String swapLocation, final OutputStream out) throws IOException {
    schema.writeTo(out);
    long contentSize = 0L;
    long maxFlowFileId = -1L;
    final List<ResourceClaim> resourceClaims = new ArrayList<>();
    for (final FlowFileRecord flowFile : toSwap) {
        contentSize += flowFile.getSize();
        if (flowFile.getId() > maxFlowFileId) {
            maxFlowFileId = flowFile.getId();
        }
        final ContentClaim contentClaim = flowFile.getContentClaim();
        if (contentClaim != null) {
            resourceClaims.add(contentClaim.getResourceClaim());
        }
    }
    final QueueSize queueSize = new QueueSize(toSwap.size(), contentSize);
    final SwapSummary swapSummary = new StandardSwapSummary(queueSize, maxFlowFileId, resourceClaims);
    final Record summaryRecord = new SwapSummaryFieldMap(swapSummary, queue.getIdentifier(), SwapSchema.SWAP_SUMMARY_SCHEMA_V1);
    final List<Record> flowFileRecords = toSwap.stream().map(flowFile -> new FlowFileRecordFieldMap(flowFile, flowFileSchema)).collect(Collectors.toList());
    // Create a simple record to hold the summary and the flowfile contents
    final RecordField summaryField = new SimpleRecordField(SwapSchema.SWAP_SUMMARY, FieldType.COMPLEX, Repetition.EXACTLY_ONE);
    final RecordField contentsField = new ComplexRecordField(SwapSchema.FLOWFILE_CONTENTS, Repetition.ZERO_OR_MORE, FlowFileSchema.FLOWFILE_SCHEMA_V2.getFields());
    final List<RecordField> fields = new ArrayList<>(2);
    fields.add(summaryField);
    fields.add(contentsField);
    final Map<RecordField, Object> swapFileMap = new LinkedHashMap<>();
    swapFileMap.put(summaryField, summaryRecord);
    swapFileMap.put(contentsField, flowFileRecords);
    final Record swapFileRecord = new FieldMapRecord(swapFileMap, new RecordSchema(fields));
    final SchemaRecordWriter writer = new SchemaRecordWriter();
    writer.writeRecord(swapFileRecord, out);
    out.flush();
}
Also used : OutputStream(java.io.OutputStream) ContentClaim(org.apache.nifi.controller.repository.claim.ContentClaim) FlowFileSchema(org.apache.nifi.controller.repository.schema.FlowFileSchema) Record(org.apache.nifi.repository.schema.Record) RecordField(org.apache.nifi.repository.schema.RecordField) ComplexRecordField(org.apache.nifi.repository.schema.ComplexRecordField) FlowFileRecord(org.apache.nifi.controller.repository.FlowFileRecord) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) QueueSize(org.apache.nifi.controller.queue.QueueSize) ArrayList(java.util.ArrayList) FieldMapRecord(org.apache.nifi.repository.schema.FieldMapRecord) RecordSchema(org.apache.nifi.repository.schema.RecordSchema) LinkedHashMap(java.util.LinkedHashMap) SwapSummary(org.apache.nifi.controller.repository.SwapSummary) FieldType(org.apache.nifi.repository.schema.FieldType) List(java.util.List) ResourceClaim(org.apache.nifi.controller.repository.claim.ResourceClaim) Map(java.util.Map) FlowFileRecordFieldMap(org.apache.nifi.controller.repository.schema.FlowFileRecordFieldMap) SimpleRecordField(org.apache.nifi.repository.schema.SimpleRecordField) Repetition(org.apache.nifi.repository.schema.Repetition) SchemaRecordWriter(org.apache.nifi.repository.schema.SchemaRecordWriter) FlowFileQueue(org.apache.nifi.controller.queue.FlowFileQueue) FieldMapRecord(org.apache.nifi.repository.schema.FieldMapRecord) RecordField(org.apache.nifi.repository.schema.RecordField) ComplexRecordField(org.apache.nifi.repository.schema.ComplexRecordField) SimpleRecordField(org.apache.nifi.repository.schema.SimpleRecordField) ArrayList(java.util.ArrayList) SwapSummary(org.apache.nifi.controller.repository.SwapSummary) LinkedHashMap(java.util.LinkedHashMap) QueueSize(org.apache.nifi.controller.queue.QueueSize) ContentClaim(org.apache.nifi.controller.repository.claim.ContentClaim) SimpleRecordField(org.apache.nifi.repository.schema.SimpleRecordField) FlowFileRecordFieldMap(org.apache.nifi.controller.repository.schema.FlowFileRecordFieldMap) ResourceClaim(org.apache.nifi.controller.repository.claim.ResourceClaim) Record(org.apache.nifi.repository.schema.Record) FlowFileRecord(org.apache.nifi.controller.repository.FlowFileRecord) FieldMapRecord(org.apache.nifi.repository.schema.FieldMapRecord) ComplexRecordField(org.apache.nifi.repository.schema.ComplexRecordField) FlowFileRecord(org.apache.nifi.controller.repository.FlowFileRecord) RecordSchema(org.apache.nifi.repository.schema.RecordSchema) SchemaRecordWriter(org.apache.nifi.repository.schema.SchemaRecordWriter)

Example 8 with RecordSchema

use of org.apache.nifi.repository.schema.RecordSchema in project nifi by apache.

the class SchemaSwapDeserializer method deserializeFlowFiles.

@Override
@SuppressWarnings("unchecked")
public SwapContents deserializeFlowFiles(final DataInputStream in, final String swapLocation, final FlowFileQueue queue, final ResourceClaimManager claimManager) throws IOException {
    final RecordSchema schema = RecordSchema.readFrom(in);
    final SchemaRecordReader reader = SchemaRecordReader.fromSchema(schema);
    final Record parentRecord = reader.readRecord(in);
    final List<Record> flowFileRecords = (List<Record>) parentRecord.getFieldValue(SwapSchema.FLOWFILE_CONTENTS);
    final List<FlowFileRecord> flowFiles = new ArrayList<>(flowFileRecords.size());
    for (final Record record : flowFileRecords) {
        flowFiles.add(FlowFileRecordFieldMap.getFlowFile(record, claimManager));
    }
    final Record summaryRecord = (Record) parentRecord.getFieldValue(SwapSchema.SWAP_SUMMARY);
    final SwapSummary swapSummary = SwapSummaryFieldMap.getSwapSummary(summaryRecord, claimManager);
    return new StandardSwapContents(swapSummary, flowFiles);
}
Also used : ArrayList(java.util.ArrayList) SwapSummary(org.apache.nifi.controller.repository.SwapSummary) Record(org.apache.nifi.repository.schema.Record) FlowFileRecord(org.apache.nifi.controller.repository.FlowFileRecord) ArrayList(java.util.ArrayList) List(java.util.List) FlowFileRecord(org.apache.nifi.controller.repository.FlowFileRecord) RecordSchema(org.apache.nifi.repository.schema.RecordSchema) SchemaRecordReader(org.apache.nifi.repository.schema.SchemaRecordReader)

Example 9 with RecordSchema

use of org.apache.nifi.repository.schema.RecordSchema in project nifi by apache.

the class SchemaRepositoryRecordSerde method serializeRecord.

@Override
public void serializeRecord(final RepositoryRecord record, final DataOutputStream out) throws IOException {
    final RecordSchema schema;
    switch(record.getType()) {
        case CREATE:
        case UPDATE:
            schema = RepositoryRecordSchema.CREATE_OR_UPDATE_SCHEMA_V2;
            break;
        case CONTENTMISSING:
        case DELETE:
            schema = RepositoryRecordSchema.DELETE_SCHEMA_V2;
            break;
        case SWAP_IN:
            schema = RepositoryRecordSchema.SWAP_IN_SCHEMA_V2;
            break;
        case SWAP_OUT:
            schema = RepositoryRecordSchema.SWAP_OUT_SCHEMA_V2;
            break;
        default:
            // won't happen.
            throw new IllegalArgumentException("Received Repository Record with unknown Update Type: " + record.getType());
    }
    serializeRecord(record, out, schema, RepositoryRecordSchema.REPOSITORY_RECORD_SCHEMA_V2);
}
Also used : RepositoryRecordSchema(org.apache.nifi.controller.repository.schema.RepositoryRecordSchema) RecordSchema(org.apache.nifi.repository.schema.RecordSchema)

Example 10 with RecordSchema

use of org.apache.nifi.repository.schema.RecordSchema in project nifi by apache.

the class TestSchemaRecordReaderWriter method createSchemaWriter.

/**
 * Creates a SchemaRecordWriter that uses a modified schema
 *
 * @param fieldModifier the callback for modifying the schema
 * @return a SchemaRecordWriter that uses the modified schema
 * @throws IOException if unable to create the writer
 */
private ByteArraySchemaRecordWriter createSchemaWriter(final Consumer<List<RecordField>> fieldModifier, final Map<RecordField, Object> fieldsToAdd) throws IOException {
    final TocWriter tocWriter = new StandardTocWriter(tocFile, false, false);
    // Create a schema that has the fields modified
    final RecordSchema schemaV1 = ProvenanceEventSchema.PROVENANCE_EVENT_SCHEMA_V1;
    final List<RecordField> fields = new ArrayList<>(schemaV1.getFields());
    fieldModifier.accept(fields);
    final RecordSchema recordSchema = new RecordSchema(fields);
    final RecordSchema contentClaimSchema = new RecordSchema(recordSchema.getField(EventFieldNames.CONTENT_CLAIM).getSubFields());
    final ByteArraySchemaRecordWriter writer = new ByteArraySchemaRecordWriter(journalFile, idGenerator, tocWriter, false, 0) {

        @Override
        public void writeHeader(long firstEventId, DataOutputStream out) throws IOException {
            final ByteArrayOutputStream baos = new ByteArrayOutputStream();
            recordSchema.writeTo(baos);
            out.writeInt(baos.size());
            baos.writeTo(out);
        }

        @Override
        protected Record createRecord(final ProvenanceEventRecord event, final long eventId) {
            final Map<RecordField, Object> values = new HashMap<>();
            final EventRecord eventRecord = new EventRecord(event, eventId, recordSchema, contentClaimSchema);
            for (final RecordField field : recordSchema.getFields()) {
                final Object value = eventRecord.getFieldValue(field);
                values.put(field, value);
            }
            values.putAll(fieldsToAdd);
            return new FieldMapRecord(values, recordSchema);
        }
    };
    return writer;
}
Also used : FieldMapRecord(org.apache.nifi.repository.schema.FieldMapRecord) RecordField(org.apache.nifi.repository.schema.RecordField) SimpleRecordField(org.apache.nifi.repository.schema.SimpleRecordField) HashMap(java.util.HashMap) DataOutputStream(java.io.DataOutputStream) ArrayList(java.util.ArrayList) ByteArrayOutputStream(java.io.ByteArrayOutputStream) StandardTocWriter(org.apache.nifi.provenance.toc.StandardTocWriter) StandardTocWriter(org.apache.nifi.provenance.toc.StandardTocWriter) NopTocWriter(org.apache.nifi.provenance.toc.NopTocWriter) TocWriter(org.apache.nifi.provenance.toc.TocWriter) EventRecord(org.apache.nifi.provenance.schema.EventRecord) RecordSchema(org.apache.nifi.repository.schema.RecordSchema)

Aggregations

RecordSchema (org.apache.nifi.repository.schema.RecordSchema)11 RecordField (org.apache.nifi.repository.schema.RecordField)8 ArrayList (java.util.ArrayList)7 Record (org.apache.nifi.repository.schema.Record)5 SimpleRecordField (org.apache.nifi.repository.schema.SimpleRecordField)4 FlowFileRecord (org.apache.nifi.controller.repository.FlowFileRecord)3 SwapSummary (org.apache.nifi.controller.repository.SwapSummary)3 FieldMapRecord (org.apache.nifi.repository.schema.FieldMapRecord)3 ByteArrayOutputStream (java.io.ByteArrayOutputStream)2 DataOutputStream (java.io.DataOutputStream)2 HashMap (java.util.HashMap)2 List (java.util.List)2 EventRecord (org.apache.nifi.provenance.schema.EventRecord)2 NopTocWriter (org.apache.nifi.provenance.toc.NopTocWriter)2 StandardTocWriter (org.apache.nifi.provenance.toc.StandardTocWriter)2 TocWriter (org.apache.nifi.provenance.toc.TocWriter)2 ComplexRecordField (org.apache.nifi.repository.schema.ComplexRecordField)2 SchemaRecordReader (org.apache.nifi.repository.schema.SchemaRecordReader)2 ByteArrayInputStream (java.io.ByteArrayInputStream)1 FileInputStream (java.io.FileInputStream)1