Search in sources :

Example 1 with FlowFileSchema

use of org.apache.nifi.controller.repository.schema.FlowFileSchema in project nifi by apache.

the class SchemaSwapSerializer method serializeFlowFiles.

@Override
public void serializeFlowFiles(final List<FlowFileRecord> toSwap, final FlowFileQueue queue, final String swapLocation, final OutputStream out) throws IOException {
    schema.writeTo(out);
    long contentSize = 0L;
    long maxFlowFileId = -1L;
    final List<ResourceClaim> resourceClaims = new ArrayList<>();
    for (final FlowFileRecord flowFile : toSwap) {
        contentSize += flowFile.getSize();
        if (flowFile.getId() > maxFlowFileId) {
            maxFlowFileId = flowFile.getId();
        }
        final ContentClaim contentClaim = flowFile.getContentClaim();
        if (contentClaim != null) {
            resourceClaims.add(contentClaim.getResourceClaim());
        }
    }
    final QueueSize queueSize = new QueueSize(toSwap.size(), contentSize);
    final SwapSummary swapSummary = new StandardSwapSummary(queueSize, maxFlowFileId, resourceClaims);
    final Record summaryRecord = new SwapSummaryFieldMap(swapSummary, queue.getIdentifier(), SwapSchema.SWAP_SUMMARY_SCHEMA_V1);
    final List<Record> flowFileRecords = toSwap.stream().map(flowFile -> new FlowFileRecordFieldMap(flowFile, flowFileSchema)).collect(Collectors.toList());
    // Create a simple record to hold the summary and the flowfile contents
    final RecordField summaryField = new SimpleRecordField(SwapSchema.SWAP_SUMMARY, FieldType.COMPLEX, Repetition.EXACTLY_ONE);
    final RecordField contentsField = new ComplexRecordField(SwapSchema.FLOWFILE_CONTENTS, Repetition.ZERO_OR_MORE, FlowFileSchema.FLOWFILE_SCHEMA_V2.getFields());
    final List<RecordField> fields = new ArrayList<>(2);
    fields.add(summaryField);
    fields.add(contentsField);
    final Map<RecordField, Object> swapFileMap = new LinkedHashMap<>();
    swapFileMap.put(summaryField, summaryRecord);
    swapFileMap.put(contentsField, flowFileRecords);
    final Record swapFileRecord = new FieldMapRecord(swapFileMap, new RecordSchema(fields));
    final SchemaRecordWriter writer = new SchemaRecordWriter();
    writer.writeRecord(swapFileRecord, out);
    out.flush();
}
Also used : OutputStream(java.io.OutputStream) ContentClaim(org.apache.nifi.controller.repository.claim.ContentClaim) FlowFileSchema(org.apache.nifi.controller.repository.schema.FlowFileSchema) Record(org.apache.nifi.repository.schema.Record) RecordField(org.apache.nifi.repository.schema.RecordField) ComplexRecordField(org.apache.nifi.repository.schema.ComplexRecordField) FlowFileRecord(org.apache.nifi.controller.repository.FlowFileRecord) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) QueueSize(org.apache.nifi.controller.queue.QueueSize) ArrayList(java.util.ArrayList) FieldMapRecord(org.apache.nifi.repository.schema.FieldMapRecord) RecordSchema(org.apache.nifi.repository.schema.RecordSchema) LinkedHashMap(java.util.LinkedHashMap) SwapSummary(org.apache.nifi.controller.repository.SwapSummary) FieldType(org.apache.nifi.repository.schema.FieldType) List(java.util.List) ResourceClaim(org.apache.nifi.controller.repository.claim.ResourceClaim) Map(java.util.Map) FlowFileRecordFieldMap(org.apache.nifi.controller.repository.schema.FlowFileRecordFieldMap) SimpleRecordField(org.apache.nifi.repository.schema.SimpleRecordField) Repetition(org.apache.nifi.repository.schema.Repetition) SchemaRecordWriter(org.apache.nifi.repository.schema.SchemaRecordWriter) FlowFileQueue(org.apache.nifi.controller.queue.FlowFileQueue) FieldMapRecord(org.apache.nifi.repository.schema.FieldMapRecord) RecordField(org.apache.nifi.repository.schema.RecordField) ComplexRecordField(org.apache.nifi.repository.schema.ComplexRecordField) SimpleRecordField(org.apache.nifi.repository.schema.SimpleRecordField) ArrayList(java.util.ArrayList) SwapSummary(org.apache.nifi.controller.repository.SwapSummary) LinkedHashMap(java.util.LinkedHashMap) QueueSize(org.apache.nifi.controller.queue.QueueSize) ContentClaim(org.apache.nifi.controller.repository.claim.ContentClaim) SimpleRecordField(org.apache.nifi.repository.schema.SimpleRecordField) FlowFileRecordFieldMap(org.apache.nifi.controller.repository.schema.FlowFileRecordFieldMap) ResourceClaim(org.apache.nifi.controller.repository.claim.ResourceClaim) Record(org.apache.nifi.repository.schema.Record) FlowFileRecord(org.apache.nifi.controller.repository.FlowFileRecord) FieldMapRecord(org.apache.nifi.repository.schema.FieldMapRecord) ComplexRecordField(org.apache.nifi.repository.schema.ComplexRecordField) FlowFileRecord(org.apache.nifi.controller.repository.FlowFileRecord) RecordSchema(org.apache.nifi.repository.schema.RecordSchema) SchemaRecordWriter(org.apache.nifi.repository.schema.SchemaRecordWriter)

Aggregations

IOException (java.io.IOException)1 OutputStream (java.io.OutputStream)1 ArrayList (java.util.ArrayList)1 LinkedHashMap (java.util.LinkedHashMap)1 List (java.util.List)1 Map (java.util.Map)1 Collectors (java.util.stream.Collectors)1 FlowFileQueue (org.apache.nifi.controller.queue.FlowFileQueue)1 QueueSize (org.apache.nifi.controller.queue.QueueSize)1 FlowFileRecord (org.apache.nifi.controller.repository.FlowFileRecord)1 SwapSummary (org.apache.nifi.controller.repository.SwapSummary)1 ContentClaim (org.apache.nifi.controller.repository.claim.ContentClaim)1 ResourceClaim (org.apache.nifi.controller.repository.claim.ResourceClaim)1 FlowFileRecordFieldMap (org.apache.nifi.controller.repository.schema.FlowFileRecordFieldMap)1 FlowFileSchema (org.apache.nifi.controller.repository.schema.FlowFileSchema)1 ComplexRecordField (org.apache.nifi.repository.schema.ComplexRecordField)1 FieldMapRecord (org.apache.nifi.repository.schema.FieldMapRecord)1 FieldType (org.apache.nifi.repository.schema.FieldType)1 Record (org.apache.nifi.repository.schema.Record)1 RecordField (org.apache.nifi.repository.schema.RecordField)1