Examples with Record org.apache.nifi.repository.schema.Record used on opensource projects

Example 6 with Record

use of org.apache.nifi.repository.schema.Record in project nifi by apache.

the class SchemaSwapDeserializer method getSwapSummary.

@Override
public SwapSummary getSwapSummary(final DataInputStream in, final String swapLocation, final ResourceClaimManager claimManager) throws IOException {
    final RecordSchema schema = RecordSchema.readFrom(in);
    final List<RecordField> summaryFields = schema.getField(SwapSchema.SWAP_SUMMARY).getSubFields();
    final RecordField summaryRecordField = new ComplexRecordField(SwapSchema.SWAP_SUMMARY, Repetition.EXACTLY_ONE, summaryFields);
    final RecordSchema summarySchema = new RecordSchema(Collections.singletonList(summaryRecordField));
    final Record summaryRecordParent = SchemaRecordReader.fromSchema(summarySchema).readRecord(in);
    final Record summaryRecord = (Record) summaryRecordParent.getFieldValue(SwapSchema.SWAP_SUMMARY);
    final SwapSummary swapSummary = SwapSummaryFieldMap.getSwapSummary(summaryRecord, claimManager);
    return swapSummary;
}

Example 7 with Record

use of org.apache.nifi.repository.schema.Record in project nifi by apache.

the class SchemaSwapSerializer method serializeFlowFiles.

@Override
public void serializeFlowFiles(final List<FlowFileRecord> toSwap, final FlowFileQueue queue, final String swapLocation, final OutputStream out) throws IOException {
    schema.writeTo(out);
    long contentSize = 0L;
    long maxFlowFileId = -1L;
    final List<ResourceClaim> resourceClaims = new ArrayList<>();
    for (final FlowFileRecord flowFile : toSwap) {
        contentSize += flowFile.getSize();
        if (flowFile.getId() > maxFlowFileId) {
            maxFlowFileId = flowFile.getId();
        }
        final ContentClaim contentClaim = flowFile.getContentClaim();
        if (contentClaim != null) {
            resourceClaims.add(contentClaim.getResourceClaim());
        }
    }
    final QueueSize queueSize = new QueueSize(toSwap.size(), contentSize);
    final SwapSummary swapSummary = new StandardSwapSummary(queueSize, maxFlowFileId, resourceClaims);
    final Record summaryRecord = new SwapSummaryFieldMap(swapSummary, queue.getIdentifier(), SwapSchema.SWAP_SUMMARY_SCHEMA_V1);
    final List<Record> flowFileRecords = toSwap.stream().map(flowFile -> new FlowFileRecordFieldMap(flowFile, flowFileSchema)).collect(Collectors.toList());
    // Create a simple record to hold the summary and the flowfile contents
    final RecordField summaryField = new SimpleRecordField(SwapSchema.SWAP_SUMMARY, FieldType.COMPLEX, Repetition.EXACTLY_ONE);
    final RecordField contentsField = new ComplexRecordField(SwapSchema.FLOWFILE_CONTENTS, Repetition.ZERO_OR_MORE, FlowFileSchema.FLOWFILE_SCHEMA_V2.getFields());
    final List<RecordField> fields = new ArrayList<>(2);
    fields.add(summaryField);
    fields.add(contentsField);
    final Map<RecordField, Object> swapFileMap = new LinkedHashMap<>();
    swapFileMap.put(summaryField, summaryRecord);
    swapFileMap.put(contentsField, flowFileRecords);
    final Record swapFileRecord = new FieldMapRecord(swapFileMap, new RecordSchema(fields));
    final SchemaRecordWriter writer = new SchemaRecordWriter();
    writer.writeRecord(swapFileRecord, out);
    out.flush();
}

Example 8 with Record

use of org.apache.nifi.repository.schema.Record in project nifi by apache.

the class SwapSummaryFieldMap method getSwapSummary.

@SuppressWarnings("unchecked")
public static SwapSummary getSwapSummary(final Record record, final ResourceClaimManager claimManager) {
    final int flowFileCount = (Integer) record.getFieldValue(SwapSchema.FLOWFILE_COUNT);
    final long flowFileSize = (Long) record.getFieldValue(SwapSchema.FLOWFILE_SIZE);
    final QueueSize queueSize = new QueueSize(flowFileCount, flowFileSize);
    final long maxFlowFileId = (Long) record.getFieldValue(SwapSchema.MAX_RECORD_ID);
    final Map<Record, Integer> resourceClaimRecords = (Map<Record, Integer>) record.getFieldValue(SwapSchema.RESOURCE_CLAIMS);
    final List<ResourceClaim> resourceClaims = new ArrayList<>();
    for (final Map.Entry<Record, Integer> entry : resourceClaimRecords.entrySet()) {
        final Record resourceClaimRecord = entry.getKey();
        final ResourceClaim claim = ResourceClaimFieldMap.getResourceClaim(resourceClaimRecord, claimManager);
        for (int i = 0; i < entry.getValue(); i++) {
            resourceClaims.add(claim);
        }
    }
    return new StandardSwapSummary(queueSize, maxFlowFileId, resourceClaims);
}

Example 9 with Record

use of org.apache.nifi.repository.schema.Record in project nifi by apache.

the class FlowFileRecordFieldMap method getFlowFile.

@SuppressWarnings("unchecked")
public static FlowFileRecord getFlowFile(final Record record, final ResourceClaimManager claimManager) {
    final StandardFlowFileRecord.Builder builder = new StandardFlowFileRecord.Builder();
    builder.id((Long) record.getFieldValue(FlowFileSchema.RECORD_ID));
    builder.entryDate((Long) record.getFieldValue(FlowFileSchema.ENTRY_DATE));
    builder.size((Long) record.getFieldValue(FlowFileSchema.FLOWFILE_SIZE));
    builder.addAttributes((Map<String, String>) record.getFieldValue(FlowFileSchema.ATTRIBUTES));
    builder.lineageStart((Long) record.getFieldValue(FlowFileSchema.LINEAGE_START_DATE), (Long) record.getFieldValue(FlowFileSchema.LINEAGE_START_INDEX));
    builder.lastQueued((Long) record.getFieldValue(FlowFileSchema.QUEUE_DATE), (Long) record.getFieldValue(FlowFileSchema.QUEUE_DATE_INDEX));
    final Record contentClaimRecord = (Record) record.getFieldValue(FlowFileSchema.CONTENT_CLAIM);
    if (contentClaimRecord != null) {
        final ContentClaim claim = ContentClaimFieldMap.getContentClaim(contentClaimRecord, claimManager);
        builder.contentClaim(claim);
        final Long offset = ContentClaimFieldMap.getContentClaimOffset(contentClaimRecord);
        if (offset != null) {
            builder.contentClaimOffset(offset);
        }
    }
    return builder.build();
}

Example 10 with Record

use of org.apache.nifi.repository.schema.Record in project nifi by apache.

the class SchemaSwapDeserializer method deserializeFlowFiles.

@Override
@SuppressWarnings("unchecked")
public SwapContents deserializeFlowFiles(final DataInputStream in, final String swapLocation, final FlowFileQueue queue, final ResourceClaimManager claimManager) throws IOException {
    final RecordSchema schema = RecordSchema.readFrom(in);
    final SchemaRecordReader reader = SchemaRecordReader.fromSchema(schema);
    final Record parentRecord = reader.readRecord(in);
    final List<Record> flowFileRecords = (List<Record>) parentRecord.getFieldValue(SwapSchema.FLOWFILE_CONTENTS);
    final List<FlowFileRecord> flowFiles = new ArrayList<>(flowFileRecords.size());
    for (final Record record : flowFileRecords) {
        flowFiles.add(FlowFileRecordFieldMap.getFlowFile(record, claimManager));
    }
    final Record summaryRecord = (Record) parentRecord.getFieldValue(SwapSchema.SWAP_SUMMARY);
    final SwapSummary swapSummary = SwapSummaryFieldMap.getSwapSummary(summaryRecord, claimManager);
    return new StandardSwapContents(swapSummary, flowFiles);
}