Examples with ResourceClaim org.apache.nifi.controller.repository.claim.ResourceClaim used on opensource projects

Example 11 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class WriteAheadFlowFileRepository method updateRepository.

private void updateRepository(final Collection<RepositoryRecord> records, final boolean sync) throws IOException {
    for (final RepositoryRecord record : records) {
        if (record.getType() != RepositoryRecordType.DELETE && record.getType() != RepositoryRecordType.CONTENTMISSING && record.getType() != RepositoryRecordType.CLEANUP_TRANSIENT_CLAIMS && record.getDestination() == null) {
            throw new IllegalArgumentException("Record " + record + " has no destination and Type is " + record.getType());
        }
    }
    // Partition records by whether or not their type is 'CLEANUP_TRANSIENT_CLAIMS'. We do this because we don't want to send
    // these types of records to the Write-Ahead Log.
    final Map<Boolean, List<RepositoryRecord>> partitionedRecords = records.stream().collect(Collectors.partitioningBy(record -> record.getType() == RepositoryRecordType.CLEANUP_TRANSIENT_CLAIMS));
    List<RepositoryRecord> recordsForWal = partitionedRecords.get(Boolean.FALSE);
    if (recordsForWal == null) {
        recordsForWal = Collections.emptyList();
    }
    // update the repository.
    final int partitionIndex = wal.update(recordsForWal, sync);
    // The below code is not entirely thread-safe, but we are OK with that because the results aren't really harmful.
    // Specifically, if two different threads call updateRepository with DELETE records for the same Content Claim,
    // it's quite possible for claimant count to be 0 below, which results in two different threads adding the Content
    // Claim to the 'claimsAwaitDestruction' map. As a result, we can call #markDestructable with the same ContentClaim
    // multiple times, and the #markDestructable method is not necessarily idempotent.
    // However, the result of this is that the FileSystem Repository may end up trying to remove the content multiple times.
    // This does not, however, cause problems, as ContentRepository should handle this
    // This does indicate that some refactoring should probably be performed, though, as this is not a very clean interface.
    final Set<ResourceClaim> claimsToAdd = new HashSet<>();
    for (final RepositoryRecord record : records) {
        if (record.getType() == RepositoryRecordType.DELETE) {
            // For any DELETE record that we have, if claim is destructible, mark it so
            if (record.getCurrentClaim() != null && isDestructable(record.getCurrentClaim())) {
                claimsToAdd.add(record.getCurrentClaim().getResourceClaim());
            }
            // If the original claim is different than the current claim and the original claim is destructible, mark it so
            if (record.getOriginalClaim() != null && !record.getOriginalClaim().equals(record.getCurrentClaim()) && isDestructable(record.getOriginalClaim())) {
                claimsToAdd.add(record.getOriginalClaim().getResourceClaim());
            }
        } else if (record.getType() == RepositoryRecordType.UPDATE) {
            // if we have an update, and the original is no longer needed, mark original as destructible
            if (record.getOriginalClaim() != null && record.getCurrentClaim() != record.getOriginalClaim() && isDestructable(record.getOriginalClaim())) {
                claimsToAdd.add(record.getOriginalClaim().getResourceClaim());
            }
        }
        final List<ContentClaim> transientClaims = record.getTransientClaims();
        if (transientClaims != null) {
            for (final ContentClaim transientClaim : transientClaims) {
                if (isDestructable(transientClaim)) {
                    claimsToAdd.add(transientClaim.getResourceClaim());
                }
            }
        }
    }
    if (!claimsToAdd.isEmpty()) {
        // Get / Register a Set<ContentClaim> for the given Partiton Index
        final Integer partitionKey = Integer.valueOf(partitionIndex);
        BlockingQueue<ResourceClaim> claimQueue = claimsAwaitingDestruction.get(partitionKey);
        if (claimQueue == null) {
            claimQueue = new LinkedBlockingQueue<>();
            final BlockingQueue<ResourceClaim> existingClaimQueue = claimsAwaitingDestruction.putIfAbsent(partitionKey, claimQueue);
            if (existingClaimQueue != null) {
                claimQueue = existingClaimQueue;
            }
        }
        claimQueue.addAll(claimsToAdd);
    }
}

Example 12 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class SchemaSwapSerializer method serializeFlowFiles.

@Override
public void serializeFlowFiles(final List<FlowFileRecord> toSwap, final FlowFileQueue queue, final String swapLocation, final OutputStream out) throws IOException {
    schema.writeTo(out);
    long contentSize = 0L;
    long maxFlowFileId = -1L;
    final List<ResourceClaim> resourceClaims = new ArrayList<>();
    for (final FlowFileRecord flowFile : toSwap) {
        contentSize += flowFile.getSize();
        if (flowFile.getId() > maxFlowFileId) {
            maxFlowFileId = flowFile.getId();
        }
        final ContentClaim contentClaim = flowFile.getContentClaim();
        if (contentClaim != null) {
            resourceClaims.add(contentClaim.getResourceClaim());
        }
    }
    final QueueSize queueSize = new QueueSize(toSwap.size(), contentSize);
    final SwapSummary swapSummary = new StandardSwapSummary(queueSize, maxFlowFileId, resourceClaims);
    final Record summaryRecord = new SwapSummaryFieldMap(swapSummary, queue.getIdentifier(), SwapSchema.SWAP_SUMMARY_SCHEMA_V1);
    final List<Record> flowFileRecords = toSwap.stream().map(flowFile -> new FlowFileRecordFieldMap(flowFile, flowFileSchema)).collect(Collectors.toList());
    // Create a simple record to hold the summary and the flowfile contents
    final RecordField summaryField = new SimpleRecordField(SwapSchema.SWAP_SUMMARY, FieldType.COMPLEX, Repetition.EXACTLY_ONE);
    final RecordField contentsField = new ComplexRecordField(SwapSchema.FLOWFILE_CONTENTS, Repetition.ZERO_OR_MORE, FlowFileSchema.FLOWFILE_SCHEMA_V2.getFields());
    final List<RecordField> fields = new ArrayList<>(2);
    fields.add(summaryField);
    fields.add(contentsField);
    final Map<RecordField, Object> swapFileMap = new LinkedHashMap<>();
    swapFileMap.put(summaryField, summaryRecord);
    swapFileMap.put(contentsField, flowFileRecords);
    final Record swapFileRecord = new FieldMapRecord(swapFileMap, new RecordSchema(fields));
    final SchemaRecordWriter writer = new SchemaRecordWriter();
    writer.writeRecord(swapFileRecord, out);
    out.flush();
}

Example 13 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class SimpleSwapSerializer method serializeFlowFiles.

@Override
public void serializeFlowFiles(final List<FlowFileRecord> toSwap, final FlowFileQueue queue, final String swapLocation, final OutputStream destination) throws IOException {
    if (toSwap == null || toSwap.isEmpty()) {
        return;
    }
    long contentSize = 0L;
    for (final FlowFileRecord record : toSwap) {
        contentSize += record.getSize();
    }
    // persist record to disk via the swap file
    final DataOutputStream out = new DataOutputStream(destination);
    try {
        out.writeInt(SWAP_ENCODING_VERSION);
        out.writeUTF(queue.getIdentifier());
        out.writeInt(toSwap.size());
        out.writeLong(contentSize);
        // get the max record id and write that out so that we know it quickly for restoration
        long maxRecordId = 0L;
        for (final FlowFileRecord flowFile : toSwap) {
            if (flowFile.getId() > maxRecordId) {
                maxRecordId = flowFile.getId();
            }
        }
        out.writeLong(maxRecordId);
        for (final FlowFileRecord flowFile : toSwap) {
            out.writeLong(flowFile.getId());
            out.writeLong(flowFile.getEntryDate());
            out.writeLong(flowFile.getLineageStartDate());
            out.writeLong(flowFile.getLineageStartIndex());
            out.writeLong(flowFile.getLastQueueDate());
            out.writeLong(flowFile.getQueueDateIndex());
            out.writeLong(flowFile.getSize());
            final ContentClaim claim = flowFile.getContentClaim();
            if (claim == null) {
                out.writeBoolean(false);
            } else {
                out.writeBoolean(true);
                final ResourceClaim resourceClaim = claim.getResourceClaim();
                out.writeUTF(resourceClaim.getId());
                out.writeUTF(resourceClaim.getContainer());
                out.writeUTF(resourceClaim.getSection());
                out.writeLong(claim.getOffset());
                out.writeLong(claim.getLength());
                out.writeLong(flowFile.getContentClaimOffset());
                out.writeBoolean(resourceClaim.isLossTolerant());
            }
            final Map<String, String> attributes = flowFile.getAttributes();
            out.writeInt(attributes.size());
            for (final Map.Entry<String, String> entry : attributes.entrySet()) {
                writeString(entry.getKey(), out);
                writeString(entry.getValue(), out);
            }
        }
    } finally {
        out.flush();
    }
    logger.info("Successfully swapped out {} FlowFiles from {} to Swap File {}", toSwap.size(), queue, swapLocation);
}

Example 14 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class SwapSummaryFieldMap method getSwapSummary.

@SuppressWarnings("unchecked")
public static SwapSummary getSwapSummary(final Record record, final ResourceClaimManager claimManager) {
    final int flowFileCount = (Integer) record.getFieldValue(SwapSchema.FLOWFILE_COUNT);
    final long flowFileSize = (Long) record.getFieldValue(SwapSchema.FLOWFILE_SIZE);
    final QueueSize queueSize = new QueueSize(flowFileCount, flowFileSize);
    final long maxFlowFileId = (Long) record.getFieldValue(SwapSchema.MAX_RECORD_ID);
    final Map<Record, Integer> resourceClaimRecords = (Map<Record, Integer>) record.getFieldValue(SwapSchema.RESOURCE_CLAIMS);
    final List<ResourceClaim> resourceClaims = new ArrayList<>();
    for (final Map.Entry<Record, Integer> entry : resourceClaimRecords.entrySet()) {
        final Record resourceClaimRecord = entry.getKey();
        final ResourceClaim claim = ResourceClaimFieldMap.getResourceClaim(resourceClaimRecord, claimManager);
        for (int i = 0; i < entry.getValue(); i++) {
            resourceClaims.add(claim);
        }
    }
    return new StandardSwapSummary(queueSize, maxFlowFileId, resourceClaims);
}

Example 15 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class WriteAheadRepositoryRecordSerde method serializeContentClaim.

private void serializeContentClaim(final ContentClaim claim, final long offset, final DataOutputStream out) throws IOException {
    if (claim == null) {
        out.write(0);
    } else {
        out.write(1);
        final ResourceClaim resourceClaim = claim.getResourceClaim();
        writeString(resourceClaim.getId(), out);
        writeString(resourceClaim.getContainer(), out);
        writeString(resourceClaim.getSection(), out);
        out.writeLong(claim.getOffset());
        out.writeLong(claim.getLength());
        out.writeLong(offset);
        out.writeBoolean(resourceClaim.isLossTolerant());
    }
}