Examples with StandardContentClaim org.apache.nifi.controller.repository.claim.StandardContentClaim used on opensource projects

Example 16 with StandardContentClaim

use of org.apache.nifi.controller.repository.claim.StandardContentClaim in project nifi by apache.

the class FileSystemRepository method create.

@Override
public ContentClaim create(final boolean lossTolerant) throws IOException {
    ResourceClaim resourceClaim;
    final long resourceOffset;
    final ClaimLengthPair pair = writableClaimQueue.poll();
    if (pair == null) {
        final long currentIndex = index.incrementAndGet();
        String containerName = null;
        boolean waitRequired = true;
        ContainerState containerState = null;
        for (long containerIndex = currentIndex; containerIndex < currentIndex + containers.size(); containerIndex++) {
            final long modulatedContainerIndex = containerIndex % containers.size();
            containerName = containerNames.get((int) modulatedContainerIndex);
            containerState = containerStateMap.get(containerName);
            if (!containerState.isWaitRequired()) {
                waitRequired = false;
                break;
            }
        }
        if (waitRequired) {
            containerState.waitForArchiveExpiration();
        }
        final long modulatedSectionIndex = currentIndex % SECTIONS_PER_CONTAINER;
        final String section = String.valueOf(modulatedSectionIndex);
        final String claimId = System.currentTimeMillis() + "-" + currentIndex;
        resourceClaim = resourceClaimManager.newResourceClaim(containerName, section, claimId, lossTolerant, true);
        resourceOffset = 0L;
        LOG.debug("Creating new Resource Claim {}", resourceClaim);
        // we always append because there may be another ContentClaim using the same resource claim.
        // However, we know that we will never write to the same claim from two different threads
        // at the same time because we will call create() to get the claim before we write to it,
        // and when we call create(), it will remove it from the Queue, which means that no other
        // thread will get the same Claim until we've finished writing to it.
        final File file = getPath(resourceClaim).toFile();
        ByteCountingOutputStream claimStream = new SynchronizedByteCountingOutputStream(new FileOutputStream(file, true), file.length());
        writableClaimStreams.put(resourceClaim, claimStream);
        incrementClaimantCount(resourceClaim, true);
    } else {
        resourceClaim = pair.getClaim();
        resourceOffset = pair.getLength();
        LOG.debug("Reusing Resource Claim {}", resourceClaim);
        incrementClaimantCount(resourceClaim, false);
    }
    final StandardContentClaim scc = new StandardContentClaim(resourceClaim, resourceOffset);
    return scc;
}

Example 17 with StandardContentClaim

use of org.apache.nifi.controller.repository.claim.StandardContentClaim in project nifi by apache.

the class SimpleSwapDeserializer method deserializeFlowFiles.

private static SwapContents deserializeFlowFiles(final DataInputStream in, final QueueSize queueSize, final Long maxRecordId, final int serializationVersion, final ResourceClaimManager claimManager, final String location) throws IOException {
    final List<FlowFileRecord> flowFiles = new ArrayList<>(queueSize.getObjectCount());
    final List<ResourceClaim> resourceClaims = new ArrayList<>(queueSize.getObjectCount());
    Long maxId = maxRecordId;
    for (int i = 0; i < queueSize.getObjectCount(); i++) {
        try {
            // legacy encoding had an "action" because it used to be couple with FlowFile Repository code
            if (serializationVersion < 3) {
                final int action = in.read();
                if (action != 1) {
                    throw new IOException("Swap File is version " + serializationVersion + " but did not contain a 'UPDATE' record type");
                }
            }
            final StandardFlowFileRecord.Builder ffBuilder = new StandardFlowFileRecord.Builder();
            final long recordId = in.readLong();
            if (maxId == null || recordId > maxId) {
                maxId = recordId;
            }
            ffBuilder.id(recordId);
            ffBuilder.entryDate(in.readLong());
            if (serializationVersion > 1) {
                // Lineage information was added in version 2
                if (serializationVersion < 10) {
                    final int numLineageIdentifiers = in.readInt();
                    for (int lineageIdIdx = 0; lineageIdIdx < numLineageIdentifiers; lineageIdIdx++) {
                        // skip each identifier
                        in.readUTF();
                    }
                }
                // version 9 adds in a 'lineage start index'
                final long lineageStartDate = in.readLong();
                final long lineageStartIndex;
                if (serializationVersion > 8) {
                    lineageStartIndex = in.readLong();
                } else {
                    lineageStartIndex = 0L;
                }
                ffBuilder.lineageStart(lineageStartDate, lineageStartIndex);
                if (serializationVersion > 5) {
                    // Version 9 adds in a 'queue date index'
                    final long lastQueueDate = in.readLong();
                    final long queueDateIndex;
                    if (serializationVersion > 8) {
                        queueDateIndex = in.readLong();
                    } else {
                        queueDateIndex = 0L;
                    }
                    ffBuilder.lastQueued(lastQueueDate, queueDateIndex);
                }
            }
            ffBuilder.size(in.readLong());
            if (serializationVersion < 3) {
                // connection Id
                readString(in);
            }
            final boolean hasClaim = in.readBoolean();
            ResourceClaim resourceClaim = null;
            if (hasClaim) {
                final String claimId;
                if (serializationVersion < 5) {
                    claimId = String.valueOf(in.readLong());
                } else {
                    claimId = in.readUTF();
                }
                final String container = in.readUTF();
                final String section = in.readUTF();
                final long resourceOffset;
                final long resourceLength;
                if (serializationVersion < 6) {
                    resourceOffset = 0L;
                    resourceLength = -1L;
                } else {
                    resourceOffset = in.readLong();
                    resourceLength = in.readLong();
                }
                final long claimOffset = in.readLong();
                final boolean lossTolerant;
                if (serializationVersion >= 4) {
                    lossTolerant = in.readBoolean();
                } else {
                    lossTolerant = false;
                }
                resourceClaim = claimManager.getResourceClaim(container, section, claimId);
                if (resourceClaim == null) {
                    logger.error("Swap file indicates that FlowFile was referencing Resource Claim at container={}, section={}, claimId={}, " + "but this Resource Claim cannot be found! Will create a temporary Resource Claim, but this may affect the framework's " + "ability to properly clean up this resource", container, section, claimId);
                    resourceClaim = claimManager.newResourceClaim(container, section, claimId, lossTolerant, true);
                }
                final StandardContentClaim claim = new StandardContentClaim(resourceClaim, resourceOffset);
                claim.setLength(resourceLength);
                ffBuilder.contentClaim(claim);
                ffBuilder.contentClaimOffset(claimOffset);
            }
            boolean attributesChanged = true;
            if (serializationVersion < 3) {
                attributesChanged = in.readBoolean();
            }
            if (attributesChanged) {
                final int numAttributes = in.readInt();
                for (int j = 0; j < numAttributes; j++) {
                    final String key = readString(in);
                    final String value = readString(in);
                    ffBuilder.addAttribute(key, value);
                }
            }
            final FlowFileRecord record = ffBuilder.build();
            if (resourceClaim != null) {
                resourceClaims.add(resourceClaim);
            }
            flowFiles.add(record);
        } catch (final EOFException eof) {
            final SwapSummary swapSummary = new StandardSwapSummary(queueSize, maxId, resourceClaims);
            final SwapContents partialContents = new StandardSwapContents(swapSummary, flowFiles);
            throw new IncompleteSwapFileException(location, partialContents);
        }
    }
    final SwapSummary swapSummary = new StandardSwapSummary(queueSize, maxId, resourceClaims);
    return new StandardSwapContents(swapSummary, flowFiles);
}

Example 18 with StandardContentClaim

use of org.apache.nifi.controller.repository.claim.StandardContentClaim in project nifi by apache.

the class FlowController method getContent.

public InputStream getContent(final ProvenanceEventRecord provEvent, final ContentDirection direction, final String requestor, final String requestUri) throws IOException {
    requireNonNull(provEvent);
    requireNonNull(direction);
    requireNonNull(requestor);
    requireNonNull(requestUri);
    final ContentClaim claim;
    final long size;
    final long offset;
    if (direction == ContentDirection.INPUT) {
        if (provEvent.getPreviousContentClaimContainer() == null || provEvent.getPreviousContentClaimSection() == null || provEvent.getPreviousContentClaimIdentifier() == null) {
            throw new IllegalArgumentException("Input Content Claim not specified");
        }
        final ResourceClaim resourceClaim = resourceClaimManager.newResourceClaim(provEvent.getPreviousContentClaimContainer(), provEvent.getPreviousContentClaimSection(), provEvent.getPreviousContentClaimIdentifier(), false, false);
        claim = new StandardContentClaim(resourceClaim, provEvent.getPreviousContentClaimOffset());
        offset = provEvent.getPreviousContentClaimOffset() == null ? 0L : provEvent.getPreviousContentClaimOffset();
        size = provEvent.getPreviousFileSize();
    } else {
        if (provEvent.getContentClaimContainer() == null || provEvent.getContentClaimSection() == null || provEvent.getContentClaimIdentifier() == null) {
            throw new IllegalArgumentException("Output Content Claim not specified");
        }
        final ResourceClaim resourceClaim = resourceClaimManager.newResourceClaim(provEvent.getContentClaimContainer(), provEvent.getContentClaimSection(), provEvent.getContentClaimIdentifier(), false, false);
        claim = new StandardContentClaim(resourceClaim, provEvent.getContentClaimOffset());
        offset = provEvent.getContentClaimOffset() == null ? 0L : provEvent.getContentClaimOffset();
        size = provEvent.getFileSize();
    }
    final InputStream rawStream = contentRepository.read(claim);
    final ResourceClaim resourceClaim = claim.getResourceClaim();
    // Register a Provenance Event to indicate that we replayed the data.
    final ProvenanceEventRecord sendEvent = new StandardProvenanceEventRecord.Builder().setEventType(ProvenanceEventType.DOWNLOAD).setFlowFileUUID(provEvent.getFlowFileUuid()).setAttributes(provEvent.getAttributes(), Collections.emptyMap()).setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), offset, size).setTransitUri(requestUri).setEventTime(System.currentTimeMillis()).setFlowFileEntryDate(provEvent.getFlowFileEntryDate()).setLineageStartDate(provEvent.getLineageStartDate()).setComponentType(getName()).setComponentId(getRootGroupId()).setDetails("Download of " + (direction == ContentDirection.INPUT ? "Input" : "Output") + " Content requested by " + requestor + " for Provenance Event " + provEvent.getEventId()).build();
    provenanceRepository.registerEvent(sendEvent);
    return new LimitedInputStream(rawStream, size);
}

Example 19 with StandardContentClaim

use of org.apache.nifi.controller.repository.claim.StandardContentClaim in project nifi by apache.

the class WriteAheadRepositoryRecordSerde method deserializeClaim.

private void deserializeClaim(final DataInputStream in, final int serializationVersion, final StandardFlowFileRecord.Builder ffBuilder) throws IOException {
    // determine current Content Claim.
    final int claimExists = in.read();
    if (claimExists == 1) {
        final String claimId;
        if (serializationVersion < 4) {
            claimId = String.valueOf(in.readLong());
        } else {
            claimId = readString(in);
        }
        final String container = readString(in);
        final String section = readString(in);
        final long resourceOffset;
        final long resourceLength;
        if (serializationVersion < 7) {
            resourceOffset = 0L;
            resourceLength = -1L;
        } else {
            resourceOffset = in.readLong();
            resourceLength = in.readLong();
        }
        final long claimOffset = in.readLong();
        final boolean lossTolerant;
        if (serializationVersion >= 3) {
            lossTolerant = in.readBoolean();
        } else {
            lossTolerant = false;
        }
        final ResourceClaim resourceClaim = claimManager.newResourceClaim(container, section, claimId, lossTolerant, false);
        final StandardContentClaim contentClaim = new StandardContentClaim(resourceClaim, resourceOffset);
        contentClaim.setLength(resourceLength);
        ffBuilder.contentClaim(contentClaim);
        ffBuilder.contentClaimOffset(claimOffset);
    } else if (claimExists == -1) {
        throw new EOFException();
    } else if (claimExists != 0) {
        throw new IOException("Claim Existence Qualifier not found in stream; found value: " + claimExists + " after successfully restoring " + recordsRestored + " records");
    }
}

Example 20 with StandardContentClaim

use of org.apache.nifi.controller.repository.claim.StandardContentClaim in project nifi by apache.

the class ContentClaimFieldMap method getContentClaim.

public static ContentClaim getContentClaim(final Record claimRecord, final ResourceClaimManager resourceClaimManager) {
    final Record resourceClaimRecord = (Record) claimRecord.getFieldValue(ContentClaimSchema.RESOURCE_CLAIM);
    final String container = (String) resourceClaimRecord.getFieldValue(ContentClaimSchema.CLAIM_CONTAINER);
    final String section = (String) resourceClaimRecord.getFieldValue(ContentClaimSchema.CLAIM_SECTION);
    final String identifier = (String) resourceClaimRecord.getFieldValue(ContentClaimSchema.CLAIM_IDENTIFIER);
    final Boolean lossTolerant = (Boolean) resourceClaimRecord.getFieldValue(ContentClaimSchema.LOSS_TOLERANT);
    final Long length = (Long) claimRecord.getFieldValue(ContentClaimSchema.CONTENT_CLAIM_LENGTH);
    final Long resourceOffset = (Long) claimRecord.getFieldValue(ContentClaimSchema.RESOURCE_CLAIM_OFFSET);
    // Make sure that we preserve the existing ResourceClaim, if there is already one held by the Resource Claim Manager
    // because we need to honor its determination of whether or not the claim is writable. If the Resource Claim Manager
    // does not have a copy of this Resource Claim, then we can go ahead and just create one and assume that it is not
    // writable (because if it were writable, then the Resource Claim Manager would know about it).
    ResourceClaim resourceClaim = resourceClaimManager.getResourceClaim(container, section, identifier);
    if (resourceClaim == null) {
        resourceClaim = resourceClaimManager.newResourceClaim(container, section, identifier, lossTolerant, false);
    }
    final StandardContentClaim contentClaim = new StandardContentClaim(resourceClaim, resourceOffset);
    contentClaim.setLength(length);
    return contentClaim;
}