Examples with ResourceClaim org.apache.nifi.controller.repository.claim.ResourceClaim used on opensource projects

Example 26 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class FileSystemRepository method getPath.

private Path getPath(final ContentClaim claim, final boolean verifyExists) throws ContentNotFoundException {
    final ResourceClaim resourceClaim = claim.getResourceClaim();
    final Path containerPath = containers.get(resourceClaim.getContainer());
    if (containerPath == null) {
        if (verifyExists) {
            throw new ContentNotFoundException(claim);
        } else {
            return null;
        }
    }
    // Create the Path that points to the data
    Path resolvedPath = containerPath.resolve(resourceClaim.getSection()).resolve(resourceClaim.getId());
    // If the data does not exist, create a Path that points to where the data would exist in the archive directory.
    if (!Files.exists(resolvedPath)) {
        resolvedPath = getArchivePath(claim.getResourceClaim());
        if (verifyExists && !Files.exists(resolvedPath)) {
            throw new ContentNotFoundException(claim);
        }
    }
    return resolvedPath;
}

Example 27 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class FileSystemRepository method create.

@Override
public ContentClaim create(final boolean lossTolerant) throws IOException {
    ResourceClaim resourceClaim;
    final long resourceOffset;
    final ClaimLengthPair pair = writableClaimQueue.poll();
    if (pair == null) {
        final long currentIndex = index.incrementAndGet();
        String containerName = null;
        boolean waitRequired = true;
        ContainerState containerState = null;
        for (long containerIndex = currentIndex; containerIndex < currentIndex + containers.size(); containerIndex++) {
            final long modulatedContainerIndex = containerIndex % containers.size();
            containerName = containerNames.get((int) modulatedContainerIndex);
            containerState = containerStateMap.get(containerName);
            if (!containerState.isWaitRequired()) {
                waitRequired = false;
                break;
            }
        }
        if (waitRequired) {
            containerState.waitForArchiveExpiration();
        }
        final long modulatedSectionIndex = currentIndex % SECTIONS_PER_CONTAINER;
        final String section = String.valueOf(modulatedSectionIndex);
        final String claimId = System.currentTimeMillis() + "-" + currentIndex;
        resourceClaim = resourceClaimManager.newResourceClaim(containerName, section, claimId, lossTolerant, true);
        resourceOffset = 0L;
        LOG.debug("Creating new Resource Claim {}", resourceClaim);
        // we always append because there may be another ContentClaim using the same resource claim.
        // However, we know that we will never write to the same claim from two different threads
        // at the same time because we will call create() to get the claim before we write to it,
        // and when we call create(), it will remove it from the Queue, which means that no other
        // thread will get the same Claim until we've finished writing to it.
        final File file = getPath(resourceClaim).toFile();
        ByteCountingOutputStream claimStream = new SynchronizedByteCountingOutputStream(new FileOutputStream(file, true), file.length());
        writableClaimStreams.put(resourceClaim, claimStream);
        incrementClaimantCount(resourceClaim, true);
    } else {
        resourceClaim = pair.getClaim();
        resourceOffset = pair.getLength();
        LOG.debug("Reusing Resource Claim {}", resourceClaim);
        incrementClaimantCount(resourceClaim, false);
    }
    final StandardContentClaim scc = new StandardContentClaim(resourceClaim, resourceOffset);
    return scc;
}

Example 28 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class SimpleSwapDeserializer method deserializeFlowFiles.

private static SwapContents deserializeFlowFiles(final DataInputStream in, final QueueSize queueSize, final Long maxRecordId, final int serializationVersion, final ResourceClaimManager claimManager, final String location) throws IOException {
    final List<FlowFileRecord> flowFiles = new ArrayList<>(queueSize.getObjectCount());
    final List<ResourceClaim> resourceClaims = new ArrayList<>(queueSize.getObjectCount());
    Long maxId = maxRecordId;
    for (int i = 0; i < queueSize.getObjectCount(); i++) {
        try {
            // legacy encoding had an "action" because it used to be couple with FlowFile Repository code
            if (serializationVersion < 3) {
                final int action = in.read();
                if (action != 1) {
                    throw new IOException("Swap File is version " + serializationVersion + " but did not contain a 'UPDATE' record type");
                }
            }
            final StandardFlowFileRecord.Builder ffBuilder = new StandardFlowFileRecord.Builder();
            final long recordId = in.readLong();
            if (maxId == null || recordId > maxId) {
                maxId = recordId;
            }
            ffBuilder.id(recordId);
            ffBuilder.entryDate(in.readLong());
            if (serializationVersion > 1) {
                // Lineage information was added in version 2
                if (serializationVersion < 10) {
                    final int numLineageIdentifiers = in.readInt();
                    for (int lineageIdIdx = 0; lineageIdIdx < numLineageIdentifiers; lineageIdIdx++) {
                        // skip each identifier
                        in.readUTF();
                    }
                }
                // version 9 adds in a 'lineage start index'
                final long lineageStartDate = in.readLong();
                final long lineageStartIndex;
                if (serializationVersion > 8) {
                    lineageStartIndex = in.readLong();
                } else {
                    lineageStartIndex = 0L;
                }
                ffBuilder.lineageStart(lineageStartDate, lineageStartIndex);
                if (serializationVersion > 5) {
                    // Version 9 adds in a 'queue date index'
                    final long lastQueueDate = in.readLong();
                    final long queueDateIndex;
                    if (serializationVersion > 8) {
                        queueDateIndex = in.readLong();
                    } else {
                        queueDateIndex = 0L;
                    }
                    ffBuilder.lastQueued(lastQueueDate, queueDateIndex);
                }
            }
            ffBuilder.size(in.readLong());
            if (serializationVersion < 3) {
                // connection Id
                readString(in);
            }
            final boolean hasClaim = in.readBoolean();
            ResourceClaim resourceClaim = null;
            if (hasClaim) {
                final String claimId;
                if (serializationVersion < 5) {
                    claimId = String.valueOf(in.readLong());
                } else {
                    claimId = in.readUTF();
                }
                final String container = in.readUTF();
                final String section = in.readUTF();
                final long resourceOffset;
                final long resourceLength;
                if (serializationVersion < 6) {
                    resourceOffset = 0L;
                    resourceLength = -1L;
                } else {
                    resourceOffset = in.readLong();
                    resourceLength = in.readLong();
                }
                final long claimOffset = in.readLong();
                final boolean lossTolerant;
                if (serializationVersion >= 4) {
                    lossTolerant = in.readBoolean();
                } else {
                    lossTolerant = false;
                }
                resourceClaim = claimManager.getResourceClaim(container, section, claimId);
                if (resourceClaim == null) {
                    logger.error("Swap file indicates that FlowFile was referencing Resource Claim at container={}, section={}, claimId={}, " + "but this Resource Claim cannot be found! Will create a temporary Resource Claim, but this may affect the framework's " + "ability to properly clean up this resource", container, section, claimId);
                    resourceClaim = claimManager.newResourceClaim(container, section, claimId, lossTolerant, true);
                }
                final StandardContentClaim claim = new StandardContentClaim(resourceClaim, resourceOffset);
                claim.setLength(resourceLength);
                ffBuilder.contentClaim(claim);
                ffBuilder.contentClaimOffset(claimOffset);
            }
            boolean attributesChanged = true;
            if (serializationVersion < 3) {
                attributesChanged = in.readBoolean();
            }
            if (attributesChanged) {
                final int numAttributes = in.readInt();
                for (int j = 0; j < numAttributes; j++) {
                    final String key = readString(in);
                    final String value = readString(in);
                    ffBuilder.addAttribute(key, value);
                }
            }
            final FlowFileRecord record = ffBuilder.build();
            if (resourceClaim != null) {
                resourceClaims.add(resourceClaim);
            }
            flowFiles.add(record);
        } catch (final EOFException eof) {
            final SwapSummary swapSummary = new StandardSwapSummary(queueSize, maxId, resourceClaims);
            final SwapContents partialContents = new StandardSwapContents(swapSummary, flowFiles);
            throw new IncompleteSwapFileException(location, partialContents);
        }
    }
    final SwapSummary swapSummary = new StandardSwapSummary(queueSize, maxId, resourceClaims);
    return new StandardSwapContents(swapSummary, flowFiles);
}

Example 29 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class FlowController method getContent.

public InputStream getContent(final ProvenanceEventRecord provEvent, final ContentDirection direction, final String requestor, final String requestUri) throws IOException {
    requireNonNull(provEvent);
    requireNonNull(direction);
    requireNonNull(requestor);
    requireNonNull(requestUri);
    final ContentClaim claim;
    final long size;
    final long offset;
    if (direction == ContentDirection.INPUT) {
        if (provEvent.getPreviousContentClaimContainer() == null || provEvent.getPreviousContentClaimSection() == null || provEvent.getPreviousContentClaimIdentifier() == null) {
            throw new IllegalArgumentException("Input Content Claim not specified");
        }
        final ResourceClaim resourceClaim = resourceClaimManager.newResourceClaim(provEvent.getPreviousContentClaimContainer(), provEvent.getPreviousContentClaimSection(), provEvent.getPreviousContentClaimIdentifier(), false, false);
        claim = new StandardContentClaim(resourceClaim, provEvent.getPreviousContentClaimOffset());
        offset = provEvent.getPreviousContentClaimOffset() == null ? 0L : provEvent.getPreviousContentClaimOffset();
        size = provEvent.getPreviousFileSize();
    } else {
        if (provEvent.getContentClaimContainer() == null || provEvent.getContentClaimSection() == null || provEvent.getContentClaimIdentifier() == null) {
            throw new IllegalArgumentException("Output Content Claim not specified");
        }
        final ResourceClaim resourceClaim = resourceClaimManager.newResourceClaim(provEvent.getContentClaimContainer(), provEvent.getContentClaimSection(), provEvent.getContentClaimIdentifier(), false, false);
        claim = new StandardContentClaim(resourceClaim, provEvent.getContentClaimOffset());
        offset = provEvent.getContentClaimOffset() == null ? 0L : provEvent.getContentClaimOffset();
        size = provEvent.getFileSize();
    }
    final InputStream rawStream = contentRepository.read(claim);
    final ResourceClaim resourceClaim = claim.getResourceClaim();
    // Register a Provenance Event to indicate that we replayed the data.
    final ProvenanceEventRecord sendEvent = new StandardProvenanceEventRecord.Builder().setEventType(ProvenanceEventType.DOWNLOAD).setFlowFileUUID(provEvent.getFlowFileUuid()).setAttributes(provEvent.getAttributes(), Collections.emptyMap()).setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), offset, size).setTransitUri(requestUri).setEventTime(System.currentTimeMillis()).setFlowFileEntryDate(provEvent.getFlowFileEntryDate()).setLineageStartDate(provEvent.getLineageStartDate()).setComponentType(getName()).setComponentId(getRootGroupId()).setDetails("Download of " + (direction == ContentDirection.INPUT ? "Input" : "Output") + " Content requested by " + requestor + " for Provenance Event " + provEvent.getEventId()).build();
    provenanceRepository.registerEvent(sendEvent);
    return new LimitedInputStream(rawStream, size);
}

Example 30 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class StandardFlowFileQueue method drop.

private QueueSize drop(final List<FlowFileRecord> flowFiles, final String requestor) throws IOException {
    // Create a Provenance Event and a FlowFile Repository record for each FlowFile
    final List<ProvenanceEventRecord> provenanceEvents = new ArrayList<>(flowFiles.size());
    final List<RepositoryRecord> flowFileRepoRecords = new ArrayList<>(flowFiles.size());
    for (final FlowFileRecord flowFile : flowFiles) {
        provenanceEvents.add(createDropEvent(flowFile, requestor));
        flowFileRepoRecords.add(createDeleteRepositoryRecord(flowFile));
    }
    long dropContentSize = 0L;
    for (final FlowFileRecord flowFile : flowFiles) {
        dropContentSize += flowFile.getSize();
        final ContentClaim contentClaim = flowFile.getContentClaim();
        if (contentClaim == null) {
            continue;
        }
        final ResourceClaim resourceClaim = contentClaim.getResourceClaim();
        if (resourceClaim == null) {
            continue;
        }
        resourceClaimManager.decrementClaimantCount(resourceClaim);
    }
    provRepository.registerEvents(provenanceEvents);
    flowFileRepository.updateRepository(flowFileRepoRecords);
    return new QueueSize(flowFiles.size(), dropContentSize);
}