Examples with ResourceClaim org.apache.nifi.controller.repository.claim.ResourceClaim used on opensource projects

Example 21 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class StandardProcessSession method enrich.

private StandardProvenanceEventRecord enrich(final ProvenanceEventRecord rawEvent, final Map<String, FlowFileRecord> flowFileRecordMap, final Map<FlowFileRecord, StandardRepositoryRecord> records, final boolean updateAttributes) {
    final StandardProvenanceEventRecord.Builder recordBuilder = new StandardProvenanceEventRecord.Builder().fromEvent(rawEvent);
    final FlowFileRecord eventFlowFile = flowFileRecordMap.get(rawEvent.getFlowFileUuid());
    if (eventFlowFile != null) {
        final StandardRepositoryRecord repoRecord = records.get(eventFlowFile);
        if (repoRecord.getCurrent() != null && repoRecord.getCurrentClaim() != null) {
            final ContentClaim currentClaim = repoRecord.getCurrentClaim();
            final long currentOffset = repoRecord.getCurrentClaimOffset();
            final long size = eventFlowFile.getSize();
            final ResourceClaim resourceClaim = currentClaim.getResourceClaim();
            recordBuilder.setCurrentContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), currentOffset + currentClaim.getOffset(), size);
        }
        if (repoRecord.getOriginal() != null && repoRecord.getOriginalClaim() != null) {
            final ContentClaim originalClaim = repoRecord.getOriginalClaim();
            final long originalOffset = repoRecord.getOriginal().getContentClaimOffset();
            final long originalSize = repoRecord.getOriginal().getSize();
            final ResourceClaim resourceClaim = originalClaim.getResourceClaim();
            recordBuilder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), originalOffset + originalClaim.getOffset(), originalSize);
        }
        final FlowFileQueue originalQueue = repoRecord.getOriginalQueue();
        if (originalQueue != null) {
            recordBuilder.setSourceQueueIdentifier(originalQueue.getIdentifier());
        }
    }
    if (updateAttributes) {
        final FlowFileRecord flowFileRecord = flowFileRecordMap.get(rawEvent.getFlowFileUuid());
        if (flowFileRecord != null) {
            final StandardRepositoryRecord record = records.get(flowFileRecord);
            if (record != null) {
                recordBuilder.setAttributes(record.getOriginalAttributes(), record.getUpdatedAttributes());
            }
        }
    }
    return recordBuilder.build();
}

Example 22 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class VolatileContentRepository method createLossTolerant.

private ContentClaim createLossTolerant() {
    final long id = idGenerator.getAndIncrement();
    final ResourceClaim resourceClaim = claimManager.newResourceClaim(CONTAINER_NAME, "section", String.valueOf(id), true, false);
    final ContentClaim claim = new StandardContentClaim(resourceClaim, 0L);
    final ContentBlock contentBlock = new ContentBlock(claim, repoSize);
    claimManager.incrementClaimantCount(resourceClaim, true);
    claimMap.put(claim, contentBlock);
    logger.debug("Created {} and mapped to {}", claim, contentBlock);
    return claim;
}

Example 23 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class WriteAheadFlowFileRepository method onSync.

@Override
public void onSync(final int partitionIndex) {
    final BlockingQueue<ResourceClaim> claimQueue = claimsAwaitingDestruction.get(Integer.valueOf(partitionIndex));
    if (claimQueue == null) {
        return;
    }
    final Set<ResourceClaim> claimsToDestroy = new HashSet<>();
    claimQueue.drainTo(claimsToDestroy);
    for (final ResourceClaim claim : claimsToDestroy) {
        markDestructable(claim);
    }
}

Example 24 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class TestSchemaSwapSerializerDeserializer method testRoundTripSerializeDeserializeSummary.

@Test
public void testRoundTripSerializeDeserializeSummary() throws IOException {
    final ResourceClaimManager resourceClaimManager = new StandardResourceClaimManager();
    final ResourceClaim firstResourceClaim = resourceClaimManager.newResourceClaim("container", "section", "id", true, false);
    resourceClaimManager.incrementClaimantCount(firstResourceClaim);
    final List<FlowFileRecord> toSwap = new ArrayList<>(10000);
    final Map<String, String> attrs = new HashMap<>();
    long size = 0L;
    final ContentClaim firstClaim = MockFlowFile.createContentClaim("id", resourceClaimManager);
    for (int i = 0; i < 10000; i++) {
        attrs.put("i", String.valueOf(i));
        final FlowFileRecord ff = i < 2 ? new MockFlowFile(attrs, i, firstClaim) : new MockFlowFile(attrs, i, resourceClaimManager);
        toSwap.add(ff);
        size += i;
    }
    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
    Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
    final String swapLocation = "target/testRoundTrip.swap";
    final File swapFile = new File(swapLocation);
    Files.deleteIfExists(swapFile.toPath());
    final SwapSerializer serializer = new SchemaSwapSerializer();
    try (final FileOutputStream fos = new FileOutputStream(swapFile)) {
        serializer.serializeFlowFiles(toSwap, flowFileQueue, swapLocation, fos);
    }
    final SwapDeserializer deserializer = new SchemaSwapDeserializer();
    final SwapSummary swapSummary;
    try (final FileInputStream fis = new FileInputStream(swapFile);
        final DataInputStream dis = new DataInputStream(fis)) {
        swapSummary = deserializer.getSwapSummary(dis, swapLocation, resourceClaimManager);
    }
    assertEquals(10000, swapSummary.getQueueSize().getObjectCount());
    assertEquals(size, swapSummary.getQueueSize().getByteCount());
    assertEquals(9999, swapSummary.getMaxFlowFileId().intValue());
    final List<ResourceClaim> resourceClaims = swapSummary.getResourceClaims();
    assertEquals(10000, resourceClaims.size());
    assertFalse(resourceClaims.stream().anyMatch(claim -> claim == null));
    assertEquals(2, resourceClaims.stream().filter(claim -> claim.getId().equals("id")).collect(Collectors.counting()).intValue());
    final Set<ResourceClaim> uniqueClaims = new HashSet<>(resourceClaims);
    assertEquals(9999, uniqueClaims.size());
}

Example 25 with ResourceClaim

use of org.apache.nifi.controller.repository.claim.ResourceClaim in project nifi by apache.

the class StandardFlowFileQueue method createDropEvent.

private ProvenanceEventRecord createDropEvent(final FlowFileRecord flowFile, final String requestor) {
    final ProvenanceEventBuilder builder = provRepository.eventBuilder();
    builder.fromFlowFile(flowFile);
    builder.setEventType(ProvenanceEventType.DROP);
    builder.setLineageStartDate(flowFile.getLineageStartDate());
    builder.setComponentId(getIdentifier());
    builder.setComponentType("Connection");
    builder.setAttributes(flowFile.getAttributes(), Collections.<String, String>emptyMap());
    builder.setDetails("FlowFile Queue emptied by " + requestor);
    builder.setSourceQueueIdentifier(getIdentifier());
    final ContentClaim contentClaim = flowFile.getContentClaim();
    if (contentClaim != null) {
        final ResourceClaim resourceClaim = contentClaim.getResourceClaim();
        builder.setPreviousContentClaim(resourceClaim.getContainer(), resourceClaim.getSection(), resourceClaim.getId(), contentClaim.getOffset(), flowFile.getSize());
    }
    return builder.build();
}