Examples with ResourceClaimManager org.apache.nifi.controller.repository.claim.ResourceClaimManager used on opensource projects

Example 6 with ResourceClaimManager

use of org.apache.nifi.controller.repository.claim.ResourceClaimManager in project nifi by apache.

the class TestFileSystemSwapManager method createSwapManager.

private FileSystemSwapManager createSwapManager() {
    final FileSystemSwapManager swapManager = new FileSystemSwapManager();
    final ResourceClaimManager resourceClaimManager = new NopResourceClaimManager();
    final FlowFileRepository flowfileRepo = Mockito.mock(FlowFileRepository.class);
    swapManager.initialize(new SwapManagerInitializationContext() {

        @Override
        public ResourceClaimManager getResourceClaimManager() {
            return resourceClaimManager;
        }

        @Override
        public FlowFileRepository getFlowFileRepository() {
            return flowfileRepo;
        }

        @Override
        public EventReporter getEventReporter() {
            return EventReporter.NO_OP;
        }
    });
    return swapManager;
}

Example 7 with ResourceClaimManager

use of org.apache.nifi.controller.repository.claim.ResourceClaimManager in project nifi by apache.

the class TestSchemaSwapSerializerDeserializer method testRoundTripSerializeDeserializeSummary.

@Test
public void testRoundTripSerializeDeserializeSummary() throws IOException {
    final ResourceClaimManager resourceClaimManager = new StandardResourceClaimManager();
    final ResourceClaim firstResourceClaim = resourceClaimManager.newResourceClaim("container", "section", "id", true, false);
    resourceClaimManager.incrementClaimantCount(firstResourceClaim);
    final List<FlowFileRecord> toSwap = new ArrayList<>(10000);
    final Map<String, String> attrs = new HashMap<>();
    long size = 0L;
    final ContentClaim firstClaim = MockFlowFile.createContentClaim("id", resourceClaimManager);
    for (int i = 0; i < 10000; i++) {
        attrs.put("i", String.valueOf(i));
        final FlowFileRecord ff = i < 2 ? new MockFlowFile(attrs, i, firstClaim) : new MockFlowFile(attrs, i, resourceClaimManager);
        toSwap.add(ff);
        size += i;
    }
    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
    Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
    final String swapLocation = "target/testRoundTrip.swap";
    final File swapFile = new File(swapLocation);
    Files.deleteIfExists(swapFile.toPath());
    final SwapSerializer serializer = new SchemaSwapSerializer();
    try (final FileOutputStream fos = new FileOutputStream(swapFile)) {
        serializer.serializeFlowFiles(toSwap, flowFileQueue, swapLocation, fos);
    }
    final SwapDeserializer deserializer = new SchemaSwapDeserializer();
    final SwapSummary swapSummary;
    try (final FileInputStream fis = new FileInputStream(swapFile);
        final DataInputStream dis = new DataInputStream(fis)) {
        swapSummary = deserializer.getSwapSummary(dis, swapLocation, resourceClaimManager);
    }
    assertEquals(10000, swapSummary.getQueueSize().getObjectCount());
    assertEquals(size, swapSummary.getQueueSize().getByteCount());
    assertEquals(9999, swapSummary.getMaxFlowFileId().intValue());
    final List<ResourceClaim> resourceClaims = swapSummary.getResourceClaims();
    assertEquals(10000, resourceClaims.size());
    assertFalse(resourceClaims.stream().anyMatch(claim -> claim == null));
    assertEquals(2, resourceClaims.stream().filter(claim -> claim.getId().equals("id")).collect(Collectors.counting()).intValue());
    final Set<ResourceClaim> uniqueClaims = new HashSet<>(resourceClaims);
    assertEquals(9999, uniqueClaims.size());
}

Example 8 with ResourceClaimManager

use of org.apache.nifi.controller.repository.claim.ResourceClaimManager in project nifi by apache.

the class TestSimpleSwapSerializerDeserializer method testWritePerformance.

@Test
@Ignore("For manual testing only. Not intended to be run as part of the automated unit tests but can " + "be convenient for determining a baseline for performance if making modifications.")
public void testWritePerformance() throws IOException, InterruptedException {
    final ResourceClaimManager resourceClaimManager = new StandardResourceClaimManager();
    final List<FlowFileRecord> toSwap = new ArrayList<>(10000);
    final Map<String, String> attrs = new HashMap<>();
    for (int i = 0; i < 10000; i++) {
        attrs.put("i", String.valueOf(i));
        final FlowFileRecord ff = new MockFlowFile(attrs, i, resourceClaimManager);
        toSwap.add(ff);
    }
    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
    Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
    final String swapLocation = "target/testRoundTrip.swap";
    final int iterations = 1000;
    final long start = System.nanoTime();
    final SwapSerializer serializer = new SimpleSwapSerializer();
    for (int i = 0; i < iterations; i++) {
        try (final OutputStream out = new NullOutputStream()) {
            serializer.serializeFlowFiles(toSwap, flowFileQueue, swapLocation, out);
        }
    }
    final long millis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - start);
    System.out.println("Wrote " + iterations + " Swap Files in " + millis + " millis");
}