Examples with ContentClaim org.apache.nifi.controller.repository.claim.ContentClaim used on opensource projects

Example 71 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class VolatileContentRepository method createLossTolerant.

private ContentClaim createLossTolerant() {
    final long id = idGenerator.getAndIncrement();
    final ResourceClaim resourceClaim = claimManager.newResourceClaim(CONTAINER_NAME, "section", String.valueOf(id), true, false);
    final ContentClaim claim = new StandardContentClaim(resourceClaim, 0L);
    final ContentBlock contentBlock = new ContentBlock(claim, repoSize);
    claimManager.incrementClaimantCount(resourceClaim, true);
    claimMap.put(claim, contentBlock);
    logger.debug("Created {} and mapped to {}", claim, contentBlock);
    return claim;
}

Example 72 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class VolatileContentRepository method merge.

@Override
public long merge(final Collection<ContentClaim> claims, final ContentClaim destination, final byte[] header, final byte[] footer, final byte[] demarcator) throws IOException {
    long bytes = 0L;
    try (final OutputStream out = write(destination)) {
        if (header != null) {
            out.write(header);
            bytes += header.length;
        }
        final Iterator<ContentClaim> itr = claims.iterator();
        while (itr.hasNext()) {
            final ContentClaim readClaim = itr.next();
            try (final InputStream in = read(readClaim)) {
                bytes += StreamUtils.copy(in, out);
            }
            if (itr.hasNext() && demarcator != null) {
                bytes += demarcator.length;
                out.write(demarcator);
            }
        }
        if (footer != null) {
            bytes += footer.length;
            out.write(footer);
        }
        return bytes;
    }
}

Example 73 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class TestVolatileContentRepository method testSimpleReadWrite.

@Test
public void testSimpleReadWrite() throws IOException {
    System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, TestVolatileContentRepository.class.getResource("/conf/nifi.properties").getFile());
    final Map<String, String> addProps = new HashMap<>();
    addProps.put(VolatileContentRepository.MAX_SIZE_PROPERTY, "11 MB");
    final NiFiProperties nifiProps = NiFiProperties.createBasicNiFiProperties(null, addProps);
    final VolatileContentRepository contentRepo = new VolatileContentRepository(nifiProps);
    contentRepo.initialize(claimManager);
    final ContentClaim claim = contentRepo.create(true);
    final OutputStream out = contentRepo.write(claim);
    final int byteCount = 2398473 * 4;
    final byte[] x = new byte[4];
    x[0] = 48;
    x[1] = 29;
    x[2] = 49;
    x[3] = 51;
    for (int i = 0; i < byteCount / 4; i++) {
        out.write(x);
    }
    out.close();
    final InputStream in = contentRepo.read(claim);
    for (int i = 0; i < byteCount; i++) {
        final int val = in.read();
        final int index = i % 4;
        final byte expectedVal = x[index];
        assertEquals(expectedVal, val);
    }
    assertEquals(-1, in.read());
}

Example 74 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class TestVolatileContentRepository method testMemoryIsFreed.

@Test
public void testMemoryIsFreed() throws IOException, InterruptedException {
    System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, TestVolatileContentRepository.class.getResource("/conf/nifi.properties").getFile());
    final Map<String, String> addProps = new HashMap<>();
    addProps.put(VolatileContentRepository.MAX_SIZE_PROPERTY, "11 MB");
    final NiFiProperties nifiProps = NiFiProperties.createBasicNiFiProperties(null, addProps);
    final VolatileContentRepository contentRepo = new VolatileContentRepository(nifiProps);
    contentRepo.initialize(claimManager);
    final byte[] oneK = new byte[1024];
    Arrays.fill(oneK, (byte) 55);
    final ExecutorService exec = Executors.newFixedThreadPool(10);
    for (int t = 0; t < 10; t++) {
        final Runnable r = new Runnable() {

            @Override
            public void run() {
                try {
                    for (int j = 0; j < 10000; j++) {
                        final ContentClaim claim = contentRepo.create(true);
                        final OutputStream out = contentRepo.write(claim);
                        // Write 1 MB to the repo
                        for (int i = 0; i < 1024; i++) {
                            out.write(oneK);
                        }
                        final int count = contentRepo.decrementClaimantCount(claim);
                        if (count <= 0) {
                            contentRepo.remove(claim);
                        }
                    }
                } catch (final Exception e) {
                    e.printStackTrace();
                }
            }
        };
        exec.submit(r);
    }
    exec.shutdown();
    exec.awaitTermination(100000, TimeUnit.MINUTES);
}

Example 75 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class TestSchemaSwapSerializerDeserializer method testRoundTripSerializeDeserializeSummary.

@Test
public void testRoundTripSerializeDeserializeSummary() throws IOException {
    final ResourceClaimManager resourceClaimManager = new StandardResourceClaimManager();
    final ResourceClaim firstResourceClaim = resourceClaimManager.newResourceClaim("container", "section", "id", true, false);
    resourceClaimManager.incrementClaimantCount(firstResourceClaim);
    final List<FlowFileRecord> toSwap = new ArrayList<>(10000);
    final Map<String, String> attrs = new HashMap<>();
    long size = 0L;
    final ContentClaim firstClaim = MockFlowFile.createContentClaim("id", resourceClaimManager);
    for (int i = 0; i < 10000; i++) {
        attrs.put("i", String.valueOf(i));
        final FlowFileRecord ff = i < 2 ? new MockFlowFile(attrs, i, firstClaim) : new MockFlowFile(attrs, i, resourceClaimManager);
        toSwap.add(ff);
        size += i;
    }
    final FlowFileQueue flowFileQueue = Mockito.mock(FlowFileQueue.class);
    Mockito.when(flowFileQueue.getIdentifier()).thenReturn("87bb99fe-412c-49f6-a441-d1b0af4e20b4");
    final String swapLocation = "target/testRoundTrip.swap";
    final File swapFile = new File(swapLocation);
    Files.deleteIfExists(swapFile.toPath());
    final SwapSerializer serializer = new SchemaSwapSerializer();
    try (final FileOutputStream fos = new FileOutputStream(swapFile)) {
        serializer.serializeFlowFiles(toSwap, flowFileQueue, swapLocation, fos);
    }
    final SwapDeserializer deserializer = new SchemaSwapDeserializer();
    final SwapSummary swapSummary;
    try (final FileInputStream fis = new FileInputStream(swapFile);
        final DataInputStream dis = new DataInputStream(fis)) {
        swapSummary = deserializer.getSwapSummary(dis, swapLocation, resourceClaimManager);
    }
    assertEquals(10000, swapSummary.getQueueSize().getObjectCount());
    assertEquals(size, swapSummary.getQueueSize().getByteCount());
    assertEquals(9999, swapSummary.getMaxFlowFileId().intValue());
    final List<ResourceClaim> resourceClaims = swapSummary.getResourceClaims();
    assertEquals(10000, resourceClaims.size());
    assertFalse(resourceClaims.stream().anyMatch(claim -> claim == null));
    assertEquals(2, resourceClaims.stream().filter(claim -> claim.getId().equals("id")).collect(Collectors.counting()).intValue());
    final Set<ResourceClaim> uniqueClaims = new HashSet<>(resourceClaims);
    assertEquals(9999, uniqueClaims.size());
}