Examples with ContentClaim org.apache.nifi.controller.repository.claim.ContentClaim used on opensource projects

Example 46 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class TestFileSystemRepository method testResourceClaimReused.

@Test
public void testResourceClaimReused() throws IOException {
    final ContentClaim claim1 = repository.create(false);
    final ContentClaim claim2 = repository.create(false);
    // should not be equal because claim1 may still be in use
    assertNotSame(claim1.getResourceClaim(), claim2.getResourceClaim());
    try (final OutputStream out = repository.write(claim1)) {
    }
    final ContentClaim claim3 = repository.create(false);
    assertEquals(claim1.getResourceClaim(), claim3.getResourceClaim());
}

Example 47 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class TestFileSystemRepository method testRemoveWhileWritingToClaim.

@Test
public void testRemoveWhileWritingToClaim() throws IOException {
    final ContentClaim claim = repository.create(false);
    final OutputStream out = repository.write(claim);
    // write at least 1 MB to the output stream so that when we close the output stream
    // the repo won't keep the stream open.
    final String maxAppendableClaimLength = nifiProperties.getMaxAppendableClaimSize();
    final int maxClaimLength = DataUnit.parseDataSize(maxAppendableClaimLength, DataUnit.B).intValue();
    final byte[] buff = new byte[maxClaimLength];
    out.write(buff);
    out.write(buff);
    // false because claimant count is still 1, so the resource claim was not removed
    assertFalse(repository.remove(claim));
    assertEquals(0, repository.decrementClaimantCount(claim));
    // false because claimant count is 0 but there is an 'active' stream for the claim
    assertFalse(repository.remove(claim));
    out.close();
    assertTrue(repository.remove(claim));
}

Example 48 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class TestFileSystemRepository method testMarkDestructableDoesNotArchiveIfStreamOpenAndNotWrittenTo.

/**
 * We have encountered a situation where the File System Repo is moving
 * files to archive and then eventually aging them off while there is still
 * an open file handle. This test is meant to replicate the conditions under
 * which this would happen and verify that it is fixed.
 *
 * The condition that caused this appears to be that a Process Session
 * created a Content Claim and then did not write to it. It then decremented
 * the claimant count (which reduced the count to 0). This was likely due to
 * creating the claim in ProcessSession.write(FlowFile, StreamCallback) and
 * then having an Exception thrown when the Process Session attempts to read
 * the current Content Claim. In this case, it would not ever get to the
 * point of calling FileSystemRepository.write().
 *
 * The above sequence of events is problematic because calling
 * FileSystemRepository.create() will remove the Resource Claim from the
 * 'writable claims queue' and expects that we will write to it. When we
 * call FileSystemRepository.write() with that Resource Claim, we return an
 * OutputStream that, when closed, will take care of adding the Resource
 * Claim back to the 'writable claims queue' or otherwise close the
 * FileOutputStream that is open for that Resource Claim. If
 * FileSystemRepository.write() is never called, or if the OutputStream
 * returned by that method is never closed, but the Content Claim is then
 * decremented to 0, we can get into a situation where we do archive the
 * content (because the claimant count is 0 and it is not in the 'writable
 * claims queue') and then eventually age it off, without ever closing the
 * OutputStream. We need to ensure that we do always close that Output
 * Stream.
 */
@Test
public void testMarkDestructableDoesNotArchiveIfStreamOpenAndNotWrittenTo() throws IOException, InterruptedException {
    FileSystemRepository repository = null;
    try {
        final List<Path> archivedPathsWithOpenStream = Collections.synchronizedList(new ArrayList<Path>());
        // We are creating our own 'local' repository in this test so shut down the one created in the setup() method
        shutdown();
        repository = new FileSystemRepository(nifiProperties) {

            @Override
            protected boolean archive(Path curPath) throws IOException {
                if (getOpenStreamCount() > 0) {
                    archivedPathsWithOpenStream.add(curPath);
                }
                return true;
            }
        };
        final StandardResourceClaimManager claimManager = new StandardResourceClaimManager();
        repository.initialize(claimManager);
        repository.purge();
        final ContentClaim claim = repository.create(false);
        assertEquals(1, claimManager.getClaimantCount(claim.getResourceClaim()));
        int claimantCount = claimManager.decrementClaimantCount(claim.getResourceClaim());
        assertEquals(0, claimantCount);
        assertTrue(archivedPathsWithOpenStream.isEmpty());
        // This would happen when FlowFile repo is checkpointed, if Resource Claim has claimant count of 0.
        // Since the Resource Claim of interest is still 'writable', we should not archive it.
        claimManager.markDestructable(claim.getResourceClaim());
        // Wait for the archive thread to have a chance to run
        long totalSleepMillis = 0;
        final long startTime = System.nanoTime();
        while (archivedPathsWithOpenStream.isEmpty() && totalSleepMillis < 5000) {
            Thread.sleep(100L);
            totalSleepMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
        }
        // Should still be empty because we have a stream open to the file so we should
        // not actually try to archive the data.
        assertTrue(archivedPathsWithOpenStream.isEmpty());
        assertEquals(0, claimManager.getClaimantCount(claim.getResourceClaim()));
    } finally {
        if (repository != null) {
            repository.shutdown();
        }
    }
}

Example 49 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class TestStandardProcessSession method testWriteAfterSessionClosesStream.

@Test
public void testWriteAfterSessionClosesStream() throws IOException {
    final ContentClaim claim = contentRepo.create(false);
    final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder().contentClaim(claim).addAttribute("uuid", "12345678-1234-1234-1234-123456789012").entryDate(System.currentTimeMillis()).build();
    flowFileQueue.put(flowFileRecord);
    FlowFile flowFile = session.get();
    assertNotNull(flowFile);
    final AtomicReference<OutputStream> outputStreamHolder = new AtomicReference<>(null);
    flowFile = session.write(flowFile, new OutputStreamCallback() {

        @Override
        public void process(final OutputStream out) throws IOException {
            outputStreamHolder.set(out);
        }
    });
    assertDisabled(outputStreamHolder.get());
}

Example 50 with ContentClaim

use of org.apache.nifi.controller.repository.claim.ContentClaim in project nifi by apache.

the class TestStandardProcessSession method testStreamAfterSessionClosesStream.

@Test
public void testStreamAfterSessionClosesStream() throws IOException {
    final ContentClaim claim = contentRepo.create(false);
    final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder().contentClaim(claim).addAttribute("uuid", "12345678-1234-1234-1234-123456789012").entryDate(System.currentTimeMillis()).build();
    flowFileQueue.put(flowFileRecord);
    FlowFile flowFile = session.get();
    assertNotNull(flowFile);
    final AtomicReference<InputStream> inputStreamHolder = new AtomicReference<>(null);
    final AtomicReference<OutputStream> outputStreamHolder = new AtomicReference<>(null);
    flowFile = session.write(flowFile, new StreamCallback() {

        @Override
        public void process(final InputStream input, final OutputStream output) throws IOException {
            inputStreamHolder.set(input);
            outputStreamHolder.set(output);
        }
    });
    assertDisabled(inputStreamHolder.get());
    assertDisabled(outputStreamHolder.get());
}