Examples with StandardResourceClaimManager org.apache.nifi.controller.repository.claim.StandardResourceClaimManager used on opensource projects

Example 11 with StandardResourceClaimManager

use of org.apache.nifi.controller.repository.claim.StandardResourceClaimManager in project nifi by apache.

the class TestFileSystemRepository method testBogusFile.

@Test
public void testBogusFile() throws IOException {
    repository.shutdown();
    System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, TestFileSystemRepository.class.getResource("/conf/nifi.properties").getFile());
    File bogus = new File(rootFile, "bogus");
    try {
        bogus.mkdir();
        bogus.setReadable(false);
        repository = new FileSystemRepository(nifiProperties);
        repository.initialize(new StandardResourceClaimManager());
    } finally {
        bogus.setReadable(true);
        assertTrue(bogus.delete());
    }
}

Example 12 with StandardResourceClaimManager

use of org.apache.nifi.controller.repository.claim.StandardResourceClaimManager in project nifi by apache.

the class TestFileSystemRepository method testWriteCannotProvideNullOutput.

@Test
public void testWriteCannotProvideNullOutput() throws IOException {
    FileSystemRepository repository = null;
    try {
        final List<Path> archivedPathsWithOpenStream = Collections.synchronizedList(new ArrayList<Path>());
        // We are creating our own 'local' repository in this test so shut down the one created in the setup() method
        shutdown();
        repository = new FileSystemRepository(nifiProperties) {

            @Override
            protected boolean archive(Path curPath) throws IOException {
                if (getOpenStreamCount() > 0) {
                    archivedPathsWithOpenStream.add(curPath);
                }
                return true;
            }
        };
        final StandardResourceClaimManager claimManager = new StandardResourceClaimManager();
        repository.initialize(claimManager);
        repository.purge();
        final ContentClaim claim = repository.create(false);
        assertEquals(1, claimManager.getClaimantCount(claim.getResourceClaim()));
        int claimantCount = claimManager.decrementClaimantCount(claim.getResourceClaim());
        assertEquals(0, claimantCount);
        assertTrue(archivedPathsWithOpenStream.isEmpty());
        OutputStream out = repository.write(claim);
        out.close();
        repository.decrementClaimantCount(claim);
        ContentClaim claim2 = repository.create(false);
        assertEquals(claim.getResourceClaim(), claim2.getResourceClaim());
        out = repository.write(claim2);
        final boolean archived = repository.archive(claim.getResourceClaim());
        assertFalse(archived);
    } finally {
        if (repository != null) {
            repository.shutdown();
        }
    }
}

Example 13 with StandardResourceClaimManager

use of org.apache.nifi.controller.repository.claim.StandardResourceClaimManager in project nifi by apache.

the class TestFileSystemRepository method testMarkDestructableDoesNotArchiveIfStreamOpenAndNotWrittenTo.

/**
 * We have encountered a situation where the File System Repo is moving
 * files to archive and then eventually aging them off while there is still
 * an open file handle. This test is meant to replicate the conditions under
 * which this would happen and verify that it is fixed.
 *
 * The condition that caused this appears to be that a Process Session
 * created a Content Claim and then did not write to it. It then decremented
 * the claimant count (which reduced the count to 0). This was likely due to
 * creating the claim in ProcessSession.write(FlowFile, StreamCallback) and
 * then having an Exception thrown when the Process Session attempts to read
 * the current Content Claim. In this case, it would not ever get to the
 * point of calling FileSystemRepository.write().
 *
 * The above sequence of events is problematic because calling
 * FileSystemRepository.create() will remove the Resource Claim from the
 * 'writable claims queue' and expects that we will write to it. When we
 * call FileSystemRepository.write() with that Resource Claim, we return an
 * OutputStream that, when closed, will take care of adding the Resource
 * Claim back to the 'writable claims queue' or otherwise close the
 * FileOutputStream that is open for that Resource Claim. If
 * FileSystemRepository.write() is never called, or if the OutputStream
 * returned by that method is never closed, but the Content Claim is then
 * decremented to 0, we can get into a situation where we do archive the
 * content (because the claimant count is 0 and it is not in the 'writable
 * claims queue') and then eventually age it off, without ever closing the
 * OutputStream. We need to ensure that we do always close that Output
 * Stream.
 */
@Test
public void testMarkDestructableDoesNotArchiveIfStreamOpenAndNotWrittenTo() throws IOException, InterruptedException {
    FileSystemRepository repository = null;
    try {
        final List<Path> archivedPathsWithOpenStream = Collections.synchronizedList(new ArrayList<Path>());
        // We are creating our own 'local' repository in this test so shut down the one created in the setup() method
        shutdown();
        repository = new FileSystemRepository(nifiProperties) {

            @Override
            protected boolean archive(Path curPath) throws IOException {
                if (getOpenStreamCount() > 0) {
                    archivedPathsWithOpenStream.add(curPath);
                }
                return true;
            }
        };
        final StandardResourceClaimManager claimManager = new StandardResourceClaimManager();
        repository.initialize(claimManager);
        repository.purge();
        final ContentClaim claim = repository.create(false);
        assertEquals(1, claimManager.getClaimantCount(claim.getResourceClaim()));
        int claimantCount = claimManager.decrementClaimantCount(claim.getResourceClaim());
        assertEquals(0, claimantCount);
        assertTrue(archivedPathsWithOpenStream.isEmpty());
        // This would happen when FlowFile repo is checkpointed, if Resource Claim has claimant count of 0.
        // Since the Resource Claim of interest is still 'writable', we should not archive it.
        claimManager.markDestructable(claim.getResourceClaim());
        // Wait for the archive thread to have a chance to run
        long totalSleepMillis = 0;
        final long startTime = System.nanoTime();
        while (archivedPathsWithOpenStream.isEmpty() && totalSleepMillis < 5000) {
            Thread.sleep(100L);
            totalSleepMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime);
        }
        // Should still be empty because we have a stream open to the file so we should
        // not actually try to archive the data.
        assertTrue(archivedPathsWithOpenStream.isEmpty());
        assertEquals(0, claimManager.getClaimantCount(claim.getResourceClaim()));
    } finally {
        if (repository != null) {
            repository.shutdown();
        }
    }
}

Example 14 with StandardResourceClaimManager

use of org.apache.nifi.controller.repository.claim.StandardResourceClaimManager in project nifi by apache.

the class TestStandardProcessSession method setup.

@Before
public void setup() throws IOException {
    resourceClaimManager = new StandardResourceClaimManager();
    System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, TestStandardProcessSession.class.getResource("/conf/nifi.properties").getFile());
    final FlowFileEventRepository flowFileEventRepo = Mockito.mock(FlowFileEventRepository.class);
    final CounterRepository counterRepo = Mockito.mock(CounterRepository.class);
    provenanceRepo = new MockProvenanceRepository();
    final Connection connection = createConnection();
    final List<Connection> connList = new ArrayList<>();
    connList.add(connection);
    final ProcessGroup procGroup = Mockito.mock(ProcessGroup.class);
    when(procGroup.getIdentifier()).thenReturn("proc-group-identifier-1");
    connectable = Mockito.mock(Connectable.class);
    when(connectable.hasIncomingConnection()).thenReturn(true);
    when(connectable.getIncomingConnections()).thenReturn(connList);
    when(connectable.getProcessGroup()).thenReturn(procGroup);
    when(connectable.getIdentifier()).thenReturn("connectable-1");
    when(connectable.getConnectableType()).thenReturn(ConnectableType.INPUT_PORT);
    when(connectable.getComponentType()).thenReturn("Unit Test Component");
    Mockito.doAnswer(new Answer<Set<Connection>>() {

        @Override
        public Set<Connection> answer(final InvocationOnMock invocation) throws Throwable {
            final Object[] arguments = invocation.getArguments();
            final Relationship relationship = (Relationship) arguments[0];
            if (relationship == Relationship.SELF) {
                return Collections.emptySet();
            } else if (relationship == FAKE_RELATIONSHIP || relationship.equals(FAKE_RELATIONSHIP)) {
                return null;
            } else {
                return new HashSet<>(connList);
            }
        }
    }).when(connectable).getConnections(Mockito.any(Relationship.class));
    when(connectable.getConnections()).thenReturn(new HashSet<>(connList));
    contentRepo = new MockContentRepository();
    contentRepo.initialize(new StandardResourceClaimManager());
    flowFileRepo = new MockFlowFileRepository();
    context = new RepositoryContext(connectable, new AtomicLong(0L), contentRepo, flowFileRepo, flowFileEventRepo, counterRepo, provenanceRepo);
    session = new StandardProcessSession(context, () -> false);
}

Example 15 with StandardResourceClaimManager

use of org.apache.nifi.controller.repository.claim.StandardResourceClaimManager in project nifi by apache.

the class TestWriteAheadFlowFileRepository method testRestartWithOneRecord.

@Test
public void testRestartWithOneRecord() throws IOException {
    final Path path = Paths.get("target/test-repo");
    if (Files.exists(path)) {
        FileUtils.deleteFile(path.toFile(), true);
    }
    final WriteAheadFlowFileRepository repo = new WriteAheadFlowFileRepository(NiFiProperties.createBasicNiFiProperties(null, null));
    repo.initialize(new StandardResourceClaimManager());
    final TestQueueProvider queueProvider = new TestQueueProvider();
    repo.loadFlowFiles(queueProvider, 0L);
    final List<FlowFileRecord> flowFileCollection = new ArrayList<>();
    final Connection connection = Mockito.mock(Connection.class);
    when(connection.getIdentifier()).thenReturn("1234");
    final FlowFileQueue queue = Mockito.mock(FlowFileQueue.class);
    when(queue.getIdentifier()).thenReturn("1234");
    doAnswer(new Answer<Object>() {

        @Override
        public Object answer(final InvocationOnMock invocation) throws Throwable {
            flowFileCollection.add((FlowFileRecord) invocation.getArguments()[0]);
            return null;
        }
    }).when(queue).put(any(FlowFileRecord.class));
    when(connection.getFlowFileQueue()).thenReturn(queue);
    queueProvider.addConnection(connection);
    StandardFlowFileRecord.Builder ffBuilder = new StandardFlowFileRecord.Builder();
    ffBuilder.id(1L);
    ffBuilder.addAttribute("abc", "xyz");
    ffBuilder.size(0L);
    final FlowFileRecord flowFileRecord = ffBuilder.build();
    final List<RepositoryRecord> records = new ArrayList<>();
    final StandardRepositoryRecord record = new StandardRepositoryRecord(null);
    record.setWorking(flowFileRecord);
    record.setDestination(connection.getFlowFileQueue());
    records.add(record);
    repo.updateRepository(records);
    // update to add new attribute
    ffBuilder = new StandardFlowFileRecord.Builder().fromFlowFile(flowFileRecord).addAttribute("hello", "world");
    final FlowFileRecord flowFileRecord2 = ffBuilder.build();
    record.setWorking(flowFileRecord2);
    repo.updateRepository(records);
    // update size but no attribute
    ffBuilder = new StandardFlowFileRecord.Builder().fromFlowFile(flowFileRecord2).size(40L);
    final FlowFileRecord flowFileRecord3 = ffBuilder.build();
    record.setWorking(flowFileRecord3);
    repo.updateRepository(records);
    repo.close();
    // restore
    final WriteAheadFlowFileRepository repo2 = new WriteAheadFlowFileRepository(NiFiProperties.createBasicNiFiProperties(null, null));
    repo2.initialize(new StandardResourceClaimManager());
    repo2.loadFlowFiles(queueProvider, 0L);
    assertEquals(1, flowFileCollection.size());
    final FlowFileRecord flowFile = flowFileCollection.get(0);
    assertEquals(1L, flowFile.getId());
    assertEquals("xyz", flowFile.getAttribute("abc"));
    assertEquals(40L, flowFile.getSize());
    assertEquals("world", flowFile.getAttribute("hello"));
    repo2.close();
}