use of org.apache.nifi.controller.repository.claim.StandardContentClaim in project nifi by apache.
the class TestStandardProcessSession method testCommitFailureRequeuesFlowFiles.
@Test
public void testCommitFailureRequeuesFlowFiles() {
final FlowFileRecord flowFileRecord = new StandardFlowFileRecord.Builder().addAttribute("uuid", "12345678-1234-1234-1234-123456789012").entryDate(System.currentTimeMillis()).contentClaim(new StandardContentClaim(resourceClaimManager.newResourceClaim("x", "x", "0", true, false), 0L)).contentClaimOffset(0L).size(0L).build();
flowFileQueue.put(flowFileRecord);
final FlowFile originalFlowFile = session.get();
assertTrue(flowFileQueue.isActiveQueueEmpty());
assertEquals(1, flowFileQueue.getUnacknowledgedQueueSize().getObjectCount());
final FlowFile modified = session.write(originalFlowFile, new OutputStreamCallback() {
@Override
public void process(OutputStream out) throws IOException {
out.write("Hello".getBytes());
}
});
session.transfer(modified);
// instruct flowfile repo to throw IOException on update
flowFileRepo.setFailOnUpdate(true);
try {
session.commit();
Assert.fail("Session commit completed, even though FlowFile Repo threw IOException");
} catch (final ProcessException pe) {
// expected behavior because FlowFile Repo will throw IOException
}
assertFalse(flowFileQueue.isActiveQueueEmpty());
assertEquals(1, flowFileQueue.size().getObjectCount());
assertEquals(0, flowFileQueue.getUnacknowledgedQueueSize().getObjectCount());
}
use of org.apache.nifi.controller.repository.claim.StandardContentClaim in project nifi by apache.
the class TestVolatileContentRepository method testRedirects.
@Test
public void testRedirects() throws IOException {
System.setProperty(NiFiProperties.PROPERTIES_FILE_PATH, TestVolatileContentRepository.class.getResource("/conf/nifi.properties").getFile());
final Map<String, String> addProps = new HashMap<>();
addProps.put(VolatileContentRepository.MAX_SIZE_PROPERTY, "10 MB");
final NiFiProperties nifiProps = NiFiProperties.createBasicNiFiProperties(null, addProps);
final VolatileContentRepository contentRepo = new VolatileContentRepository(nifiProps);
contentRepo.initialize(claimManager);
final ContentClaim claim = contentRepo.create(true);
final OutputStream out = contentRepo.write(claim);
final byte[] oneK = new byte[1024];
Arrays.fill(oneK, (byte) 55);
// Write 10 MB to the repo
for (int i = 0; i < 10240; i++) {
out.write(oneK);
}
try {
out.write(1);
Assert.fail("Expected to be out of space on content repo");
} catch (final IOException e) {
}
try {
out.write(1);
Assert.fail("Expected to be out of space on content repo");
} catch (final IOException e) {
}
final ContentRepository mockRepo = Mockito.mock(ContentRepository.class);
contentRepo.setBackupRepository(mockRepo);
final ResourceClaim resourceClaim = claimManager.newResourceClaim("container", "section", "1000", true, false);
final ContentClaim contentClaim = new StandardContentClaim(resourceClaim, 0L);
Mockito.when(mockRepo.create(Matchers.anyBoolean())).thenReturn(contentClaim);
final ByteArrayOutputStream overflowStream = new ByteArrayOutputStream();
Mockito.when(mockRepo.write(Matchers.any(ContentClaim.class))).thenReturn(overflowStream);
out.write(10);
assertEquals(1024 * 1024 * 10 + 1, overflowStream.size());
final byte[] overflowBuffer = overflowStream.toByteArray();
assertEquals(55, overflowBuffer[0]);
for (int i = 0; i < overflowBuffer.length; i++) {
if (i == overflowBuffer.length - 1) {
assertEquals(10, overflowBuffer[i]);
} else {
assertEquals(55, overflowBuffer[i]);
}
}
}
use of org.apache.nifi.controller.repository.claim.StandardContentClaim in project nifi by apache.
the class TestWriteAheadFlowFileRepository method testResourceClaimsIncremented.
@Test
public void testResourceClaimsIncremented() throws IOException {
final ResourceClaimManager claimManager = new StandardResourceClaimManager();
final TestQueueProvider queueProvider = new TestQueueProvider();
final Connection connection = Mockito.mock(Connection.class);
when(connection.getIdentifier()).thenReturn("1234");
when(connection.getDestination()).thenReturn(Mockito.mock(Connectable.class));
final FlowFileSwapManager swapMgr = new MockFlowFileSwapManager();
final FlowFileQueue queue = new StandardFlowFileQueue("1234", connection, null, null, claimManager, null, swapMgr, null, 10000);
when(connection.getFlowFileQueue()).thenReturn(queue);
queueProvider.addConnection(connection);
final ResourceClaim resourceClaim1 = claimManager.newResourceClaim("container", "section", "1", false, false);
final ContentClaim claim1 = new StandardContentClaim(resourceClaim1, 0L);
final ResourceClaim resourceClaim2 = claimManager.newResourceClaim("container", "section", "2", false, false);
final ContentClaim claim2 = new StandardContentClaim(resourceClaim2, 0L);
// resource claims' counts should be updated for both the swapped out FlowFile and the non-swapped out FlowFile
try (final WriteAheadFlowFileRepository repo = new WriteAheadFlowFileRepository(NiFiProperties.createBasicNiFiProperties(null, null))) {
repo.initialize(claimManager);
repo.loadFlowFiles(queueProvider, -1L);
// Create a Repository Record that indicates that a FlowFile was created
final FlowFileRecord flowFile1 = new StandardFlowFileRecord.Builder().id(1L).addAttribute("uuid", "11111111-1111-1111-1111-111111111111").contentClaim(claim1).build();
final StandardRepositoryRecord rec1 = new StandardRepositoryRecord(queue);
rec1.setWorking(flowFile1);
rec1.setDestination(queue);
// Create a Record that we can swap out
final FlowFileRecord flowFile2 = new StandardFlowFileRecord.Builder().id(2L).addAttribute("uuid", "11111111-1111-1111-1111-111111111112").contentClaim(claim2).build();
final StandardRepositoryRecord rec2 = new StandardRepositoryRecord(queue);
rec2.setWorking(flowFile2);
rec2.setDestination(queue);
final List<RepositoryRecord> records = new ArrayList<>();
records.add(rec1);
records.add(rec2);
repo.updateRepository(records);
final String swapLocation = swapMgr.swapOut(Collections.singletonList(flowFile2), queue);
repo.swapFlowFilesOut(Collections.singletonList(flowFile2), queue, swapLocation);
}
final ResourceClaimManager recoveryClaimManager = new StandardResourceClaimManager();
try (final WriteAheadFlowFileRepository repo = new WriteAheadFlowFileRepository(NiFiProperties.createBasicNiFiProperties(null, null))) {
repo.initialize(recoveryClaimManager);
final long largestId = repo.loadFlowFiles(queueProvider, 0L);
// largest ID known is 1 because this doesn't take into account the FlowFiles that have been swapped out
assertEquals(1, largestId);
}
// resource claim 1 will have a single claimant count while resource claim 2 will have no claimant counts
// because resource claim 2 is referenced only by flowfiles that are swapped out.
assertEquals(1, recoveryClaimManager.getClaimantCount(resourceClaim1));
assertEquals(0, recoveryClaimManager.getClaimantCount(resourceClaim2));
final SwapSummary summary = queue.recoverSwappedFlowFiles();
assertNotNull(summary);
assertEquals(2, summary.getMaxFlowFileId().intValue());
assertEquals(new QueueSize(1, 0L), summary.getQueueSize());
final List<ResourceClaim> swappedOutClaims = summary.getResourceClaims();
assertNotNull(swappedOutClaims);
assertEquals(1, swappedOutClaims.size());
assertEquals(claim2.getResourceClaim(), swappedOutClaims.get(0));
}
use of org.apache.nifi.controller.repository.claim.StandardContentClaim in project nifi by apache.
the class MockFlowFile method createContentClaim.
public static ContentClaim createContentClaim(final String id, final ResourceClaimManager claimManager) {
final ResourceClaim resourceClaim = claimManager.newResourceClaim("container", "section", id, false, false);
claimManager.incrementClaimantCount(resourceClaim);
return new StandardContentClaim(resourceClaim, 3L);
}
use of org.apache.nifi.controller.repository.claim.StandardContentClaim in project nifi by apache.
the class TestFileSystemRepository method testReadWithNoContent.
@Test(expected = ContentNotFoundException.class)
public void testReadWithNoContent() throws IOException {
final ContentClaim claim = new StandardContentClaim(new StandardResourceClaim(claimManager, "container1", "section 1", "1", false), 0L);
final InputStream in = repository.read(claim);
in.close();
}
Aggregations