Examples with FlowFileRecord org.apache.nifi.controller.repository.FlowFileRecord used on opensource projects

Example 6 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class TestStandardFlowFileQueue method testSwapInWhenThresholdIsLessThanSwapSize.

@Test
public void testSwapInWhenThresholdIsLessThanSwapSize() {
    // create a queue where the swap threshold is less than 10k
    queue = new StandardFlowFileQueue("id", connection, flowFileRepo, provRepo, claimManager, scheduler, swapManager, null, 1000);
    for (int i = 1; i <= 20000; i++) {
        queue.put(new TestFlowFile());
    }
    assertEquals(1, swapManager.swappedOut.size());
    queue.put(new TestFlowFile());
    assertEquals(1, swapManager.swappedOut.size());
    final Set<FlowFileRecord> exp = new HashSet<>();
    for (int i = 0; i < 999; i++) {
        // 
        final FlowFileRecord flowFile = queue.poll(exp);
        assertNotNull(flowFile);
        assertEquals(1, queue.getUnacknowledgedQueueSize().getObjectCount());
        assertEquals(1, queue.getUnacknowledgedQueueSize().getByteCount());
        queue.acknowledge(Collections.singleton(flowFile));
        assertEquals(0, queue.getUnacknowledgedQueueSize().getObjectCount());
        assertEquals(0, queue.getUnacknowledgedQueueSize().getByteCount());
    }
    assertEquals(0, swapManager.swapInCalledCount);
    assertEquals(1, queue.getActiveQueueSize().getObjectCount());
    assertNotNull(queue.poll(exp));
    assertEquals(0, swapManager.swapInCalledCount);
    assertEquals(0, queue.getActiveQueueSize().getObjectCount());
    assertEquals(1, swapManager.swapOutCalledCount);
    // this should trigger a swap-in of 10,000 records, and then pull 1 off the top.
    assertNotNull(queue.poll(exp));
    assertEquals(1, swapManager.swapInCalledCount);
    assertEquals(9999, queue.getActiveQueueSize().getObjectCount());
    assertTrue(swapManager.swappedOut.isEmpty());
    queue.poll(exp);
}

Example 7 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class TestStandardFlowFileQueue method testListFlowFilesResultsLimitedCollection.

@Test(timeout = 5000)
public void testListFlowFilesResultsLimitedCollection() throws InterruptedException {
    Collection<FlowFileRecord> tff = new ArrayList<>();
    // Swap Size is 10000 records, so 30000 is equal to 3 swap files.
    for (int i = 0; i < 30000; i++) {
        tff.add(new TestFlowFile());
    }
    queue.putAll(tff);
    final ListFlowFileStatus status = queue.listFlowFiles(UUID.randomUUID().toString(), 100);
    assertNotNull(status);
    assertEquals(30000, status.getQueueSize().getObjectCount());
    while (status.getState() != ListFlowFileState.COMPLETE) {
        Thread.sleep(100);
    }
    assertEquals(100, status.getFlowFileSummaries().size());
    assertEquals(100, status.getCompletionPercentage());
    assertNull(status.getFailureReason());
}

Example 8 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class TestStandardFlowFileQueue method testOOMEFollowedBySuccessfulSwapIn.

@Test
public void testOOMEFollowedBySuccessfulSwapIn() {
    final List<FlowFileRecord> flowFiles = new ArrayList<>();
    for (int i = 0; i < 50000; i++) {
        flowFiles.add(new TestFlowFile());
    }
    queue.putAll(flowFiles);
    swapManager.failSwapInAfterN = 2;
    swapManager.setSwapInFailure(new OutOfMemoryError("Intentional OOME for unit test"));
    final Set<FlowFileRecord> expiredRecords = new HashSet<>();
    for (int i = 0; i < 30000; i++) {
        final FlowFileRecord polled = queue.poll(expiredRecords);
        assertNotNull(polled);
    }
    // verify that unexpected ERROR's are handled in such a way that we keep retrying
    for (int i = 0; i < 3; i++) {
        try {
            queue.poll(expiredRecords);
            Assert.fail("Expected OOME to be thrown");
        } catch (final OutOfMemoryError oome) {
        // expected
        }
    }
    // verify that unexpected Runtime Exceptions are handled in such a way that we keep retrying
    swapManager.setSwapInFailure(new NullPointerException("Intentional OOME for unit test"));
    for (int i = 0; i < 3; i++) {
        try {
            queue.poll(expiredRecords);
            Assert.fail("Expected NPE to be thrown");
        } catch (final NullPointerException npe) {
        // expected
        }
    }
    swapManager.failSwapInAfterN = -1;
    for (int i = 0; i < 20000; i++) {
        final FlowFileRecord polled = queue.poll(expiredRecords);
        assertNotNull(polled);
    }
    queue.acknowledge(flowFiles);
    assertNull(queue.poll(expiredRecords));
    assertEquals(0, queue.getActiveQueueSize().getObjectCount());
    assertEquals(0, queue.size().getObjectCount());
    assertTrue(swapManager.swappedOut.isEmpty());
}

Example 9 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class StandardFlowFileQueue method setPriorities.

@Override
public void setPriorities(final List<FlowFilePrioritizer> newPriorities) {
    writeLock.lock();
    try {
        final PriorityQueue<FlowFileRecord> newQueue = new PriorityQueue<>(Math.max(20, activeQueue.size()), new Prioritizer(newPriorities));
        newQueue.addAll(activeQueue);
        activeQueue = newQueue;
        priorities.clear();
        priorities.addAll(newPriorities);
    } finally {
        writeLock.unlock("setPriorities");
    }
}

Example 10 with FlowFileRecord

use of org.apache.nifi.controller.repository.FlowFileRecord in project nifi by apache.

the class StandardFlowFileQueue method poll.

@Override
public FlowFileRecord poll(final Set<FlowFileRecord> expiredRecords) {
    FlowFileRecord flowFile = null;
    // First check if we have any records Pre-Fetched.
    final long expirationMillis = expirationPeriod.get().getMillis();
    writeLock.lock();
    try {
        flowFile = doPoll(expiredRecords, expirationMillis);
        return flowFile;
    } finally {
        writeLock.unlock("poll(Set)");
        if (flowFile != null) {
            incrementUnacknowledgedQueueSize(1, flowFile.getSize());
        }
    }
}