Examples with FirstInFirstOutPrioritizer org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer used on opensource projects

Example 1 with FirstInFirstOutPrioritizer

use of org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer in project nifi by apache.

the class TestEnforceOrder method testDuplicatedOrder.

@Test
public void testDuplicatedOrder() {
    final TestRunner runner = TestRunners.newTestRunner(EnforceOrder.class);
    runner.setProperty(EnforceOrder.GROUP_IDENTIFIER, "${group}");
    runner.setProperty(EnforceOrder.ORDER_ATTRIBUTE, "index");
    runner.setProperty(EnforceOrder.INITIAL_ORDER, "1");
    runner.assertValid();
    Ordered.enqueue(runner, "b", 1);
    Ordered.enqueue(runner, "a", 2);
    Ordered.enqueue(runner, "a", 1);
    Ordered.enqueue(runner, "a", 2);
    Ordered.enqueue(runner, "a", 3);
    runner.run();
    final List<MockFlowFile> succeeded = runner.getFlowFilesForRelationship(EnforceOrder.REL_SUCCESS);
    assertEquals(4, succeeded.size());
    succeeded.sort(new FirstInFirstOutPrioritizer());
    succeeded.get(0).assertContentEquals("a.1");
    succeeded.get(1).assertContentEquals("a.2");
    succeeded.get(2).assertContentEquals("a.3");
    succeeded.get(3).assertContentEquals("b.1");
    // It's not possible to distinguish skipped and duplicated, since we only tracks target order number.
    final List<MockFlowFile> skipped = runner.getFlowFilesForRelationship(EnforceOrder.REL_SKIPPED);
    assertEquals(1, skipped.size());
    skipped.get(0).assertContentEquals("a.2");
    skipped.get(0).assertAttributeEquals(EnforceOrder.ATTR_EXPECTED_ORDER, "3");
}

Example 2 with FirstInFirstOutPrioritizer

use of org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer in project nifi by apache.

the class TestEnforceOrder method testMaxOrder.

@Test
public void testMaxOrder() {
    final TestRunner runner = TestRunners.newTestRunner(EnforceOrder.class);
    runner.setProperty(EnforceOrder.GROUP_IDENTIFIER, "${fragment.identifier}");
    runner.setProperty(EnforceOrder.ORDER_ATTRIBUTE, "index");
    runner.setProperty(EnforceOrder.INITIAL_ORDER, "1");
    runner.setProperty(EnforceOrder.MAX_ORDER, "${fragment.count}");
    runner.assertValid();
    runner.enqueue("b.1", Ordered.i(1).put("fragment.identifier", "b").put("fragment.count", "3").map());
    runner.enqueue("a.2", Ordered.i(2).put("fragment.identifier", "a").put("fragment.count", "2").map());
    runner.enqueue("without max order", Ordered.i(1).put("fragment.identifier", "c").map());
    runner.enqueue("illegal max order", Ordered.i(1).put("fragment.identifier", "d").put("fragment.count", "X").map());
    runner.enqueue("a.1", Ordered.i(1).put("fragment.identifier", "a").put("fragment.count", "2").map());
    // Exceed max
    runner.enqueue("a.3", Ordered.i(3).put("fragment.identifier", "a").put("fragment.count", "2").map());
    runner.run();
    final List<MockFlowFile> succeeded = runner.getFlowFilesForRelationship(EnforceOrder.REL_SUCCESS);
    succeeded.sort(new FirstInFirstOutPrioritizer());
    assertEquals(3, succeeded.size());
    succeeded.get(0).assertContentEquals("a.1");
    succeeded.get(1).assertContentEquals("a.2");
    succeeded.get(2).assertContentEquals("b.1");
    final List<MockFlowFile> failed = runner.getFlowFilesForRelationship(EnforceOrder.REL_FAILURE);
    assertEquals(3, failed.size());
    failed.get(0).assertContentEquals("without max order");
    failed.get(1).assertContentEquals("illegal max order");
    // exceeds max order
    failed.get(2).assertContentEquals("a.3");
    final MockStateManager stateManager = runner.getStateManager();
    stateManager.assertStateEquals("a.target", "2", Scope.LOCAL);
    stateManager.assertStateEquals("a.max", "2", Scope.LOCAL);
}

Example 3 with FirstInFirstOutPrioritizer

use of org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer in project nifi by apache.

the class TestEnforceOrder method testWaitOvertakeSkip.

@Test
public void testWaitOvertakeSkip() throws Exception {
    final TestRunner runner = TestRunners.newTestRunner(EnforceOrder.class);
    runner.setProperty(EnforceOrder.GROUP_IDENTIFIER, "${group}");
    runner.setProperty(EnforceOrder.ORDER_ATTRIBUTE, "index");
    runner.setProperty(EnforceOrder.INITIAL_ORDER, "1");
    runner.setProperty(EnforceOrder.MAX_ORDER, "10");
    runner.assertValid();
    Ordered.enqueue(runner, "b", 1);
    Ordered.enqueue(runner, "a", 2);
    Ordered.enqueue(runner, "a", 1);
    // waits for a.3 and a.4
    Ordered.enqueue(runner, "a", 5);
    // waits for b.2
    Ordered.enqueue(runner, "b", 3);
    // waits for c.1 to 8
    Ordered.enqueue(runner, "c", 9);
    // waits for d.1 to 9
    Ordered.enqueue(runner, "d", 10);
    runner.run();
    List<MockFlowFile> succeeded = runner.getFlowFilesForRelationship(EnforceOrder.REL_SUCCESS);
    assertEquals(3, succeeded.size());
    final FirstInFirstOutPrioritizer fifo = new FirstInFirstOutPrioritizer();
    succeeded.sort(fifo);
    succeeded.get(0).assertContentEquals("a.1");
    succeeded.get(1).assertContentEquals("a.2");
    succeeded.get(2).assertContentEquals("b.1");
    List<MockFlowFile> waiting = runner.getFlowFilesForRelationship(EnforceOrder.REL_WAIT);
    assertEquals(4, waiting.size());
    waiting.get(0).assertContentEquals("a.5");
    waiting.get(1).assertContentEquals("b.3");
    waiting.get(2).assertContentEquals("c.9");
    waiting.get(3).assertContentEquals("d.10");
    waiting.get(0).assertAttributeExists("EnforceOrder.startedAt");
    waiting.get(1).assertAttributeExists("EnforceOrder.startedAt");
    waiting.get(2).assertAttributeExists("EnforceOrder.startedAt");
    waiting.get(3).assertAttributeExists("EnforceOrder.startedAt");
    final MockStateManager stateManager = runner.getStateManager();
    stateManager.assertStateEquals("a.target", "3", Scope.LOCAL);
    stateManager.assertStateEquals("b.target", "2", Scope.LOCAL);
    stateManager.assertStateEquals("c.target", "1", Scope.LOCAL);
    stateManager.assertStateEquals("d.target", "1", Scope.LOCAL);
    stateManager.assertStateSet("a.updatedAt", Scope.LOCAL);
    stateManager.assertStateSet("b.updatedAt", Scope.LOCAL);
    stateManager.assertStateSet("c.updatedAt", Scope.LOCAL);
    stateManager.assertStateSet("d.updatedAt", Scope.LOCAL);
    // Run it again with waiting files.
    runner.clearTransferState();
    waiting.forEach(runner::enqueue);
    runner.run();
    runner.assertAllFlowFilesTransferred(EnforceOrder.REL_WAIT, 4);
    waiting = runner.getFlowFilesForRelationship(EnforceOrder.REL_WAIT);
    // Run it again with shorter wait timeout to make overtaking happen.
    runner.clearTransferState();
    runner.setProperty(EnforceOrder.WAIT_TIMEOUT, "10 ms");
    Thread.sleep(20);
    waiting.forEach(runner::enqueue);
    // arrived in time
    Ordered.enqueue(runner, "b", 2);
    // a.4 and a.5 have not arrived yet
    Ordered.enqueue(runner, "a", 6);
    runner.run();
    succeeded = runner.getFlowFilesForRelationship(EnforceOrder.REL_SUCCESS);
    succeeded.sort(fifo);
    assertEquals(3, succeeded.size());
    // This is ok because a.5 was there.
    succeeded.get(0).assertContentEquals("a.6");
    succeeded.get(1).assertContentEquals("b.2");
    succeeded.get(2).assertContentEquals("b.3");
    List<MockFlowFile> overtook = runner.getFlowFilesForRelationship(EnforceOrder.REL_OVERTOOK);
    assertEquals(3, overtook.size());
    // overtook a.3.
    overtook.get(0).assertContentEquals("a.5");
    overtook.get(0).assertAttributeEquals(EnforceOrder.ATTR_EXPECTED_ORDER, "3");
    // overtook c.1 - 8.
    overtook.get(1).assertContentEquals("c.9");
    overtook.get(1).assertAttributeEquals(EnforceOrder.ATTR_EXPECTED_ORDER, "1");
    // overtook d.1 - 9.
    overtook.get(2).assertContentEquals("d.10");
    overtook.get(2).assertAttributeEquals(EnforceOrder.ATTR_EXPECTED_ORDER, "1");
    stateManager.assertStateEquals("a.target", "7", Scope.LOCAL);
    stateManager.assertStateEquals("b.target", "4", Scope.LOCAL);
    // it was c.9, so +1
    stateManager.assertStateEquals("c.target", "10", Scope.LOCAL);
    // it was d.10 (max) so don't +1
    stateManager.assertStateEquals("d.target", "10", Scope.LOCAL);
    // Simulate a.3 and a.4 arrive but too late..
    runner.clearTransferState();
    Ordered.enqueue(runner, "a", 3);
    Ordered.enqueue(runner, "a", 4);
    runner.run();
    runner.assertAllFlowFilesTransferred(EnforceOrder.REL_SKIPPED, 2);
    final List<MockFlowFile> skipped = runner.getFlowFilesForRelationship(EnforceOrder.REL_SKIPPED);
    skipped.get(0).assertContentEquals("a.3");
    skipped.get(0).assertAttributeExists(EnforceOrder.ATTR_DETAIL);
    skipped.get(1).assertContentEquals("a.4");
    skipped.get(1).assertAttributeExists(EnforceOrder.ATTR_DETAIL);
}

Example 4 with FirstInFirstOutPrioritizer

use of org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer in project nifi by apache.

the class TestEnforceOrder method testIllegalOrderValue.

@Test
public void testIllegalOrderValue() {
    final TestRunner runner = TestRunners.newTestRunner(EnforceOrder.class);
    runner.setProperty(EnforceOrder.GROUP_IDENTIFIER, "${group}");
    runner.setProperty(EnforceOrder.ORDER_ATTRIBUTE, "index");
    runner.setProperty(EnforceOrder.INITIAL_ORDER, "1");
    runner.assertValid();
    Ordered.enqueue(runner, "b", 1);
    Ordered.enqueue(runner, "a", 2);
    runner.enqueue("illegal order", Ordered.i("a", 1).put("index", "non-integer").map());
    Ordered.enqueue(runner, "a", 1);
    runner.run();
    final List<MockFlowFile> succeeded = runner.getFlowFilesForRelationship(EnforceOrder.REL_SUCCESS);
    assertEquals(3, succeeded.size());
    succeeded.sort(new FirstInFirstOutPrioritizer());
    succeeded.get(0).assertContentEquals("a.1");
    succeeded.get(1).assertContentEquals("a.2");
    succeeded.get(2).assertContentEquals("b.1");
    final List<MockFlowFile> failed = runner.getFlowFilesForRelationship(EnforceOrder.REL_FAILURE);
    assertEquals(1, failed.size());
    failed.get(0).assertAttributeExists(EnforceOrder.ATTR_DETAIL);
    failed.get(0).assertContentEquals("illegal order");
}

Example 5 with FirstInFirstOutPrioritizer

use of org.apache.nifi.prioritizer.FirstInFirstOutPrioritizer in project nifi by apache.

the class TestEnforceOrder method testNoGroupIdentifier.

@Test
public void testNoGroupIdentifier() {
    final TestRunner runner = TestRunners.newTestRunner(EnforceOrder.class);
    runner.setProperty(EnforceOrder.GROUP_IDENTIFIER, "${group}");
    runner.setProperty(EnforceOrder.ORDER_ATTRIBUTE, "index");
    runner.setProperty(EnforceOrder.INITIAL_ORDER, "1");
    runner.assertValid();
    Ordered.enqueue(runner, "b", 1);
    Ordered.enqueue(runner, "a", 2);
    // without group attribute
    runner.enqueue("no group id", Ordered.i(1).map());
    Ordered.enqueue(runner, "a", 1);
    runner.run();
    final List<MockFlowFile> succeeded = runner.getFlowFilesForRelationship(EnforceOrder.REL_SUCCESS);
    assertEquals(3, succeeded.size());
    succeeded.sort(new FirstInFirstOutPrioritizer());
    succeeded.get(0).assertContentEquals("a.1");
    succeeded.get(1).assertContentEquals("a.2");
    succeeded.get(2).assertContentEquals("b.1");
    final List<MockFlowFile> failed = runner.getFlowFilesForRelationship(EnforceOrder.REL_FAILURE);
    assertEquals(1, failed.size());
    failed.get(0).assertAttributeExists(EnforceOrder.ATTR_DETAIL);
}