Examples with ProgramController co.cask.cdap.app.runtime.ProgramController used on opensource projects

Example 36 with ProgramController

use of co.cask.cdap.app.runtime.ProgramController in project cdap by caskdata.

the class FlowTest method testFlowPendingMetric.

@Test
public void testFlowPendingMetric() throws Exception {
    final ApplicationWithPrograms app = AppFabricTestHelper.deployApplicationWithManager(PendingMetricTestApp.class, TEMP_FOLDER_SUPPLIER);
    File tempFolder = TEMP_FOLDER_SUPPLIER.get();
    ProgramController controller = null;
    for (ProgramDescriptor programDescriptor : app.getPrograms()) {
        // running mapreduce is out of scope of this tests (there's separate unit-test for that)
        if (programDescriptor.getProgramId().getType() == ProgramType.FLOW) {
            Arguments args = new BasicArguments(ImmutableMap.of("temp", tempFolder.getAbsolutePath(), "count", "4"));
            controller = AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), args, TEMP_FOLDER_SUPPLIER);
        }
    }
    Assert.assertNotNull(controller);
    Map<String, String> tagsForSourceToOne = metricTagsForQueue("source", "ints", "forward-one");
    Map<String, String> tagsForSourceToTwo = metricTagsForQueue("source", null, "forward-two");
    Map<String, String> tagsForSourceToTwoInts = metricTagsForQueue("source", "ints", "forward-two");
    Map<String, String> tagsForSourceToTwoStrings = metricTagsForQueue("source", "strings", "forward-two");
    Map<String, String> tagsForOneToSink = metricTagsForQueue("forward-one", "queue", "sink");
    Map<String, String> tagsForTwoToSink = metricTagsForQueue("forward-two", "queue", "sink");
    Map<String, String> tagsForAllToOne = metricTagsForQueue(null, null, "forward-one");
    Map<String, String> tagsForAllToTwo = metricTagsForQueue(null, null, "forward-two");
    Map<String, String> tagsForAllToSink = metricTagsForQueue(null, null, "sink");
    Map<String, String> tagsForAll = metricTagsForQueue(null, null, null);
    try {
        // source emits 4, then forward-one reads 1, hence 3 should be pending
        // wait a little longer as flow needs to start
        waitForPending(tagsForSourceToOne, 3, 5000);
        // wait a little longer as flow needs to start
        waitForPending(tagsForAllToOne, 3, 100);
        // forward-two receives each of the 4 as a string and an int, but could have read 1 at most per each queue
        // so there should be either 3 + 4 = 7 pending or 3 + 3 = 6 pending, or 4 + 4 = 8 pending
        // but we don't know whether the queue pending count will be 4, 3 or 3, 4 or 3, 3 or 4, 4
        long intPending = waitForPending(tagsForSourceToTwoInts, 3, 4L, 1000);
        long stringPending = waitForPending(tagsForSourceToTwoStrings, 3, 4L, 1000);
        long totalPending = intPending + stringPending;
        Assert.assertTrue(String.format("Expected the pending events count to be 6, 7 or 8. But it was %d", totalPending), totalPending == 6 || totalPending == 7 || totalPending == 8);
        waitForPending(tagsForSourceToTwo, 7, 6L, 500);
        waitForPending(tagsForAllToTwo, 7, 6L, 100);
        // neither one nor two have emitted, so the total pending should be = 12 - 1 (forward-one) - 1 or 2 (forward-two)
        // => 10 or 9 events
        waitForPending(tagsForAll, 10, 9L, 100);
        // kick on forward-one, it should now consume all its events
        Assert.assertTrue(new File(tempFolder, "one").createNewFile());
        waitForPending(tagsForSourceToOne, 0, 2000);
        waitForPending(tagsForAllToOne, 0, 100);
        // sink has received 4 but started to read 1, so it has 3 pending
        waitForPending(tagsForOneToSink, 3, 1000);
        waitForPending(tagsForAllToSink, 3, 100);
        // kick-off forward-two, it should now consume all its integer and string events
        Assert.assertTrue(new File(tempFolder, "two-i").createNewFile());
        Assert.assertTrue(new File(tempFolder, "two-s").createNewFile());
        // pending events for all of forward-two's queues should go to zero
        waitForPending(tagsForSourceToTwoInts, 0, 2000);
        waitForPending(tagsForSourceToTwoStrings, 0, 1000);
        waitForPending(tagsForSourceToTwo, 0, 1000);
        waitForPending(tagsForAllToTwo, 0, 100);
        // but now sink should have 8 more events waiting
        waitForPending(tagsForOneToSink, 3, 1000);
        waitForPending(tagsForTwoToSink, 8, 1000);
        waitForPending(tagsForAllToSink, 11, 100);
        // kick off sink, its pending events should now go to zero
        Assert.assertTrue(new File(tempFolder, "three").createNewFile());
        waitForPending(tagsForOneToSink, 0, 2000);
        waitForPending(tagsForTwoToSink, 0, 2000);
        waitForPending(tagsForAllToSink, 0, 100);
    } finally {
        controller.stop();
    }
}

Example 37 with ProgramController

use of co.cask.cdap.app.runtime.ProgramController in project cdap by caskdata.

the class MultiConsumerTest method testMulti.

@Test
public void testMulti() throws Exception {
    // TODO: Fix this test case to really test with numGroups settings.
    final ApplicationWithPrograms app = AppFabricTestHelper.deployApplicationWithManager(MultiApp.class, TEMP_FOLDER_SUPPLIER);
    List<ProgramController> controllers = Lists.newArrayList();
    for (ProgramDescriptor programDescriptor : app.getPrograms()) {
        controllers.add(AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), new BasicArguments(), TEMP_FOLDER_SUPPLIER));
    }
    DatasetFramework datasetFramework = AppFabricTestHelper.getInjector().getInstance(DatasetFramework.class);
    DynamicDatasetCache datasetCache = new SingleThreadDatasetCache(new SystemDatasetInstantiator(datasetFramework, getClass().getClassLoader(), null), AppFabricTestHelper.getInjector().getInstance(TransactionSystemClient.class), NamespaceId.DEFAULT, DatasetDefinition.NO_ARGUMENTS, null, null);
    final KeyValueTable accumulated = datasetCache.getDataset("accumulated");
    TransactionExecutorFactory txExecutorFactory = AppFabricTestHelper.getInjector().getInstance(TransactionExecutorFactory.class);
    // Try to get accumulated result and verify it. Expect result appear in max of 60 seconds.
    int trial = 0;
    while (trial < 60) {
        try {
            Transactions.createTransactionExecutor(txExecutorFactory, accumulated).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    byte[] value = accumulated.read(MultiApp.KEY);
                    // Sum(1..100) * 3
                    Assert.assertEquals(((1 + 99) * 99 / 2) * 3, Longs.fromByteArray(value));
                }
            });
            break;
        } catch (TransactionFailureException e) {
            // No-op
            trial++;
            TimeUnit.SECONDS.sleep(1);
        }
    }
    Assert.assertTrue(trial < 60);
    for (ProgramController controller : controllers) {
        controller.stop().get();
    }
}