Examples with BasicArguments co.cask.cdap.internal.app.runtime.BasicArguments used on opensource projects

Example 26 with BasicArguments

use of co.cask.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class FlowTest method testFlowPendingMetric.

@Test
public void testFlowPendingMetric() throws Exception {
    final ApplicationWithPrograms app = AppFabricTestHelper.deployApplicationWithManager(PendingMetricTestApp.class, TEMP_FOLDER_SUPPLIER);
    File tempFolder = TEMP_FOLDER_SUPPLIER.get();
    ProgramController controller = null;
    for (ProgramDescriptor programDescriptor : app.getPrograms()) {
        // running mapreduce is out of scope of this tests (there's separate unit-test for that)
        if (programDescriptor.getProgramId().getType() == ProgramType.FLOW) {
            Arguments args = new BasicArguments(ImmutableMap.of("temp", tempFolder.getAbsolutePath(), "count", "4"));
            controller = AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), args, TEMP_FOLDER_SUPPLIER);
        }
    }
    Assert.assertNotNull(controller);
    Map<String, String> tagsForSourceToOne = metricTagsForQueue("source", "ints", "forward-one");
    Map<String, String> tagsForSourceToTwo = metricTagsForQueue("source", null, "forward-two");
    Map<String, String> tagsForSourceToTwoInts = metricTagsForQueue("source", "ints", "forward-two");
    Map<String, String> tagsForSourceToTwoStrings = metricTagsForQueue("source", "strings", "forward-two");
    Map<String, String> tagsForOneToSink = metricTagsForQueue("forward-one", "queue", "sink");
    Map<String, String> tagsForTwoToSink = metricTagsForQueue("forward-two", "queue", "sink");
    Map<String, String> tagsForAllToOne = metricTagsForQueue(null, null, "forward-one");
    Map<String, String> tagsForAllToTwo = metricTagsForQueue(null, null, "forward-two");
    Map<String, String> tagsForAllToSink = metricTagsForQueue(null, null, "sink");
    Map<String, String> tagsForAll = metricTagsForQueue(null, null, null);
    try {
        // source emits 4, then forward-one reads 1, hence 3 should be pending
        // wait a little longer as flow needs to start
        waitForPending(tagsForSourceToOne, 3, 5000);
        // wait a little longer as flow needs to start
        waitForPending(tagsForAllToOne, 3, 100);
        // forward-two receives each of the 4 as a string and an int, but could have read 1 at most per each queue
        // so there should be either 3 + 4 = 7 pending or 3 + 3 = 6 pending, or 4 + 4 = 8 pending
        // but we don't know whether the queue pending count will be 4, 3 or 3, 4 or 3, 3 or 4, 4
        long intPending = waitForPending(tagsForSourceToTwoInts, 3, 4L, 1000);
        long stringPending = waitForPending(tagsForSourceToTwoStrings, 3, 4L, 1000);
        long totalPending = intPending + stringPending;
        Assert.assertTrue(String.format("Expected the pending events count to be 6, 7 or 8. But it was %d", totalPending), totalPending == 6 || totalPending == 7 || totalPending == 8);
        waitForPending(tagsForSourceToTwo, 7, 6L, 500);
        waitForPending(tagsForAllToTwo, 7, 6L, 100);
        // neither one nor two have emitted, so the total pending should be = 12 - 1 (forward-one) - 1 or 2 (forward-two)
        // => 10 or 9 events
        waitForPending(tagsForAll, 10, 9L, 100);
        // kick on forward-one, it should now consume all its events
        Assert.assertTrue(new File(tempFolder, "one").createNewFile());
        waitForPending(tagsForSourceToOne, 0, 2000);
        waitForPending(tagsForAllToOne, 0, 100);
        // sink has received 4 but started to read 1, so it has 3 pending
        waitForPending(tagsForOneToSink, 3, 1000);
        waitForPending(tagsForAllToSink, 3, 100);
        // kick-off forward-two, it should now consume all its integer and string events
        Assert.assertTrue(new File(tempFolder, "two-i").createNewFile());
        Assert.assertTrue(new File(tempFolder, "two-s").createNewFile());
        // pending events for all of forward-two's queues should go to zero
        waitForPending(tagsForSourceToTwoInts, 0, 2000);
        waitForPending(tagsForSourceToTwoStrings, 0, 1000);
        waitForPending(tagsForSourceToTwo, 0, 1000);
        waitForPending(tagsForAllToTwo, 0, 100);
        // but now sink should have 8 more events waiting
        waitForPending(tagsForOneToSink, 3, 1000);
        waitForPending(tagsForTwoToSink, 8, 1000);
        waitForPending(tagsForAllToSink, 11, 100);
        // kick off sink, its pending events should now go to zero
        Assert.assertTrue(new File(tempFolder, "three").createNewFile());
        waitForPending(tagsForOneToSink, 0, 2000);
        waitForPending(tagsForTwoToSink, 0, 2000);
        waitForPending(tagsForAllToSink, 0, 100);
    } finally {
        controller.stop();
    }
}

Example 27 with BasicArguments

use of co.cask.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class MultiConsumerTest method testMulti.

@Test
public void testMulti() throws Exception {
    // TODO: Fix this test case to really test with numGroups settings.
    final ApplicationWithPrograms app = AppFabricTestHelper.deployApplicationWithManager(MultiApp.class, TEMP_FOLDER_SUPPLIER);
    List<ProgramController> controllers = Lists.newArrayList();
    for (ProgramDescriptor programDescriptor : app.getPrograms()) {
        controllers.add(AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), new BasicArguments(), TEMP_FOLDER_SUPPLIER));
    }
    DatasetFramework datasetFramework = AppFabricTestHelper.getInjector().getInstance(DatasetFramework.class);
    DynamicDatasetCache datasetCache = new SingleThreadDatasetCache(new SystemDatasetInstantiator(datasetFramework, getClass().getClassLoader(), null), AppFabricTestHelper.getInjector().getInstance(TransactionSystemClient.class), NamespaceId.DEFAULT, DatasetDefinition.NO_ARGUMENTS, null, null);
    final KeyValueTable accumulated = datasetCache.getDataset("accumulated");
    TransactionExecutorFactory txExecutorFactory = AppFabricTestHelper.getInjector().getInstance(TransactionExecutorFactory.class);
    // Try to get accumulated result and verify it. Expect result appear in max of 60 seconds.
    int trial = 0;
    while (trial < 60) {
        try {
            Transactions.createTransactionExecutor(txExecutorFactory, accumulated).execute(new TransactionExecutor.Subroutine() {

                @Override
                public void apply() throws Exception {
                    byte[] value = accumulated.read(MultiApp.KEY);
                    // Sum(1..100) * 3
                    Assert.assertEquals(((1 + 99) * 99 / 2) * 3, Longs.fromByteArray(value));
                }
            });
            break;
        } catch (TransactionFailureException e) {
            // No-op
            trial++;
            TimeUnit.SECONDS.sleep(1);
        }
    }
    Assert.assertTrue(trial < 60);
    for (ProgramController controller : controllers) {
        controller.stop().get();
    }
}

Example 28 with BasicArguments

use of co.cask.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class WorkerProgramRunnerTest method startProgram.

private ProgramController startProgram(ApplicationWithPrograms app, Class<?> programClass) throws Throwable {
    final AtomicReference<Throwable> errorCause = new AtomicReference<>();
    final ProgramController controller = AppFabricTestHelper.submit(app, programClass.getName(), new BasicArguments(), TEMP_FOLDER_SUPPLIER);
    runningPrograms.add(controller);
    controller.addListener(new AbstractListener() {

        @Override
        public void error(Throwable cause) {
            errorCause.set(cause);
        }

        @Override
        public void killed() {
            errorCause.set(new RuntimeException("Killed"));
        }
    }, Threads.SAME_THREAD_EXECUTOR);
    Tasks.waitFor(ProgramController.State.ALIVE, new Callable<ProgramController.State>() {

        @Override
        public ProgramController.State call() throws Exception {
            Throwable t = errorCause.get();
            if (t != null) {
                Throwables.propagateIfInstanceOf(t, Exception.class);
                throw Throwables.propagate(t);
            }
            return controller.getState();
        }
    }, 30, TimeUnit.SECONDS);
    return controller;
}

Example 29 with BasicArguments

use of co.cask.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class AbstractProgramRuntimeServiceTest method testScopingRuntimeArguments.

@Test
public void testScopingRuntimeArguments() throws Exception {
    Map<ProgramId, Arguments> argumentsMap = new ConcurrentHashMap<>();
    ProgramRunnerFactory runnerFactory = createProgramRunnerFactory(argumentsMap);
    final Program program = createDummyProgram();
    final ProgramRuntimeService runtimeService = new AbstractProgramRuntimeService(CConfiguration.create(), runnerFactory, null) {

        @Override
        public ProgramLiveInfo getLiveInfo(ProgramId programId) {
            return new ProgramLiveInfo(programId, "runtime") {
            };
        }

        @Override
        protected Program createProgram(CConfiguration cConf, ProgramRunner programRunner, ProgramDescriptor programDescriptor, ArtifactDetail artifactDetail, File tempDir) throws IOException {
            return program;
        }

        @Override
        protected ArtifactDetail getArtifactDetail(ArtifactId artifactId) throws IOException, ArtifactNotFoundException {
            co.cask.cdap.api.artifact.ArtifactId id = new co.cask.cdap.api.artifact.ArtifactId("dummy", new ArtifactVersion("1.0"), ArtifactScope.USER);
            return new ArtifactDetail(new ArtifactDescriptor(id, Locations.toLocation(TEMP_FOLDER.newFile())), new ArtifactMeta(ArtifactClasses.builder().build()));
        }
    };
    runtimeService.startAndWait();
    try {
        try {
            ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, null);
            // Set of scopes to test
            String programScope = program.getType().getScope();
            String clusterName = "c1";
            List<String> scopes = Arrays.asList("cluster.*.", "cluster." + clusterName + ".", "cluster." + clusterName + ".app.*.", "app.*.", "app." + program.getApplicationId() + ".", "app." + program.getApplicationId() + "." + programScope + ".*.", "app." + program.getApplicationId() + "." + programScope + "." + program.getName() + ".", programScope + ".*.", programScope + "." + program.getName() + ".", "");
            for (String scope : scopes) {
                ProgramOptions programOptions = new SimpleProgramOptions(program.getName(), new BasicArguments(Collections.singletonMap(Constants.CLUSTER_NAME, clusterName)), new BasicArguments(Collections.singletonMap(scope + "size", Integer.toString(scope.length()))));
                final ProgramController controller = runtimeService.run(descriptor, programOptions).getController();
                Tasks.waitFor(ProgramController.State.COMPLETED, new Callable<ProgramController.State>() {

                    @Override
                    public ProgramController.State call() throws Exception {
                        return controller.getState();
                    }
                }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
                // Should get an argument
                Arguments args = argumentsMap.get(program.getId());
                Assert.assertNotNull(args);
                Assert.assertEquals(scope.length(), Integer.parseInt(args.getOption("size")));
            }
        } finally {
            runtimeService.stopAndWait();
        }
    } finally {
        runtimeService.stopAndWait();
    }
}

Example 30 with BasicArguments

use of co.cask.cdap.internal.app.runtime.BasicArguments in project cdap by caskdata.

the class WorkflowTest method testWorkflow.

@Test(timeout = 120 * 1000L)
public void testWorkflow() throws Exception {
    final ApplicationWithPrograms app = AppFabricTestHelper.deployApplicationWithManager(WorkflowApp.class, TEMP_FOLDER_SUPPLIER);
    final Injector injector = AppFabricTestHelper.getInjector();
    final ProgramDescriptor programDescriptor = Iterators.filter(app.getPrograms().iterator(), new Predicate<ProgramDescriptor>() {

        @Override
        public boolean apply(ProgramDescriptor input) {
            return input.getProgramId().getType() == ProgramType.WORKFLOW;
        }
    }).next();
    String inputPath = createInput();
    String outputPath = new File(tmpFolder.newFolder(), "output").getAbsolutePath();
    BasicArguments userArgs = new BasicArguments(ImmutableMap.of("inputPath", inputPath, "outputPath", outputPath));
    final SettableFuture<String> completion = SettableFuture.create();
    final ProgramController controller = AppFabricTestHelper.submit(app, programDescriptor.getSpecification().getClassName(), userArgs, TEMP_FOLDER_SUPPLIER);
    controller.addListener(new AbstractListener() {

        @Override
        public void init(ProgramController.State currentState, @Nullable Throwable cause) {
            LOG.info("Starting");
            injector.getInstance(Store.class).setStart(controller.getProgramRunId().getParent(), controller.getProgramRunId().getRun(), System.currentTimeMillis());
        }

        @Override
        public void completed() {
            LOG.info("Completed");
            completion.set("Completed");
        }

        @Override
        public void error(Throwable cause) {
            LOG.info("Error", cause);
            completion.setException(cause);
        }
    }, Threads.SAME_THREAD_EXECUTOR);
    completion.get();
}