Examples with ProgramController co.cask.cdap.app.runtime.ProgramController used on opensource projects

Example 11 with ProgramController

use of co.cask.cdap.app.runtime.ProgramController in project cdap by caskdata.

the class InMemoryFlowProgramRunner method createFlowlets.

/**
 * Starts all flowlets in the flow program.
 * @param program Program to run
 * @param flowSpec The {@link FlowSpecification}.
 * @return A {@link Table} with row as flowlet id, column as instance id, cell as the {@link ProgramController}
 *         for the flowlet.
 */
private Table<String, Integer, ProgramController> createFlowlets(Program program, ProgramOptions options, FlowSpecification flowSpec) {
    Table<String, Integer, ProgramController> flowlets = HashBasedTable.create();
    try {
        for (Map.Entry<String, FlowletDefinition> entry : flowSpec.getFlowlets().entrySet()) {
            ProgramOptions flowletOptions = resolveFlowletOptions(options, entry.getKey());
            int instanceCount = entry.getValue().getInstances();
            for (int instanceId = 0; instanceId < instanceCount; instanceId++) {
                flowlets.put(entry.getKey(), instanceId, startFlowlet(program, createFlowletOptions(instanceId, instanceCount, flowletOptions)));
            }
        }
    } catch (Throwable t) {
        try {
            // Need to stop all started flowlets
            Futures.successfulAsList(Iterables.transform(flowlets.values(), new Function<ProgramController, ListenableFuture<?>>() {

                @Override
                public ListenableFuture<?> apply(ProgramController controller) {
                    return controller.stop();
                }
            })).get();
        } catch (Exception e) {
            LOG.error("Fail to stop all flowlets on failure.");
        }
        throw Throwables.propagate(t);
    }
    return flowlets;
}

Example 12 with ProgramController

use of co.cask.cdap.app.runtime.ProgramController in project cdap by caskdata.

the class WorkerProgramRunnerTest method testWorkerWithMisbehavedDataset.

@Test
public void testWorkerWithMisbehavedDataset() throws Throwable {
    final ApplicationWithPrograms app = AppFabricTestHelper.deployApplicationWithManager(AppWithMisbehavedDataset.class, TEMP_FOLDER_SUPPLIER);
    final ProgramController controller = startProgram(app, AppWithMisbehavedDataset.TableWriter.class);
    Tasks.waitFor(ProgramController.State.COMPLETED, new Callable<ProgramController.State>() {

        @Override
        public ProgramController.State call() throws Exception {
            return controller.getState();
        }
    }, 30, TimeUnit.SECONDS);
    // validate worker was able to execute its second transaction
    final TransactionExecutor executor = txExecutorFactory.createExecutor(datasetCache);
    executor.execute(new TransactionExecutor.Subroutine() {

        @Override
        public void apply() throws Exception {
            Table table = datasetCache.getDataset(AppWithMisbehavedDataset.TABLE);
            Row result = table.get(new Get(AppWithMisbehavedDataset.ROW, AppWithMisbehavedDataset.COLUMN));
            Assert.assertEquals(AppWithMisbehavedDataset.VALUE, result.getString(AppWithMisbehavedDataset.COLUMN));
        }
    });
}

Example 13 with ProgramController

use of co.cask.cdap.app.runtime.ProgramController in project cdap by caskdata.

the class MapReduceProgramRunner method run.

@Override
public ProgramController run(final Program program, ProgramOptions options) {
    // Extract and verify parameters
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type.");
    Preconditions.checkArgument(processorType == ProgramType.MAPREDUCE, "Only MAPREDUCE process type is supported.");
    MapReduceSpecification spec = appSpec.getMapReduce().get(program.getName());
    Preconditions.checkNotNull(spec, "Missing MapReduceSpecification for %s", program.getName());
    Arguments arguments = options.getArguments();
    RunId runId = ProgramRunners.getRunId(options);
    WorkflowProgramInfo workflowInfo = WorkflowProgramInfo.create(arguments);
    DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, appSpec);
    // Setup dataset framework context, if required
    if (programDatasetFramework instanceof ProgramContextAware) {
        ProgramId programId = program.getId();
        ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programId.run(runId)));
    }
    MapReduce mapReduce;
    try {
        mapReduce = new InstantiatorFactory(false).get(TypeToken.of(program.<MapReduce>getMainClass())).create();
    } catch (Exception e) {
        LOG.error("Failed to instantiate MapReduce class for {}", spec.getClassName(), e);
        throw Throwables.propagate(e);
    }
    // List of all Closeable resources that needs to be cleanup
    List<Closeable> closeables = new ArrayList<>();
    try {
        PluginInstantiator pluginInstantiator = createPluginInstantiator(options, program.getClassLoader());
        if (pluginInstantiator != null) {
            closeables.add(pluginInstantiator);
        }
        final BasicMapReduceContext context = new BasicMapReduceContext(program, options, cConf, spec, workflowInfo, discoveryServiceClient, metricsCollectionService, txSystemClient, programDatasetFramework, streamAdmin, getPluginArchive(options), pluginInstantiator, secureStore, secureStoreManager, messagingService);
        closeables.add(context);
        Reflections.visit(mapReduce, mapReduce.getClass(), new PropertyFieldSetter(context.getSpecification().getProperties()), new MetricsFieldSetter(context.getMetrics()), new DataSetFieldSetter(context));
        // note: this sets logging context on the thread level
        LoggingContextAccessor.setLoggingContext(context.getLoggingContext());
        // Set the job queue to hConf if it is provided
        Configuration hConf = new Configuration(this.hConf);
        String schedulerQueue = options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE);
        if (schedulerQueue != null && !schedulerQueue.isEmpty()) {
            hConf.set(JobContext.QUEUE_NAME, schedulerQueue);
        }
        Service mapReduceRuntimeService = new MapReduceRuntimeService(injector, cConf, hConf, mapReduce, spec, context, program.getJarLocation(), locationFactory, streamAdmin, authorizationEnforcer, authenticationContext);
        mapReduceRuntimeService.addListener(createRuntimeServiceListener(closeables), Threads.SAME_THREAD_EXECUTOR);
        ProgramController controller = new MapReduceProgramController(mapReduceRuntimeService, context);
        LOG.debug("Starting MapReduce Job: {}", context);
        // be running the job, but the data directory will be owned by cdap.
        if (MapReduceTaskContextProvider.isLocal(hConf) || UserGroupInformation.isSecurityEnabled()) {
            mapReduceRuntimeService.start();
        } else {
            ProgramRunners.startAsUser(cConf.get(Constants.CFG_HDFS_USER), mapReduceRuntimeService);
        }
        return controller;
    } catch (Exception e) {
        closeAllQuietly(closeables);
        throw Throwables.propagate(e);
    }
}

Example 14 with ProgramController

use of co.cask.cdap.app.runtime.ProgramController in project cdap by caskdata.

the class ScheduleTaskRunner method execute.

/**
 * Executes a program without blocking until its completion.
 *
 * @return a {@link ListenableFuture} object that completes when the program completes
 */
public ListenableFuture<?> execute(final ProgramId id, Map<String, String> sysArgs, Map<String, String> userArgs) throws Exception {
    ProgramRuntimeService.RuntimeInfo runtimeInfo;
    String originalUserId = SecurityRequestContext.getUserId();
    try {
        // if the program has a namespace user configured then set that user in the security request context.
        // See: CDAP-7396
        String nsPrincipal = namespaceQueryAdmin.get(id.getNamespaceId()).getConfig().getPrincipal();
        if (nsPrincipal != null && SecurityUtil.isKerberosEnabled(cConf)) {
            SecurityRequestContext.setUserId(new KerberosName(nsPrincipal).getServiceName());
        }
        runtimeInfo = lifecycleService.startInternal(id, sysArgs, userArgs, false);
    } catch (ProgramNotFoundException | ApplicationNotFoundException e) {
        throw new TaskExecutionException(String.format(UserMessages.getMessage(UserErrors.PROGRAM_NOT_FOUND), id), e, false);
    } finally {
        SecurityRequestContext.setUserId(originalUserId);
    }
    final ProgramController controller = runtimeInfo.getController();
    final CountDownLatch latch = new CountDownLatch(1);
    controller.addListener(new AbstractListener() {

        @Override
        public void init(ProgramController.State state, @Nullable Throwable cause) {
            if (state == ProgramController.State.COMPLETED) {
                completed();
            }
            if (state == ProgramController.State.ERROR) {
                error(controller.getFailureCause());
            }
        }

        @Override
        public void killed() {
            latch.countDown();
        }

        @Override
        public void completed() {
            latch.countDown();
        }

        @Override
        public void error(Throwable cause) {
            latch.countDown();
        }
    }, Threads.SAME_THREAD_EXECUTOR);
    return executorService.submit(new Callable<Void>() {

        @Override
        public Void call() throws Exception {
            latch.await();
            return null;
        }
    });
}

Example 15 with ProgramController

use of co.cask.cdap.app.runtime.ProgramController in project cdap by caskdata.

the class ServiceProgramRunner method run.

@Override
public ProgramController run(Program program, ProgramOptions options) {
    int instanceId = Integer.parseInt(options.getArguments().getOption(ProgramOptionConstants.INSTANCE_ID, "-1"));
    Preconditions.checkArgument(instanceId >= 0, "Missing instance Id");
    int instanceCount = Integer.parseInt(options.getArguments().getOption(ProgramOptionConstants.INSTANCES, "0"));
    Preconditions.checkArgument(instanceCount > 0, "Invalid or missing instance count");
    RunId runId = ProgramRunners.getRunId(options);
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType programType = program.getType();
    Preconditions.checkNotNull(programType, "Missing processor type.");
    Preconditions.checkArgument(programType == ProgramType.SERVICE, "Only Service process type is supported.");
    ServiceSpecification spec = appSpec.getServices().get(program.getName());
    String host = options.getArguments().getOption(ProgramOptionConstants.HOST);
    Preconditions.checkArgument(host != null, "No hostname is provided");
    // Setup dataset framework context, if required
    if (datasetFramework instanceof ProgramContextAware) {
        ProgramId programId = program.getId();
        ((ProgramContextAware) datasetFramework).setContext(new BasicProgramContext(programId.run(runId)));
    }
    final PluginInstantiator pluginInstantiator = createPluginInstantiator(options, program.getClassLoader());
    try {
        RetryStrategy retryStrategy = SystemArguments.getRetryStrategy(options.getUserArguments().asMap(), program.getType(), cConf);
        ArtifactManager artifactManager = artifactManagerFactory.create(program.getId().getNamespaceId(), retryStrategy);
        ServiceHttpServer component = new ServiceHttpServer(host, program, options, cConf, spec, instanceId, instanceCount, serviceAnnouncer, metricsCollectionService, datasetFramework, txClient, discoveryServiceClient, pluginInstantiator, secureStore, secureStoreManager, messagingService, artifactManager);
        // Add a service listener to make sure the plugin instantiator is closed when the http server is finished.
        component.addListener(createRuntimeServiceListener(Collections.singleton((Closeable) pluginInstantiator)), Threads.SAME_THREAD_EXECUTOR);
        ProgramController controller = new ServiceProgramControllerAdapter(component, program.getId().run(runId), spec.getName() + "-" + instanceId);
        component.start();
        return controller;
    } catch (Throwable t) {
        Closeables.closeQuietly(pluginInstantiator);
        throw t;
    }
}