Examples with ProgramOptions co.cask.cdap.app.runtime.ProgramOptions used on opensource projects

Example 1 with ProgramOptions

use of co.cask.cdap.app.runtime.ProgramOptions in project cdap by caskdata.

the class FlowProgramRunner method createFlowlets.

/**
   * Starts all flowlets in the flow program.
   * @param program Program to run
   * @param flowSpec The {@link FlowSpecification}.
   * @return A {@link Table} with row as flowlet id, column as instance id, cell as the {@link ProgramController}
   *         for the flowlet.
   */
private Table<String, Integer, ProgramController> createFlowlets(Program program, ProgramOptions options, FlowSpecification flowSpec) {
    Table<String, Integer, ProgramController> flowlets = HashBasedTable.create();
    try {
        for (Map.Entry<String, FlowletDefinition> entry : flowSpec.getFlowlets().entrySet()) {
            ProgramOptions flowletOptions = resolveFlowletOptions(options, entry.getKey());
            int instanceCount = entry.getValue().getInstances();
            for (int instanceId = 0; instanceId < instanceCount; instanceId++) {
                flowlets.put(entry.getKey(), instanceId, startFlowlet(program, createFlowletOptions(entry.getKey(), instanceId, instanceCount, flowletOptions)));
            }
        }
    } catch (Throwable t) {
        try {
            // Need to stop all started flowlets
            Futures.successfulAsList(Iterables.transform(flowlets.values(), new Function<ProgramController, ListenableFuture<?>>() {

                @Override
                public ListenableFuture<?> apply(ProgramController controller) {
                    return controller.stop();
                }
            })).get();
        } catch (Exception e) {
            LOG.error("Fail to stop all flowlets on failure.");
        }
        throw Throwables.propagate(t);
    }
    return flowlets;
}

Example 2 with ProgramOptions

use of co.cask.cdap.app.runtime.ProgramOptions in project cdap by caskdata.

the class AbstractProgramTwillRunnable method createProgramOptions.

/**
 * Creates program options. It contains program and user arguments as passed form the distributed program runner.
 * Extra program arguments are inserted based on the environment information (e.g. host, instance id). Also all
 * configs available through the TwillRunnable configs are also available through program arguments.
 */
private ProgramOptions createProgramOptions(CommandLine cmdLine, TwillContext context, Map<String, String> configs) {
    ProgramOptions original = GSON.fromJson(cmdLine.getOptionValue(RunnableOptions.PROGRAM_OPTIONS), ProgramOptions.class);
    // Overwrite them with environmental information
    Map<String, String> arguments = Maps.newHashMap(original.getArguments().asMap());
    arguments.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(context.getInstanceId()));
    arguments.put(ProgramOptionConstants.INSTANCES, Integer.toString(context.getInstanceCount()));
    arguments.put(ProgramOptionConstants.TWILL_RUN_ID, context.getApplicationRunId().getId());
    arguments.put(ProgramOptionConstants.HOST, context.getHost().getCanonicalHostName());
    arguments.putAll(configs);
    // Use the name passed in by the constructor as the program name to construct the ProgramId
    ProgramId originalProgramId = original.getProgramId();
    ProgramId runnableProgramId = originalProgramId.getParent().program(originalProgramId.getType(), name);
    return new SimpleProgramOptions(runnableProgramId, new BasicArguments(arguments), resolveScope(original.getUserArguments()), original.isDebug());
}

Example 3 with ProgramOptions

use of co.cask.cdap.app.runtime.ProgramOptions in project cdap by caskdata.

the class InMemoryFlowProgramRunner method createFlowlets.

/**
 * Starts all flowlets in the flow program.
 * @param program Program to run
 * @param flowSpec The {@link FlowSpecification}.
 * @return A {@link Table} with row as flowlet id, column as instance id, cell as the {@link ProgramController}
 *         for the flowlet.
 */
private Table<String, Integer, ProgramController> createFlowlets(Program program, ProgramOptions options, FlowSpecification flowSpec) {
    Table<String, Integer, ProgramController> flowlets = HashBasedTable.create();
    try {
        for (Map.Entry<String, FlowletDefinition> entry : flowSpec.getFlowlets().entrySet()) {
            ProgramOptions flowletOptions = resolveFlowletOptions(options, entry.getKey());
            int instanceCount = entry.getValue().getInstances();
            for (int instanceId = 0; instanceId < instanceCount; instanceId++) {
                flowlets.put(entry.getKey(), instanceId, startFlowlet(program, createFlowletOptions(instanceId, instanceCount, flowletOptions)));
            }
        }
    } catch (Throwable t) {
        try {
            // Need to stop all started flowlets
            Futures.successfulAsList(Iterables.transform(flowlets.values(), new Function<ProgramController, ListenableFuture<?>>() {

                @Override
                public ListenableFuture<?> apply(ProgramController controller) {
                    return controller.stop();
                }
            })).get();
        } catch (Exception e) {
            LOG.error("Fail to stop all flowlets on failure.");
        }
        throw Throwables.propagate(t);
    }
    return flowlets;
}

Example 4 with ProgramOptions

use of co.cask.cdap.app.runtime.ProgramOptions in project cdap by caskdata.

the class ProgramNotificationSubscriberServiceTest method testAppSpecNotRequiredToWriteState.

@Test
public void testAppSpecNotRequiredToWriteState() throws Exception {
    Injector injector = AppFabricTestHelper.getInjector();
    CConfiguration cConf = injector.getInstance(CConfiguration.class);
    ProgramNotificationSubscriberService programNotificationSubscriberService = injector.getInstance(ProgramNotificationSubscriberService.class);
    programNotificationSubscriberService.startAndWait();
    DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
    TransactionExecutorFactory txExecutorFactory = injector.getInstance(TransactionExecutorFactory.class);
    DatasetId storeTable = NamespaceId.SYSTEM.dataset(Constants.AppMetaStore.TABLE);
    Table table = DatasetsUtil.getOrCreateDataset(datasetFramework, storeTable, Table.class.getName(), DatasetProperties.EMPTY, Collections.<String, String>emptyMap());
    final AppMetadataStore metadataStoreDataset = new AppMetadataStore(table, cConf, new AtomicBoolean(false));
    final TransactionExecutor txnl = txExecutorFactory.createExecutor(Collections.singleton((TransactionAware) metadataStoreDataset));
    ProgramStateWriter programStateWriter = injector.getInstance(ProgramStateWriter.class);
    ProgramId programId = NamespaceId.DEFAULT.app("someapp").program(ProgramType.SERVICE, "s");
    ProgramOptions programOptions = new SimpleProgramOptions(programId);
    final ProgramRunId runId = programId.run(RunIds.generate());
    programStateWriter.start(runId, programOptions, null);
    Tasks.waitFor(ProgramRunStatus.STARTING, () -> txnl.execute(() -> {
        RunRecordMeta meta = metadataStoreDataset.getRun(runId);
        return meta == null ? null : meta.getStatus();
    }), 10, TimeUnit.SECONDS);
    programStateWriter.running(runId, UUID.randomUUID().toString());
    Tasks.waitFor(ProgramRunStatus.RUNNING, () -> txnl.execute(() -> {
        RunRecordMeta meta = metadataStoreDataset.getRun(runId);
        return meta == null ? null : meta.getStatus();
    }), 10, TimeUnit.SECONDS);
    programStateWriter.killed(runId);
    Tasks.waitFor(ProgramRunStatus.KILLED, () -> txnl.execute(() -> {
        RunRecordMeta meta = metadataStoreDataset.getRun(runId);
        return meta == null ? null : meta.getStatus();
    }), 10, TimeUnit.SECONDS);
}

Example 5 with ProgramOptions

use of co.cask.cdap.app.runtime.ProgramOptions in project cdap by caskdata.

the class MapReduceTaskContextProvider method createCacheLoader.

/**
 * Creates a {@link CacheLoader} for the task context cache.
 */
private CacheLoader<ContextCacheKey, BasicMapReduceTaskContext> createCacheLoader(final Injector injector) {
    final DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
    final DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
    final SecureStore secureStore = injector.getInstance(SecureStore.class);
    final SecureStoreManager secureStoreManager = injector.getInstance(SecureStoreManager.class);
    final MessagingService messagingService = injector.getInstance(MessagingService.class);
    // Multiple instances of BasicMapReduceTaskContext can share the same program.
    final AtomicReference<Program> programRef = new AtomicReference<>();
    return new CacheLoader<ContextCacheKey, BasicMapReduceTaskContext>() {

        @Override
        public BasicMapReduceTaskContext load(ContextCacheKey key) throws Exception {
            TaskAttemptID taskAttemptId = key.getTaskAttemptID();
            // taskAttemptId could be null if used from a org.apache.hadoop.mapreduce.Partitioner or
            // from a org.apache.hadoop.io.RawComparator, in which case we can get the JobId from the conf. Note that the
            // JobId isn't in the conf for the OutputCommitter#setupJob method, in which case we use the taskAttemptId
            Path txFile = MainOutputCommitter.getTxFile(key.getConfiguration(), taskAttemptId != null ? taskAttemptId.getJobID() : null);
            FileSystem fs = txFile.getFileSystem(key.getConfiguration());
            Preconditions.checkArgument(fs.exists(txFile));
            Transaction tx;
            try (FSDataInputStream txFileInputStream = fs.open(txFile)) {
                byte[] txByteArray = ByteStreams.toByteArray(txFileInputStream);
                tx = new TransactionCodec().decode(txByteArray);
            }
            MapReduceContextConfig contextConfig = new MapReduceContextConfig(key.getConfiguration());
            MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(key.getConfiguration());
            Program program = programRef.get();
            if (program == null) {
                // Creation of program is relatively cheap, so just create and do compare and set.
                programRef.compareAndSet(null, createProgram(contextConfig, classLoader.getProgramClassLoader()));
                program = programRef.get();
            }
            WorkflowProgramInfo workflowInfo = contextConfig.getWorkflowProgramInfo();
            DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, program.getApplicationSpecification());
            // Setup dataset framework context, if required
            if (programDatasetFramework instanceof ProgramContextAware) {
                ProgramRunId programRunId = program.getId().run(ProgramRunners.getRunId(contextConfig.getProgramOptions()));
                ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programRunId));
            }
            MapReduceSpecification spec = program.getApplicationSpecification().getMapReduce().get(program.getName());
            MetricsCollectionService metricsCollectionService = null;
            MapReduceMetrics.TaskType taskType = null;
            String taskId = null;
            ProgramOptions options = contextConfig.getProgramOptions();
            // from a org.apache.hadoop.io.RawComparator
            if (taskAttemptId != null) {
                taskId = taskAttemptId.getTaskID().toString();
                if (MapReduceMetrics.TaskType.hasType(taskAttemptId.getTaskType())) {
                    taskType = MapReduceMetrics.TaskType.from(taskAttemptId.getTaskType());
                    // if this is not for a mapper or a reducer, we don't need the metrics collection service
                    metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
                    options = new SimpleProgramOptions(options.getProgramId(), options.getArguments(), new BasicArguments(RuntimeArguments.extractScope("task", taskType.toString().toLowerCase(), contextConfig.getProgramOptions().getUserArguments().asMap())), options.isDebug());
                }
            }
            CConfiguration cConf = injector.getInstance(CConfiguration.class);
            TransactionSystemClient txClient = injector.getInstance(TransactionSystemClient.class);
            return new BasicMapReduceTaskContext(program, options, cConf, taskType, taskId, spec, workflowInfo, discoveryServiceClient, metricsCollectionService, txClient, tx, programDatasetFramework, classLoader.getPluginInstantiator(), contextConfig.getLocalizedResources(), secureStore, secureStoreManager, authorizationEnforcer, authenticationContext, messagingService, mapReduceClassLoader);
        }
    };
}