Examples with ProgramOptions io.cdap.cdap.app.runtime.ProgramOptions used on opensource projects

Example 16 with ProgramOptions

use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by caskdata.

the class ProgramStateWriterWithHeartBeatTest method testHeartBeatThread.

private void testHeartBeatThread(ProgramRunStatus terminalState) throws Exception {
    // configure program state writer to emit heart beat every second
    MockProgramStatePublisher programStatePublisher = new MockProgramStatePublisher();
    AtomicReference<ProgramRunStatus> completionState = new AtomicReference<>();
    ProgramStateWriter programStateWriter = new NoOpProgramStateWriter() {

        @Override
        public void completed(ProgramRunId programRunId) {
            super.completed(programRunId);
            completionState.set(ProgramRunStatus.COMPLETED);
        }

        @Override
        public void killed(ProgramRunId programRunId) {
            super.killed(programRunId);
            completionState.set(ProgramRunStatus.KILLED);
        }

        @Override
        public void error(ProgramRunId programRunId, Throwable failureCause) {
            super.error(programRunId, failureCause);
            completionState.set(ProgramRunStatus.FAILED);
        }
    };
    // mock program configurations
    ProgramId programId = NamespaceId.DEFAULT.app("someapp").program(ProgramType.SERVICE, "s");
    Map<String, String> systemArguments = new HashMap<>();
    systemArguments.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
    ProgramOptions programOptions = new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments());
    ProgramRunId runId = programId.run(RunIds.generate());
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    ProgramStateWriterWithHeartBeat programStateWriterWithHeartBeat = new ProgramStateWriterWithHeartBeat(runId, programStateWriter, 1, programStatePublisher);
    ApplicationSpecification appSpec = new DefaultApplicationSpecification("name", "1.0.0", ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec);
    // start the program and ensure heart beat is 0 before we call running
    programStateWriter.start(runId, programOptions, null, programDescriptor);
    Assert.assertEquals(0, programStatePublisher.getHeartBeatCount());
    programStateWriterWithHeartBeat.running(null);
    // on running, we start receiving heart beat messages, verify if the heartbeat count goes to 2.
    Tasks.waitFor(true, () -> programStatePublisher.getHeartBeatCount() > 1, 10, TimeUnit.SECONDS, "Didn't receive expected heartbeat after 10 seconds");
    // Terminate the program and make sure the heart beat thread also gets stopped
    switch(terminalState) {
        case COMPLETED:
            programStateWriterWithHeartBeat.completed();
            break;
        case FAILED:
            programStateWriterWithHeartBeat.error(new RuntimeException());
            break;
        case KILLED:
            programStateWriterWithHeartBeat.killed();
            break;
        default:
            throw new IllegalStateException("The terminal state must one of COMPLETED, FAILED, or KILLED");
    }
    Tasks.waitFor(false, programStateWriterWithHeartBeat::isHeartBeatThreadAlive, 5, TimeUnit.SECONDS, "Heartbeat thread did not stop after 5 seconds");
    Assert.assertEquals(terminalState, completionState.get());
}

Example 17 with ProgramOptions

use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by cdapio.

the class RuntimeServiceMainTest method testRuntimeService.

@Test
public void testRuntimeService() throws Exception {
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("test", "1.0");
    ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").worker("worker").run(RunIds.generate());
    Map<String, String> systemArgs = ImmutableMap.of(SystemArguments.PROFILE_PROVISIONER, NativeProvisioner.SPEC.getName(), SystemArguments.PROFILE_NAME, "default");
    ProgramOptions programOptions = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments());
    ProgramDescriptor programDescriptor = new ProgramDescriptor(programRunId.getParent(), null, artifactId);
    // Write out program state events to simulate program start
    Injector appFabricInjector = getServiceMainInstance(AppFabricServiceMain.class).getInjector();
    CConfiguration cConf = appFabricInjector.getInstance(CConfiguration.class);
    ProgramStatePublisher programStatePublisher = new MessagingProgramStatePublisher(appFabricInjector.getInstance(MessagingService.class), NamespaceId.SYSTEM.topic(cConf.get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC)), RetryStrategies.fromConfiguration(cConf, "system.program.state."));
    new MessagingProgramStateWriter(programStatePublisher).start(programRunId, programOptions, null, programDescriptor);
    Injector injector = getServiceMainInstance(RuntimeServiceMain.class).getInjector();
    TransactionRunner txRunner = injector.getInstance(TransactionRunner.class);
    // Should see a STARTING record in the runtime store
    Tasks.waitFor(ProgramRunStatus.STARTING, () -> {
        RunRecordDetail detail = TransactionRunners.run(txRunner, context -> {
            return AppMetadataStore.create(context).getRun(programRunId);
        });
        return detail == null ? null : detail.getStatus();
    }, 5, TimeUnit.SECONDS);
    ProgramStateWriter programStateWriter = createProgramStateWriter(injector, programRunId);
    // Write a running state. We should see a RUNNING record in the runtime store
    programStateWriter.running(programRunId, null);
    Tasks.waitFor(ProgramRunStatus.RUNNING, () -> {
        RunRecordDetail detail = TransactionRunners.run(txRunner, context -> {
            return AppMetadataStore.create(context).getRun(programRunId);
        });
        return detail == null ? null : detail.getStatus();
    }, 5, TimeUnit.SECONDS);
    // Write a complete state. The run record should be removed in the runtime store
    programStateWriter.completed(programRunId);
    Tasks.waitFor(true, () -> TransactionRunners.run(txRunner, context -> AppMetadataStore.create(context).getRun(programRunId) == null), 5, TimeUnit.SECONDS);
}

Example 18 with ProgramOptions

use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by cdapio.

the class DistributedWorkflowProgramRunnerTest method setupWorkflowRuntime.

/**
 * Setup the {@link ProgramLaunchConfig} for the given workflow.
 */
private ProgramLaunchConfig setupWorkflowRuntime(String workflowName, Map<String, String> runtimeArgs) throws IOException {
    // Create the distributed workflow program runner
    ProgramRunner programRunner = programRunnerFactory.create(ProgramType.WORKFLOW);
    Assert.assertTrue(programRunner instanceof DistributedWorkflowProgramRunner);
    DistributedWorkflowProgramRunner workflowRunner = (DistributedWorkflowProgramRunner) programRunner;
    // Create the Workflow Program
    Program workflowProgram = createWorkflowProgram(cConf, programRunner, workflowName);
    ProgramLaunchConfig launchConfig = new ProgramLaunchConfig();
    ProgramOptions programOpts = new SimpleProgramOptions(workflowProgram.getId(), new BasicArguments(), new BasicArguments(runtimeArgs));
    // Setup the launching config
    workflowRunner.setupLaunchConfig(launchConfig, workflowProgram, programOpts, cConf, new Configuration(), TEMP_FOLDER.newFolder());
    return launchConfig;
}

Example 19 with ProgramOptions

use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by cdapio.

the class AbstractProgramTwillRunnable method createProgramOptions.

/**
 * Creates a {@link ProgramOptions} by deserializing the given json file.
 */
private ProgramOptions createProgramOptions(File programOptionsFile) throws IOException {
    ProgramOptions original = readJsonFile(programOptionsFile, ProgramOptions.class);
    // Overwrite them with environmental information
    Map<String, String> arguments = new HashMap<>(original.getArguments().asMap());
    arguments.putAll(getExtraSystemArguments());
    // Use the name passed in by the constructor as the program name to construct the ProgramId
    return new SimpleProgramOptions(original.getProgramId(), new BasicArguments(arguments), original.getUserArguments(), original.isDebug());
}

Example 20 with ProgramOptions

use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by cdapio.

the class DistributedProgramRunner method run.

@Override
public final ProgramController run(final Program program, ProgramOptions oldOptions) {
    validateOptions(program, oldOptions);
    CConfiguration cConf = CConfiguration.copy(this.cConf);
    // Reload config for log extension jar update (CDAP-15091)
    cConf.reloadConfiguration();
    File tempDir = DirUtils.createTempDir(new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile());
    try {
        // For runs from a tethered instance, load additional resources
        if (oldOptions.getArguments().hasOption(ProgramOptionConstants.PEER_NAME)) {
            loadAdditionalResources(oldOptions.getArguments(), cConf, tempDir);
        }
        ProgramLaunchConfig launchConfig = new ProgramLaunchConfig();
        if (clusterMode == ClusterMode.ISOLATED) {
            // For isolated mode, the hadoop classes comes from the hadoop classpath in the target cluster directly
            launchConfig.addExtraClasspath(Collections.singletonList("$HADOOP_CLASSPATH"));
        }
        setupLaunchConfig(launchConfig, program, oldOptions, cConf, hConf, tempDir);
        // Add extra localize resources needed by the program runner
        final Map<String, LocalizeResource> localizeResources = new HashMap<>(launchConfig.getExtraResources());
        final List<String> additionalClassPaths = new ArrayList<>();
        addContainerJars(cConf, localizeResources, additionalClassPaths);
        addAdditionalLogAppenderJars(cConf, tempDir, localizeResources, SystemArguments.getProfileProvisioner(oldOptions.getArguments().asMap()));
        prepareHBaseDDLExecutorResources(tempDir, cConf, localizeResources);
        List<URI> configResources = localizeConfigs(createContainerCConf(cConf), createContainerHConf(this.hConf), tempDir, localizeResources);
        // Localize the program jar
        Location programJarLocation = program.getJarLocation();
        final String programJarName = programJarLocation.getName();
        localizeResources.put(programJarName, new LocalizeResource(programJarLocation.toURI(), false));
        // Localize the app spec
        localizeResources.put(APP_SPEC_FILE_NAME, new LocalizeResource(saveJsonFile(program.getApplicationSpecification(), ApplicationSpecification.class, File.createTempFile("appSpec", ".json", tempDir))));
        URI logbackURI = getLogBackURI(program, oldOptions.getArguments(), tempDir);
        if (logbackURI != null) {
            // Localize the logback xml
            localizeResources.put(LOGBACK_FILE_NAME, new LocalizeResource(logbackURI, false));
        }
        // Update the ProgramOptions to carry program and runtime information necessary to reconstruct the program
        // and runs it in the remote container
        Map<String, String> extraSystemArgs = new HashMap<>(launchConfig.getExtraSystemArguments());
        extraSystemArgs.put(ProgramOptionConstants.PROGRAM_JAR, programJarName);
        extraSystemArgs.put(ProgramOptionConstants.HADOOP_CONF_FILE, HADOOP_CONF_FILE_NAME);
        extraSystemArgs.put(ProgramOptionConstants.CDAP_CONF_FILE, CDAP_CONF_FILE_NAME);
        extraSystemArgs.put(ProgramOptionConstants.APP_SPEC_FILE, APP_SPEC_FILE_NAME);
        ProgramOptions options = updateProgramOptions(oldOptions, localizeResources, DirUtils.createTempDir(tempDir), extraSystemArgs);
        ProgramRunId programRunId = program.getId().run(ProgramRunners.getRunId(options));
        // Localize the serialized program options
        localizeResources.put(PROGRAM_OPTIONS_FILE_NAME, new LocalizeResource(saveJsonFile(options, ProgramOptions.class, File.createTempFile("program.options", ".json", tempDir))));
        Callable<ProgramController> callable = () -> {
            ProgramTwillApplication twillApplication = new ProgramTwillApplication(programRunId, options, launchConfig.getRunnables(), launchConfig.getLaunchOrder(), localizeResources, createEventHandler(cConf, programRunId, options));
            TwillPreparer twillPreparer = twillRunner.prepare(twillApplication);
            // Also add the configuration files to container classpath so that the
            // TwillAppLifecycleEventHandler can get it. This can be removed when TWILL-246 is fixed.
            // Only ON_PREMISE mode will be using EventHandler
            twillPreparer.withResources(configResources);
            Map<String, String> userArgs = options.getUserArguments().asMap();
            // Setup log level
            twillPreparer.setLogLevels(transformLogLevels(SystemArguments.getLogLevels(userArgs)));
            // Set the configuration for the twill application
            Map<String, String> twillConfigs = new HashMap<>();
            if (DistributedProgramRunner.this instanceof LongRunningDistributedProgramRunner) {
                twillConfigs.put(Configs.Keys.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL, cConf.get(Constants.AppFabric.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL));
            } else {
                // For non long running program type, set the max attempts to 1 to avoid YARN retry.
                // If the AM container dies, the program execution will be marked as failure.
                // Note that this setting is only applicable to the Twill YARN application
                // (e.g. workflow, Spark client, MR client, etc), but not to the actual Spark / MR job.
                twillConfigs.put(Configs.Keys.YARN_MAX_APP_ATTEMPTS, Integer.toString(1));
            }
            // Add twill configurations coming from the runtime arguments
            twillConfigs.putAll(SystemArguments.getNamespaceConfigs(options.getArguments().asMap()));
            twillConfigs.putAll(SystemArguments.getTwillApplicationConfigs(userArgs));
            twillPreparer.withConfiguration(twillConfigs);
            // Setup per runnable configurations
            Set<String> runnables = new HashSet<>();
            for (Map.Entry<String, RunnableDefinition> entry : launchConfig.getRunnables().entrySet()) {
                String runnable = entry.getKey();
                runnables.add(runnable);
                RunnableDefinition runnableDefinition = entry.getValue();
                if (runnableDefinition.getMaxRetries() != null) {
                    twillPreparer.withMaxRetries(runnable, runnableDefinition.getMaxRetries());
                }
                twillPreparer.setLogLevels(runnable, transformLogLevels(runnableDefinition.getLogLevels()));
                twillPreparer.withConfiguration(runnable, runnableDefinition.getTwillRunnableConfigs());
                // Add cdap-security.xml if using secrets, and set the runnable identity.
                if (twillPreparer instanceof SecureTwillPreparer) {
                    String twillSystemIdentity = cConf.get(Constants.Twill.Security.IDENTITY_SYSTEM);
                    if (twillSystemIdentity != null) {
                        SecurityContext securityContext = new SecurityContext.Builder().withIdentity(twillSystemIdentity).build();
                        twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecurityContext(runnable, securityContext);
                    }
                    String securityName = cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_NAME);
                    String securityPath = cConf.get(Constants.Twill.Security.MASTER_SECRET_DISK_PATH);
                    twillPreparer = ((SecureTwillPreparer) twillPreparer).withSecretDisk(runnable, new SecretDisk(securityName, securityPath));
                }
            }
            if (options.isDebug()) {
                twillPreparer.enableDebugging();
            }
            logProgramStart(program, options);
            // Add scheduler queue name if defined
            String schedulerQueueName = options.getArguments().getOption(Constants.AppFabric.APP_SCHEDULER_QUEUE);
            if (schedulerQueueName != null && !schedulerQueueName.isEmpty()) {
                LOG.info("Setting scheduler queue for app {} as {}", program.getId(), schedulerQueueName);
                twillPreparer.setSchedulerQueue(schedulerQueueName);
            }
            // Set JVM options based on configuration
            String jvmOpts = cConf.get(Constants.AppFabric.PROGRAM_JVM_OPTS);
            if (!Strings.isNullOrEmpty(jvmOpts)) {
                twillPreparer.addJVMOptions(jvmOpts);
            }
            // Overwrite JVM options if specified
            LauncherUtils.overrideJVMOpts(cConf, twillPreparer, runnables);
            if (logbackURI != null) {
                twillPreparer.addJVMOptions("-Dlogback.configurationFile=" + LOGBACK_FILE_NAME);
            }
            addLogHandler(twillPreparer, cConf);
            // Setup the environment for the container logback.xml
            twillPreparer.withEnv(Collections.singletonMap("CDAP_LOG_DIR", ApplicationConstants.LOG_DIR_EXPANSION_VAR));
            // Add dependencies
            Set<Class<?>> extraDependencies = addExtraDependencies(cConf, new HashSet<>(launchConfig.getExtraDependencies()));
            twillPreparer.withDependencies(extraDependencies);
            // Add the additional classes to the classpath that comes from the container jar setting
            twillPreparer.withClassPaths(additionalClassPaths);
            twillPreparer.withClassPaths(launchConfig.getExtraClasspath());
            twillPreparer.withEnv(launchConfig.getExtraEnv());
            // Add the YARN_APPLICATION_CLASSPATH so that yarn classpath are included in the twill container.
            // The Yarn app classpath goes last
            List<String> yarnAppClassPath = Arrays.asList(hConf.getTrimmedStrings(YarnConfiguration.YARN_APPLICATION_CLASSPATH, YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH));
            twillPreparer.withApplicationClassPaths(yarnAppClassPath).withClassPaths(yarnAppClassPath);
            twillPreparer.withBundlerClassAcceptor(launchConfig.getClassAcceptor()).withApplicationArguments(PROGRAM_OPTIONS_FILE_NAME).setClassLoader(MainClassLoader.class.getName());
            TwillController twillController;
            // Change the context classloader to the combine classloader of this ProgramRunner and
            // all the classloaders of the dependencies classes so that Twill can trace classes.
            ClassLoader oldClassLoader = ClassLoaders.setContextClassLoader(new CombineClassLoader(DistributedProgramRunner.this.getClass().getClassLoader(), extraDependencies.stream().map(Class::getClassLoader)::iterator));
            try {
                twillController = twillPreparer.start(cConf.getLong(Constants.AppFabric.PROGRAM_MAX_START_SECONDS), TimeUnit.SECONDS);
                // Block on the twill controller until it is in running state or terminated (due to failure)
                CountDownLatch latch = new CountDownLatch(1);
                twillController.onRunning(latch::countDown, Threads.SAME_THREAD_EXECUTOR);
                twillController.onTerminated(latch::countDown, Threads.SAME_THREAD_EXECUTOR);
                latch.await(cConf.getLong(Constants.AppFabric.PROGRAM_MAX_START_SECONDS), TimeUnit.SECONDS);
            } finally {
                ClassLoaders.setContextClassLoader(oldClassLoader);
            }
            return createProgramController(programRunId, addCleanupListener(twillController, program, tempDir));
        };
        return impersonator.doAs(programRunId, callable);
    } catch (Exception e) {
        deleteDirectory(tempDir);
        throw Throwables.propagate(e);
    }
}