Examples with ProgramOptions io.cdap.cdap.app.runtime.ProgramOptions used on opensource projects

Example 26 with ProgramOptions

use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by cdapio.

the class ProgramNotificationSubscriberServiceTest method testAppSpecNotRequiredToWriteState.

@Test
public void testAppSpecNotRequiredToWriteState() throws Exception {
    ProgramId programId = NamespaceId.DEFAULT.app("someapp").program(ProgramType.SERVICE, "s");
    Map<String, String> systemArguments = new HashMap<>();
    systemArguments.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
    systemArguments.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
    ProgramOptions programOptions = new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments());
    ProgramRunId runId = programId.run(RunIds.generate());
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    ApplicationSpecification appSpec = new DefaultApplicationSpecification("name", "1.0.0", ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec);
    programStateWriter.start(runId, programOptions, null, programDescriptor);
    Tasks.waitFor(ProgramRunStatus.STARTING, () -> TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        RunRecordDetail meta = metadataStoreDataset.getRun(runId);
        if (meta == null) {
            return null;
        }
        Assert.assertEquals(artifactId, meta.getArtifactId());
        return meta.getStatus();
    }), 10, TimeUnit.SECONDS);
    programStateWriter.completed(runId);
}

Example 27 with ProgramOptions

use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by cdapio.

the class ProgramNotificationSubscriberServiceTest method testHeartBeatStoreForProgramStatusMessages.

@Test
public void testHeartBeatStoreForProgramStatusMessages() throws Exception {
    ProgramId programId = NamespaceId.DEFAULT.app("someapp", "1.0-SNAPSHOT").program(ProgramType.SERVICE, "s");
    Map<String, String> systemArguments = new HashMap<>();
    systemArguments.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
    systemArguments.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
    ProgramOptions programOptions = new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments());
    ProgramRunId runId = programId.run(RunIds.generate());
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    ApplicationSpecification appSpec = new DefaultApplicationSpecification("name", "1.0.0", ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec);
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.start(runId, programOptions, null, programDescriptor);
    });
    checkProgramStatus(artifactId, runId, ProgramRunStatus.STARTING);
    long startTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.running(runId, null);
    });
    // perform scan on heart beat store - ensure latest message notification is running
    checkProgramStatus(artifactId, runId, ProgramRunStatus.RUNNING);
    heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
    long suspendTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.suspend(runId);
    });
    // perform scan on heart beat store - ensure latest message notification is suspended
    checkProgramStatus(artifactId, runId, ProgramRunStatus.SUSPENDED);
    heartbeatDatasetStatusCheck(suspendTime, ProgramRunStatus.SUSPENDED);
    long resumeTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.resume(runId);
    });
    // app metadata records as RUNNING
    checkProgramStatus(artifactId, runId, ProgramRunStatus.RUNNING);
    // heart beat messages wont have been sent due to high interval. resuming program will be recorded as running
    // in run record by app meta
    heartbeatDatasetStatusCheck(resumeTime, ProgramRunStatus.RUNNING);
    // killed status check after error
    long stopTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.error(runId, new Throwable("Testing"));
    });
    checkProgramStatus(artifactId, runId, ProgramRunStatus.FAILED);
    heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.FAILED);
    ProgramRunId runId2 = programId.run(RunIds.generate());
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.start(runId2, programOptions, null, programDescriptor);
    });
    checkProgramStatus(artifactId, runId2, ProgramRunStatus.STARTING);
    startTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.running(runId2, null);
    });
    // perform scan on heart beat store - ensure latest message notification is running
    checkProgramStatus(artifactId, runId2, ProgramRunStatus.RUNNING);
    heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
    // completed status check
    stopTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.completed(runId2);
    });
    checkProgramStatus(artifactId, runId2, ProgramRunStatus.COMPLETED);
    heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.COMPLETED);
    ProgramRunId runId3 = programId.run(RunIds.generate());
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.start(runId3, programOptions, null, programDescriptor);
    });
    checkProgramStatus(artifactId, runId3, ProgramRunStatus.STARTING);
    startTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.running(runId3, null);
    });
    // perform scan on heart beat store - ensure latest message notification is running
    checkProgramStatus(artifactId, runId3, ProgramRunStatus.RUNNING);
    heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
    // completed status check
    stopTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.stop(runId3, 10);
    });
    checkProgramStatus(artifactId, runId3, ProgramRunStatus.STOPPING);
    heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.KILLED);
}

Example 28 with ProgramOptions

use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by cdapio.

the class ProgramLifecycleServiceTest method testCreateProgramOptions.

@Test
public void testCreateProgramOptions() throws Exception {
    deploy(AllProgramsApp.class, 200);
    ProgramId programId = NamespaceId.DEFAULT.app(AllProgramsApp.NAME).program(ProgramType.SPARK, AllProgramsApp.NoOpSpark.NAME);
    ProgramOptions options = programLifecycleService.createProgramOptions(programId, Collections.emptyMap(), Collections.emptyMap(), false);
    Assert.assertEquals(ProjectInfo.getVersion().toString(), options.getArguments().getOption(Constants.APP_CDAP_VERSION));
}

Example 29 with ProgramOptions

use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by cdapio.

the class SparkRuntimeContextProvider method createIfNotExists.

/**
 * Creates a singleton {@link SparkRuntimeContext}.
 * It has assumption on file location that are localized by the SparkRuntimeService.
 */
private static synchronized SparkRuntimeContext createIfNotExists() {
    if (sparkRuntimeContext != null) {
        return sparkRuntimeContext;
    }
    try {
        CConfiguration cConf = createCConf();
        Configuration hConf = createHConf();
        SparkRuntimeContextConfig contextConfig = new SparkRuntimeContextConfig(hConf);
        ProgramOptions programOptions = contextConfig.getProgramOptions();
        ClusterMode clusterMode = ProgramRunners.getClusterMode(programOptions);
        if (masterEnvName != null) {
            MasterEnvironment masterEnv = MasterEnvironments.create(cConf, masterEnvName);
            MasterEnvironmentContext context = MasterEnvironments.createContext(cConf, hConf, masterEnv.getName());
            masterEnv.initialize(context);
            MasterEnvironments.setMasterEnvironment(masterEnv);
        }
        // Create the program
        Program program = createProgram(cConf, contextConfig);
        ProgramRunId programRunId = program.getId().run(ProgramRunners.getRunId(programOptions));
        Injector injector = createInjector(cConf, hConf, contextConfig.getProgramId(), programOptions);
        LogAppenderInitializer logAppenderInitializer = injector.getInstance(LogAppenderInitializer.class);
        logAppenderInitializer.initialize();
        SystemArguments.setLogLevel(programOptions.getUserArguments(), logAppenderInitializer);
        ProxySelector oldProxySelector = ProxySelector.getDefault();
        if (clusterMode == ClusterMode.ISOLATED) {
            RuntimeMonitors.setupMonitoring(injector, programOptions);
        }
        MetricsCollectionService metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
        SparkServiceAnnouncer serviceAnnouncer = injector.getInstance(SparkServiceAnnouncer.class);
        Deque<Service> coreServices = new LinkedList<>();
        if (clusterMode == ClusterMode.ON_PREMISE && masterEnvName == null) {
            // Add ZK for discovery and Kafka
            coreServices.add(injector.getInstance(ZKClientService.class));
            // Add the Kafka client for logs collection
            coreServices.add(injector.getInstance(KafkaClientService.class));
        }
        coreServices.add(metricsCollectionService);
        coreServices.add(serviceAnnouncer);
        for (Service coreService : coreServices) {
            coreService.startAndWait();
        }
        AtomicBoolean closed = new AtomicBoolean();
        Closeable closeable = () -> {
            if (!closed.compareAndSet(false, true)) {
                return;
            }
            // Close to flush out all important logs
            logAppenderInitializer.close();
            // Stop all services. Reverse the order.
            for (Service service : (Iterable<Service>) coreServices::descendingIterator) {
                try {
                    service.stopAndWait();
                } catch (Exception e) {
                    LOG.warn("Exception raised when stopping service {} during program termination.", service, e);
                }
            }
            Authenticator.setDefault(null);
            ProxySelector.setDefault(oldProxySelector);
            LOG.debug("Spark runtime services shutdown completed");
        };
        // Constructor the DatasetFramework
        DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
        WorkflowProgramInfo workflowInfo = contextConfig.getWorkflowProgramInfo();
        DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, contextConfig.getApplicationSpecification());
        // Setup dataset framework context, if required
        if (programDatasetFramework instanceof ProgramContextAware) {
            ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programRunId));
        }
        PluginInstantiator pluginInstantiator = createPluginInstantiator(cConf, contextConfig, program.getClassLoader());
        // Create the context object
        sparkRuntimeContext = new SparkRuntimeContext(contextConfig.getConfiguration(), program, programOptions, cConf, getHostname(), injector.getInstance(TransactionSystemClient.class), programDatasetFramework, metricsCollectionService, contextConfig.getWorkflowProgramInfo(), pluginInstantiator, injector.getInstance(SecureStore.class), injector.getInstance(SecureStoreManager.class), injector.getInstance(AccessEnforcer.class), injector.getInstance(AuthenticationContext.class), injector.getInstance(MessagingService.class), serviceAnnouncer, injector.getInstance(PluginFinder.class), injector.getInstance(LocationFactory.class), injector.getInstance(MetadataReader.class), injector.getInstance(MetadataPublisher.class), injector.getInstance(NamespaceQueryAdmin.class), injector.getInstance(FieldLineageWriter.class), injector.getInstance(RemoteClientFactory.class), closeable);
        LoggingContextAccessor.setLoggingContext(sparkRuntimeContext.getLoggingContext());
        return sparkRuntimeContext;
    } catch (Exception e) {
        throw Throwables.propagate(e);
    }
}

Example 30 with ProgramOptions

use of io.cdap.cdap.app.runtime.ProgramOptions in project cdap by cdapio.

the class ProgramStateWriterWithHeartBeatTest method testHeartBeatThread.

private void testHeartBeatThread(ProgramRunStatus terminalState) throws Exception {
    // configure program state writer to emit heart beat every second
    MockProgramStatePublisher programStatePublisher = new MockProgramStatePublisher();
    AtomicReference<ProgramRunStatus> completionState = new AtomicReference<>();
    ProgramStateWriter programStateWriter = new NoOpProgramStateWriter() {

        @Override
        public void completed(ProgramRunId programRunId) {
            super.completed(programRunId);
            completionState.set(ProgramRunStatus.COMPLETED);
        }

        @Override
        public void killed(ProgramRunId programRunId) {
            super.killed(programRunId);
            completionState.set(ProgramRunStatus.KILLED);
        }

        @Override
        public void error(ProgramRunId programRunId, Throwable failureCause) {
            super.error(programRunId, failureCause);
            completionState.set(ProgramRunStatus.FAILED);
        }
    };
    // mock program configurations
    ProgramId programId = NamespaceId.DEFAULT.app("someapp").program(ProgramType.SERVICE, "s");
    Map<String, String> systemArguments = new HashMap<>();
    systemArguments.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
    ProgramOptions programOptions = new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments());
    ProgramRunId runId = programId.run(RunIds.generate());
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    ProgramStateWriterWithHeartBeat programStateWriterWithHeartBeat = new ProgramStateWriterWithHeartBeat(runId, programStateWriter, 1, programStatePublisher);
    ApplicationSpecification appSpec = new DefaultApplicationSpecification("name", "1.0.0", ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec);
    // start the program and ensure heart beat is 0 before we call running
    programStateWriter.start(runId, programOptions, null, programDescriptor);
    Assert.assertEquals(0, programStatePublisher.getHeartBeatCount());
    programStateWriterWithHeartBeat.running(null);
    // on running, we start receiving heart beat messages, verify if the heartbeat count goes to 2.
    Tasks.waitFor(true, () -> programStatePublisher.getHeartBeatCount() > 1, 10, TimeUnit.SECONDS, "Didn't receive expected heartbeat after 10 seconds");
    // Terminate the program and make sure the heart beat thread also gets stopped
    switch(terminalState) {
        case COMPLETED:
            programStateWriterWithHeartBeat.completed();
            break;
        case FAILED:
            programStateWriterWithHeartBeat.error(new RuntimeException());
            break;
        case KILLED:
            programStateWriterWithHeartBeat.killed();
            break;
        default:
            throw new IllegalStateException("The terminal state must one of COMPLETED, FAILED, or KILLED");
    }
    Tasks.waitFor(false, programStateWriterWithHeartBeat::isHeartBeatThreadAlive, 5, TimeUnit.SECONDS, "Heartbeat thread did not stop after 5 seconds");
    Assert.assertEquals(terminalState, completionState.get());
}