Examples with SimpleProgramOptions io.cdap.cdap.internal.app.runtime.SimpleProgramOptions used on opensource projects

Example 21 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by cdapio.

the class AbstractProgramRuntimeServiceTest method testDeadlock.

@Test(timeout = 5000)
public void testDeadlock() throws IOException, ExecutionException, InterruptedException, TimeoutException {
    // This test is for testing condition in (CDAP-3579)
    // The race condition is if a program finished very fast such that inside the AbstractProgramRuntimeService is
    // still in the run method, it holds the object lock, making the callback from the listener block forever.
    ProgramRunnerFactory runnerFactory = createProgramRunnerFactory();
    Program program = createDummyProgram();
    ProgramRuntimeService runtimeService = new TestProgramRuntimeService(CConfiguration.create(), runnerFactory, program, null, null);
    runtimeService.startAndWait();
    try {
        ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, NamespaceId.DEFAULT.artifact("test", "1.0"));
        ProgramController controller = runtimeService.run(descriptor, new SimpleProgramOptions(program.getId()), RunIds.generate()).getController();
        Tasks.waitFor(ProgramController.State.COMPLETED, controller::getState, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        Tasks.waitFor(true, () -> runtimeService.list(ProgramType.WORKER).isEmpty(), 5, TimeUnit.SECONDS, 100, TimeUnit.MICROSECONDS);
    } finally {
        runtimeService.stopAndWait();
    }
}

Example 22 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by cdapio.

the class AbstractInMemoryProgramRunner method createComponentOptions.

private ProgramOptions createComponentOptions(int instanceId, int instances, RunId runId, ProgramOptions options) {
    Map<String, String> systemOptions = Maps.newHashMap();
    systemOptions.putAll(options.getArguments().asMap());
    systemOptions.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId));
    systemOptions.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances));
    systemOptions.put(ProgramOptionConstants.RUN_ID, runId.getId());
    systemOptions.put(ProgramOptionConstants.HOST, host);
    return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemOptions), options.getUserArguments());
}

Example 23 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by cdapio.

the class ProgramNotificationSubscriberServiceTest method testWorkflowInnerPrograms.

@Test
public void testWorkflowInnerPrograms() throws Exception {
    AppFabricTestHelper.deployApplication(Id.Namespace.DEFAULT, ProgramStateWorkflowApp.class, null, cConf);
    ProgramRunId workflowRunId = NamespaceId.DEFAULT.app(ProgramStateWorkflowApp.class.getSimpleName()).workflow(ProgramStateWorkflowApp.ProgramStateWorkflow.class.getSimpleName()).run(RunIds.generate());
    ApplicationSpecification appSpec = TransactionRunners.run(transactionRunner, context -> {
        return AppMetadataStore.create(context).getApplication(workflowRunId.getParent().getParent()).getSpec();
    });
    ProgramDescriptor programDescriptor = new ProgramDescriptor(workflowRunId.getParent(), appSpec);
    // Start and run the workflow
    Map<String, String> systemArgs = new HashMap<>();
    systemArgs.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
    systemArgs.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
    programStateWriter.start(workflowRunId, new SimpleProgramOptions(workflowRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments()), null, programDescriptor);
    programStateWriter.running(workflowRunId, null);
    ProgramRunId mrRunId = workflowRunId.getParent().getParent().mr(ProgramStateWorkflowApp.ProgramStateMR.class.getSimpleName()).run(RunIds.generate());
    ProgramRunId sparkRunId = workflowRunId.getParent().getParent().spark(ProgramStateWorkflowApp.ProgramStateSpark.class.getSimpleName()).run(RunIds.generate());
    ProgramId sparkId2 = workflowRunId.getParent().getParent().spark(ProgramStateWorkflowApp.ProgramStateSpark2.class.getSimpleName());
    // Start and run the MR and Spark inside
    for (ProgramRunId programRunId : Arrays.asList(mrRunId, sparkRunId)) {
        workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail(programRunId.getProgram(), NodeStatus.STARTING));
        workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail(programRunId.getProgram(), NodeStatus.RUNNING));
        systemArgs = new HashMap<>(systemArgs);
        systemArgs.put(ProgramOptionConstants.RUN_ID, programRunId.getRun());
        systemArgs.put(ProgramOptionConstants.WORKFLOW_NAME, workflowRunId.getProgram());
        systemArgs.put(ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId.getRun());
        systemArgs.put(ProgramOptionConstants.WORKFLOW_NODE_ID, programRunId.getProgram());
        systemArgs.put(ProgramOptionConstants.PROGRAM_NAME_IN_WORKFLOW, programRunId.getProgram());
        programStateWriter.start(programRunId, new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments()), null, programDescriptor);
        programStateWriter.running(programRunId, null);
        // Wait for the inner program running
        Tasks.waitFor(ProgramRunStatus.RUNNING, () -> TransactionRunners.run(transactionRunner, context -> {
            AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
            RunRecordDetail meta = metadataStoreDataset.getRun(programRunId);
            if (meta == null) {
                return null;
            }
            return meta.getStatus();
        }), 10, TimeUnit.SECONDS);
    }
    // Stop the Spark normally
    programStateWriter.completed(sparkRunId);
    // Error out the Workflow without stopping the MR
    programStateWriter.error(workflowRunId, new IllegalStateException("Explicitly error out"));
    // Wait for the Workflow state changed to failed
    Tasks.waitFor(ProgramRunStatus.FAILED, () -> TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        RunRecordDetail meta = metadataStoreDataset.getRun(workflowRunId);
        if (meta == null) {
            return null;
        }
        return meta.getStatus();
    }), 10000, TimeUnit.SECONDS);
    // The MR run record should be changed to ERROR state as well (without race)
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        RunRecordDetail meta = metadataStoreDataset.getRun(mrRunId);
        Assert.assertNotNull(meta);
        Assert.assertEquals(ProgramRunStatus.FAILED, meta.getStatus());
    });
    // The Spark run record should stay as COMPLETED
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        RunRecordDetail meta = metadataStoreDataset.getRun(sparkRunId);
        Assert.assertNotNull(meta);
        Assert.assertEquals(ProgramRunStatus.COMPLETED, meta.getStatus());
    });
    // Since the Spark2 program hasn't been executed, there should be no run record
    TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        Map<ProgramRunId, RunRecordDetail> runs = metadataStoreDataset.getRuns(sparkId2, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, 100, null);
        Assert.assertTrue(runs.isEmpty());
    });
}

Example 24 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by cdapio.

the class ProgramNotificationSubscriberServiceTest method testHeartBeatStoreForProgramStatusMessages.

@Test
public void testHeartBeatStoreForProgramStatusMessages() throws Exception {
    ProgramId programId = NamespaceId.DEFAULT.app("someapp", "1.0-SNAPSHOT").program(ProgramType.SERVICE, "s");
    Map<String, String> systemArguments = new HashMap<>();
    systemArguments.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
    systemArguments.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
    ProgramOptions programOptions = new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments());
    ProgramRunId runId = programId.run(RunIds.generate());
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    ApplicationSpecification appSpec = new DefaultApplicationSpecification("name", "1.0.0", ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec);
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.start(runId, programOptions, null, programDescriptor);
    });
    checkProgramStatus(artifactId, runId, ProgramRunStatus.STARTING);
    long startTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.running(runId, null);
    });
    // perform scan on heart beat store - ensure latest message notification is running
    checkProgramStatus(artifactId, runId, ProgramRunStatus.RUNNING);
    heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
    long suspendTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.suspend(runId);
    });
    // perform scan on heart beat store - ensure latest message notification is suspended
    checkProgramStatus(artifactId, runId, ProgramRunStatus.SUSPENDED);
    heartbeatDatasetStatusCheck(suspendTime, ProgramRunStatus.SUSPENDED);
    long resumeTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.resume(runId);
    });
    // app metadata records as RUNNING
    checkProgramStatus(artifactId, runId, ProgramRunStatus.RUNNING);
    // heart beat messages wont have been sent due to high interval. resuming program will be recorded as running
    // in run record by app meta
    heartbeatDatasetStatusCheck(resumeTime, ProgramRunStatus.RUNNING);
    // killed status check after error
    long stopTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.error(runId, new Throwable("Testing"));
    });
    checkProgramStatus(artifactId, runId, ProgramRunStatus.FAILED);
    heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.FAILED);
    ProgramRunId runId2 = programId.run(RunIds.generate());
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.start(runId2, programOptions, null, programDescriptor);
    });
    checkProgramStatus(artifactId, runId2, ProgramRunStatus.STARTING);
    startTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.running(runId2, null);
    });
    // perform scan on heart beat store - ensure latest message notification is running
    checkProgramStatus(artifactId, runId2, ProgramRunStatus.RUNNING);
    heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
    // completed status check
    stopTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.completed(runId2);
    });
    checkProgramStatus(artifactId, runId2, ProgramRunStatus.COMPLETED);
    heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.COMPLETED);
    ProgramRunId runId3 = programId.run(RunIds.generate());
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.start(runId3, programOptions, null, programDescriptor);
    });
    checkProgramStatus(artifactId, runId3, ProgramRunStatus.STARTING);
    startTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.running(runId3, null);
    });
    // perform scan on heart beat store - ensure latest message notification is running
    checkProgramStatus(artifactId, runId3, ProgramRunStatus.RUNNING);
    heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
    // completed status check
    stopTime = System.currentTimeMillis();
    TransactionRunners.run(transactionRunner, context -> {
        programStateWriter.stop(runId3, 10);
    });
    checkProgramStatus(artifactId, runId3, ProgramRunStatus.STOPPING);
    heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.STOPPING);
}

Example 25 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by cdapio.

the class ProgramNotificationSubscriberServiceTest method testAppSpecNotRequiredToWriteState.

@Test
public void testAppSpecNotRequiredToWriteState() throws Exception {
    ProgramId programId = NamespaceId.DEFAULT.app("someapp").program(ProgramType.SERVICE, "s");
    Map<String, String> systemArguments = new HashMap<>();
    systemArguments.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
    systemArguments.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
    ProgramOptions programOptions = new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments());
    ProgramRunId runId = programId.run(RunIds.generate());
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
    ApplicationSpecification appSpec = new DefaultApplicationSpecification("name", "1.0.0", ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
    ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec);
    programStateWriter.start(runId, programOptions, null, programDescriptor);
    Tasks.waitFor(ProgramRunStatus.STARTING, () -> TransactionRunners.run(transactionRunner, context -> {
        AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
        RunRecordDetail meta = metadataStoreDataset.getRun(runId);
        if (meta == null) {
            return null;
        }
        Assert.assertEquals(artifactId, meta.getArtifactId());
        return meta.getStatus();
    }), 10, TimeUnit.SECONDS);
    programStateWriter.completed(runId);
}