use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.
the class CoreSchedulerServiceTest method testProgramEvents.
@Test
@Category(XSlowTests.class)
public void testProgramEvents() throws Exception {
// Deploy the app
deploy(AppWithMultipleSchedules.class, 200);
CConfiguration cConf = getInjector().getInstance(CConfiguration.class);
TopicId programEventTopic = NamespaceId.SYSTEM.topic(cConf.get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC));
ProgramStateWriter programStateWriter = new MessagingProgramStateWriter(cConf, messagingService);
// These notifications should not trigger the program
ProgramRunId anotherWorkflowRun = ANOTHER_WORKFLOW.run(RunIds.generate());
ArtifactId artifactId = ANOTHER_WORKFLOW.getNamespaceId().artifact("test", "1.0").toApiArtifactId();
ApplicationSpecification appSpec = new DefaultApplicationSpecification(AppWithMultipleSchedules.NAME, ApplicationId.DEFAULT_VERSION, ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
ProgramDescriptor programDescriptor = new ProgramDescriptor(anotherWorkflowRun.getParent(), appSpec);
BasicArguments systemArgs = new BasicArguments(ImmutableMap.of(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString()));
ProgramOptions programOptions = new SimpleProgramOptions(anotherWorkflowRun.getParent(), systemArgs, new BasicArguments(), false);
programStateWriter.start(anotherWorkflowRun, programOptions, null, programDescriptor);
programStateWriter.running(anotherWorkflowRun, null);
long lastProcessed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
programStateWriter.error(anotherWorkflowRun, null);
waitUntilProcessed(programEventTopic, lastProcessed);
ProgramRunId someWorkflowRun = SOME_WORKFLOW.run(RunIds.generate());
programDescriptor = new ProgramDescriptor(someWorkflowRun.getParent(), appSpec);
programStateWriter.start(someWorkflowRun, new SimpleProgramOptions(someWorkflowRun.getParent(), systemArgs, new BasicArguments()), null, programDescriptor);
programStateWriter.running(someWorkflowRun, null);
lastProcessed = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis());
programStateWriter.killed(someWorkflowRun);
waitUntilProcessed(programEventTopic, lastProcessed);
Assert.assertEquals(0, getRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.ALL));
// Enable the schedule
scheduler.enableSchedule(APP_MULT_ID.schedule(AppWithMultipleSchedules.WORKFLOW_COMPLETED_SCHEDULE));
// Start a program with user arguments
startProgram(ANOTHER_WORKFLOW, ImmutableMap.of(AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_KEY, AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_VALUE), 200);
// Wait for a completed run record
waitForCompleteRuns(1, TRIGGERED_WORKFLOW);
assertProgramRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.COMPLETED, 1);
RunRecord run = getProgramRuns(TRIGGERED_WORKFLOW, ProgramRunStatus.COMPLETED).get(0);
Map<String, List<WorkflowTokenDetail.NodeValueDetail>> tokenData = getWorkflowToken(TRIGGERED_WORKFLOW, run.getPid(), null, null).getTokenData();
// There should be 2 entries in tokenData
Assert.assertEquals(2, tokenData.size());
// The value of TRIGGERED_RUNTIME_ARG_KEY should be ANOTHER_RUNTIME_ARG_VALUE from the triggering workflow
Assert.assertEquals(AppWithMultipleSchedules.ANOTHER_RUNTIME_ARG_VALUE, tokenData.get(AppWithMultipleSchedules.TRIGGERED_RUNTIME_ARG_KEY).get(0).getValue());
// The value of TRIGGERED_TOKEN_KEY should be ANOTHER_TOKEN_VALUE from the triggering workflow
Assert.assertEquals(AppWithMultipleSchedules.ANOTHER_TOKEN_VALUE, tokenData.get(AppWithMultipleSchedules.TRIGGERED_TOKEN_KEY).get(0).getValue());
}
use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.
the class MapReduceTaskContextProvider method createCacheLoader.
/**
* Creates a {@link CacheLoader} for the task context cache.
*/
private CacheLoader<ContextCacheKey, BasicMapReduceTaskContext> createCacheLoader(final Injector injector) {
DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
SecureStore secureStore = injector.getInstance(SecureStore.class);
SecureStoreManager secureStoreManager = injector.getInstance(SecureStoreManager.class);
MessagingService messagingService = injector.getInstance(MessagingService.class);
// Multiple instances of BasicMapReduceTaskContext can share the same program.
AtomicReference<Program> programRef = new AtomicReference<>();
MetadataReader metadataReader = injector.getInstance(MetadataReader.class);
MetadataPublisher metadataPublisher = injector.getInstance(MetadataPublisher.class);
FieldLineageWriter fieldLineageWriter = injector.getInstance(FieldLineageWriter.class);
RemoteClientFactory remoteClientFactory = injector.getInstance(RemoteClientFactory.class);
return new CacheLoader<ContextCacheKey, BasicMapReduceTaskContext>() {
@Override
public BasicMapReduceTaskContext load(ContextCacheKey key) throws Exception {
TaskAttemptID taskAttemptId = key.getTaskAttemptID();
// taskAttemptId could be null if used from a org.apache.hadoop.mapreduce.Partitioner or
// from a org.apache.hadoop.io.RawComparator, in which case we can get the JobId from the conf. Note that the
// JobId isn't in the conf for the OutputCommitter#setupJob method, in which case we use the taskAttemptId
Path txFile = MainOutputCommitter.getTxFile(key.getConfiguration(), taskAttemptId != null ? taskAttemptId.getJobID() : null);
FileSystem fs = txFile.getFileSystem(key.getConfiguration());
Transaction transaction = null;
if (fs.exists(txFile)) {
try (FSDataInputStream txFileInputStream = fs.open(txFile)) {
transaction = new TransactionCodec().decode(ByteStreams.toByteArray(txFileInputStream));
}
}
MapReduceContextConfig contextConfig = new MapReduceContextConfig(key.getConfiguration());
MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(key.getConfiguration());
Program program = programRef.get();
if (program == null) {
// Creation of program is relatively cheap, so just create and do compare and set.
programRef.compareAndSet(null, createProgram(contextConfig, classLoader.getProgramClassLoader()));
program = programRef.get();
}
WorkflowProgramInfo workflowInfo = contextConfig.getWorkflowProgramInfo();
DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, program.getApplicationSpecification());
// Setup dataset framework context, if required
if (programDatasetFramework instanceof ProgramContextAware) {
ProgramRunId programRunId = program.getId().run(ProgramRunners.getRunId(contextConfig.getProgramOptions()));
((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programRunId));
}
MapReduceSpecification spec = program.getApplicationSpecification().getMapReduce().get(program.getName());
MetricsCollectionService metricsCollectionService = null;
MapReduceMetrics.TaskType taskType = null;
String taskId = null;
ProgramOptions options = contextConfig.getProgramOptions();
// from a org.apache.hadoop.io.RawComparator
if (taskAttemptId != null) {
taskId = taskAttemptId.getTaskID().toString();
if (MapReduceMetrics.TaskType.hasType(taskAttemptId.getTaskType())) {
taskType = MapReduceMetrics.TaskType.from(taskAttemptId.getTaskType());
// if this is not for a mapper or a reducer, we don't need the metrics collection service
metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
options = new SimpleProgramOptions(options.getProgramId(), options.getArguments(), new BasicArguments(RuntimeArguments.extractScope("task", taskType.toString().toLowerCase(), contextConfig.getProgramOptions().getUserArguments().asMap())), options.isDebug());
}
}
CConfiguration cConf = injector.getInstance(CConfiguration.class);
TransactionSystemClient txClient = injector.getInstance(TransactionSystemClient.class);
NamespaceQueryAdmin namespaceQueryAdmin = injector.getInstance(NamespaceQueryAdmin.class);
return new BasicMapReduceTaskContext(program, options, cConf, taskType, taskId, spec, workflowInfo, discoveryServiceClient, metricsCollectionService, txClient, transaction, programDatasetFramework, classLoader.getPluginInstantiator(), contextConfig.getLocalizedResources(), secureStore, secureStoreManager, accessEnforcer, authenticationContext, messagingService, mapReduceClassLoader, metadataReader, metadataPublisher, namespaceQueryAdmin, fieldLineageWriter, remoteClientFactory);
}
};
}
use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.
the class ProgramNotificationSubscriberServiceTest method testWorkflowInnerPrograms.
@Test
public void testWorkflowInnerPrograms() throws Exception {
AppFabricTestHelper.deployApplication(Id.Namespace.DEFAULT, ProgramStateWorkflowApp.class, null, cConf);
ProgramRunId workflowRunId = NamespaceId.DEFAULT.app(ProgramStateWorkflowApp.class.getSimpleName()).workflow(ProgramStateWorkflowApp.ProgramStateWorkflow.class.getSimpleName()).run(RunIds.generate());
ApplicationSpecification appSpec = TransactionRunners.run(transactionRunner, context -> {
return AppMetadataStore.create(context).getApplication(workflowRunId.getParent().getParent()).getSpec();
});
ProgramDescriptor programDescriptor = new ProgramDescriptor(workflowRunId.getParent(), appSpec);
// Start and run the workflow
Map<String, String> systemArgs = new HashMap<>();
systemArgs.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
systemArgs.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
programStateWriter.start(workflowRunId, new SimpleProgramOptions(workflowRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments()), null, programDescriptor);
programStateWriter.running(workflowRunId, null);
ProgramRunId mrRunId = workflowRunId.getParent().getParent().mr(ProgramStateWorkflowApp.ProgramStateMR.class.getSimpleName()).run(RunIds.generate());
ProgramRunId sparkRunId = workflowRunId.getParent().getParent().spark(ProgramStateWorkflowApp.ProgramStateSpark.class.getSimpleName()).run(RunIds.generate());
ProgramId sparkId2 = workflowRunId.getParent().getParent().spark(ProgramStateWorkflowApp.ProgramStateSpark2.class.getSimpleName());
// Start and run the MR and Spark inside
for (ProgramRunId programRunId : Arrays.asList(mrRunId, sparkRunId)) {
workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail(programRunId.getProgram(), NodeStatus.STARTING));
workflowStateWriter.addWorkflowNodeState(workflowRunId, new WorkflowNodeStateDetail(programRunId.getProgram(), NodeStatus.RUNNING));
systemArgs = new HashMap<>(systemArgs);
systemArgs.put(ProgramOptionConstants.RUN_ID, programRunId.getRun());
systemArgs.put(ProgramOptionConstants.WORKFLOW_NAME, workflowRunId.getProgram());
systemArgs.put(ProgramOptionConstants.WORKFLOW_RUN_ID, workflowRunId.getRun());
systemArgs.put(ProgramOptionConstants.WORKFLOW_NODE_ID, programRunId.getProgram());
systemArgs.put(ProgramOptionConstants.PROGRAM_NAME_IN_WORKFLOW, programRunId.getProgram());
programStateWriter.start(programRunId, new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments()), null, programDescriptor);
programStateWriter.running(programRunId, null);
// Wait for the inner program running
Tasks.waitFor(ProgramRunStatus.RUNNING, () -> TransactionRunners.run(transactionRunner, context -> {
AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
RunRecordDetail meta = metadataStoreDataset.getRun(programRunId);
if (meta == null) {
return null;
}
return meta.getStatus();
}), 10, TimeUnit.SECONDS);
}
// Stop the Spark normally
programStateWriter.completed(sparkRunId);
// Error out the Workflow without stopping the MR
programStateWriter.error(workflowRunId, new IllegalStateException("Explicitly error out"));
// Wait for the Workflow state changed to failed
Tasks.waitFor(ProgramRunStatus.FAILED, () -> TransactionRunners.run(transactionRunner, context -> {
AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
RunRecordDetail meta = metadataStoreDataset.getRun(workflowRunId);
if (meta == null) {
return null;
}
return meta.getStatus();
}), 10000, TimeUnit.SECONDS);
// The MR run record should be changed to ERROR state as well (without race)
TransactionRunners.run(transactionRunner, context -> {
AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
RunRecordDetail meta = metadataStoreDataset.getRun(mrRunId);
Assert.assertNotNull(meta);
Assert.assertEquals(ProgramRunStatus.FAILED, meta.getStatus());
});
// The Spark run record should stay as COMPLETED
TransactionRunners.run(transactionRunner, context -> {
AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
RunRecordDetail meta = metadataStoreDataset.getRun(sparkRunId);
Assert.assertNotNull(meta);
Assert.assertEquals(ProgramRunStatus.COMPLETED, meta.getStatus());
});
// Since the Spark2 program hasn't been executed, there should be no run record
TransactionRunners.run(transactionRunner, context -> {
AppMetadataStore metadataStoreDataset = AppMetadataStore.create(context);
Map<ProgramRunId, RunRecordDetail> runs = metadataStoreDataset.getRuns(sparkId2, ProgramRunStatus.ALL, 0, Long.MAX_VALUE, 100, null);
Assert.assertTrue(runs.isEmpty());
});
}
use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.
the class ProgramNotificationSubscriberServiceTest method testHeartBeatStoreForProgramStatusMessages.
@Test
public void testHeartBeatStoreForProgramStatusMessages() throws Exception {
ProgramId programId = NamespaceId.DEFAULT.app("someapp", "1.0-SNAPSHOT").program(ProgramType.SERVICE, "s");
Map<String, String> systemArguments = new HashMap<>();
systemArguments.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
systemArguments.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
ProgramOptions programOptions = new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments());
ProgramRunId runId = programId.run(RunIds.generate());
ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
ApplicationSpecification appSpec = new DefaultApplicationSpecification("name", "1.0.0", ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec);
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.start(runId, programOptions, null, programDescriptor);
});
checkProgramStatus(artifactId, runId, ProgramRunStatus.STARTING);
long startTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.running(runId, null);
});
// perform scan on heart beat store - ensure latest message notification is running
checkProgramStatus(artifactId, runId, ProgramRunStatus.RUNNING);
heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
long suspendTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.suspend(runId);
});
// perform scan on heart beat store - ensure latest message notification is suspended
checkProgramStatus(artifactId, runId, ProgramRunStatus.SUSPENDED);
heartbeatDatasetStatusCheck(suspendTime, ProgramRunStatus.SUSPENDED);
long resumeTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.resume(runId);
});
// app metadata records as RUNNING
checkProgramStatus(artifactId, runId, ProgramRunStatus.RUNNING);
// heart beat messages wont have been sent due to high interval. resuming program will be recorded as running
// in run record by app meta
heartbeatDatasetStatusCheck(resumeTime, ProgramRunStatus.RUNNING);
// killed status check after error
long stopTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.error(runId, new Throwable("Testing"));
});
checkProgramStatus(artifactId, runId, ProgramRunStatus.FAILED);
heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.FAILED);
ProgramRunId runId2 = programId.run(RunIds.generate());
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.start(runId2, programOptions, null, programDescriptor);
});
checkProgramStatus(artifactId, runId2, ProgramRunStatus.STARTING);
startTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.running(runId2, null);
});
// perform scan on heart beat store - ensure latest message notification is running
checkProgramStatus(artifactId, runId2, ProgramRunStatus.RUNNING);
heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
// completed status check
stopTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.completed(runId2);
});
checkProgramStatus(artifactId, runId2, ProgramRunStatus.COMPLETED);
heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.COMPLETED);
ProgramRunId runId3 = programId.run(RunIds.generate());
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.start(runId3, programOptions, null, programDescriptor);
});
checkProgramStatus(artifactId, runId3, ProgramRunStatus.STARTING);
startTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.running(runId3, null);
});
// perform scan on heart beat store - ensure latest message notification is running
checkProgramStatus(artifactId, runId3, ProgramRunStatus.RUNNING);
heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
// completed status check
stopTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.stop(runId3, 10);
});
checkProgramStatus(artifactId, runId3, ProgramRunStatus.STOPPING);
heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.STOPPING);
}
use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.
the class ProgramStateWriterWithHeartBeatTest method testHeartBeatThread.
@Test
public void testHeartBeatThread() throws InterruptedException, ExecutionException, TimeoutException {
// configure program state writer to emit heart beat every second
ProgramStatePublisher programStatePublisher = new MockProgramStatePublisher();
NoOpProgramStateWriter programStateWriter = new NoOpProgramStateWriter();
// mock program configurations
ProgramId programId = NamespaceId.DEFAULT.app("someapp").program(ProgramType.SERVICE, "s");
Map<String, String> systemArguments = new HashMap<>();
systemArguments.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
ProgramOptions programOptions = new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments());
ProgramRunId runId = programId.run(RunIds.generate());
ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
ProgramStateWriterWithHeartBeat programStateWriterWithHeartBeat = new ProgramStateWriterWithHeartBeat(runId, programStateWriter, 1, programStatePublisher);
ApplicationSpecification appSpec = new DefaultApplicationSpecification("name", "1.0.0", ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec);
// start the program and ensure heart beat is 0 before we call running
programStateWriterWithHeartBeat.start(programOptions, null, programDescriptor);
Assert.assertEquals(0, ((MockProgramStatePublisher) programStatePublisher).getHeartBeatCount());
programStateWriterWithHeartBeat.running(null);
// on running, we start receiving heart beat messages, verify if we heart beat count goes to 2.
Tasks.waitFor(true, () -> ((MockProgramStatePublisher) programStatePublisher).getHeartBeatCount() > 1, 10, TimeUnit.SECONDS, "Didn't receive expected heartbeat after 10 seconds");
// make sure suspending program suspended the heartbeat thread
programStateWriterWithHeartBeat.suspend();
Tasks.waitFor(false, () -> programStateWriterWithHeartBeat.isHeartBeatThreadAlive(), 5, TimeUnit.SECONDS, "Heartbeat thread did not stop after 5 seconds");
long heartBeatAfterSuspend = ((MockProgramStatePublisher) programStatePublisher).getHeartBeatCount();
// resume the program and make sure that the heart beat messages goes up after resuming program
programStateWriterWithHeartBeat.resume();
long expected = heartBeatAfterSuspend + 1;
Tasks.waitFor(true, () -> ((MockProgramStatePublisher) programStatePublisher).getHeartBeatCount() > expected, 10, TimeUnit.SECONDS, "Didn't receive expected heartbeat after 10 seconds after resuming program");
// kill the program and make sure the heart beat thread also gets stopped
programStateWriterWithHeartBeat.killed();
Tasks.waitFor(false, () -> programStateWriterWithHeartBeat.isHeartBeatThreadAlive(), 5, TimeUnit.SECONDS, "Heartbeat thread did not stop after 5 seconds");
}
Aggregations