use of io.cdap.cdap.proto.id.ProgramId in project cdap by caskdata.
the class ProgramNotificationSubscriberServiceTest method testHeartBeatStoreForProgramStatusMessages.
@Test
public void testHeartBeatStoreForProgramStatusMessages() throws Exception {
ProgramId programId = NamespaceId.DEFAULT.app("someapp", "1.0-SNAPSHOT").program(ProgramType.SERVICE, "s");
Map<String, String> systemArguments = new HashMap<>();
systemArguments.put(ProgramOptionConstants.SKIP_PROVISIONING, Boolean.TRUE.toString());
systemArguments.put(SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
ProgramOptions programOptions = new SimpleProgramOptions(programId, new BasicArguments(systemArguments), new BasicArguments());
ProgramRunId runId = programId.run(RunIds.generate());
ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
ApplicationSpecification appSpec = new DefaultApplicationSpecification("name", "1.0.0", ProjectInfo.getVersion().toString(), "desc", null, artifactId, Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap(), Collections.emptyMap());
ProgramDescriptor programDescriptor = new ProgramDescriptor(programId, appSpec);
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.start(runId, programOptions, null, programDescriptor);
});
checkProgramStatus(artifactId, runId, ProgramRunStatus.STARTING);
long startTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.running(runId, null);
});
// perform scan on heart beat store - ensure latest message notification is running
checkProgramStatus(artifactId, runId, ProgramRunStatus.RUNNING);
heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
long suspendTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.suspend(runId);
});
// perform scan on heart beat store - ensure latest message notification is suspended
checkProgramStatus(artifactId, runId, ProgramRunStatus.SUSPENDED);
heartbeatDatasetStatusCheck(suspendTime, ProgramRunStatus.SUSPENDED);
long resumeTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.resume(runId);
});
// app metadata records as RUNNING
checkProgramStatus(artifactId, runId, ProgramRunStatus.RUNNING);
// heart beat messages wont have been sent due to high interval. resuming program will be recorded as running
// in run record by app meta
heartbeatDatasetStatusCheck(resumeTime, ProgramRunStatus.RUNNING);
// killed status check after error
long stopTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.error(runId, new Throwable("Testing"));
});
checkProgramStatus(artifactId, runId, ProgramRunStatus.FAILED);
heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.FAILED);
ProgramRunId runId2 = programId.run(RunIds.generate());
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.start(runId2, programOptions, null, programDescriptor);
});
checkProgramStatus(artifactId, runId2, ProgramRunStatus.STARTING);
startTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.running(runId2, null);
});
// perform scan on heart beat store - ensure latest message notification is running
checkProgramStatus(artifactId, runId2, ProgramRunStatus.RUNNING);
heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
// completed status check
stopTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.completed(runId2);
});
checkProgramStatus(artifactId, runId2, ProgramRunStatus.COMPLETED);
heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.COMPLETED);
ProgramRunId runId3 = programId.run(RunIds.generate());
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.start(runId3, programOptions, null, programDescriptor);
});
checkProgramStatus(artifactId, runId3, ProgramRunStatus.STARTING);
startTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.running(runId3, null);
});
// perform scan on heart beat store - ensure latest message notification is running
checkProgramStatus(artifactId, runId3, ProgramRunStatus.RUNNING);
heartbeatDatasetStatusCheck(startTime, ProgramRunStatus.RUNNING);
// completed status check
stopTime = System.currentTimeMillis();
TransactionRunners.run(transactionRunner, context -> {
programStateWriter.stop(runId3, 10);
});
checkProgramStatus(artifactId, runId3, ProgramRunStatus.STOPPING);
heartbeatDatasetStatusCheck(stopTime, ProgramRunStatus.STOPPING);
}
use of io.cdap.cdap.proto.id.ProgramId in project cdap by caskdata.
the class RunRecordCorrectorServiceTest method testFixProgram.
@Test
public void testFixProgram() throws Exception {
final AtomicInteger sourceId = new AtomicInteger(0);
// Write 10 services with starting state
// Write 10 workers with running state
Map<ProgramRunId, ProgramRunStatus> expectedStates = new HashMap<>();
ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
for (int i = 0; i < 10; i++) {
ProgramRunId serviceId = NamespaceId.DEFAULT.app("test").service("service" + i).run(randomRunId());
store.setProvisioning(serviceId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
store.setProvisioned(serviceId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
store.setStart(serviceId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
expectedStates.put(serviceId, ProgramRunStatus.FAILED);
ProgramRunId workerId = new NamespaceId("ns").app("test").worker("worker" + i).run(randomRunId());
store.setProvisioning(workerId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
store.setProvisioned(workerId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
store.setStart(workerId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
store.setRunning(workerId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
expectedStates.put(workerId, ProgramRunStatus.FAILED);
}
// Write a service with suspend state
ProgramRunId flowId = new NamespaceId("ns").app("test").service("flow").run(randomRunId());
store.setProvisioning(flowId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
store.setProvisioned(flowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
store.setStart(flowId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
store.setRunning(flowId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
store.setSuspend(flowId, Bytes.toBytes(sourceId.getAndIncrement()), -1);
expectedStates.put(flowId, ProgramRunStatus.SUSPENDED);
// Write two MR in starting state. One with workflow information, one without.
ProgramRunId mrId = NamespaceId.DEFAULT.app("app").mr("mr").run(randomRunId());
store.setProvisioning(mrId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
store.setProvisioned(mrId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
store.setStart(mrId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
expectedStates.put(mrId, ProgramRunStatus.FAILED);
ProgramRunId workflowId = NamespaceId.DEFAULT.app("app").workflow("workflow").run(randomRunId());
ProgramRunId mrInWorkflowId = workflowId.getParent().getParent().mr("mrInWorkflow").run(randomRunId());
store.setProvisioning(mrInWorkflowId, Collections.emptyMap(), ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NAME, workflowId.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, workflowId.getRun(), ProgramOptionConstants.WORKFLOW_NODE_ID, "mr", SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName()), Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
store.setProvisioned(mrInWorkflowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
store.setStart(mrInWorkflowId, null, ImmutableMap.of(ProgramOptionConstants.WORKFLOW_NAME, workflowId.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, workflowId.getRun(), ProgramOptionConstants.WORKFLOW_NODE_ID, "mr"), Bytes.toBytes(sourceId.getAndIncrement()));
expectedStates.put(workflowId, ProgramRunStatus.STARTING);
// Write the workflow in RUNNING state.
store.setProvisioning(workflowId, Collections.emptyMap(), SINGLETON_PROFILE_MAP, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
store.setProvisioned(workflowId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
store.setStart(workflowId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
store.setRunning(workflowId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
expectedStates.put(workflowId, ProgramRunStatus.RUNNING);
// Use a ProgramRuntimeService that only reports running state based on a set of know ids
final Map<ProgramId, RunId> runningSet = new HashMap<>();
ProgramRuntimeService programRuntimeService = new AbstractProgramRuntimeService(cConf, null, null, new NoOpProgramStateWriter(), null) {
@Override
public ProgramLiveInfo getLiveInfo(ProgramId programId) {
return new NotRunningProgramLiveInfo(programId);
}
@Override
public Map<RunId, RuntimeInfo> list(ProgramId program) {
RunId runId = runningSet.get(program);
if (runId != null) {
RuntimeInfo runtimeInfo = new SimpleRuntimeInfo(null, program);
return Collections.singletonMap(runId, runtimeInfo);
}
return Collections.emptyMap();
}
};
// Have both flow and workflow running
runningSet.put(flowId.getParent(), RunIds.fromString(flowId.getRun()));
runningSet.put(workflowId.getParent(), RunIds.fromString(workflowId.getRun()));
ProgramStateWriter programStateWriter = new NoOpProgramStateWriter() {
@Override
public void error(ProgramRunId programRunId, Throwable failureCause) {
store.setStop(programRunId, System.currentTimeMillis(), ProgramRunStatus.FAILED, new BasicThrowable(failureCause), Bytes.toBytes(sourceId.getAndIncrement()));
}
};
// Create a run record fixer.
// Set the start buffer time to -1 so that it fixes right away.
// Also use a small tx batch size to validate the batching logic.
RunRecordCorrectorService fixer = new RunRecordCorrectorService(cConf, store, programStateWriter, programRuntimeService, namespaceAdmin, datasetFramework, -1L, 5) {
};
fixer.fixRunRecords();
// Validates all expected states
for (Map.Entry<ProgramRunId, ProgramRunStatus> entry : expectedStates.entrySet()) {
validateExpectedState(entry.getKey(), entry.getValue());
}
// Remove the workflow from the running set and mark it as completed
runningSet.remove(workflowId.getParent());
store.setStop(workflowId, System.currentTimeMillis(), ProgramRunStatus.COMPLETED, Bytes.toBytes(sourceId.getAndIncrement()));
fixer.fixRunRecords();
// Both the workflow and the MR in workflow should be changed to failed state
expectedStates.put(workflowId, ProgramRunStatus.COMPLETED);
expectedStates.put(mrInWorkflowId, ProgramRunStatus.FAILED);
// Validates all expected states again
for (Map.Entry<ProgramRunId, ProgramRunStatus> entry : expectedStates.entrySet()) {
validateExpectedState(entry.getKey(), entry.getValue());
}
}
use of io.cdap.cdap.proto.id.ProgramId in project cdap by caskdata.
the class RunRecordCorrectorServiceTest method testFixUnderlyingProgramInWorkflow.
@Test
public void testFixUnderlyingProgramInWorkflow() throws Exception {
AtomicInteger sourceId = new AtomicInteger(0);
ArtifactId artifactId = NamespaceId.DEFAULT.artifact("testArtifact", "1.0").toApiArtifactId();
// set up a workflow
Map<String, String> wfSystemArg = ImmutableMap.of(ProgramOptionConstants.CLUSTER_MODE, ClusterMode.ISOLATED.name(), SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName());
ProgramRunId wfId = NamespaceId.DEFAULT.app("test").workflow("testWF").run(randomRunId());
store.setProvisioning(wfId, Collections.emptyMap(), wfSystemArg, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
store.setProvisioned(wfId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
store.setStart(wfId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
store.setRunning(wfId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
// set up a spark program inside workflow, and in ISOLATED mode
Map<String, String> spSystemArgs = ImmutableMap.of(ProgramOptionConstants.CLUSTER_MODE, ClusterMode.ISOLATED.name(), SystemArguments.PROFILE_NAME, ProfileId.NATIVE.getScopedName(), ProgramOptionConstants.WORKFLOW_NAME, wfId.getProgram(), ProgramOptionConstants.WORKFLOW_RUN_ID, wfId.getRun(), ProgramOptionConstants.WORKFLOW_NODE_ID, "spark");
ProgramRunId spId = NamespaceId.DEFAULT.app("test").spark("phase-1").run(randomRunId());
store.setProvisioning(spId, Collections.emptyMap(), spSystemArgs, Bytes.toBytes(sourceId.getAndIncrement()), artifactId);
store.setProvisioned(spId, 0, Bytes.toBytes(sourceId.getAndIncrement()));
store.setStart(spId, null, Collections.emptyMap(), Bytes.toBytes(sourceId.getAndIncrement()));
store.setRunning(spId, System.currentTimeMillis(), null, Bytes.toBytes(sourceId.getAndIncrement()));
// mark the workflow as finished
store.setStop(wfId, System.currentTimeMillis(), ProgramRunStatus.COMPLETED, Bytes.toBytes(sourceId.getAndIncrement()));
ProgramStateWriter programStateWriter = new NoOpProgramStateWriter() {
@Override
public void error(ProgramRunId programRunId, Throwable failureCause) {
store.setStop(programRunId, System.currentTimeMillis(), ProgramRunStatus.FAILED, new BasicThrowable(failureCause), Bytes.toBytes(sourceId.getAndIncrement()));
}
};
ProgramRuntimeService noOpRuntimeSerivce = new AbstractProgramRuntimeService(cConf, null, null, new NoOpProgramStateWriter(), null) {
@Override
public ProgramLiveInfo getLiveInfo(ProgramId programId) {
return new NotRunningProgramLiveInfo(programId);
}
@Override
public Map<RunId, RuntimeInfo> list(ProgramId program) {
return Collections.emptyMap();
}
};
// Create a run record fixer.
// Set the start buffer time to -1 so that it fixes right away.
RunRecordCorrectorService fixer = new RunRecordCorrectorService(cConf, store, programStateWriter, noOpRuntimeSerivce, namespaceAdmin, datasetFramework, -1L, 5) {
};
fixer.fixRunRecords();
// check the record is fixed for spark program
Assert.assertEquals(ProgramRunStatus.FAILED, store.getRun(spId).getStatus());
}
use of io.cdap.cdap.proto.id.ProgramId in project cdap by caskdata.
the class ApplicationLifecycleServiceTest method testDeployArtifactAndApplicationCleansUpArtifactOnFailure.
// test that the call to deploy an artifact and application in a single step will delete the artifact
// if the application could not be created
@Test(expected = ArtifactNotFoundException.class)
public void testDeployArtifactAndApplicationCleansUpArtifactOnFailure() throws Exception {
Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, "missing-mr", "1.0.0-SNAPSHOT");
Location appJar = AppJarHelper.createDeploymentJar(locationFactory, MissingMapReduceWorkflowApp.class);
File appJarFile = new File(tmpFolder.newFolder(), String.format("%s-%s.jar", artifactId.getName(), artifactId.getVersion().getVersion()));
Locations.linkOrCopyOverwrite(appJar, appJarFile);
appJar.delete();
try {
applicationLifecycleService.deployAppAndArtifact(NamespaceId.DEFAULT, "appName", artifactId, appJarFile, null, null, programId -> {
}, true);
Assert.fail("expected application deployment to fail.");
} catch (Exception e) {
// expected
}
// the artifact should have been cleaned up, and this should throw a not found exception
artifactRepository.getArtifact(artifactId);
}
use of io.cdap.cdap.proto.id.ProgramId in project cdap by caskdata.
the class ApplicationLifecycleServiceTest method testCapabilityMetaDataDeletion.
@Test
public void testCapabilityMetaDataDeletion() throws Exception {
Class<CapabilityAppWithWorkflow> appWithWorkflowClass = CapabilityAppWithWorkflow.class;
Requirements declaredAnnotation = appWithWorkflowClass.getDeclaredAnnotation(Requirements.class);
Set<String> expected = Arrays.stream(declaredAnnotation.capabilities()).collect(Collectors.toSet());
Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, appWithWorkflowClass.getSimpleName(), "1.0.0-SNAPSHOT");
Location appJar = AppJarHelper.createDeploymentJar(locationFactory, appWithWorkflowClass);
File appJarFile = new File(tmpFolder.newFolder(), String.format("%s-%s.jar", artifactId.getName(), artifactId.getVersion().getVersion()));
Locations.linkOrCopyOverwrite(appJar, appJarFile);
appJar.delete();
// deploy app
try {
applicationLifecycleService.deployAppAndArtifact(NamespaceId.DEFAULT, appWithWorkflowClass.getSimpleName(), artifactId, appJarFile, null, null, programId -> {
}, true);
Assert.fail("Expecting exception");
} catch (CapabilityNotAvailableException ex) {
// expected
}
for (String capability : declaredAnnotation.capabilities()) {
CapabilityConfig capabilityConfig = new CapabilityConfig("Test", CapabilityStatus.ENABLED, capability, Collections.emptyList(), Collections.emptyList(), Collections.emptyList());
capabilityWriter.addOrUpdateCapability(capability, CapabilityStatus.ENABLED, capabilityConfig);
}
applicationLifecycleService.deployAppAndArtifact(NamespaceId.DEFAULT, appWithWorkflowClass.getSimpleName(), artifactId, appJarFile, null, null, programId -> {
}, true);
// Check for the capability metadata
ApplicationId appId = NamespaceId.DEFAULT.app(appWithWorkflowClass.getSimpleName());
MetadataEntity appMetadataId = appId.toMetadataEntity();
Assert.assertFalse(metadataStorage.read(new Read(appMetadataId, MetadataScope.SYSTEM, MetadataKind.PROPERTY)).isEmpty());
Map<String, String> metadataProperties = metadataStorage.read(new Read(appMetadataId)).getProperties(MetadataScope.SYSTEM);
String capabilityMetaData = metadataProperties.get(AppSystemMetadataWriter.CAPABILITY_TAG);
Set<String> actual = Arrays.stream(capabilityMetaData.split(AppSystemMetadataWriter.CAPABILITY_DELIMITER)).collect(Collectors.toSet());
Assert.assertEquals(expected, actual);
// Remove the application and verify that all metadata is removed
applicationLifecycleService.removeApplication(appId);
Assert.assertTrue(metadataStorage.read(new Read(appMetadataId)).isEmpty());
}
Aggregations