use of co.cask.cdap.proto.id.ArtifactId in project cdap by caskdata.
the class AbstractProgramRuntimeService method run.
@Override
public final RuntimeInfo run(ProgramDescriptor programDescriptor, ProgramOptions options) {
ProgramId programId = programDescriptor.getProgramId();
ProgramRunner runner = programRunnerFactory.create(programId.getType());
RunId runId = RunIds.generate();
File tempDir = createTempDirectory(programId, runId);
Runnable cleanUpTask = createCleanupTask(tempDir, runner);
try {
// Get the artifact details and save it into the program options.
ArtifactId artifactId = programDescriptor.getArtifactId();
ArtifactDetail artifactDetail = getArtifactDetail(artifactId);
ProgramOptions runtimeProgramOptions = updateProgramOptions(programId, options, runId);
// Take a snapshot of all the plugin artifacts used by the program
ProgramOptions optionsWithPlugins = createPluginSnapshot(runtimeProgramOptions, programId, tempDir, programDescriptor.getApplicationSpecification());
// Create and run the program
Program executableProgram = createProgram(cConf, runner, programDescriptor, artifactDetail, tempDir);
cleanUpTask = createCleanupTask(cleanUpTask, executableProgram);
RuntimeInfo runtimeInfo = createRuntimeInfo(runner.run(executableProgram, optionsWithPlugins), programId);
monitorProgram(runtimeInfo, cleanUpTask);
return runtimeInfo;
} catch (Exception e) {
cleanUpTask.run();
LOG.error("Exception while trying to run program", e);
throw Throwables.propagate(e);
}
}
use of co.cask.cdap.proto.id.ArtifactId in project cdap by caskdata.
the class AbstractProgramRuntimeServiceTest method testDeadlock.
@Test(timeout = 5000)
public void testDeadlock() throws IOException, ExecutionException, InterruptedException, TimeoutException {
// This test is for testing condition in (CDAP-3579)
// The race condition is if a program finished very fast such that inside the AbstractProgramRuntimeService is
// still in the run method, it holds the object lock, making the callback from the listener block forever.
ProgramRunnerFactory runnerFactory = createProgramRunnerFactory();
final Program program = createDummyProgram();
final ProgramRuntimeService runtimeService = new AbstractProgramRuntimeService(CConfiguration.create(), runnerFactory, null) {
@Override
public ProgramLiveInfo getLiveInfo(ProgramId programId) {
return new ProgramLiveInfo(programId, "runtime") {
};
}
@Override
protected Program createProgram(CConfiguration cConf, ProgramRunner programRunner, ProgramDescriptor programDescriptor, ArtifactDetail artifactDetail, File tempDir) throws IOException {
return program;
}
@Override
protected ArtifactDetail getArtifactDetail(ArtifactId artifactId) throws IOException, ArtifactNotFoundException {
co.cask.cdap.api.artifact.ArtifactId id = new co.cask.cdap.api.artifact.ArtifactId("dummy", new ArtifactVersion("1.0"), ArtifactScope.USER);
return new ArtifactDetail(new ArtifactDescriptor(id, Locations.toLocation(TEMP_FOLDER.newFile())), new ArtifactMeta(ArtifactClasses.builder().build()));
}
};
runtimeService.startAndWait();
try {
ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, null);
final ProgramController controller = runtimeService.run(descriptor, new SimpleProgramOptions(program.getId())).getController();
Tasks.waitFor(ProgramController.State.COMPLETED, new Callable<ProgramController.State>() {
@Override
public ProgramController.State call() throws Exception {
return controller.getState();
}
}, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
Tasks.waitFor(true, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
return runtimeService.list(ProgramType.WORKER).isEmpty();
}
}, 5, TimeUnit.SECONDS, 100, TimeUnit.MICROSECONDS);
} finally {
runtimeService.stopAndWait();
}
}
use of co.cask.cdap.proto.id.ArtifactId in project cdap by caskdata.
the class AbstractProgramRuntimeServiceTest method testScopingRuntimeArguments.
@Test
public void testScopingRuntimeArguments() throws Exception {
Map<ProgramId, Arguments> argumentsMap = new ConcurrentHashMap<>();
ProgramRunnerFactory runnerFactory = createProgramRunnerFactory(argumentsMap);
final Program program = createDummyProgram();
final ProgramRuntimeService runtimeService = new AbstractProgramRuntimeService(CConfiguration.create(), runnerFactory, null) {
@Override
public ProgramLiveInfo getLiveInfo(ProgramId programId) {
return new ProgramLiveInfo(programId, "runtime") {
};
}
@Override
protected Program createProgram(CConfiguration cConf, ProgramRunner programRunner, ProgramDescriptor programDescriptor, ArtifactDetail artifactDetail, File tempDir) throws IOException {
return program;
}
@Override
protected ArtifactDetail getArtifactDetail(ArtifactId artifactId) throws IOException, ArtifactNotFoundException {
co.cask.cdap.api.artifact.ArtifactId id = new co.cask.cdap.api.artifact.ArtifactId("dummy", new ArtifactVersion("1.0"), ArtifactScope.USER);
return new ArtifactDetail(new ArtifactDescriptor(id, Locations.toLocation(TEMP_FOLDER.newFile())), new ArtifactMeta(ArtifactClasses.builder().build()));
}
};
runtimeService.startAndWait();
try {
try {
ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, null);
// Set of scopes to test
String programScope = program.getType().getScope();
String clusterName = "c1";
List<String> scopes = Arrays.asList("cluster.*.", "cluster." + clusterName + ".", "cluster." + clusterName + ".app.*.", "app.*.", "app." + program.getApplicationId() + ".", "app." + program.getApplicationId() + "." + programScope + ".*.", "app." + program.getApplicationId() + "." + programScope + "." + program.getName() + ".", programScope + ".*.", programScope + "." + program.getName() + ".", "");
for (String scope : scopes) {
ProgramOptions programOptions = new SimpleProgramOptions(program.getName(), new BasicArguments(Collections.singletonMap(Constants.CLUSTER_NAME, clusterName)), new BasicArguments(Collections.singletonMap(scope + "size", Integer.toString(scope.length()))));
final ProgramController controller = runtimeService.run(descriptor, programOptions).getController();
Tasks.waitFor(ProgramController.State.COMPLETED, new Callable<ProgramController.State>() {
@Override
public ProgramController.State call() throws Exception {
return controller.getState();
}
}, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
// Should get an argument
Arguments args = argumentsMap.get(program.getId());
Assert.assertNotNull(args);
Assert.assertEquals(scope.length(), Integer.parseInt(args.getOption("size")));
}
} finally {
runtimeService.stopAndWait();
}
} finally {
runtimeService.stopAndWait();
}
}
use of co.cask.cdap.proto.id.ArtifactId in project cdap by caskdata.
the class HydratorTestBase method setupBatchArtifacts.
protected static void setupBatchArtifacts(ArtifactId artifactId, Class<?> appClass) throws Exception {
// add the app artifact
addAppArtifact(artifactId, appClass, BatchSource.class.getPackage().getName(), Action.class.getPackage().getName(), PipelineConfigurable.class.getPackage().getName(), "org.apache.avro.mapred", "org.apache.avro", "org.apache.avro.generic", "org.apache.avro.io");
// add plugins artifact
addPluginArtifact(new ArtifactId(artifactId.getNamespace(), artifactId.getArtifact() + "-mocks", "1.0.0"), artifactId, BATCH_MOCK_PLUGINS, co.cask.cdap.etl.mock.batch.MockSource.class, co.cask.cdap.etl.mock.batch.MockSink.class, MockExternalSource.class, MockExternalSink.class, DoubleTransform.class, AllErrorTransform.class, IdentityTransform.class, IntValueFilterTransform.class, StringValueFilterTransform.class, FieldCountAggregator.class, IdentityAggregator.class, FieldsPrefixTransform.class, StringValueFilterCompute.class, NodeStatesAction.class, LookupTransform.class);
}
use of co.cask.cdap.proto.id.ArtifactId in project cdap by caskdata.
the class HydratorTestBase method setupStreamingArtifacts.
protected static void setupStreamingArtifacts(ArtifactId artifactId, Class<?> appClass) throws Exception {
// add the app artifact
addAppArtifact(artifactId, appClass, StreamingSource.class.getPackage().getName(), Transform.class.getPackage().getName(), SparkCompute.class.getPackage().getName(), PipelineConfigurable.class.getPackage().getName());
// add plugins artifact
addPluginArtifact(new ArtifactId(artifactId.getNamespace(), artifactId.getArtifact() + "-mocks", "1.0.0"), artifactId, STREAMING_MOCK_PLUGINS, co.cask.cdap.etl.mock.spark.streaming.MockSource.class, co.cask.cdap.etl.mock.batch.MockSink.class, co.cask.cdap.etl.mock.spark.streaming.MockSink.class, DoubleTransform.class, AllErrorTransform.class, IdentityTransform.class, IntValueFilterTransform.class, StringValueFilterTransform.class, StringValueFilterCompute.class, Window.class);
}
Aggregations