Search in sources :

Example 56 with ArtifactId

use of co.cask.cdap.proto.id.ArtifactId in project cdap by caskdata.

the class AbstractProgramRuntimeService method run.

@Override
public final RuntimeInfo run(ProgramDescriptor programDescriptor, ProgramOptions options) {
    ProgramId programId = programDescriptor.getProgramId();
    ProgramRunner runner = programRunnerFactory.create(programId.getType());
    RunId runId = RunIds.generate();
    File tempDir = createTempDirectory(programId, runId);
    Runnable cleanUpTask = createCleanupTask(tempDir, runner);
    try {
        // Get the artifact details and save it into the program options.
        ArtifactId artifactId = programDescriptor.getArtifactId();
        ArtifactDetail artifactDetail = getArtifactDetail(artifactId);
        ProgramOptions runtimeProgramOptions = updateProgramOptions(programId, options, runId);
        // Take a snapshot of all the plugin artifacts used by the program
        ProgramOptions optionsWithPlugins = createPluginSnapshot(runtimeProgramOptions, programId, tempDir, programDescriptor.getApplicationSpecification());
        // Create and run the program
        Program executableProgram = createProgram(cConf, runner, programDescriptor, artifactDetail, tempDir);
        cleanUpTask = createCleanupTask(cleanUpTask, executableProgram);
        RuntimeInfo runtimeInfo = createRuntimeInfo(runner.run(executableProgram, optionsWithPlugins), programId);
        monitorProgram(runtimeInfo, cleanUpTask);
        return runtimeInfo;
    } catch (Exception e) {
        cleanUpTask.run();
        LOG.error("Exception while trying to run program", e);
        throw Throwables.propagate(e);
    }
}
Also used : Program(co.cask.cdap.app.program.Program) ArtifactId(co.cask.cdap.proto.id.ArtifactId) SimpleRuntimeInfo(co.cask.cdap.internal.app.runtime.service.SimpleRuntimeInfo) ProgramId(co.cask.cdap.proto.id.ProgramId) RunId(org.apache.twill.api.RunId) File(java.io.File) SimpleProgramOptions(co.cask.cdap.internal.app.runtime.SimpleProgramOptions) ArtifactNotFoundException(co.cask.cdap.common.ArtifactNotFoundException) IOException(java.io.IOException) ArtifactDetail(co.cask.cdap.internal.app.runtime.artifact.ArtifactDetail)

Example 57 with ArtifactId

use of co.cask.cdap.proto.id.ArtifactId in project cdap by caskdata.

the class AbstractProgramRuntimeServiceTest method testDeadlock.

@Test(timeout = 5000)
public void testDeadlock() throws IOException, ExecutionException, InterruptedException, TimeoutException {
    // This test is for testing condition in (CDAP-3579)
    // The race condition is if a program finished very fast such that inside the AbstractProgramRuntimeService is
    // still in the run method, it holds the object lock, making the callback from the listener block forever.
    ProgramRunnerFactory runnerFactory = createProgramRunnerFactory();
    final Program program = createDummyProgram();
    final ProgramRuntimeService runtimeService = new AbstractProgramRuntimeService(CConfiguration.create(), runnerFactory, null) {

        @Override
        public ProgramLiveInfo getLiveInfo(ProgramId programId) {
            return new ProgramLiveInfo(programId, "runtime") {
            };
        }

        @Override
        protected Program createProgram(CConfiguration cConf, ProgramRunner programRunner, ProgramDescriptor programDescriptor, ArtifactDetail artifactDetail, File tempDir) throws IOException {
            return program;
        }

        @Override
        protected ArtifactDetail getArtifactDetail(ArtifactId artifactId) throws IOException, ArtifactNotFoundException {
            co.cask.cdap.api.artifact.ArtifactId id = new co.cask.cdap.api.artifact.ArtifactId("dummy", new ArtifactVersion("1.0"), ArtifactScope.USER);
            return new ArtifactDetail(new ArtifactDescriptor(id, Locations.toLocation(TEMP_FOLDER.newFile())), new ArtifactMeta(ArtifactClasses.builder().build()));
        }
    };
    runtimeService.startAndWait();
    try {
        ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, null);
        final ProgramController controller = runtimeService.run(descriptor, new SimpleProgramOptions(program.getId())).getController();
        Tasks.waitFor(ProgramController.State.COMPLETED, new Callable<ProgramController.State>() {

            @Override
            public ProgramController.State call() throws Exception {
                return controller.getState();
            }
        }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        Tasks.waitFor(true, new Callable<Boolean>() {

            @Override
            public Boolean call() throws Exception {
                return runtimeService.list(ProgramType.WORKER).isEmpty();
            }
        }, 5, TimeUnit.SECONDS, 100, TimeUnit.MICROSECONDS);
    } finally {
        runtimeService.stopAndWait();
    }
}
Also used : ArtifactId(co.cask.cdap.proto.id.ArtifactId) ArtifactVersion(co.cask.cdap.api.artifact.ArtifactVersion) ArtifactDescriptor(co.cask.cdap.internal.app.runtime.artifact.ArtifactDescriptor) ProgramDescriptor(co.cask.cdap.app.program.ProgramDescriptor) ArtifactMeta(co.cask.cdap.internal.app.runtime.artifact.ArtifactMeta) Program(co.cask.cdap.app.program.Program) ProgramId(co.cask.cdap.proto.id.ProgramId) CConfiguration(co.cask.cdap.common.conf.CConfiguration) TimeoutException(java.util.concurrent.TimeoutException) ArtifactNotFoundException(co.cask.cdap.common.ArtifactNotFoundException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) SimpleProgramOptions(co.cask.cdap.internal.app.runtime.SimpleProgramOptions) ProgramLiveInfo(co.cask.cdap.proto.ProgramLiveInfo) File(java.io.File) ArtifactDetail(co.cask.cdap.internal.app.runtime.artifact.ArtifactDetail) Test(org.junit.Test)

Example 58 with ArtifactId

use of co.cask.cdap.proto.id.ArtifactId in project cdap by caskdata.

the class AbstractProgramRuntimeServiceTest method testScopingRuntimeArguments.

@Test
public void testScopingRuntimeArguments() throws Exception {
    Map<ProgramId, Arguments> argumentsMap = new ConcurrentHashMap<>();
    ProgramRunnerFactory runnerFactory = createProgramRunnerFactory(argumentsMap);
    final Program program = createDummyProgram();
    final ProgramRuntimeService runtimeService = new AbstractProgramRuntimeService(CConfiguration.create(), runnerFactory, null) {

        @Override
        public ProgramLiveInfo getLiveInfo(ProgramId programId) {
            return new ProgramLiveInfo(programId, "runtime") {
            };
        }

        @Override
        protected Program createProgram(CConfiguration cConf, ProgramRunner programRunner, ProgramDescriptor programDescriptor, ArtifactDetail artifactDetail, File tempDir) throws IOException {
            return program;
        }

        @Override
        protected ArtifactDetail getArtifactDetail(ArtifactId artifactId) throws IOException, ArtifactNotFoundException {
            co.cask.cdap.api.artifact.ArtifactId id = new co.cask.cdap.api.artifact.ArtifactId("dummy", new ArtifactVersion("1.0"), ArtifactScope.USER);
            return new ArtifactDetail(new ArtifactDescriptor(id, Locations.toLocation(TEMP_FOLDER.newFile())), new ArtifactMeta(ArtifactClasses.builder().build()));
        }
    };
    runtimeService.startAndWait();
    try {
        try {
            ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, null);
            // Set of scopes to test
            String programScope = program.getType().getScope();
            String clusterName = "c1";
            List<String> scopes = Arrays.asList("cluster.*.", "cluster." + clusterName + ".", "cluster." + clusterName + ".app.*.", "app.*.", "app." + program.getApplicationId() + ".", "app." + program.getApplicationId() + "." + programScope + ".*.", "app." + program.getApplicationId() + "." + programScope + "." + program.getName() + ".", programScope + ".*.", programScope + "." + program.getName() + ".", "");
            for (String scope : scopes) {
                ProgramOptions programOptions = new SimpleProgramOptions(program.getName(), new BasicArguments(Collections.singletonMap(Constants.CLUSTER_NAME, clusterName)), new BasicArguments(Collections.singletonMap(scope + "size", Integer.toString(scope.length()))));
                final ProgramController controller = runtimeService.run(descriptor, programOptions).getController();
                Tasks.waitFor(ProgramController.State.COMPLETED, new Callable<ProgramController.State>() {

                    @Override
                    public ProgramController.State call() throws Exception {
                        return controller.getState();
                    }
                }, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
                // Should get an argument
                Arguments args = argumentsMap.get(program.getId());
                Assert.assertNotNull(args);
                Assert.assertEquals(scope.length(), Integer.parseInt(args.getOption("size")));
            }
        } finally {
            runtimeService.stopAndWait();
        }
    } finally {
        runtimeService.stopAndWait();
    }
}
Also used : ArtifactId(co.cask.cdap.proto.id.ArtifactId) ArtifactVersion(co.cask.cdap.api.artifact.ArtifactVersion) ArtifactDescriptor(co.cask.cdap.internal.app.runtime.artifact.ArtifactDescriptor) ProgramDescriptor(co.cask.cdap.app.program.ProgramDescriptor) BasicArguments(co.cask.cdap.internal.app.runtime.BasicArguments) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ArtifactMeta(co.cask.cdap.internal.app.runtime.artifact.ArtifactMeta) Program(co.cask.cdap.app.program.Program) BasicArguments(co.cask.cdap.internal.app.runtime.BasicArguments) ProgramId(co.cask.cdap.proto.id.ProgramId) CConfiguration(co.cask.cdap.common.conf.CConfiguration) SimpleProgramOptions(co.cask.cdap.internal.app.runtime.SimpleProgramOptions) TimeoutException(java.util.concurrent.TimeoutException) ArtifactNotFoundException(co.cask.cdap.common.ArtifactNotFoundException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) SimpleProgramOptions(co.cask.cdap.internal.app.runtime.SimpleProgramOptions) ProgramLiveInfo(co.cask.cdap.proto.ProgramLiveInfo) File(java.io.File) ArtifactDetail(co.cask.cdap.internal.app.runtime.artifact.ArtifactDetail) Test(org.junit.Test)

Example 59 with ArtifactId

use of co.cask.cdap.proto.id.ArtifactId in project cdap by caskdata.

the class HydratorTestBase method setupBatchArtifacts.

protected static void setupBatchArtifacts(ArtifactId artifactId, Class<?> appClass) throws Exception {
    // add the app artifact
    addAppArtifact(artifactId, appClass, BatchSource.class.getPackage().getName(), Action.class.getPackage().getName(), PipelineConfigurable.class.getPackage().getName(), "org.apache.avro.mapred", "org.apache.avro", "org.apache.avro.generic", "org.apache.avro.io");
    // add plugins artifact
    addPluginArtifact(new ArtifactId(artifactId.getNamespace(), artifactId.getArtifact() + "-mocks", "1.0.0"), artifactId, BATCH_MOCK_PLUGINS, co.cask.cdap.etl.mock.batch.MockSource.class, co.cask.cdap.etl.mock.batch.MockSink.class, MockExternalSource.class, MockExternalSink.class, DoubleTransform.class, AllErrorTransform.class, IdentityTransform.class, IntValueFilterTransform.class, StringValueFilterTransform.class, FieldCountAggregator.class, IdentityAggregator.class, FieldsPrefixTransform.class, StringValueFilterCompute.class, NodeStatesAction.class, LookupTransform.class);
}
Also used : ArtifactId(co.cask.cdap.proto.id.ArtifactId)

Example 60 with ArtifactId

use of co.cask.cdap.proto.id.ArtifactId in project cdap by caskdata.

the class HydratorTestBase method setupStreamingArtifacts.

protected static void setupStreamingArtifacts(ArtifactId artifactId, Class<?> appClass) throws Exception {
    // add the app artifact
    addAppArtifact(artifactId, appClass, StreamingSource.class.getPackage().getName(), Transform.class.getPackage().getName(), SparkCompute.class.getPackage().getName(), PipelineConfigurable.class.getPackage().getName());
    // add plugins artifact
    addPluginArtifact(new ArtifactId(artifactId.getNamespace(), artifactId.getArtifact() + "-mocks", "1.0.0"), artifactId, STREAMING_MOCK_PLUGINS, co.cask.cdap.etl.mock.spark.streaming.MockSource.class, co.cask.cdap.etl.mock.batch.MockSink.class, co.cask.cdap.etl.mock.spark.streaming.MockSink.class, DoubleTransform.class, AllErrorTransform.class, IdentityTransform.class, IntValueFilterTransform.class, StringValueFilterTransform.class, StringValueFilterCompute.class, Window.class);
}
Also used : ArtifactId(co.cask.cdap.proto.id.ArtifactId)

Aggregations

ArtifactId (co.cask.cdap.proto.id.ArtifactId)82 Test (org.junit.Test)35 NamespaceId (co.cask.cdap.proto.id.NamespaceId)31 Id (co.cask.cdap.proto.Id)24 IOException (java.io.IOException)24 Path (javax.ws.rs.Path)24 ApplicationId (co.cask.cdap.proto.id.ApplicationId)23 ArtifactSummary (co.cask.cdap.api.artifact.ArtifactSummary)16 ProgramId (co.cask.cdap.proto.id.ProgramId)16 AppRequest (co.cask.cdap.proto.artifact.AppRequest)13 PluginClass (co.cask.cdap.api.plugin.PluginClass)11 ArtifactNotFoundException (co.cask.cdap.common.ArtifactNotFoundException)11 File (java.io.File)10 ArtifactVersion (co.cask.cdap.api.artifact.ArtifactVersion)9 ArtifactDetail (co.cask.cdap.internal.app.runtime.artifact.ArtifactDetail)8 DELETE (javax.ws.rs.DELETE)8 GET (javax.ws.rs.GET)8 ArtifactRange (co.cask.cdap.api.artifact.ArtifactRange)7 BadRequestException (co.cask.cdap.common.BadRequestException)7 AuditPolicy (co.cask.cdap.common.security.AuditPolicy)6