Search in sources :

Example 16 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.

the class AbstractProgramRuntimeServiceTest method testDeadlock.

@Test(timeout = 5000)
public void testDeadlock() throws IOException, ExecutionException, InterruptedException, TimeoutException {
    // This test is for testing condition in (CDAP-3579)
    // The race condition is if a program finished very fast such that inside the AbstractProgramRuntimeService is
    // still in the run method, it holds the object lock, making the callback from the listener block forever.
    ProgramRunnerFactory runnerFactory = createProgramRunnerFactory();
    Program program = createDummyProgram();
    ProgramRuntimeService runtimeService = new TestProgramRuntimeService(CConfiguration.create(), runnerFactory, program, null, null);
    runtimeService.startAndWait();
    try {
        ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, NamespaceId.DEFAULT.artifact("test", "1.0"));
        ProgramController controller = runtimeService.run(descriptor, new SimpleProgramOptions(program.getId()), RunIds.generate()).getController();
        Tasks.waitFor(ProgramController.State.COMPLETED, controller::getState, 5, TimeUnit.SECONDS, 100, TimeUnit.MILLISECONDS);
        Tasks.waitFor(true, () -> runtimeService.list(ProgramType.WORKER).isEmpty(), 5, TimeUnit.SECONDS, 100, TimeUnit.MICROSECONDS);
    } finally {
        runtimeService.stopAndWait();
    }
}
Also used : Program(io.cdap.cdap.app.program.Program) ProgramDescriptor(io.cdap.cdap.app.program.ProgramDescriptor) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) Test(org.junit.Test)

Example 17 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.

the class AbstractProgramRuntimeServiceTest method testConcurrentStartLimit.

@Test
public void testConcurrentStartLimit() throws Exception {
    Semaphore proceed = new Semaphore(0);
    Set<String> threadNames = ConcurrentHashMap.newKeySet();
    // Creates a runner factory that block start until a signal is received
    ProgramRunnerFactory runnerFactory = programType -> (program, options) -> {
        threadNames.add(Thread.currentThread().getName());
        proceed.acquireUninterruptibly();
        ProgramId programId = program.getId();
        Service service = new FastService();
        ProgramController controller = new ProgramControllerServiceAdapter(service, programId.run(RunIds.generate()));
        service.start();
        return controller;
    };
    Program program = createDummyProgram();
    ProgramDescriptor descriptor = new ProgramDescriptor(program.getId(), null, NamespaceId.DEFAULT.artifact("test", "1.0"));
    CConfiguration cConf = CConfiguration.create();
    cConf.setInt(Constants.AppFabric.PROGRAM_LAUNCH_THREADS, 2);
    ProgramRuntimeService runtimeService = new TestProgramRuntimeService(cConf, runnerFactory, program, null, null);
    runtimeService.startAndWait();
    try {
        List<ProgramController> controllers = new ArrayList<>();
        for (int i = 0; i < 5; i++) {
            controllers.add(runtimeService.run(descriptor, new SimpleProgramOptions(program.getId()), RunIds.generate()).getController());
        }
        // There should be 2 program start threads running only
        Tasks.waitFor(2, proceed::getQueueLength, 2, TimeUnit.SECONDS);
        try {
            // Shouldn't never have more than 2
            Tasks.waitFor(true, () -> proceed.getQueueLength() > 2, 2, TimeUnit.SECONDS);
            Assert.fail("Shouldn't have more than 2 program starting");
        } catch (TimeoutException e) {
        // expected
        }
        // Let all start to proceed. They should have all finished pretty quickly
        proceed.release(5);
        Tasks.waitFor(true, () -> controllers.stream().map(ProgramController::getState).allMatch(ProgramController.State.COMPLETED::equals), 5, TimeUnit.SECONDS);
        Assert.assertEquals(2, threadNames.size());
    } finally {
        runtimeService.stopAndWait();
    }
}
Also used : AbstractExecutionThreadService(com.google.common.util.concurrent.AbstractExecutionThreadService) Arrays(java.util.Arrays) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) ArtifactClasses(io.cdap.cdap.api.artifact.ArtifactClasses) TimeoutException(java.util.concurrent.TimeoutException) ProgramControllerServiceAdapter(io.cdap.cdap.internal.app.runtime.ProgramControllerServiceAdapter) ArtifactMeta(io.cdap.cdap.internal.app.runtime.artifact.ArtifactMeta) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) Map(java.util.Map) RunId(org.apache.twill.api.RunId) ClassRule(org.junit.ClassRule) Tasks(io.cdap.cdap.common.utils.Tasks) ArtifactDescriptor(io.cdap.cdap.internal.app.runtime.artifact.ArtifactDescriptor) ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) List(java.util.List) Constants(io.cdap.cdap.common.conf.Constants) ApplicationId(io.cdap.cdap.proto.id.ApplicationId) ArtifactScope(io.cdap.cdap.api.artifact.ArtifactScope) Location(org.apache.twill.filesystem.Location) Program(io.cdap.cdap.app.program.Program) HashMap(java.util.HashMap) ProgramType(io.cdap.cdap.proto.ProgramType) ArrayList(java.util.ArrayList) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) AbstractIdleService(com.google.common.util.concurrent.AbstractIdleService) Locations(io.cdap.cdap.common.io.Locations) ArtifactId(io.cdap.cdap.proto.id.ArtifactId) ArtifactDetail(io.cdap.cdap.internal.app.runtime.artifact.ArtifactDetail) Nullable(javax.annotation.Nullable) ProgramLiveInfo(io.cdap.cdap.proto.ProgramLiveInfo) RunIds(io.cdap.cdap.common.app.RunIds) Semaphore(java.util.concurrent.Semaphore) ProgramId(io.cdap.cdap.proto.id.ProgramId) ProgramDescriptor(io.cdap.cdap.app.program.ProgramDescriptor) Test(org.junit.Test) IOException(java.io.IOException) File(java.io.File) Service(com.google.common.util.concurrent.Service) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) ArtifactVersion(io.cdap.cdap.api.artifact.ArtifactVersion) ArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository) Assert(org.junit.Assert) Collections(java.util.Collections) TemporaryFolder(org.junit.rules.TemporaryFolder) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) Program(io.cdap.cdap.app.program.Program) ArrayList(java.util.ArrayList) AbstractExecutionThreadService(com.google.common.util.concurrent.AbstractExecutionThreadService) AbstractIdleService(com.google.common.util.concurrent.AbstractIdleService) Service(com.google.common.util.concurrent.Service) Semaphore(java.util.concurrent.Semaphore) ProgramId(io.cdap.cdap.proto.id.ProgramId) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) ProgramControllerServiceAdapter(io.cdap.cdap.internal.app.runtime.ProgramControllerServiceAdapter) ProgramDescriptor(io.cdap.cdap.app.program.ProgramDescriptor) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) TimeoutException(java.util.concurrent.TimeoutException) Test(org.junit.Test)

Example 18 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.

the class RuntimeServiceMainTest method testRuntimeService.

@Test
public void testRuntimeService() throws Exception {
    ArtifactId artifactId = NamespaceId.DEFAULT.artifact("test", "1.0");
    ProgramRunId programRunId = NamespaceId.DEFAULT.app("app").worker("worker").run(RunIds.generate());
    Map<String, String> systemArgs = ImmutableMap.of(SystemArguments.PROFILE_PROVISIONER, NativeProvisioner.SPEC.getName(), SystemArguments.PROFILE_NAME, "default");
    ProgramOptions programOptions = new SimpleProgramOptions(programRunId.getParent(), new BasicArguments(systemArgs), new BasicArguments());
    ProgramDescriptor programDescriptor = new ProgramDescriptor(programRunId.getParent(), null, artifactId);
    // Write out program state events to simulate program start
    Injector appFabricInjector = getServiceMainInstance(AppFabricServiceMain.class).getInjector();
    CConfiguration cConf = appFabricInjector.getInstance(CConfiguration.class);
    ProgramStatePublisher programStatePublisher = new MessagingProgramStatePublisher(appFabricInjector.getInstance(MessagingService.class), NamespaceId.SYSTEM.topic(cConf.get(Constants.AppFabric.PROGRAM_STATUS_RECORD_EVENT_TOPIC)), RetryStrategies.fromConfiguration(cConf, "system.program.state."));
    new MessagingProgramStateWriter(programStatePublisher).start(programRunId, programOptions, null, programDescriptor);
    Injector injector = getServiceMainInstance(RuntimeServiceMain.class).getInjector();
    TransactionRunner txRunner = injector.getInstance(TransactionRunner.class);
    // Should see a STARTING record in the runtime store
    Tasks.waitFor(ProgramRunStatus.STARTING, () -> {
        RunRecordDetail detail = TransactionRunners.run(txRunner, context -> {
            return AppMetadataStore.create(context).getRun(programRunId);
        });
        return detail == null ? null : detail.getStatus();
    }, 5, TimeUnit.SECONDS);
    ProgramStateWriter programStateWriter = createProgramStateWriter(injector, programRunId);
    // Write a running state. We should see a RUNNING record in the runtime store
    programStateWriter.running(programRunId, null);
    Tasks.waitFor(ProgramRunStatus.RUNNING, () -> {
        RunRecordDetail detail = TransactionRunners.run(txRunner, context -> {
            return AppMetadataStore.create(context).getRun(programRunId);
        });
        return detail == null ? null : detail.getStatus();
    }, 5, TimeUnit.SECONDS);
    // Write a complete state. The run record should be removed in the runtime store
    programStateWriter.completed(programRunId);
    Tasks.waitFor(true, () -> TransactionRunners.run(txRunner, context -> AppMetadataStore.create(context).getRun(programRunId) == null), 5, TimeUnit.SECONDS);
}
Also used : Retries(io.cdap.cdap.common.service.Retries) RetryStrategy(io.cdap.cdap.common.service.RetryStrategy) RunRecordDetail(io.cdap.cdap.internal.app.store.RunRecordDetail) TransactionRunners(io.cdap.cdap.spi.data.transaction.TransactionRunners) BeforeClass(org.junit.BeforeClass) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) MessagingProgramStateWriter(io.cdap.cdap.internal.app.program.MessagingProgramStateWriter) Notification(io.cdap.cdap.proto.Notification) Bytes(io.cdap.cdap.api.common.Bytes) RetryStrategies(io.cdap.cdap.common.service.RetryStrategies) TopicId(io.cdap.cdap.proto.id.TopicId) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) ProgramStateWriter(io.cdap.cdap.app.runtime.ProgramStateWriter) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) Gson(com.google.gson.Gson) ProgramOptions(io.cdap.cdap.app.runtime.ProgramOptions) Map(java.util.Map) SystemArguments(io.cdap.cdap.internal.app.runtime.SystemArguments) NativeProvisioner(io.cdap.cdap.internal.provision.NativeProvisioner) ArtifactId(io.cdap.cdap.proto.id.ArtifactId) AppMetadataStore(io.cdap.cdap.internal.app.store.AppMetadataStore) Tasks(io.cdap.cdap.common.utils.Tasks) Message(io.cdap.cdap.api.messaging.Message) ImmutableMap(com.google.common.collect.ImmutableMap) RunIds(io.cdap.cdap.common.app.RunIds) ProgramStatePublisher(io.cdap.cdap.internal.app.program.ProgramStatePublisher) RetryableException(io.cdap.cdap.api.retry.RetryableException) MessagingService(io.cdap.cdap.messaging.MessagingService) ProgramDescriptor(io.cdap.cdap.app.program.ProgramDescriptor) MessagingProgramStatePublisher(io.cdap.cdap.internal.app.program.MessagingProgramStatePublisher) RuntimeClient(io.cdap.cdap.internal.app.runtime.monitor.RuntimeClient) Test(org.junit.Test) IOException(java.io.IOException) ProgramRunStatus(io.cdap.cdap.proto.ProgramRunStatus) StandardCharsets(java.nio.charset.StandardCharsets) MessageId(io.cdap.cdap.messaging.data.MessageId) Injector(com.google.inject.Injector) TimeUnit(java.util.concurrent.TimeUnit) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) TransactionRunner(io.cdap.cdap.spi.data.transaction.TransactionRunner) Constants(io.cdap.cdap.common.conf.Constants) Collections(java.util.Collections) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) ArtifactId(io.cdap.cdap.proto.id.ArtifactId) RunRecordDetail(io.cdap.cdap.internal.app.store.RunRecordDetail) MessagingProgramStateWriter(io.cdap.cdap.internal.app.program.MessagingProgramStateWriter) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) ProgramOptions(io.cdap.cdap.app.runtime.ProgramOptions) MessagingService(io.cdap.cdap.messaging.MessagingService) MessagingProgramStatePublisher(io.cdap.cdap.internal.app.program.MessagingProgramStatePublisher) MessagingProgramStateWriter(io.cdap.cdap.internal.app.program.MessagingProgramStateWriter) ProgramStateWriter(io.cdap.cdap.app.runtime.ProgramStateWriter) Injector(com.google.inject.Injector) TransactionRunner(io.cdap.cdap.spi.data.transaction.TransactionRunner) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) ProgramDescriptor(io.cdap.cdap.app.program.ProgramDescriptor) ProgramStatePublisher(io.cdap.cdap.internal.app.program.ProgramStatePublisher) MessagingProgramStatePublisher(io.cdap.cdap.internal.app.program.MessagingProgramStatePublisher) Test(org.junit.Test)

Example 19 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.

the class DistributedWorkflowProgramRunnerTest method setupWorkflowRuntime.

/**
 * Setup the {@link ProgramLaunchConfig} for the given workflow.
 */
private ProgramLaunchConfig setupWorkflowRuntime(String workflowName, Map<String, String> runtimeArgs) throws IOException {
    // Create the distributed workflow program runner
    ProgramRunner programRunner = programRunnerFactory.create(ProgramType.WORKFLOW);
    Assert.assertTrue(programRunner instanceof DistributedWorkflowProgramRunner);
    DistributedWorkflowProgramRunner workflowRunner = (DistributedWorkflowProgramRunner) programRunner;
    // Create the Workflow Program
    Program workflowProgram = createWorkflowProgram(cConf, programRunner, workflowName);
    ProgramLaunchConfig launchConfig = new ProgramLaunchConfig();
    ProgramOptions programOpts = new SimpleProgramOptions(workflowProgram.getId(), new BasicArguments(), new BasicArguments(runtimeArgs));
    // Setup the launching config
    workflowRunner.setupLaunchConfig(launchConfig, workflowProgram, programOpts, cConf, new Configuration(), TEMP_FOLDER.newFolder());
    return launchConfig;
}
Also used : Program(io.cdap.cdap.app.program.Program) Configuration(org.apache.hadoop.conf.Configuration) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) ProgramRunner(io.cdap.cdap.app.runtime.ProgramRunner) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) ProgramOptions(io.cdap.cdap.app.runtime.ProgramOptions)

Example 20 with SimpleProgramOptions

use of io.cdap.cdap.internal.app.runtime.SimpleProgramOptions in project cdap by caskdata.

the class AbstractProgramRuntimeService method createPluginSnapshot.

/**
 * Return the copy of the {@link ProgramOptions} including locations of plugin artifacts in it.
 * @param options the {@link ProgramOptions} in which the locations of plugin artifacts needs to be included
 * @param programId Id of the Program
 * @param tempDir Temporary Directory to create the plugin artifact snapshot
 * @param appSpec program's Application Specification
 * @return the copy of the program options with locations of plugin artifacts included in them
 */
private ProgramOptions createPluginSnapshot(ProgramOptions options, ProgramId programId, File tempDir, @Nullable ApplicationSpecification appSpec) throws Exception {
    // appSpec is null in an unit test
    if (appSpec == null) {
        return options;
    }
    Set<String> files = Sets.newHashSet();
    ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
    builder.putAll(options.getArguments().asMap());
    for (Map.Entry<String, Plugin> pluginEntry : appSpec.getPlugins().entrySet()) {
        Plugin plugin = pluginEntry.getValue();
        File destFile = new File(tempDir, Artifacts.getFileName(plugin.getArtifactId()));
        // Skip if the file has already been copied.
        if (!files.add(destFile.getName())) {
            continue;
        }
        try {
            ArtifactId artifactId = Artifacts.toProtoArtifactId(programId.getNamespaceId(), plugin.getArtifactId());
            copyArtifact(artifactId, noAuthArtifactRepository.getArtifact(Id.Artifact.fromEntityId(artifactId)), destFile);
        } catch (ArtifactNotFoundException e) {
            throw new IllegalArgumentException(String.format("Artifact %s could not be found", plugin.getArtifactId()), e);
        }
    }
    LOG.debug("Plugin artifacts of {} copied to {}", programId, tempDir.getAbsolutePath());
    builder.put(ProgramOptionConstants.PLUGIN_DIR, tempDir.getAbsolutePath());
    return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(builder.build()), options.getUserArguments(), options.isDebug());
}
Also used : ArtifactId(io.cdap.cdap.proto.id.ArtifactId) ImmutableMap(com.google.common.collect.ImmutableMap) SimpleProgramOptions(io.cdap.cdap.internal.app.runtime.SimpleProgramOptions) BasicArguments(io.cdap.cdap.internal.app.runtime.BasicArguments) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) File(java.io.File) ArtifactNotFoundException(io.cdap.cdap.common.ArtifactNotFoundException) Plugin(io.cdap.cdap.api.plugin.Plugin)

Aggregations

SimpleProgramOptions (io.cdap.cdap.internal.app.runtime.SimpleProgramOptions)30 BasicArguments (io.cdap.cdap.internal.app.runtime.BasicArguments)28 ProgramOptions (io.cdap.cdap.app.runtime.ProgramOptions)18 ProgramDescriptor (io.cdap.cdap.app.program.ProgramDescriptor)15 ProgramRunId (io.cdap.cdap.proto.id.ProgramRunId)15 HashMap (java.util.HashMap)15 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)12 ProgramId (io.cdap.cdap.proto.id.ProgramId)11 Test (org.junit.Test)10 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)9 Map (java.util.Map)9 Program (io.cdap.cdap.app.program.Program)8 SystemArguments (io.cdap.cdap.internal.app.runtime.SystemArguments)8 ImmutableMap (com.google.common.collect.ImmutableMap)6 ProgramStateWriter (io.cdap.cdap.app.runtime.ProgramStateWriter)6 Constants (io.cdap.cdap.common.conf.Constants)6 IOException (java.io.IOException)6 Collections (java.util.Collections)6 List (java.util.List)6 Injector (com.google.inject.Injector)5