Search in sources :

Example 1 with ProgramType

use of io.cdap.cdap.proto.ProgramType in project cdap by caskdata.

the class RemoteExecutionDiscoveryService method discover.

@Override
public ServiceDiscovered discover(String name) {
    // In Remote runtime, we don't support program discovery except for the SYSTEM namespace.
    for (ProgramType programType : ProgramType.values()) {
        if (programType.isDiscoverable() && name.startsWith(programType.getDiscoverableTypeName() + ".") && !name.startsWith(programType.getDiscoverableTypeName() + "." + NamespaceId.SYSTEM.getNamespace() + ".")) {
            return new DefaultServiceDiscovered(name);
        }
    }
    // Discovery for system services
    DefaultServiceDiscovered serviceDiscovered = services.get(name);
    String key = Constants.RuntimeMonitor.DISCOVERY_SERVICE_PREFIX + name;
    // Use a while loop to resolve races between discover and cancellable of register
    while (serviceDiscovered == null) {
        // Try to add the ServiceDiscovered object
        serviceDiscovered = new DefaultServiceDiscovered(name);
        synchronized (this) {
            updateServiceDiscovered(serviceDiscovered, cConf, key);
            services.putIfAbsent(name, serviceDiscovered);
        }
        serviceDiscovered = services.get(name);
    }
    return serviceDiscovered;
}
Also used : ProgramType(io.cdap.cdap.proto.ProgramType) DefaultServiceDiscovered(io.cdap.cdap.master.spi.discovery.DefaultServiceDiscovered)

Example 2 with ProgramType

use of io.cdap.cdap.proto.ProgramType in project cdap by caskdata.

the class DistributedMapReduceProgramRunner method validateOptions.

@Override
protected void validateOptions(Program program, ProgramOptions options) {
    super.validateOptions(program, options);
    // Extract and verify parameters
    ApplicationSpecification appSpec = program.getApplicationSpecification();
    Preconditions.checkNotNull(appSpec, "Missing application specification.");
    ProgramType processorType = program.getType();
    Preconditions.checkNotNull(processorType, "Missing processor type.");
    Preconditions.checkArgument(processorType == ProgramType.MAPREDUCE, "Only MapReduce process type is supported.");
    MapReduceSpecification spec = appSpec.getMapReduce().get(program.getName());
    Preconditions.checkNotNull(spec, "Missing MapReduceSpecification for %s", program.getName());
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) MapReduceSpecification(io.cdap.cdap.api.mapreduce.MapReduceSpecification) ProgramType(io.cdap.cdap.proto.ProgramType)

Example 3 with ProgramType

use of io.cdap.cdap.proto.ProgramType in project cdap by caskdata.

the class DistributedProgramRuntimeService method list.

@Override
public Map<RunId, RuntimeInfo> list(ProgramType type) {
    Map<RunId, RuntimeInfo> result = new HashMap<>(super.list(type));
    // Table holds the Twill RunId and TwillController associated with the program matching the input type
    Table<ProgramId, RunId, TwillController> twillProgramInfo = HashBasedTable.create();
    List<RuntimeInfo> runtimeInfos = getRuntimeInfos();
    // Goes through all live application and fill the twillProgramInfo table
    for (TwillRunner.LiveInfo liveInfo : twillRunner.lookupLive()) {
        String appName = liveInfo.getApplicationName();
        ProgramId programId = TwillAppNames.fromTwillAppName(appName, false);
        if (programId == null) {
            continue;
        }
        if (!type.equals(programId.getType())) {
            continue;
        }
        for (TwillController controller : liveInfo.getControllers()) {
            RunId twillRunId = controller.getRunId();
            // If it already in the runtime info, no need to lookup
            if (runtimeInfos.stream().anyMatch(info -> twillRunId.equals(info.getTwillRunId()))) {
                continue;
            }
            twillProgramInfo.put(programId, twillRunId, controller);
        }
    }
    if (twillProgramInfo.isEmpty()) {
        return ImmutableMap.copyOf(result);
    }
    final Set<RunId> twillRunIds = twillProgramInfo.columnKeySet();
    Collection<RunRecordDetail> activeRunRecords;
    synchronized (this) {
        activeRunRecords = store.getRuns(ProgramRunStatus.RUNNING, record -> record.getTwillRunId() != null && twillRunIds.contains(org.apache.twill.internal.RunIds.fromString(record.getTwillRunId()))).values();
    }
    for (RunRecordDetail record : activeRunRecords) {
        String twillRunId = record.getTwillRunId();
        if (twillRunId == null) {
            // This is unexpected. Just log and ignore the run record
            LOG.warn("No twill runId for in run record {}.", record);
            continue;
        }
        RunId twillRunIdFromRecord = org.apache.twill.internal.RunIds.fromString(twillRunId);
        // Get the CDAP RunId from RunRecord
        RunId runId = RunIds.fromString(record.getPid());
        // Get the Program and TwillController for the current twillRunId
        Map<ProgramId, TwillController> mapForTwillId = twillProgramInfo.columnMap().get(twillRunIdFromRecord);
        Map.Entry<ProgramId, TwillController> entry = mapForTwillId.entrySet().iterator().next();
        // Create RuntimeInfo for the current Twill RunId
        if (result.computeIfAbsent(runId, rid -> createRuntimeInfo(entry.getKey(), rid, entry.getValue())) == null) {
            LOG.warn("Unable to create runtime info for program {} with run id {}", entry.getKey(), runId);
        }
    }
    return ImmutableMap.copyOf(result);
}
Also used : RunRecordDetail(io.cdap.cdap.internal.app.store.RunRecordDetail) Inject(com.google.inject.Inject) LoggerFactory(org.slf4j.LoggerFactory) DistributedProgramLiveInfo(io.cdap.cdap.proto.DistributedProgramLiveInfo) HashBasedTable(com.google.common.collect.HashBasedTable) TwillAppNames(io.cdap.cdap.common.twill.TwillAppNames) ProgramRunnerFactory(io.cdap.cdap.app.runtime.ProgramRunnerFactory) ResourceReport(org.apache.twill.api.ResourceReport) ProgramStateWriter(io.cdap.cdap.app.runtime.ProgramStateWriter) AppFabricServiceRuntimeModule(io.cdap.cdap.app.guice.AppFabricServiceRuntimeModule) Map(java.util.Map) RunId(org.apache.twill.api.RunId) TwillController(org.apache.twill.api.TwillController) Threads(org.apache.twill.common.Threads) Containers(io.cdap.cdap.proto.Containers) ImmutableMap(com.google.common.collect.ImmutableMap) Collection(java.util.Collection) Set(java.util.Set) ProgramRunStatus(io.cdap.cdap.proto.ProgramRunStatus) List(java.util.List) SimpleRuntimeInfo(io.cdap.cdap.internal.app.runtime.service.SimpleRuntimeInfo) TwillRunResources(org.apache.twill.api.TwillRunResources) HashMap(java.util.HashMap) ProgramType(io.cdap.cdap.proto.ProgramType) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) TwillRunner(org.apache.twill.api.TwillRunner) AbstractProgramRuntimeService(io.cdap.cdap.app.runtime.AbstractProgramRuntimeService) Locations(io.cdap.cdap.common.io.Locations) ArtifactId(io.cdap.cdap.proto.id.ArtifactId) ArtifactDetail(io.cdap.cdap.internal.app.runtime.artifact.ArtifactDetail) Nullable(javax.annotation.Nullable) ProgramController(io.cdap.cdap.app.runtime.ProgramController) ProgramLiveInfo(io.cdap.cdap.proto.ProgramLiveInfo) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) ConfiguratorFactory(io.cdap.cdap.internal.app.deploy.ConfiguratorFactory) RunIds(io.cdap.cdap.common.app.RunIds) ContainerInfo(io.cdap.cdap.proto.Containers.ContainerInfo) ProgramId(io.cdap.cdap.proto.id.ProgramId) Throwables(com.google.common.base.Throwables) Impersonator(io.cdap.cdap.security.impersonation.Impersonator) IOException(java.io.IOException) FileAlreadyExistsException(java.nio.file.FileAlreadyExistsException) File(java.io.File) Delegator(io.cdap.cdap.common.lang.Delegator) Store(io.cdap.cdap.app.store.Store) AbstractListener(io.cdap.cdap.internal.app.runtime.AbstractListener) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) ArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository) Named(com.google.inject.name.Named) Table(com.google.common.collect.Table) SimpleRuntimeInfo(io.cdap.cdap.internal.app.runtime.service.SimpleRuntimeInfo) HashMap(java.util.HashMap) TwillRunner(org.apache.twill.api.TwillRunner) RunRecordDetail(io.cdap.cdap.internal.app.store.RunRecordDetail) ProgramId(io.cdap.cdap.proto.id.ProgramId) TwillController(org.apache.twill.api.TwillController) RunId(org.apache.twill.api.RunId) ProgramRunId(io.cdap.cdap.proto.id.ProgramRunId) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap)

Example 4 with ProgramType

use of io.cdap.cdap.proto.ProgramType in project cdap by caskdata.

the class WorkflowTwillRunnable method createModule.

@Override
protected Module createModule(CConfiguration cConf, Configuration hConf, ProgramOptions programOptions, ProgramRunId programRunId) {
    List<Module> modules = new ArrayList<>();
    modules.add(super.createModule(cConf, hConf, programOptions, programRunId));
    if (ProgramRunners.getClusterMode(programOptions) == ClusterMode.ON_PREMISE) {
        modules.add(new DistributedArtifactManagerModule());
    } else {
        modules.add(new AbstractModule() {

            @Override
            protected void configure() {
                bind(PluginFinder.class).to(UnsupportedPluginFinder.class);
            }
        });
    }
    modules.add(new PrivateModule() {

        @Override
        protected void configure() {
            // Bind ProgramRunner for MR, which is used by Workflow.
            // The ProgramRunner for Spark is provided by the DefaultProgramRunnerFactory through the extension mechanism
            MapBinder<ProgramType, ProgramRunner> runnerFactoryBinder = MapBinder.newMapBinder(binder(), ProgramType.class, ProgramRunner.class);
            runnerFactoryBinder.addBinding(ProgramType.MAPREDUCE).to(MapReduceProgramRunner.class);
            // It uses local mode factory because for Workflow we launch the job from the Workflow container directly.
            // The actual execution mode of the job is governed by the framework configuration
            // For mapreduce, it's in the mapred-site.xml
            // for spark, it's in the hConf we shipped from DistributedWorkflowProgramRunner
            bind(ProgramRuntimeProvider.Mode.class).toInstance(ProgramRuntimeProvider.Mode.LOCAL);
            bind(ProgramRunnerFactory.class).to(DefaultProgramRunnerFactory.class).in(Scopes.SINGLETON);
            expose(ProgramRunnerFactory.class);
        }
    });
    return Modules.combine(modules);
}
Also used : ArrayList(java.util.ArrayList) UnsupportedPluginFinder(io.cdap.cdap.app.guice.UnsupportedPluginFinder) DefaultProgramRunnerFactory(io.cdap.cdap.app.guice.DefaultProgramRunnerFactory) ProgramRunnerFactory(io.cdap.cdap.app.runtime.ProgramRunnerFactory) AbstractModule(com.google.inject.AbstractModule) ProgramRuntimeProvider(io.cdap.cdap.app.runtime.ProgramRuntimeProvider) DistributedArtifactManagerModule(io.cdap.cdap.app.guice.DistributedArtifactManagerModule) MapReduceProgramRunner(io.cdap.cdap.internal.app.runtime.batch.MapReduceProgramRunner) MapBinder(com.google.inject.multibindings.MapBinder) ProgramType(io.cdap.cdap.proto.ProgramType) Module(com.google.inject.Module) PrivateModule(com.google.inject.PrivateModule) DistributedArtifactManagerModule(io.cdap.cdap.app.guice.DistributedArtifactManagerModule) AbstractModule(com.google.inject.AbstractModule) WorkflowProgramRunner(io.cdap.cdap.internal.app.runtime.workflow.WorkflowProgramRunner) ProgramRunner(io.cdap.cdap.app.runtime.ProgramRunner) MapReduceProgramRunner(io.cdap.cdap.internal.app.runtime.batch.MapReduceProgramRunner) PrivateModule(com.google.inject.PrivateModule)

Example 5 with ProgramType

use of io.cdap.cdap.proto.ProgramType in project cdap by caskdata.

the class DefaultRuntimeJob method createModules.

/**
 * Returns list of guice modules used to start the program run.
 */
@VisibleForTesting
List<Module> createModules(RuntimeJobEnvironment runtimeJobEnv, CConfiguration cConf, ProgramRunId programRunId, ProgramOptions programOpts) {
    List<Module> modules = new ArrayList<>();
    modules.add(new ConfigModule(cConf));
    RuntimeMonitorType runtimeMonitorType = SystemArguments.getRuntimeMonitorType(cConf, programOpts);
    modules.add(RuntimeMonitors.getRemoteAuthenticatorModule(runtimeMonitorType, programOpts));
    modules.add(new IOModule());
    modules.add(new TMSLogAppenderModule());
    modules.add(new RemoteExecutionDiscoveryModule());
    modules.add(new AuthenticationContextModules().getProgramContainerModule(cConf));
    modules.add(new MetricsClientRuntimeModule().getDistributedModules());
    modules.add(new MessagingServerRuntimeModule().getStandaloneModules());
    modules.add(new AbstractModule() {

        @Override
        protected void configure() {
            bind(ClusterMode.class).toInstance(ClusterMode.ISOLATED);
            bind(UGIProvider.class).to(CurrentUGIProvider.class).in(Scopes.SINGLETON);
            // Bindings from the environment
            bind(TwillRunner.class).annotatedWith(Constants.AppFabric.ProgramRunner.class).toInstance(runtimeJobEnv.getTwillRunner());
            bind(LocationFactory.class).toInstance(runtimeJobEnv.getLocationFactory());
            MapBinder<ProgramType, ProgramRunner> defaultProgramRunnerBinder = MapBinder.newMapBinder(binder(), ProgramType.class, ProgramRunner.class);
            bind(ProgramRuntimeProvider.Mode.class).toInstance(ProgramRuntimeProvider.Mode.DISTRIBUTED);
            bind(ProgramRunnerFactory.class).annotatedWith(Constants.AppFabric.ProgramRunner.class).to(DefaultProgramRunnerFactory.class).in(Scopes.SINGLETON);
            bind(ProgramStateWriter.class).to(MessagingProgramStateWriter.class).in(Scopes.SINGLETON);
            defaultProgramRunnerBinder.addBinding(ProgramType.MAPREDUCE).to(DistributedMapReduceProgramRunner.class);
            defaultProgramRunnerBinder.addBinding(ProgramType.WORKFLOW).to(DistributedWorkflowProgramRunner.class);
            defaultProgramRunnerBinder.addBinding(ProgramType.WORKER).to(DistributedWorkerProgramRunner.class);
            bind(ProgramRunnerFactory.class).to(DefaultProgramRunnerFactory.class).in(Scopes.SINGLETON);
            bind(ProgramRunId.class).toInstance(programRunId);
            bind(RuntimeMonitorType.class).toInstance(runtimeMonitorType);
            install(new FactoryModuleBuilder().implement(Configurator.class, InMemoryConfigurator.class).build(ConfiguratorFactory.class));
            bind(String.class).annotatedWith(Names.named(RemoteIsolatedPluginFinder.ISOLATED_PLUGIN_DIR)).toInstance(programOpts.getArguments().getOption(ProgramOptionConstants.PLUGIN_DIR, DistributedProgramRunner.PLUGIN_DIR));
            bind(PluginFinder.class).to(RemoteIsolatedPluginFinder.class);
            bind(ArtifactRepositoryReader.class).to(RemoteArtifactRepositoryReader.class).in(Scopes.SINGLETON);
            bind(ArtifactRepository.class).to(RemoteArtifactRepository.class);
        }
    });
    return modules;
}
Also used : IOModule(io.cdap.cdap.common.guice.IOModule) RuntimeMonitorType(io.cdap.cdap.runtime.spi.RuntimeMonitorType) ConfigModule(io.cdap.cdap.common.guice.ConfigModule) FactoryModuleBuilder(com.google.inject.assistedinject.FactoryModuleBuilder) InMemoryConfigurator(io.cdap.cdap.internal.app.deploy.InMemoryConfigurator) Configurator(io.cdap.cdap.app.deploy.Configurator) UGIProvider(io.cdap.cdap.security.impersonation.UGIProvider) CurrentUGIProvider(io.cdap.cdap.security.impersonation.CurrentUGIProvider) TwillRunner(org.apache.twill.api.TwillRunner) ArrayList(java.util.ArrayList) MessagingServerRuntimeModule(io.cdap.cdap.messaging.guice.MessagingServerRuntimeModule) MetricsClientRuntimeModule(io.cdap.cdap.metrics.guice.MetricsClientRuntimeModule) RemoteExecutionDiscoveryModule(io.cdap.cdap.app.guice.RemoteExecutionDiscoveryModule) ProgramRunnerFactory(io.cdap.cdap.app.runtime.ProgramRunnerFactory) DefaultProgramRunnerFactory(io.cdap.cdap.app.guice.DefaultProgramRunnerFactory) ArtifactRepositoryReader(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepositoryReader) RemoteArtifactRepositoryReader(io.cdap.cdap.internal.app.runtime.artifact.RemoteArtifactRepositoryReader) TMSLogAppenderModule(io.cdap.cdap.logging.guice.TMSLogAppenderModule) ProgramStateWriter(io.cdap.cdap.app.runtime.ProgramStateWriter) MessagingProgramStateWriter(io.cdap.cdap.internal.app.program.MessagingProgramStateWriter) RemoteArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.RemoteArtifactRepository) ProgramType(io.cdap.cdap.proto.ProgramType) DistributedWorkflowProgramRunner(io.cdap.cdap.internal.app.runtime.distributed.DistributedWorkflowProgramRunner) DistributedProgramRunner(io.cdap.cdap.internal.app.runtime.distributed.DistributedProgramRunner) DistributedMapReduceProgramRunner(io.cdap.cdap.internal.app.runtime.distributed.DistributedMapReduceProgramRunner) DistributedWorkerProgramRunner(io.cdap.cdap.internal.app.runtime.distributed.DistributedWorkerProgramRunner) ProgramRunner(io.cdap.cdap.app.runtime.ProgramRunner) DistributedWorkflowProgramRunner(io.cdap.cdap.internal.app.runtime.distributed.DistributedWorkflowProgramRunner) AuthenticationContextModules(io.cdap.cdap.security.auth.context.AuthenticationContextModules) Constants(io.cdap.cdap.common.conf.Constants) ProgramOptionConstants(io.cdap.cdap.internal.app.runtime.ProgramOptionConstants) RemoteIsolatedPluginFinder(io.cdap.cdap.internal.app.runtime.artifact.RemoteIsolatedPluginFinder) DistributedMapReduceProgramRunner(io.cdap.cdap.internal.app.runtime.distributed.DistributedMapReduceProgramRunner) AbstractModule(com.google.inject.AbstractModule) ProgramRuntimeProvider(io.cdap.cdap.app.runtime.ProgramRuntimeProvider) MapBinder(com.google.inject.multibindings.MapBinder) DistributedWorkerProgramRunner(io.cdap.cdap.internal.app.runtime.distributed.DistributedWorkerProgramRunner) MessagingServerRuntimeModule(io.cdap.cdap.messaging.guice.MessagingServerRuntimeModule) TMSLogAppenderModule(io.cdap.cdap.logging.guice.TMSLogAppenderModule) IOModule(io.cdap.cdap.common.guice.IOModule) AbstractModule(com.google.inject.AbstractModule) Module(com.google.inject.Module) ConfigModule(io.cdap.cdap.common.guice.ConfigModule) RemoteExecutionDiscoveryModule(io.cdap.cdap.app.guice.RemoteExecutionDiscoveryModule) MetricsClientRuntimeModule(io.cdap.cdap.metrics.guice.MetricsClientRuntimeModule) InMemoryConfigurator(io.cdap.cdap.internal.app.deploy.InMemoryConfigurator) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Aggregations

ProgramType (io.cdap.cdap.proto.ProgramType)76 ProgramId (io.cdap.cdap.proto.id.ProgramId)42 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)27 ApplicationId (io.cdap.cdap.proto.id.ApplicationId)27 Path (javax.ws.rs.Path)23 GET (javax.ws.rs.GET)21 ArrayList (java.util.ArrayList)18 ProgramRunId (io.cdap.cdap.proto.id.ProgramRunId)16 BadRequestException (io.cdap.cdap.common.BadRequestException)14 List (java.util.List)14 Map (java.util.Map)14 HashMap (java.util.HashMap)13 Set (java.util.Set)13 Nullable (javax.annotation.Nullable)13 RunRecordDetail (io.cdap.cdap.internal.app.store.RunRecordDetail)12 ImmutableMap (com.google.common.collect.ImmutableMap)11 NamespaceNotFoundException (io.cdap.cdap.common.NamespaceNotFoundException)11 NotFoundException (io.cdap.cdap.common.NotFoundException)11 Constants (io.cdap.cdap.common.conf.Constants)11 HashSet (java.util.HashSet)11