Search in sources :

Example 1 with PluginFinder

use of io.cdap.cdap.internal.app.runtime.artifact.PluginFinder in project cdap by caskdata.

the class DefaultAppConfigurer method addSpark.

@Override
public void addSpark(Spark spark) {
    Preconditions.checkArgument(spark != null, "Spark cannot be null.");
    DefaultSparkConfigurer configurer = null;
    // It is a bit hacky here to look for the DefaultExtendedSparkConfigurer implementation through the
    // SparkRunnerClassloader directly (CDAP-11797)
    ClassLoader sparkRunnerClassLoader = ClassLoaders.findByName(spark.getClass().getClassLoader(), "io.cdap.cdap.app.runtime.spark.classloader.SparkRunnerClassLoader");
    if (sparkRunnerClassLoader != null) {
        try {
            configurer = (DefaultSparkConfigurer) sparkRunnerClassLoader.loadClass("io.cdap.cdap.app.deploy.spark.DefaultExtendedSparkConfigurer").getConstructor(Spark.class, Id.Namespace.class, Id.Artifact.class, PluginFinder.class, PluginInstantiator.class, AppDeploymentRuntimeInfo.class, FeatureFlagsProvider.class).newInstance(spark, deployNamespace, artifactId, pluginFinder, pluginInstantiator, runtimeInfo, getFeatureFlagsProvider());
        } catch (Exception e) {
            // Ignore it and the configurer will be defaulted to DefaultSparkConfigurer
            LOG.trace("No DefaultExtendedSparkConfigurer found. Fallback to DefaultSparkConfigurer.", e);
        }
    }
    if (configurer == null) {
        configurer = new DefaultSparkConfigurer(spark, deployNamespace, artifactId, pluginFinder, pluginInstantiator, runtimeInfo, getFeatureFlagsProvider());
    }
    spark.configure(configurer);
    addDatasetsAndPlugins(configurer);
    SparkSpecification spec = configurer.createSpecification();
    sparks.put(spec.getName(), spec);
}
Also used : SparkSpecification(io.cdap.cdap.api.spark.SparkSpecification) DefaultSparkConfigurer(io.cdap.cdap.internal.app.spark.DefaultSparkConfigurer) PluginFinder(io.cdap.cdap.internal.app.runtime.artifact.PluginFinder) PluginInstantiator(io.cdap.cdap.internal.app.runtime.plugin.PluginInstantiator) Spark(io.cdap.cdap.api.spark.Spark) FeatureFlagsProvider(io.cdap.cdap.api.feature.FeatureFlagsProvider) AppDeploymentRuntimeInfo(io.cdap.cdap.internal.app.deploy.pipeline.AppDeploymentRuntimeInfo)

Example 2 with PluginFinder

use of io.cdap.cdap.internal.app.runtime.artifact.PluginFinder in project cdap by caskdata.

the class SystemAppTask method buildTaskSystemAppContext.

private SystemAppTaskContext buildTaskSystemAppContext(Injector injector, String systemAppNamespace, ArtifactId artifactId, ClassLoader artifactClassLoader) {
    PreferencesFetcher preferencesFetcher = injector.getInstance(PreferencesFetcher.class);
    PluginFinder pluginFinder = injector.getInstance(PluginFinder.class);
    SecureStore secureStore = injector.getInstance(SecureStore.class);
    ArtifactManagerFactory artifactManagerFactory = injector.getInstance(ArtifactManagerFactory.class);
    RemoteClientFactory remoteClientFactory = injector.getInstance(RemoteClientFactory.class);
    return new DefaultSystemAppTaskContext(injector.getInstance(CConfiguration.class), preferencesFetcher, pluginFinder, secureStore, systemAppNamespace, artifactId, artifactClassLoader, artifactManagerFactory, Constants.Service.TASK_WORKER, remoteClientFactory);
}
Also used : RemoteClientFactory(io.cdap.cdap.common.internal.remote.RemoteClientFactory) PluginFinder(io.cdap.cdap.internal.app.runtime.artifact.PluginFinder) ArtifactManagerFactory(io.cdap.cdap.internal.app.runtime.artifact.ArtifactManagerFactory) PreferencesFetcher(io.cdap.cdap.metadata.PreferencesFetcher) SecureStore(io.cdap.cdap.api.security.store.SecureStore) CConfiguration(io.cdap.cdap.common.conf.CConfiguration)

Example 3 with PluginFinder

use of io.cdap.cdap.internal.app.runtime.artifact.PluginFinder in project cdap by caskdata.

the class ConfiguratorTest method testAppWithConfig.

@Test
public void testAppWithConfig() throws Exception {
    LocationFactory locationFactory = new LocalLocationFactory(TMP_FOLDER.newFolder());
    Location appJar = AppJarHelper.createDeploymentJar(locationFactory, ConfigTestApp.class);
    Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, ConfigTestApp.class.getSimpleName(), "1.0.0");
    CConfiguration cConf = CConfiguration.create();
    ArtifactRepository baseArtifactRepo = new DefaultArtifactRepository(conf, null, null, null, new DummyProgramRunnerFactory(), new DefaultImpersonator(cConf, null));
    ArtifactRepository artifactRepo = new AuthorizationArtifactRepository(baseArtifactRepo, authEnforcer, authenticationContext);
    PluginFinder pluginFinder = new LocalPluginFinder(artifactRepo);
    ConfigTestApp.ConfigClass config = new ConfigTestApp.ConfigClass("myTable");
    AppDeploymentInfo appDeploymentInfo = new AppDeploymentInfo(artifactId.toEntityId(), appJar, NamespaceId.DEFAULT, new ApplicationClass(ConfigTestApp.class.getName(), "", null), null, null, new Gson().toJson(config));
    // Create a configurator that is testable. Provide it an application.
    Configurator configurator = new InMemoryConfigurator(conf, pluginFinder, new DefaultImpersonator(cConf, null), artifactRepo, null, appDeploymentInfo);
    ListenableFuture<ConfigResponse> result = configurator.config();
    ConfigResponse response = result.get(10, TimeUnit.SECONDS);
    Assert.assertNotNull(response);
    AppSpecInfo appSpecInfo = response.getAppSpecInfo();
    if (appSpecInfo == null) {
        throw new IllegalStateException("Failed to deploy application");
    }
    ApplicationSpecification specification = appSpecInfo.getAppSpec();
    Assert.assertNotNull(specification);
    Assert.assertEquals(1, specification.getDatasets().size());
    Assert.assertTrue(specification.getDatasets().containsKey("myTable"));
    // Create a deployment info without the app configuration
    appDeploymentInfo = new AppDeploymentInfo(artifactId.toEntityId(), appJar, NamespaceId.DEFAULT, new ApplicationClass(ConfigTestApp.class.getName(), "", null), null, null, null);
    Configurator configuratorWithoutConfig = new InMemoryConfigurator(conf, pluginFinder, new DefaultImpersonator(cConf, null), artifactRepo, null, appDeploymentInfo);
    result = configuratorWithoutConfig.config();
    response = result.get(10, TimeUnit.SECONDS);
    Assert.assertNotNull(response);
    appSpecInfo = response.getAppSpecInfo();
    if (appSpecInfo == null) {
        throw new IllegalStateException("Failed to deploy application");
    }
    specification = appSpecInfo.getAppSpec();
    Assert.assertNotNull(specification);
    Assert.assertEquals(1, specification.getDatasets().size());
    Assert.assertTrue(specification.getDatasets().containsKey(ConfigTestApp.DEFAULT_TABLE));
    Assert.assertNotNull(specification.getProgramSchedules().get(ConfigTestApp.SCHEDULE_NAME));
    ProgramStatusTrigger trigger = (ProgramStatusTrigger) specification.getProgramSchedules().get(ConfigTestApp.SCHEDULE_NAME).getTrigger();
    Assert.assertEquals(trigger.getProgramId().getProgram(), ConfigTestApp.WORKFLOW_NAME);
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) LocalPluginFinder(io.cdap.cdap.internal.app.runtime.artifact.LocalPluginFinder) Configurator(io.cdap.cdap.app.deploy.Configurator) DefaultArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.DefaultArtifactRepository) Gson(com.google.gson.Gson) ConfigResponse(io.cdap.cdap.app.deploy.ConfigResponse) AppDeploymentInfo(io.cdap.cdap.internal.app.deploy.pipeline.AppDeploymentInfo) PluginFinder(io.cdap.cdap.internal.app.runtime.artifact.PluginFinder) LocalPluginFinder(io.cdap.cdap.internal.app.runtime.artifact.LocalPluginFinder) ProgramStatusTrigger(io.cdap.cdap.internal.app.runtime.schedule.trigger.ProgramStatusTrigger) LocalLocationFactory(org.apache.twill.filesystem.LocalLocationFactory) ConfigTestApp(io.cdap.cdap.ConfigTestApp) AuthorizationArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.AuthorizationArtifactRepository) DummyProgramRunnerFactory(io.cdap.cdap.app.runtime.DummyProgramRunnerFactory) AppSpecInfo(io.cdap.cdap.internal.app.deploy.pipeline.AppSpecInfo) ApplicationClass(io.cdap.cdap.api.artifact.ApplicationClass) AuthorizationArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.AuthorizationArtifactRepository) DefaultArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.DefaultArtifactRepository) ArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) DefaultImpersonator(io.cdap.cdap.security.impersonation.DefaultImpersonator) LocalLocationFactory(org.apache.twill.filesystem.LocalLocationFactory) LocationFactory(org.apache.twill.filesystem.LocationFactory) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Id(io.cdap.cdap.common.id.Id) Location(org.apache.twill.filesystem.Location) Test(org.junit.Test)

Example 4 with PluginFinder

use of io.cdap.cdap.internal.app.runtime.artifact.PluginFinder in project cdap by caskdata.

the class ConfiguratorTest method testInMemoryConfigurator.

@Test
public void testInMemoryConfigurator() throws Exception {
    LocationFactory locationFactory = new LocalLocationFactory(TMP_FOLDER.newFolder());
    Location appJar = AppJarHelper.createDeploymentJar(locationFactory, AllProgramsApp.class);
    Id.Artifact artifactId = Id.Artifact.from(Id.Namespace.DEFAULT, AllProgramsApp.class.getSimpleName(), "1.0.0");
    CConfiguration cConf = CConfiguration.create();
    ArtifactRepository baseArtifactRepo = new DefaultArtifactRepository(conf, null, null, null, new DummyProgramRunnerFactory(), new DefaultImpersonator(cConf, null));
    ArtifactRepository artifactRepo = new AuthorizationArtifactRepository(baseArtifactRepo, authEnforcer, authenticationContext);
    PluginFinder pluginFinder = new LocalPluginFinder(artifactRepo);
    AppDeploymentInfo appDeploymentInfo = new AppDeploymentInfo(artifactId.toEntityId(), appJar, NamespaceId.DEFAULT, new ApplicationClass(AllProgramsApp.class.getName(), "", null), null, null, null);
    // Create a configurator that is testable. Provide it a application.
    Configurator configurator = new InMemoryConfigurator(conf, pluginFinder, new DefaultImpersonator(cConf, null), artifactRepo, null, appDeploymentInfo);
    // Extract response from the configurator.
    ListenableFuture<ConfigResponse> result = configurator.config();
    ConfigResponse response = result.get(10, TimeUnit.SECONDS);
    Assert.assertNotNull(response);
    AppSpecInfo appSpecInfo = response.getAppSpecInfo();
    if (appSpecInfo == null) {
        throw new IllegalStateException("Failed to deploy application");
    }
    ApplicationSpecification specification = appSpecInfo.getAppSpec();
    Assert.assertNotNull(specification);
    // Simple checks.
    Assert.assertEquals(AllProgramsApp.NAME, specification.getName());
    ApplicationSpecification expectedSpec = Specifications.from(new AllProgramsApp());
    for (ProgramType programType : ProgramType.values()) {
        Assert.assertEquals(expectedSpec.getProgramsByType(programType), specification.getProgramsByType(programType));
    }
    Assert.assertEquals(expectedSpec.getDatasets(), specification.getDatasets());
}
Also used : ApplicationSpecification(io.cdap.cdap.api.app.ApplicationSpecification) AuthorizationArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.AuthorizationArtifactRepository) DummyProgramRunnerFactory(io.cdap.cdap.app.runtime.DummyProgramRunnerFactory) LocalPluginFinder(io.cdap.cdap.internal.app.runtime.artifact.LocalPluginFinder) AppSpecInfo(io.cdap.cdap.internal.app.deploy.pipeline.AppSpecInfo) Configurator(io.cdap.cdap.app.deploy.Configurator) DefaultArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.DefaultArtifactRepository) ApplicationClass(io.cdap.cdap.api.artifact.ApplicationClass) AllProgramsApp(io.cdap.cdap.AllProgramsApp) AuthorizationArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.AuthorizationArtifactRepository) DefaultArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.DefaultArtifactRepository) ArtifactRepository(io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository) ConfigResponse(io.cdap.cdap.app.deploy.ConfigResponse) CConfiguration(io.cdap.cdap.common.conf.CConfiguration) DefaultImpersonator(io.cdap.cdap.security.impersonation.DefaultImpersonator) LocalLocationFactory(org.apache.twill.filesystem.LocalLocationFactory) LocationFactory(org.apache.twill.filesystem.LocationFactory) AppDeploymentInfo(io.cdap.cdap.internal.app.deploy.pipeline.AppDeploymentInfo) PluginFinder(io.cdap.cdap.internal.app.runtime.artifact.PluginFinder) LocalPluginFinder(io.cdap.cdap.internal.app.runtime.artifact.LocalPluginFinder) NamespaceId(io.cdap.cdap.proto.id.NamespaceId) Id(io.cdap.cdap.common.id.Id) ProgramType(io.cdap.cdap.api.app.ProgramType) LocalLocationFactory(org.apache.twill.filesystem.LocalLocationFactory) Location(org.apache.twill.filesystem.Location) Test(org.junit.Test)

Example 5 with PluginFinder

use of io.cdap.cdap.internal.app.runtime.artifact.PluginFinder in project cdap by caskdata.

the class DistributedArtifactManagerModule method configure.

@Override
protected void configure() {
    // Bind the PluginFinder implementation
    bind(PluginFinder.class).to(RemotePluginFinder.class);
    // ArtifactLocalizerClient is only used for task worker
    OptionalBinder.newOptionalBinder(binder(), ArtifactLocalizerClient.class).setBinding().toProvider(Providers.of(null));
    // Bind the ArtifactManager implementation
    install(new FactoryModuleBuilder().implement(ArtifactManager.class, RemoteArtifactManager.class).build(ArtifactManagerFactory.class));
}
Also used : RemotePluginFinder(io.cdap.cdap.internal.app.runtime.artifact.RemotePluginFinder) PluginFinder(io.cdap.cdap.internal.app.runtime.artifact.PluginFinder) FactoryModuleBuilder(com.google.inject.assistedinject.FactoryModuleBuilder) ArtifactManagerFactory(io.cdap.cdap.internal.app.runtime.artifact.ArtifactManagerFactory)

Aggregations

PluginFinder (io.cdap.cdap.internal.app.runtime.artifact.PluginFinder)5 CConfiguration (io.cdap.cdap.common.conf.CConfiguration)3 ApplicationSpecification (io.cdap.cdap.api.app.ApplicationSpecification)2 ApplicationClass (io.cdap.cdap.api.artifact.ApplicationClass)2 ConfigResponse (io.cdap.cdap.app.deploy.ConfigResponse)2 Configurator (io.cdap.cdap.app.deploy.Configurator)2 DummyProgramRunnerFactory (io.cdap.cdap.app.runtime.DummyProgramRunnerFactory)2 Id (io.cdap.cdap.common.id.Id)2 AppDeploymentInfo (io.cdap.cdap.internal.app.deploy.pipeline.AppDeploymentInfo)2 AppSpecInfo (io.cdap.cdap.internal.app.deploy.pipeline.AppSpecInfo)2 ArtifactManagerFactory (io.cdap.cdap.internal.app.runtime.artifact.ArtifactManagerFactory)2 ArtifactRepository (io.cdap.cdap.internal.app.runtime.artifact.ArtifactRepository)2 AuthorizationArtifactRepository (io.cdap.cdap.internal.app.runtime.artifact.AuthorizationArtifactRepository)2 DefaultArtifactRepository (io.cdap.cdap.internal.app.runtime.artifact.DefaultArtifactRepository)2 LocalPluginFinder (io.cdap.cdap.internal.app.runtime.artifact.LocalPluginFinder)2 NamespaceId (io.cdap.cdap.proto.id.NamespaceId)2 DefaultImpersonator (io.cdap.cdap.security.impersonation.DefaultImpersonator)2 LocalLocationFactory (org.apache.twill.filesystem.LocalLocationFactory)2 Location (org.apache.twill.filesystem.Location)2 LocationFactory (org.apache.twill.filesystem.LocationFactory)2