Search in sources :

Example 1 with DiscoveryServiceClient

use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.

the class MapReduceTaskContextProvider method createCacheLoader.

/**
   * Creates a {@link CacheLoader} for the task context cache.
   */
private CacheLoader<ContextCacheKey, BasicMapReduceTaskContext> createCacheLoader(final Injector injector) {
    final DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
    final DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
    final SecureStore secureStore = injector.getInstance(SecureStore.class);
    final SecureStoreManager secureStoreManager = injector.getInstance(SecureStoreManager.class);
    final MessagingService messagingService = injector.getInstance(MessagingService.class);
    // Multiple instances of BasicMapReduceTaskContext can shares the same program.
    final AtomicReference<Program> programRef = new AtomicReference<>();
    return new CacheLoader<ContextCacheKey, BasicMapReduceTaskContext>() {

        @Override
        public BasicMapReduceTaskContext load(ContextCacheKey key) throws Exception {
            MapReduceContextConfig contextConfig = new MapReduceContextConfig(key.getConfiguration());
            MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(key.getConfiguration());
            Program program = programRef.get();
            if (program == null) {
                // Creation of program is relatively cheap, so just create and do compare and set.
                programRef.compareAndSet(null, createProgram(contextConfig, classLoader.getProgramClassLoader()));
                program = programRef.get();
            }
            WorkflowProgramInfo workflowInfo = contextConfig.getWorkflowProgramInfo();
            DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, program.getApplicationSpecification());
            // Setup dataset framework context, if required
            if (programDatasetFramework instanceof ProgramContextAware) {
                ProgramRunId programRunId = program.getId().run(ProgramRunners.getRunId(contextConfig.getProgramOptions()));
                ((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programRunId));
            }
            MapReduceSpecification spec = program.getApplicationSpecification().getMapReduce().get(program.getName());
            MetricsCollectionService metricsCollectionService = null;
            MapReduceMetrics.TaskType taskType = null;
            String taskId = null;
            TaskAttemptID taskAttemptId = key.getTaskAttemptID();
            // from a org.apache.hadoop.io.RawComparator
            if (taskAttemptId != null) {
                taskId = taskAttemptId.getTaskID().toString();
                if (MapReduceMetrics.TaskType.hasType(taskAttemptId.getTaskType())) {
                    taskType = MapReduceMetrics.TaskType.from(taskAttemptId.getTaskType());
                    // if this is not for a mapper or a reducer, we don't need the metrics collection service
                    metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
                }
            }
            CConfiguration cConf = injector.getInstance(CConfiguration.class);
            TransactionSystemClient txClient = injector.getInstance(TransactionSystemClient.class);
            return new BasicMapReduceTaskContext(program, contextConfig.getProgramOptions(), cConf, taskType, taskId, spec, workflowInfo, discoveryServiceClient, metricsCollectionService, txClient, contextConfig.getTx(), programDatasetFramework, classLoader.getPluginInstantiator(), contextConfig.getLocalizedResources(), secureStore, secureStoreManager, authorizationEnforcer, authenticationContext, messagingService);
        }
    };
}
Also used : DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) TaskAttemptID(org.apache.hadoop.mapreduce.TaskAttemptID) DatasetFramework(co.cask.cdap.data2.dataset2.DatasetFramework) NameMappedDatasetFramework(co.cask.cdap.internal.app.runtime.workflow.NameMappedDatasetFramework) TransactionSystemClient(org.apache.tephra.TransactionSystemClient) SecureStoreManager(co.cask.cdap.api.security.store.SecureStoreManager) MapReduceMetrics(co.cask.cdap.app.metrics.MapReduceMetrics) Program(co.cask.cdap.app.program.Program) DefaultProgram(co.cask.cdap.app.program.DefaultProgram) MetricsCollectionService(co.cask.cdap.api.metrics.MetricsCollectionService) MapReduceSpecification(co.cask.cdap.api.mapreduce.MapReduceSpecification) AtomicReference(java.util.concurrent.atomic.AtomicReference) BasicProgramContext(co.cask.cdap.internal.app.runtime.BasicProgramContext) SecureStore(co.cask.cdap.api.security.store.SecureStore) CConfiguration(co.cask.cdap.common.conf.CConfiguration) MessagingService(co.cask.cdap.messaging.MessagingService) WorkflowProgramInfo(co.cask.cdap.internal.app.runtime.workflow.WorkflowProgramInfo) CacheLoader(com.google.common.cache.CacheLoader) ProgramRunId(co.cask.cdap.proto.id.ProgramRunId) ProgramContextAware(co.cask.cdap.data.ProgramContextAware)

Example 2 with DiscoveryServiceClient

use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.

the class ArtifactHttpHandlerTest method setup.

@BeforeClass
public static void setup() throws IOException {
    artifactRepository = getInjector().getInstance(ArtifactRepository.class);
    DiscoveryServiceClient discoveryClient = getInjector().getInstance(DiscoveryServiceClient.class);
    ServiceDiscovered metadataHttpDiscovered = discoveryClient.discover(Constants.Service.METADATA_SERVICE);
    EndpointStrategy endpointStrategy = new RandomEndpointStrategy(metadataHttpDiscovered);
    Discoverable discoverable = endpointStrategy.pick(1, TimeUnit.SECONDS);
    Assert.assertNotNull(discoverable);
    String host = "127.0.0.1";
    int port = discoverable.getSocketAddress().getPort();
    ConnectionConfig connectionConfig = ConnectionConfig.builder().setHostname(host).setPort(port).build();
    clientConfig = ClientConfig.builder().setConnectionConfig(connectionConfig).build();
    metadataClient = new MetadataClient(clientConfig);
}
Also used : Discoverable(org.apache.twill.discovery.Discoverable) MetadataClient(co.cask.cdap.client.MetadataClient) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) EndpointStrategy(co.cask.cdap.common.discovery.EndpointStrategy) RandomEndpointStrategy(co.cask.cdap.common.discovery.RandomEndpointStrategy) ArtifactRepository(co.cask.cdap.internal.app.runtime.artifact.ArtifactRepository) ServiceDiscovered(org.apache.twill.discovery.ServiceDiscovered) ConnectionConfig(co.cask.cdap.client.config.ConnectionConfig) RandomEndpointStrategy(co.cask.cdap.common.discovery.RandomEndpointStrategy) BeforeClass(org.junit.BeforeClass)

Example 3 with DiscoveryServiceClient

use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.

the class DefaultSecureStoreServiceTest method setup.

@BeforeClass
public static void setup() throws Exception {
    SConfiguration sConf = SConfiguration.create();
    sConf.set(Constants.Security.Store.FILE_PASSWORD, "secret");
    final Injector injector = AppFabricTestHelper.getInjector(createCConf(), sConf, new AbstractModule() {

        @Override
        protected void configure() {
        // no overrides
        }
    });
    discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
    appFabricServer = injector.getInstance(AppFabricServer.class);
    appFabricServer.startAndWait();
    waitForService(Constants.Service.DATASET_MANAGER);
    secureStore = injector.getInstance(SecureStore.class);
    secureStoreManager = injector.getInstance(SecureStoreManager.class);
    authorizer = injector.getInstance(AuthorizerInstantiator.class).get();
    authorizer.grant(NamespaceId.DEFAULT, ALICE, Collections.singleton(Action.READ));
    Tasks.waitFor(true, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return injector.getInstance(NamespaceAdmin.class).exists(NamespaceId.DEFAULT);
        }
    }, 5, TimeUnit.SECONDS);
    authorizer.revoke(NamespaceId.DEFAULT, ALICE, Collections.singleton(Action.READ));
}
Also used : DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) Injector(com.google.inject.Injector) SConfiguration(co.cask.cdap.common.conf.SConfiguration) SecureStoreManager(co.cask.cdap.api.security.store.SecureStoreManager) SecureStore(co.cask.cdap.api.security.store.SecureStore) UnauthorizedException(co.cask.cdap.security.spi.authorization.UnauthorizedException) AbstractModule(com.google.inject.AbstractModule) BeforeClass(org.junit.BeforeClass)

Example 4 with DiscoveryServiceClient

use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.

the class AppFabricTestBase method initializeAndStartServices.

protected static void initializeAndStartServices(CConfiguration cConf, @Nullable SConfiguration sConf) throws Exception {
    injector = Guice.createInjector(Modules.override(new AppFabricTestModule(cConf, sConf)).with(new AbstractModule() {

        @Override
        protected void configure() {
            // needed because we set Kerberos to true in DefaultNamespaceAdminTest
            bind(UGIProvider.class).to(CurrentUGIProvider.class);
        }
    }));
    messagingService = injector.getInstance(MessagingService.class);
    if (messagingService instanceof Service) {
        ((Service) messagingService).startAndWait();
    }
    txManager = injector.getInstance(TransactionManager.class);
    txManager.startAndWait();
    dsOpService = injector.getInstance(DatasetOpExecutor.class);
    dsOpService.startAndWait();
    remoteSysOpService = injector.getInstance(RemoteSystemOperationsService.class);
    remoteSysOpService.startAndWait();
    datasetService = injector.getInstance(DatasetService.class);
    datasetService.startAndWait();
    appFabricServer = injector.getInstance(AppFabricServer.class);
    appFabricServer.startAndWait();
    DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
    ServiceDiscovered appFabricHttpDiscovered = discoveryClient.discover(Constants.Service.APP_FABRIC_HTTP);
    EndpointStrategy endpointStrategy = new RandomEndpointStrategy(appFabricHttpDiscovered);
    port = endpointStrategy.pick(1, TimeUnit.SECONDS).getSocketAddress().getPort();
    txClient = injector.getInstance(TransactionSystemClient.class);
    metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
    metricsCollectionService.startAndWait();
    metricsService = injector.getInstance(MetricsQueryService.class);
    metricsService.startAndWait();
    streamService = injector.getInstance(StreamService.class);
    streamService.startAndWait();
    serviceStore = injector.getInstance(ServiceStore.class);
    serviceStore.startAndWait();
    metadataService = injector.getInstance(MetadataService.class);
    metadataService.startAndWait();
    locationFactory = getInjector().getInstance(LocationFactory.class);
    streamClient = new StreamClient(getClientConfig(discoveryClient, Constants.Service.STREAMS));
    datasetClient = new DatasetClient(getClientConfig(discoveryClient, Constants.Service.DATASET_MANAGER));
    createNamespaces();
}
Also used : StreamService(co.cask.cdap.data.stream.service.StreamService) DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) MetricsQueryService(co.cask.cdap.metrics.query.MetricsQueryService) MetricsCollectionService(co.cask.cdap.api.metrics.MetricsCollectionService) AppFabricServer(co.cask.cdap.internal.app.services.AppFabricServer) MetricsCollectionService(co.cask.cdap.api.metrics.MetricsCollectionService) MetadataService(co.cask.cdap.metadata.MetadataService) Service(com.google.common.util.concurrent.Service) MessagingService(co.cask.cdap.messaging.MessagingService) MetricsQueryService(co.cask.cdap.metrics.query.MetricsQueryService) StreamService(co.cask.cdap.data.stream.service.StreamService) RemoteSystemOperationsService(co.cask.cdap.gateway.handlers.meta.RemoteSystemOperationsService) DatasetService(co.cask.cdap.data2.datafabric.dataset.service.DatasetService) DatasetService(co.cask.cdap.data2.datafabric.dataset.service.DatasetService) DatasetOpExecutor(co.cask.cdap.data2.datafabric.dataset.service.executor.DatasetOpExecutor) RemoteSystemOperationsService(co.cask.cdap.gateway.handlers.meta.RemoteSystemOperationsService) ServiceDiscovered(org.apache.twill.discovery.ServiceDiscovered) MetadataService(co.cask.cdap.metadata.MetadataService) AbstractModule(com.google.inject.AbstractModule) MessagingService(co.cask.cdap.messaging.MessagingService) LocationFactory(org.apache.twill.filesystem.LocationFactory) TransactionSystemClient(org.apache.tephra.TransactionSystemClient) DatasetClient(co.cask.cdap.client.DatasetClient) ServiceStore(co.cask.cdap.app.store.ServiceStore) RandomEndpointStrategy(co.cask.cdap.common.discovery.RandomEndpointStrategy) EndpointStrategy(co.cask.cdap.common.discovery.EndpointStrategy) StreamClient(co.cask.cdap.client.StreamClient) TransactionManager(org.apache.tephra.TransactionManager) CurrentUGIProvider(co.cask.cdap.security.impersonation.CurrentUGIProvider) AppFabricTestModule(co.cask.cdap.internal.guice.AppFabricTestModule) RandomEndpointStrategy(co.cask.cdap.common.discovery.RandomEndpointStrategy)

Example 5 with DiscoveryServiceClient

use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.

the class AuthorizationBootstrapperTest method setup.

@BeforeClass
public static void setup() throws Exception {
    CConfiguration cConf = CConfiguration.create();
    cConf.set(Constants.CFG_LOCAL_DATA_DIR, TMP_FOLDER.newFolder().getAbsolutePath());
    cConf.setBoolean(Constants.Security.ENABLED, true);
    cConf.setBoolean(Constants.Security.KERBEROS_ENABLED, false);
    cConf.setBoolean(Constants.Security.Authorization.ENABLED, true);
    Location deploymentJar = AppJarHelper.createDeploymentJar(new LocalLocationFactory(TMP_FOLDER.newFolder()), InMemoryAuthorizer.class);
    cConf.set(Constants.Security.Authorization.EXTENSION_JAR_PATH, deploymentJar.toURI().getPath());
    // make Alice an admin user, so she can create namespaces
    cConf.set(Constants.Security.Authorization.ADMIN_USERS, ADMIN_USER.getName());
    instanceId = new InstanceId(cConf.get(Constants.INSTANCE_NAME));
    // setup a system artifact
    File systemArtifactsDir = TMP_FOLDER.newFolder();
    cConf.set(Constants.AppFabric.SYSTEM_ARTIFACTS_DIR, systemArtifactsDir.getAbsolutePath());
    createSystemArtifact(systemArtifactsDir);
    Injector injector = Guice.createInjector(new AppFabricTestModule(cConf));
    namespaceQueryAdmin = injector.getInstance(NamespaceQueryAdmin.class);
    namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
    defaultNamespaceEnsurer = new DefaultNamespaceEnsurer(namespaceAdmin);
    discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
    txManager = injector.getInstance(TransactionManager.class);
    datasetService = injector.getInstance(DatasetService.class);
    systemArtifactLoader = injector.getInstance(SystemArtifactLoader.class);
    authorizationBootstrapper = injector.getInstance(AuthorizationBootstrapper.class);
    artifactRepository = injector.getInstance(ArtifactRepository.class);
    dsFramework = injector.getInstance(DatasetFramework.class);
    authorizationEnforcer = injector.getInstance(AuthorizationEnforcer.class);
}
Also used : DiscoveryServiceClient(org.apache.twill.discovery.DiscoveryServiceClient) InstanceId(co.cask.cdap.proto.id.InstanceId) NamespaceAdmin(co.cask.cdap.common.namespace.NamespaceAdmin) DatasetService(co.cask.cdap.data2.datafabric.dataset.service.DatasetService) ArtifactRepository(co.cask.cdap.internal.app.runtime.artifact.ArtifactRepository) AuthorizationEnforcer(co.cask.cdap.security.spi.authorization.AuthorizationEnforcer) DefaultNamespaceEnsurer(co.cask.cdap.internal.app.namespace.DefaultNamespaceEnsurer) CConfiguration(co.cask.cdap.common.conf.CConfiguration) DatasetFramework(co.cask.cdap.data2.dataset2.DatasetFramework) AuthorizationBootstrapper(co.cask.cdap.security.authorization.AuthorizationBootstrapper) Injector(com.google.inject.Injector) TransactionManager(org.apache.tephra.TransactionManager) SystemArtifactLoader(co.cask.cdap.internal.app.runtime.artifact.SystemArtifactLoader) NamespaceQueryAdmin(co.cask.cdap.common.namespace.NamespaceQueryAdmin) AppFabricTestModule(co.cask.cdap.internal.guice.AppFabricTestModule) LocalLocationFactory(org.apache.twill.filesystem.LocalLocationFactory) File(java.io.File) Location(org.apache.twill.filesystem.Location) BeforeClass(org.junit.BeforeClass)

Aggregations

DiscoveryServiceClient (org.apache.twill.discovery.DiscoveryServiceClient)21 Injector (com.google.inject.Injector)13 RandomEndpointStrategy (co.cask.cdap.common.discovery.RandomEndpointStrategy)12 AbstractModule (com.google.inject.AbstractModule)10 CConfiguration (co.cask.cdap.common.conf.CConfiguration)9 DiscoveryRuntimeModule (co.cask.cdap.common.guice.DiscoveryRuntimeModule)9 TransactionManager (org.apache.tephra.TransactionManager)9 EndpointStrategy (co.cask.cdap.common.discovery.EndpointStrategy)8 DatasetService (co.cask.cdap.data2.datafabric.dataset.service.DatasetService)8 ConfigModule (co.cask.cdap.common.guice.ConfigModule)7 MetricsCollectionService (co.cask.cdap.api.metrics.MetricsCollectionService)6 AuthorizationEnforcementModule (co.cask.cdap.security.authorization.AuthorizationEnforcementModule)6 Discoverable (org.apache.twill.discovery.Discoverable)6 BeforeClass (org.junit.BeforeClass)6 Test (org.junit.Test)6 AuthenticationContextModules (co.cask.cdap.security.auth.context.AuthenticationContextModules)5 AuthorizationTestModule (co.cask.cdap.security.authorization.AuthorizationTestModule)5 TransactionSystemClient (org.apache.tephra.TransactionSystemClient)5 SConfiguration (co.cask.cdap.common.conf.SConfiguration)4 IOModule (co.cask.cdap.common.guice.IOModule)4