use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.
the class MapReduceTaskContextProvider method createCacheLoader.
/**
* Creates a {@link CacheLoader} for the task context cache.
*/
private CacheLoader<ContextCacheKey, BasicMapReduceTaskContext> createCacheLoader(final Injector injector) {
final DiscoveryServiceClient discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
final DatasetFramework datasetFramework = injector.getInstance(DatasetFramework.class);
final SecureStore secureStore = injector.getInstance(SecureStore.class);
final SecureStoreManager secureStoreManager = injector.getInstance(SecureStoreManager.class);
final MessagingService messagingService = injector.getInstance(MessagingService.class);
// Multiple instances of BasicMapReduceTaskContext can shares the same program.
final AtomicReference<Program> programRef = new AtomicReference<>();
return new CacheLoader<ContextCacheKey, BasicMapReduceTaskContext>() {
@Override
public BasicMapReduceTaskContext load(ContextCacheKey key) throws Exception {
MapReduceContextConfig contextConfig = new MapReduceContextConfig(key.getConfiguration());
MapReduceClassLoader classLoader = MapReduceClassLoader.getFromConfiguration(key.getConfiguration());
Program program = programRef.get();
if (program == null) {
// Creation of program is relatively cheap, so just create and do compare and set.
programRef.compareAndSet(null, createProgram(contextConfig, classLoader.getProgramClassLoader()));
program = programRef.get();
}
WorkflowProgramInfo workflowInfo = contextConfig.getWorkflowProgramInfo();
DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework : NameMappedDatasetFramework.createFromWorkflowProgramInfo(datasetFramework, workflowInfo, program.getApplicationSpecification());
// Setup dataset framework context, if required
if (programDatasetFramework instanceof ProgramContextAware) {
ProgramRunId programRunId = program.getId().run(ProgramRunners.getRunId(contextConfig.getProgramOptions()));
((ProgramContextAware) programDatasetFramework).setContext(new BasicProgramContext(programRunId));
}
MapReduceSpecification spec = program.getApplicationSpecification().getMapReduce().get(program.getName());
MetricsCollectionService metricsCollectionService = null;
MapReduceMetrics.TaskType taskType = null;
String taskId = null;
TaskAttemptID taskAttemptId = key.getTaskAttemptID();
// from a org.apache.hadoop.io.RawComparator
if (taskAttemptId != null) {
taskId = taskAttemptId.getTaskID().toString();
if (MapReduceMetrics.TaskType.hasType(taskAttemptId.getTaskType())) {
taskType = MapReduceMetrics.TaskType.from(taskAttemptId.getTaskType());
// if this is not for a mapper or a reducer, we don't need the metrics collection service
metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
}
}
CConfiguration cConf = injector.getInstance(CConfiguration.class);
TransactionSystemClient txClient = injector.getInstance(TransactionSystemClient.class);
return new BasicMapReduceTaskContext(program, contextConfig.getProgramOptions(), cConf, taskType, taskId, spec, workflowInfo, discoveryServiceClient, metricsCollectionService, txClient, contextConfig.getTx(), programDatasetFramework, classLoader.getPluginInstantiator(), contextConfig.getLocalizedResources(), secureStore, secureStoreManager, authorizationEnforcer, authenticationContext, messagingService);
}
};
}
use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.
the class ArtifactHttpHandlerTest method setup.
@BeforeClass
public static void setup() throws IOException {
artifactRepository = getInjector().getInstance(ArtifactRepository.class);
DiscoveryServiceClient discoveryClient = getInjector().getInstance(DiscoveryServiceClient.class);
ServiceDiscovered metadataHttpDiscovered = discoveryClient.discover(Constants.Service.METADATA_SERVICE);
EndpointStrategy endpointStrategy = new RandomEndpointStrategy(metadataHttpDiscovered);
Discoverable discoverable = endpointStrategy.pick(1, TimeUnit.SECONDS);
Assert.assertNotNull(discoverable);
String host = "127.0.0.1";
int port = discoverable.getSocketAddress().getPort();
ConnectionConfig connectionConfig = ConnectionConfig.builder().setHostname(host).setPort(port).build();
clientConfig = ClientConfig.builder().setConnectionConfig(connectionConfig).build();
metadataClient = new MetadataClient(clientConfig);
}
use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.
the class DefaultSecureStoreServiceTest method setup.
@BeforeClass
public static void setup() throws Exception {
SConfiguration sConf = SConfiguration.create();
sConf.set(Constants.Security.Store.FILE_PASSWORD, "secret");
final Injector injector = AppFabricTestHelper.getInjector(createCConf(), sConf, new AbstractModule() {
@Override
protected void configure() {
// no overrides
}
});
discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
appFabricServer = injector.getInstance(AppFabricServer.class);
appFabricServer.startAndWait();
waitForService(Constants.Service.DATASET_MANAGER);
secureStore = injector.getInstance(SecureStore.class);
secureStoreManager = injector.getInstance(SecureStoreManager.class);
authorizer = injector.getInstance(AuthorizerInstantiator.class).get();
authorizer.grant(NamespaceId.DEFAULT, ALICE, Collections.singleton(Action.READ));
Tasks.waitFor(true, new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
return injector.getInstance(NamespaceAdmin.class).exists(NamespaceId.DEFAULT);
}
}, 5, TimeUnit.SECONDS);
authorizer.revoke(NamespaceId.DEFAULT, ALICE, Collections.singleton(Action.READ));
}
use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.
the class AppFabricTestBase method initializeAndStartServices.
protected static void initializeAndStartServices(CConfiguration cConf, @Nullable SConfiguration sConf) throws Exception {
injector = Guice.createInjector(Modules.override(new AppFabricTestModule(cConf, sConf)).with(new AbstractModule() {
@Override
protected void configure() {
// needed because we set Kerberos to true in DefaultNamespaceAdminTest
bind(UGIProvider.class).to(CurrentUGIProvider.class);
}
}));
messagingService = injector.getInstance(MessagingService.class);
if (messagingService instanceof Service) {
((Service) messagingService).startAndWait();
}
txManager = injector.getInstance(TransactionManager.class);
txManager.startAndWait();
dsOpService = injector.getInstance(DatasetOpExecutor.class);
dsOpService.startAndWait();
remoteSysOpService = injector.getInstance(RemoteSystemOperationsService.class);
remoteSysOpService.startAndWait();
datasetService = injector.getInstance(DatasetService.class);
datasetService.startAndWait();
appFabricServer = injector.getInstance(AppFabricServer.class);
appFabricServer.startAndWait();
DiscoveryServiceClient discoveryClient = injector.getInstance(DiscoveryServiceClient.class);
ServiceDiscovered appFabricHttpDiscovered = discoveryClient.discover(Constants.Service.APP_FABRIC_HTTP);
EndpointStrategy endpointStrategy = new RandomEndpointStrategy(appFabricHttpDiscovered);
port = endpointStrategy.pick(1, TimeUnit.SECONDS).getSocketAddress().getPort();
txClient = injector.getInstance(TransactionSystemClient.class);
metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
metricsCollectionService.startAndWait();
metricsService = injector.getInstance(MetricsQueryService.class);
metricsService.startAndWait();
streamService = injector.getInstance(StreamService.class);
streamService.startAndWait();
serviceStore = injector.getInstance(ServiceStore.class);
serviceStore.startAndWait();
metadataService = injector.getInstance(MetadataService.class);
metadataService.startAndWait();
locationFactory = getInjector().getInstance(LocationFactory.class);
streamClient = new StreamClient(getClientConfig(discoveryClient, Constants.Service.STREAMS));
datasetClient = new DatasetClient(getClientConfig(discoveryClient, Constants.Service.DATASET_MANAGER));
createNamespaces();
}
use of org.apache.twill.discovery.DiscoveryServiceClient in project cdap by caskdata.
the class AuthorizationBootstrapperTest method setup.
@BeforeClass
public static void setup() throws Exception {
CConfiguration cConf = CConfiguration.create();
cConf.set(Constants.CFG_LOCAL_DATA_DIR, TMP_FOLDER.newFolder().getAbsolutePath());
cConf.setBoolean(Constants.Security.ENABLED, true);
cConf.setBoolean(Constants.Security.KERBEROS_ENABLED, false);
cConf.setBoolean(Constants.Security.Authorization.ENABLED, true);
Location deploymentJar = AppJarHelper.createDeploymentJar(new LocalLocationFactory(TMP_FOLDER.newFolder()), InMemoryAuthorizer.class);
cConf.set(Constants.Security.Authorization.EXTENSION_JAR_PATH, deploymentJar.toURI().getPath());
// make Alice an admin user, so she can create namespaces
cConf.set(Constants.Security.Authorization.ADMIN_USERS, ADMIN_USER.getName());
instanceId = new InstanceId(cConf.get(Constants.INSTANCE_NAME));
// setup a system artifact
File systemArtifactsDir = TMP_FOLDER.newFolder();
cConf.set(Constants.AppFabric.SYSTEM_ARTIFACTS_DIR, systemArtifactsDir.getAbsolutePath());
createSystemArtifact(systemArtifactsDir);
Injector injector = Guice.createInjector(new AppFabricTestModule(cConf));
namespaceQueryAdmin = injector.getInstance(NamespaceQueryAdmin.class);
namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
defaultNamespaceEnsurer = new DefaultNamespaceEnsurer(namespaceAdmin);
discoveryServiceClient = injector.getInstance(DiscoveryServiceClient.class);
txManager = injector.getInstance(TransactionManager.class);
datasetService = injector.getInstance(DatasetService.class);
systemArtifactLoader = injector.getInstance(SystemArtifactLoader.class);
authorizationBootstrapper = injector.getInstance(AuthorizationBootstrapper.class);
artifactRepository = injector.getInstance(ArtifactRepository.class);
dsFramework = injector.getInstance(DatasetFramework.class);
authorizationEnforcer = injector.getInstance(AuthorizationEnforcer.class);
}
Aggregations