use of io.cdap.cdap.explore.executor.ExploreExecutorService in project cdap by caskdata.
the class TestBase method finish.
@AfterClass
public static void finish() throws Exception {
if (--nestedStartCount != 0) {
return;
}
if (previewRunnerManager instanceof Service) {
((Service) previewRunnerManager).stopAndWait();
}
previewHttpServer.stopAndWait();
if (cConf.getBoolean(Constants.Security.Authorization.ENABLED)) {
InstanceId instance = new InstanceId(cConf.get(Constants.INSTANCE_NAME));
Principal principal = new Principal(System.getProperty("user.name"), Principal.PrincipalType.USER);
accessControllerInstantiator.get().grant(Authorizable.fromEntityId(instance), principal, ImmutableSet.of(StandardPermission.UPDATE));
accessControllerInstantiator.get().grant(Authorizable.fromEntityId(NamespaceId.DEFAULT), principal, ImmutableSet.of(StandardPermission.UPDATE));
}
namespaceAdmin.delete(NamespaceId.DEFAULT);
accessControllerInstantiator.close();
if (programScheduler instanceof Service) {
((Service) programScheduler).stopAndWait();
}
metricsCollectionService.stopAndWait();
if (scheduler instanceof Service) {
((Service) scheduler).stopAndWait();
}
Closeables.closeQuietly(exploreClient);
if (exploreExecutorService != null) {
exploreExecutorService.stopAndWait();
}
datasetService.stopAndWait();
dsOpService.stopAndWait();
metadataService.stopAndWait();
metadataSubscriberService.stopAndWait();
Closeables.closeQuietly(metadataStorage);
txService.stopAndWait();
if (messagingService instanceof Service) {
((Service) messagingService).stopAndWait();
}
appFabricServer.stopAndWait();
supportBundleInternalService.stopAndWait();
appFabricHealthCheckService.stopAndWait();
}
use of io.cdap.cdap.explore.executor.ExploreExecutorService in project cdap by caskdata.
the class TestBase method initialize.
@BeforeClass
public static void initialize() throws Exception {
if (nestedStartCount++ > 0) {
return;
}
File localDataDir = TMP_FOLDER.newFolder();
cConf = createCConf(localDataDir);
CConfiguration previewCConf = createPreviewConf(cConf);
LevelDBTableService previewLevelDBTableService = new LevelDBTableService();
previewLevelDBTableService.setConfiguration(previewCConf);
// enable default services
File capabilityFolder = new File(localDataDir.toString(), "capability");
capabilityFolder.mkdir();
cConf.set(Constants.Capability.CONFIG_DIR, capabilityFolder.getAbsolutePath());
cConf.setInt(Constants.Capability.AUTO_INSTALL_THREADS, 5);
org.apache.hadoop.conf.Configuration hConf = new org.apache.hadoop.conf.Configuration();
hConf.addResource("mapred-site-local.xml");
hConf.reloadConfiguration();
hConf.set(Constants.CFG_LOCAL_DATA_DIR, localDataDir.getAbsolutePath());
hConf.set(Constants.AppFabric.OUTPUT_DIR, cConf.get(Constants.AppFabric.OUTPUT_DIR));
hConf.set("hadoop.tmp.dir", new File(localDataDir, cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsolutePath());
// Windows specific requirements
if (OSDetector.isWindows()) {
File tmpDir = TMP_FOLDER.newFolder();
File binDir = new File(tmpDir, "bin");
Assert.assertTrue(binDir.mkdirs());
copyTempFile("hadoop.dll", tmpDir);
copyTempFile("winutils.exe", binDir);
System.setProperty("hadoop.home.dir", tmpDir.getAbsolutePath());
System.load(new File(tmpDir, "hadoop.dll").getAbsolutePath());
}
injector = Guice.createInjector(createDataFabricModule(), new TransactionExecutorModule(), new DataSetsModules().getStandaloneModules(), new DataSetServiceModules().getInMemoryModules(), new ConfigModule(cConf, hConf), RemoteAuthenticatorModules.getNoOpModule(), new IOModule(), new LocalLocationModule(), new InMemoryDiscoveryModule(), new AppFabricServiceRuntimeModule(cConf).getInMemoryModules(), new MonitorHandlerModule(false), new AuthenticationContextModules().getMasterModule(), new AuthorizationModule(), new AuthorizationEnforcementModule().getInMemoryModules(), new ProgramRunnerRuntimeModule().getInMemoryModules(), new SecureStoreServerModule(), new MetadataReaderWriterModules().getInMemoryModules(), new MetadataServiceModule(), new AbstractModule() {
@Override
protected void configure() {
bind(MetricsManager.class).toProvider(MetricsManagerProvider.class);
}
}, new MetricsClientRuntimeModule().getInMemoryModules(), new LocalLogAppenderModule(), new LogReaderRuntimeModules().getInMemoryModules(), new ExploreRuntimeModule().getInMemoryModules(), new ExploreClientModule(), new MessagingServerRuntimeModule().getInMemoryModules(), new PreviewConfigModule(cConf, new Configuration(), SConfiguration.create()), new PreviewManagerModule(false), new PreviewRunnerManagerModule().getInMemoryModules(), new SupportBundleServiceModule(), new MockProvisionerModule(), new AbstractModule() {
@Override
protected void configure() {
install(new FactoryModuleBuilder().implement(ApplicationManager.class, DefaultApplicationManager.class).build(ApplicationManagerFactory.class));
install(new FactoryModuleBuilder().implement(ArtifactManager.class, DefaultArtifactManager.class).build(ArtifactManagerFactory.class));
bind(TemporaryFolder.class).toInstance(TMP_FOLDER);
bind(AuthorizationHandler.class).in(Scopes.SINGLETON);
// Needed by MonitorHandlerModuler
bind(TwillRunner.class).to(NoopTwillRunnerService.class);
bind(MetadataSubscriberService.class).in(Scopes.SINGLETON);
}
});
messagingService = injector.getInstance(MessagingService.class);
if (messagingService instanceof Service) {
((Service) messagingService).startAndWait();
}
txService = injector.getInstance(TransactionManager.class);
txService.startAndWait();
metadataSubscriberService = injector.getInstance(MetadataSubscriberService.class);
metadataStorage = injector.getInstance(MetadataStorage.class);
metadataAdmin = injector.getInstance(MetadataAdmin.class);
metadataStorage.createIndex();
metadataService = injector.getInstance(MetadataService.class);
metadataService.startAndWait();
// Define all StructuredTable before starting any services that need StructuredTable
StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
dsOpService = injector.getInstance(DatasetOpExecutorService.class);
dsOpService.startAndWait();
datasetService = injector.getInstance(DatasetService.class);
datasetService.startAndWait();
metricsCollectionService = injector.getInstance(MetricsCollectionService.class);
metricsCollectionService.startAndWait();
if (cConf.getBoolean(Constants.Explore.EXPLORE_ENABLED)) {
exploreExecutorService = injector.getInstance(ExploreExecutorService.class);
exploreExecutorService.startAndWait();
// wait for explore service to be discoverable
DiscoveryServiceClient discoveryService = injector.getInstance(DiscoveryServiceClient.class);
EndpointStrategy endpointStrategy = new RandomEndpointStrategy(() -> discoveryService.discover(Constants.Service.EXPLORE_HTTP_USER_SERVICE));
Preconditions.checkNotNull(endpointStrategy.pick(5, TimeUnit.SECONDS), "%s service is not up after 5 seconds", Constants.Service.EXPLORE_HTTP_USER_SERVICE);
exploreClient = injector.getInstance(ExploreClient.class);
}
programScheduler = injector.getInstance(Scheduler.class);
if (programScheduler instanceof Service) {
((Service) programScheduler).startAndWait();
}
testManager = injector.getInstance(UnitTestManager.class);
metricsManager = injector.getInstance(MetricsManager.class);
accessControllerInstantiator = injector.getInstance(AccessControllerInstantiator.class);
// This is needed so the logged-in user can successfully create the default namespace
if (cConf.getBoolean(Constants.Security.Authorization.ENABLED)) {
String user = System.getProperty("user.name");
SecurityRequestContext.setUserId(user);
InstanceId instance = new InstanceId(cConf.get(Constants.INSTANCE_NAME));
Principal principal = new Principal(user, Principal.PrincipalType.USER);
accessControllerInstantiator.get().grant(Authorizable.fromEntityId(instance), principal, EnumSet.allOf(StandardPermission.class));
accessControllerInstantiator.get().grant(Authorizable.fromEntityId(NamespaceId.DEFAULT), principal, EnumSet.allOf(StandardPermission.class));
}
namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
if (firstInit) {
// only create the default namespace on first test. if multiple tests are run in the same JVM,
// then any time after the first time, the default namespace already exists. That is because
// the namespaceAdmin.delete(Id.Namespace.DEFAULT) in finish() only clears the default namespace
// but does not remove it entirely
namespaceAdmin.create(NamespaceMeta.DEFAULT);
ProfileService profileService = injector.getInstance(ProfileService.class);
profileService.saveProfile(ProfileId.NATIVE, Profile.NATIVE);
}
secureStore = injector.getInstance(SecureStore.class);
secureStoreManager = injector.getInstance(SecureStoreManager.class);
messagingContext = new MultiThreadMessagingContext(messagingService);
firstInit = false;
previewHttpServer = injector.getInstance(PreviewHttpServer.class);
previewHttpServer.startAndWait();
fieldLineageAdmin = injector.getInstance(FieldLineageAdmin.class);
lineageAdmin = injector.getInstance(LineageAdmin.class);
metadataSubscriberService.startAndWait();
previewRunnerManager = injector.getInstance(PreviewRunnerManager.class);
if (previewRunnerManager instanceof Service) {
((Service) previewRunnerManager).startAndWait();
}
appFabricServer = injector.getInstance(AppFabricServer.class);
appFabricServer.startAndWait();
preferencesService = injector.getInstance(PreferencesService.class);
scheduler = injector.getInstance(Scheduler.class);
if (scheduler instanceof Service) {
((Service) scheduler).startAndWait();
}
if (scheduler instanceof CoreSchedulerService) {
((CoreSchedulerService) scheduler).waitUntilFunctional(10, TimeUnit.SECONDS);
}
supportBundleInternalService = injector.getInstance(SupportBundleInternalService.class);
supportBundleInternalService.startAndWait();
appFabricHealthCheckService = injector.getInstance(HealthCheckService.class);
appFabricHealthCheckService.helper(Constants.AppFabricHealthCheck.APP_FABRIC_HEALTH_CHECK_SERVICE, cConf, Constants.Service.MASTER_SERVICES_BIND_ADDRESS);
appFabricHealthCheckService.startAndWait();
}
use of io.cdap.cdap.explore.executor.ExploreExecutorService in project cdap by caskdata.
the class BaseHiveExploreServiceTest method initialize.
protected static void initialize(CConfiguration cConf, TemporaryFolder tmpFolder, boolean enableAuthorization) throws Exception {
if (!runBefore) {
return;
}
Configuration hConf = new Configuration();
if (enableAuthorization) {
LocationFactory locationFactory = new LocalLocationFactory(tmpFolder.newFolder());
Location authExtensionJar = AppJarHelper.createDeploymentJar(locationFactory, InMemoryAccessController.class);
cConf.setBoolean(Constants.Security.ENABLED, true);
cConf.setBoolean(Constants.Security.Authorization.ENABLED, true);
cConf.set(Constants.Security.Authorization.EXTENSION_JAR_PATH, authExtensionJar.toURI().getPath());
cConf.setBoolean(Constants.Security.KERBEROS_ENABLED, false);
cConf.setInt(Constants.Security.Authorization.CACHE_MAX_ENTRIES, 0);
}
List<Module> modules = createInMemoryModules(cConf, hConf, tmpFolder);
injector = Guice.createInjector(modules);
transactionManager = injector.getInstance(TransactionManager.class);
transactionManager.startAndWait();
transactionSystemClient = injector.getInstance(TransactionSystemClient.class);
StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
dsOpService = injector.getInstance(DatasetOpExecutorService.class);
dsOpService.startAndWait();
datasetService = injector.getInstance(DatasetService.class);
datasetService.startAndWait();
exploreExecutorService = injector.getInstance(ExploreExecutorService.class);
exploreExecutorService.startAndWait();
datasetFramework = injector.getInstance(DatasetFramework.class);
exploreClient = injector.getInstance(DiscoveryExploreClient.class);
exploreService = injector.getInstance(ExploreService.class);
exploreClient.ping();
exploreTableManager = injector.getInstance(ExploreTableManager.class);
namespaceAdmin = injector.getInstance(NamespaceAdmin.class);
namespacePathLocator = injector.getInstance(NamespacePathLocator.class);
// create namespaces
// This happens when you create a namespace via REST APIs. However, since we do not start AppFabricServer in
// Explore tests, simulating that scenario by explicitly calling DatasetFramework APIs.
createNamespace(NamespaceId.DEFAULT);
createNamespace(NAMESPACE_ID);
createNamespace(OTHER_NAMESPACE_ID);
}
use of io.cdap.cdap.explore.executor.ExploreExecutorService in project cdap by caskdata.
the class StandaloneMain method startUp.
/**
* Start the service.
*/
public void startUp() throws Exception {
// Workaround for release of file descriptors opened by URLClassLoader - https://issues.cask.co/browse/CDAP-2841
URLConnections.setDefaultUseCaches(false);
ConfigurationLogger.logImportantConfig(cConf);
if (messagingService instanceof Service) {
((Service) messagingService).startAndWait();
}
// TODO: CDAP-7688, remove next line after the issue is resolved
injector.getInstance(MessagingHttpService.class).startAndWait();
if (txService != null) {
txService.startAndWait();
}
// Define all StructuredTable before starting any services that need StructuredTable
StoreDefinition.createAllTables(injector.getInstance(StructuredTableAdmin.class));
metadataStorage.createIndex();
metricsCollectionService.startAndWait();
datasetOpExecutorService.startAndWait();
datasetService.startAndWait();
serviceStore.startAndWait();
remoteExecutionTwillRunnerService.start();
metadataSubscriberService.startAndWait();
// Validate the logging pipeline configuration.
// Do it explicitly as Standalone doesn't have a separate master check phase as the distributed does.
new LogPipelineLoader(cConf).validate();
// It is recommended to initialize log appender after datasetService is started,
// since log appender instantiates a dataset.
logAppenderInitializer.initialize();
runtimeServer.startAndWait();
Service.State state = appFabricServer.startAndWait();
if (state != Service.State.RUNNING) {
throw new Exception("Failed to start Application Fabric");
}
previewHttpServer.startAndWait();
previewRunnerManager.startAndWait();
metricsQueryService.startAndWait();
logQueryService.startAndWait();
router.startAndWait();
if (userInterfaceService != null) {
userInterfaceService.startAndWait();
}
if (SecurityUtil.isManagedSecurity(cConf)) {
externalAuthenticationServer.startAndWait();
}
if (exploreExecutorService != null) {
exploreExecutorService.startAndWait();
}
metadataService.startAndWait();
operationalStatsService.startAndWait();
secureStoreService.startAndWait();
supportBundleInternalService.startAndWait();
appFabricHealthCheckService.startAndWait();
String protocol = sslEnabled ? "https" : "http";
int dashboardPort = sslEnabled ? cConf.getInt(Constants.Dashboard.SSL_BIND_PORT) : cConf.getInt(Constants.Dashboard.BIND_PORT);
System.out.println("CDAP Sandbox started successfully.");
System.out.printf("Connect to the CDAP UI at %s://%s:%d\n", protocol, "localhost", dashboardPort);
}
use of io.cdap.cdap.explore.executor.ExploreExecutorService in project cdap by caskdata.
the class StandaloneMain method shutDown.
/**
* Shutdown the service.
*/
public void shutDown() {
LOG.info("Shutting down Standalone CDAP");
boolean halt = false;
try {
// order matters: first shut down UI 'cause it will stop working after router is down
if (userInterfaceService != null) {
userInterfaceService.stopAndWait();
}
// shut down router to stop all incoming traffic
router.stopAndWait();
secureStoreService.stopAndWait();
supportBundleInternalService.stopAndWait();
operationalStatsService.stopAndWait();
// Stop all services that requires tx service
metadataSubscriberService.stopAndWait();
if (exploreExecutorService != null) {
exploreExecutorService.stopAndWait();
}
exploreClient.close();
metadataService.stopAndWait();
remoteExecutionTwillRunnerService.stop();
serviceStore.stopAndWait();
previewRunnerManager.stopAndWait();
previewHttpServer.stopAndWait();
// app fabric will also stop all programs
appFabricServer.stopAndWait();
appFabricHealthCheckService.stopAndWait();
runtimeServer.stopAndWait();
// all programs are stopped: dataset service, metrics, transactions can stop now
datasetService.stopAndWait();
datasetOpExecutorService.stopAndWait();
logQueryService.stopAndWait();
metricsCollectionService.stopAndWait();
metricsQueryService.stopAndWait();
if (txService != null) {
txService.stopAndWait();
}
if (SecurityUtil.isManagedSecurity(cConf)) {
// auth service is on the side anyway
externalAuthenticationServer.stopAndWait();
}
// TODO: CDAP-7688, remove next line after the issue is resolved
injector.getInstance(MessagingHttpService.class).startAndWait();
if (messagingService instanceof Service) {
((Service) messagingService).stopAndWait();
}
logAppenderInitializer.close();
accessControllerInstantiator.close();
metadataStorage.close();
levelDBTableService.close();
} catch (Throwable e) {
halt = true;
LOG.error("Exception during shutdown", e);
} finally {
File tmpDir = new File(cConf.get(Constants.CFG_LOCAL_DATA_DIR), cConf.get(Constants.AppFabric.TEMP_DIR)).getAbsoluteFile();
cleanupTempDir(tmpDir);
}
// Therefore System.exit() won't do it, we need to force a halt.
if (halt) {
Runtime.getRuntime().halt(1);
}
}
Aggregations