Search in sources :

Example 1 with TempStorageModule

use of com.facebook.presto.storage.TempStorageModule in project presto by prestodb.

the class PrestoSparkInjectorFactory method create.

public Injector create() {
    // TODO: migrate docker containers to a newer JVM, then re-enable it
    // verifyJvmRequirements();
    verifySystemTimeIsReasonable();
    ImmutableList.Builder<Module> modules = ImmutableList.builder();
    modules.add(new JsonModule(), new SmileModule(), new EventListenerModule(), new PrestoSparkModule(sparkProcessType, sqlParserOptions), new WarningCollectorModule());
    if (isForTesting) {
        modules.add(binder -> {
            binder.bind(TestingAccessControlManager.class).in(Scopes.SINGLETON);
            binder.bind(AccessControlManager.class).to(TestingAccessControlManager.class).in(Scopes.SINGLETON);
            binder.bind(AccessControl.class).to(AccessControlManager.class).in(Scopes.SINGLETON);
            binder.bind(TestingTempStorageManager.class).in(Scopes.SINGLETON);
            binder.bind(TempStorageManager.class).to(TestingTempStorageManager.class).in(Scopes.SINGLETON);
            newSetBinder(binder, PrestoSparkServiceWaitTimeMetrics.class).addBinding().to(PrestoSparkTestingServiceWaitTimeMetrics.class).in(Scopes.SINGLETON);
        });
    } else {
        modules.add(new AccessControlModule());
        modules.add(new TempStorageModule());
    }
    modules.addAll(additionalModules);
    Bootstrap app = new Bootstrap(modules.build());
    // Stream redirect doesn't work well with spark logging
    app.doNotInitializeLogging();
    Map<String, String> requiredProperties = new HashMap<>();
    requiredProperties.put("node.environment", "spark");
    requiredProperties.putAll(configProperties);
    app.setRequiredConfigurationProperties(ImmutableMap.copyOf(requiredProperties));
    Injector injector = app.initialize();
    try {
        injector.getInstance(PluginManager.class).loadPlugins();
        injector.getInstance(StaticCatalogStore.class).loadCatalogs(catalogProperties);
        injector.getInstance(ResourceGroupManager.class).loadConfigurationManager();
        injector.getInstance(PasswordAuthenticatorManager.class).loadPasswordAuthenticator();
        eventListenerProperties.ifPresent(properties -> injector.getInstance(EventListenerManager.class).loadConfiguredEventListener(properties));
        if (!isForTesting) {
            if (accessControlProperties.isPresent()) {
                injector.getInstance(AccessControlManager.class).loadSystemAccessControl(accessControlProperties.get());
            } else {
                injector.getInstance(AccessControlManager.class).loadSystemAccessControl();
            }
            if (tempStorageProperties.isPresent()) {
                injector.getInstance(TempStorageManager.class).loadTempStorages(tempStorageProperties.get());
            } else {
                injector.getInstance(TempStorageManager.class).loadTempStorages();
            }
        }
        if ((sparkProcessType.equals(DRIVER))) {
            if (sessionPropertyConfigurationProperties.isPresent()) {
                injector.getInstance(SessionPropertyDefaults.class).loadConfigurationManager(sessionPropertyConfigurationProperties.get());
            } else {
                injector.getInstance(SessionPropertyDefaults.class).loadConfigurationManager();
            }
        }
        if (sparkProcessType.equals(DRIVER) || !injector.getInstance(FeaturesConfig.class).isInlineSqlFunctions()) {
            if (functionNamespaceProperties.isPresent()) {
                injector.getInstance(StaticFunctionNamespaceStore.class).loadFunctionNamespaceManagers(functionNamespaceProperties.get());
            } else {
                injector.getInstance(StaticFunctionNamespaceStore.class).loadFunctionNamespaceManagers();
            }
        }
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    return injector;
}
Also used : AccessControlManager(com.facebook.presto.security.AccessControlManager) TestingAccessControlManager(com.facebook.presto.testing.TestingAccessControlManager) WarningCollectorModule(com.facebook.presto.execution.warnings.WarningCollectorModule) HashMap(java.util.HashMap) ImmutableList(com.google.common.collect.ImmutableList) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) PasswordAuthenticatorManager(com.facebook.presto.server.security.PasswordAuthenticatorManager) PluginManager(com.facebook.presto.server.PluginManager) TempStorageModule(com.facebook.presto.storage.TempStorageModule) TempStorageManager(com.facebook.presto.storage.TempStorageManager) TestingTempStorageManager(com.facebook.presto.testing.TestingTempStorageManager) Injector(com.google.inject.Injector) Bootstrap(com.facebook.airlift.bootstrap.Bootstrap) EventListenerModule(com.facebook.presto.eventlistener.EventListenerModule) AccessControlModule(com.facebook.presto.security.AccessControlModule) StaticCatalogStore(com.facebook.presto.metadata.StaticCatalogStore) SessionPropertyDefaults(com.facebook.presto.server.SessionPropertyDefaults) StaticFunctionNamespaceStore(com.facebook.presto.metadata.StaticFunctionNamespaceStore) ResourceGroupManager(com.facebook.presto.execution.resourceGroups.ResourceGroupManager) JsonModule(com.facebook.airlift.json.JsonModule) TestingAccessControlManager(com.facebook.presto.testing.TestingAccessControlManager) TestingTempStorageManager(com.facebook.presto.testing.TestingTempStorageManager) SmileModule(com.facebook.airlift.json.smile.SmileModule) Module(com.google.inject.Module) WarningCollectorModule(com.facebook.presto.execution.warnings.WarningCollectorModule) AccessControlModule(com.facebook.presto.security.AccessControlModule) JsonModule(com.facebook.airlift.json.JsonModule) EventListenerModule(com.facebook.presto.eventlistener.EventListenerModule) SmileModule(com.facebook.airlift.json.smile.SmileModule) TempStorageModule(com.facebook.presto.storage.TempStorageModule)

Example 2 with TempStorageModule

use of com.facebook.presto.storage.TempStorageModule in project presto by prestodb.

the class PrestoServer method run.

@Override
public void run() {
    verifyJvmRequirements();
    verifySystemTimeIsReasonable();
    Logger log = Logger.get(PrestoServer.class);
    ImmutableList.Builder<Module> modules = ImmutableList.builder();
    modules.add(new NodeModule(), new DiscoveryModule(), new HttpServerModule(), new JsonModule(), installModuleIf(FeaturesConfig.class, FeaturesConfig::isJsonSerdeCodeGenerationEnabled, binder -> jsonBinder(binder).addModuleBinding().to(AfterburnerModule.class)), new SmileModule(), new JaxrsModule(true), new MBeanModule(), new JmxModule(), new JmxHttpModule(), new LogJmxModule(), new TraceTokenModule(), new JsonEventModule(), new HttpEventModule(), new ServerSecurityModule(), new AccessControlModule(), new EventListenerModule(), new ServerMainModule(sqlParserOptions), new GracefulShutdownModule(), new WarningCollectorModule(), new TempStorageModule(), new QueryPrerequisitesManagerModule(), new NodeTtlFetcherManagerModule(), new ClusterTtlProviderManagerModule());
    modules.addAll(getAdditionalModules());
    Bootstrap app = new Bootstrap(modules.build());
    try {
        Injector injector = app.initialize();
        injector.getInstance(PluginManager.class).loadPlugins();
        ServerConfig serverConfig = injector.getInstance(ServerConfig.class);
        if (!serverConfig.isResourceManager()) {
            injector.getInstance(StaticCatalogStore.class).loadCatalogs();
        }
        // TODO: remove this huge hack
        updateConnectorIds(injector.getInstance(Announcer.class), injector.getInstance(CatalogManager.class), injector.getInstance(ServerConfig.class), injector.getInstance(NodeSchedulerConfig.class));
        // TODO: thrift server port should be announced by discovery server similar to http/https ports
        updateThriftServerPort(injector.getInstance(Announcer.class), injector.getInstance(DriftServer.class));
        injector.getInstance(StaticFunctionNamespaceStore.class).loadFunctionNamespaceManagers();
        injector.getInstance(SessionPropertyDefaults.class).loadConfigurationManager();
        injector.getInstance(ResourceGroupManager.class).loadConfigurationManager();
        if (!serverConfig.isResourceManager()) {
            injector.getInstance(AccessControlManager.class).loadSystemAccessControl();
        }
        injector.getInstance(PasswordAuthenticatorManager.class).loadPasswordAuthenticator();
        injector.getInstance(EventListenerManager.class).loadConfiguredEventListener();
        injector.getInstance(TempStorageManager.class).loadTempStorages();
        injector.getInstance(QueryPrerequisitesManager.class).loadQueryPrerequisites();
        injector.getInstance(NodeTtlFetcherManager.class).loadNodeTtlFetcher();
        injector.getInstance(ClusterTtlProviderManager.class).loadClusterTtlProvider();
        injector.getInstance(Announcer.class).start();
        log.info("======== SERVER STARTED ========");
    } catch (Throwable e) {
        log.error(e);
        System.exit(1);
    }
}
Also used : Module(com.google.inject.Module) SqlParserOptions(com.facebook.presto.sql.parser.SqlParserOptions) DriftServer(com.facebook.drift.server.DriftServer) MBeanModule(org.weakref.jmx.guice.MBeanModule) ClusterTtlProviderManagerModule(com.facebook.presto.ttl.clusterttlprovidermanagers.ClusterTtlProviderManagerModule) JaxrsModule(com.facebook.airlift.jaxrs.JaxrsModule) JmxHttpModule(com.facebook.airlift.jmx.JmxHttpModule) EventListenerManager(com.facebook.presto.eventlistener.EventListenerManager) DriftNettyServerTransport(com.facebook.drift.transport.netty.server.DriftNettyServerTransport) WarningCollectorModule(com.facebook.presto.execution.warnings.WarningCollectorModule) PrestoSystemRequirements.verifySystemTimeIsReasonable(com.facebook.presto.server.PrestoSystemRequirements.verifySystemTimeIsReasonable) Map(java.util.Map) QueryPrerequisitesManagerModule(com.facebook.presto.dispatcher.QueryPrerequisitesManagerModule) Splitter(com.google.common.base.Splitter) ServiceAnnouncement(com.facebook.airlift.discovery.client.ServiceAnnouncement) ClusterTtlProviderManager(com.facebook.presto.ttl.clusterttlprovidermanagers.ClusterTtlProviderManager) ServiceAnnouncement.serviceAnnouncement(com.facebook.airlift.discovery.client.ServiceAnnouncement.serviceAnnouncement) JsonModule(com.facebook.airlift.json.JsonModule) ConditionalModule.installModuleIf(com.facebook.airlift.configuration.ConditionalModule.installModuleIf) SmileModule(com.facebook.airlift.json.smile.SmileModule) Set(java.util.Set) JsonBinder.jsonBinder(com.facebook.airlift.json.JsonBinder.jsonBinder) ServiceAnnouncementBuilder(com.facebook.airlift.discovery.client.ServiceAnnouncement.ServiceAnnouncementBuilder) TempStorageModule(com.facebook.presto.storage.TempStorageModule) AfterburnerModule(com.fasterxml.jackson.module.afterburner.AfterburnerModule) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) HttpEventModule(com.facebook.airlift.event.client.HttpEventModule) List(java.util.List) Catalog(com.facebook.presto.metadata.Catalog) PasswordAuthenticatorManager(com.facebook.presto.server.security.PasswordAuthenticatorManager) NodeTtlFetcherManager(com.facebook.presto.ttl.nodettlfetchermanagers.NodeTtlFetcherManager) JsonEventModule(com.facebook.airlift.event.client.JsonEventModule) JmxModule(com.facebook.airlift.jmx.JmxModule) Joiner(com.google.common.base.Joiner) NodeSchedulerConfig(com.facebook.presto.execution.scheduler.NodeSchedulerConfig) Bootstrap(com.facebook.airlift.bootstrap.Bootstrap) Logger(com.facebook.airlift.log.Logger) Strings.nullToEmpty(com.google.common.base.Strings.nullToEmpty) QueryPrerequisitesManager(com.facebook.presto.dispatcher.QueryPrerequisitesManager) AccessControlManager(com.facebook.presto.security.AccessControlManager) TempStorageManager(com.facebook.presto.storage.TempStorageManager) Announcer(com.facebook.airlift.discovery.client.Announcer) TraceTokenModule(com.facebook.airlift.tracetoken.TraceTokenModule) LinkedHashMap(java.util.LinkedHashMap) StaticCatalogStore(com.facebook.presto.metadata.StaticCatalogStore) ImmutableList(com.google.common.collect.ImmutableList) Objects.requireNonNull(java.util.Objects.requireNonNull) AccessControlModule(com.facebook.presto.security.AccessControlModule) PrestoSystemRequirements.verifyJvmRequirements(com.facebook.presto.server.PrestoSystemRequirements.verifyJvmRequirements) LogJmxModule(com.facebook.airlift.log.LogJmxModule) ResourceGroupManager(com.facebook.presto.execution.resourceGroups.ResourceGroupManager) LinkedHashSet(java.util.LinkedHashSet) EventListenerModule(com.facebook.presto.eventlistener.EventListenerModule) ServerSecurityModule(com.facebook.presto.server.security.ServerSecurityModule) HttpServerModule(com.facebook.airlift.http.server.HttpServerModule) CatalogManager(com.facebook.presto.metadata.CatalogManager) Injector(com.google.inject.Injector) NodeTtlFetcherManagerModule(com.facebook.presto.ttl.nodettlfetchermanagers.NodeTtlFetcherManagerModule) DiscoveryModule(com.facebook.airlift.discovery.client.DiscoveryModule) StaticFunctionNamespaceStore(com.facebook.presto.metadata.StaticFunctionNamespaceStore) NodeModule(com.facebook.airlift.node.NodeModule) AccessControlManager(com.facebook.presto.security.AccessControlManager) JmxModule(com.facebook.airlift.jmx.JmxModule) LogJmxModule(com.facebook.airlift.log.LogJmxModule) DiscoveryModule(com.facebook.airlift.discovery.client.DiscoveryModule) ClusterTtlProviderManagerModule(com.facebook.presto.ttl.clusterttlprovidermanagers.ClusterTtlProviderManagerModule) JsonEventModule(com.facebook.airlift.event.client.JsonEventModule) PasswordAuthenticatorManager(com.facebook.presto.server.security.PasswordAuthenticatorManager) TempStorageModule(com.facebook.presto.storage.TempStorageModule) TempStorageManager(com.facebook.presto.storage.TempStorageManager) Announcer(com.facebook.airlift.discovery.client.Announcer) Injector(com.google.inject.Injector) QueryPrerequisitesManager(com.facebook.presto.dispatcher.QueryPrerequisitesManager) StaticCatalogStore(com.facebook.presto.metadata.StaticCatalogStore) NodeModule(com.facebook.airlift.node.NodeModule) StaticFunctionNamespaceStore(com.facebook.presto.metadata.StaticFunctionNamespaceStore) ResourceGroupManager(com.facebook.presto.execution.resourceGroups.ResourceGroupManager) LogJmxModule(com.facebook.airlift.log.LogJmxModule) JsonModule(com.facebook.airlift.json.JsonModule) EventListenerManager(com.facebook.presto.eventlistener.EventListenerManager) TraceTokenModule(com.facebook.airlift.tracetoken.TraceTokenModule) HttpServerModule(com.facebook.airlift.http.server.HttpServerModule) Module(com.google.inject.Module) MBeanModule(org.weakref.jmx.guice.MBeanModule) ClusterTtlProviderManagerModule(com.facebook.presto.ttl.clusterttlprovidermanagers.ClusterTtlProviderManagerModule) JaxrsModule(com.facebook.airlift.jaxrs.JaxrsModule) JmxHttpModule(com.facebook.airlift.jmx.JmxHttpModule) WarningCollectorModule(com.facebook.presto.execution.warnings.WarningCollectorModule) QueryPrerequisitesManagerModule(com.facebook.presto.dispatcher.QueryPrerequisitesManagerModule) JsonModule(com.facebook.airlift.json.JsonModule) SmileModule(com.facebook.airlift.json.smile.SmileModule) TempStorageModule(com.facebook.presto.storage.TempStorageModule) AfterburnerModule(com.fasterxml.jackson.module.afterburner.AfterburnerModule) HttpEventModule(com.facebook.airlift.event.client.HttpEventModule) JsonEventModule(com.facebook.airlift.event.client.JsonEventModule) JmxModule(com.facebook.airlift.jmx.JmxModule) TraceTokenModule(com.facebook.airlift.tracetoken.TraceTokenModule) AccessControlModule(com.facebook.presto.security.AccessControlModule) LogJmxModule(com.facebook.airlift.log.LogJmxModule) EventListenerModule(com.facebook.presto.eventlistener.EventListenerModule) ServerSecurityModule(com.facebook.presto.server.security.ServerSecurityModule) HttpServerModule(com.facebook.airlift.http.server.HttpServerModule) NodeTtlFetcherManagerModule(com.facebook.presto.ttl.nodettlfetchermanagers.NodeTtlFetcherManagerModule) DiscoveryModule(com.facebook.airlift.discovery.client.DiscoveryModule) NodeModule(com.facebook.airlift.node.NodeModule) JmxHttpModule(com.facebook.airlift.jmx.JmxHttpModule) MBeanModule(org.weakref.jmx.guice.MBeanModule) WarningCollectorModule(com.facebook.presto.execution.warnings.WarningCollectorModule) ImmutableList(com.google.common.collect.ImmutableList) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) NodeSchedulerConfig(com.facebook.presto.execution.scheduler.NodeSchedulerConfig) Logger(com.facebook.airlift.log.Logger) NodeTtlFetcherManagerModule(com.facebook.presto.ttl.nodettlfetchermanagers.NodeTtlFetcherManagerModule) HttpEventModule(com.facebook.airlift.event.client.HttpEventModule) ClusterTtlProviderManager(com.facebook.presto.ttl.clusterttlprovidermanagers.ClusterTtlProviderManager) ServerSecurityModule(com.facebook.presto.server.security.ServerSecurityModule) Bootstrap(com.facebook.airlift.bootstrap.Bootstrap) DriftServer(com.facebook.drift.server.DriftServer) QueryPrerequisitesManagerModule(com.facebook.presto.dispatcher.QueryPrerequisitesManagerModule) EventListenerModule(com.facebook.presto.eventlistener.EventListenerModule) AccessControlModule(com.facebook.presto.security.AccessControlModule) JaxrsModule(com.facebook.airlift.jaxrs.JaxrsModule) CatalogManager(com.facebook.presto.metadata.CatalogManager) SmileModule(com.facebook.airlift.json.smile.SmileModule) NodeTtlFetcherManager(com.facebook.presto.ttl.nodettlfetchermanagers.NodeTtlFetcherManager)

Aggregations

Bootstrap (com.facebook.airlift.bootstrap.Bootstrap)2 JsonModule (com.facebook.airlift.json.JsonModule)2 SmileModule (com.facebook.airlift.json.smile.SmileModule)2 EventListenerModule (com.facebook.presto.eventlistener.EventListenerModule)2 ResourceGroupManager (com.facebook.presto.execution.resourceGroups.ResourceGroupManager)2 WarningCollectorModule (com.facebook.presto.execution.warnings.WarningCollectorModule)2 StaticCatalogStore (com.facebook.presto.metadata.StaticCatalogStore)2 StaticFunctionNamespaceStore (com.facebook.presto.metadata.StaticFunctionNamespaceStore)2 AccessControlManager (com.facebook.presto.security.AccessControlManager)2 AccessControlModule (com.facebook.presto.security.AccessControlModule)2 PasswordAuthenticatorManager (com.facebook.presto.server.security.PasswordAuthenticatorManager)2 FeaturesConfig (com.facebook.presto.sql.analyzer.FeaturesConfig)2 TempStorageManager (com.facebook.presto.storage.TempStorageManager)2 TempStorageModule (com.facebook.presto.storage.TempStorageModule)2 ImmutableList (com.google.common.collect.ImmutableList)2 Injector (com.google.inject.Injector)2 Module (com.google.inject.Module)2 ConditionalModule.installModuleIf (com.facebook.airlift.configuration.ConditionalModule.installModuleIf)1 Announcer (com.facebook.airlift.discovery.client.Announcer)1 DiscoveryModule (com.facebook.airlift.discovery.client.DiscoveryModule)1