Examples with HeuristicIndexerManager io.prestosql.heuristicindex.HeuristicIndexerManager used on opensource projects

Example 1 with HeuristicIndexerManager

use of io.prestosql.heuristicindex.HeuristicIndexerManager in project hetu-core by openlookeng.

the class PrestoServer method run.

@Override
public void run() {
    verifyJvmRequirements();
    verifySystemTimeIsReasonable();
    Logger log = Logger.get(PrestoServer.class);
    ImmutableList.Builder<Module> modules = ImmutableList.builder();
    modules.add(new NodeModule(), Modules.override(new DiscoveryModule()).with(new HetuDiscoveryModule()), Modules.override(new HttpServerModule()).with(new HetuHttpServerModule()), new JsonModule(), new SmileModule(), new JaxrsModule(), new MBeanModule(), new PrefixObjectNameGeneratorModule("io.prestosql"), new HetuJmxModule(), new JmxHttpModule(), new LogJmxModule(), new TraceTokenModule(), new JsonEventModule(), new HttpEventModule(), new ServerSecurityModule(), new AccessControlModule(), new PasswordSecurityModule(), new EventListenerModule(), new ServerMainModule(sqlParserOptions), new NodeStateChangeModule(), new WarningCollectorModule());
    modules.addAll(getAdditionalModules());
    Bootstrap app = new Bootstrap(modules.build());
    try {
        Injector injector = app.strictConfig().initialize();
        logLocation(log, "Working directory", Paths.get("."));
        logLocation(log, "Etc directory", Paths.get("etc"));
        injector.getInstance(PluginManager.class).loadPlugins();
        FileSystemClientManager fileSystemClientManager = injector.getInstance(FileSystemClientManager.class);
        fileSystemClientManager.loadFactoryConfigs();
        injector.getInstance(SeedStoreManager.class).loadSeedStore();
        if (injector.getInstance(SeedStoreManager.class).isSeedStoreOnYarnEnabled()) {
            addSeedOnYarnInformation(injector.getInstance(ServerConfig.class), injector.getInstance(SeedStoreManager.class), (HetuHttpServerInfo) injector.getInstance(HttpServerInfo.class));
        }
        launchEmbeddedStateStore(injector.getInstance(HetuConfig.class), injector.getInstance(StateStoreLauncher.class));
        injector.getInstance(StateStoreProvider.class).loadStateStore();
        // relies on state-store
        injector.getInstance(HetuMetaStoreManager.class).loadHetuMetastore(fileSystemClientManager);
        // relies on metastore
        injector.getInstance(HeuristicIndexerManager.class).buildIndexClient();
        injector.getInstance(StaticFunctionNamespaceStore.class).loadFunctionNamespaceManagers();
        injector.getInstance(StaticCatalogStore.class).loadCatalogs();
        injector.getInstance(DynamicCatalogStore.class).loadCatalogStores(fileSystemClientManager);
        injector.getInstance(DynamicCatalogScanner.class).start();
        injector.getInstance(SessionPropertyDefaults.class).loadConfigurationManager();
        injector.getInstance(ResourceGroupManager.class).loadConfigurationManager();
        injector.getInstance(AccessControlManager.class).loadSystemAccessControl();
        injector.getInstance(PasswordAuthenticatorManager.class).loadPasswordAuthenticator();
        injector.getInstance(EventListenerManager.class).loadConfiguredEventListener();
        injector.getInstance(GroupProviderManager.class).loadConfiguredGroupProvider();
        // preload index (on coordinator only)
        if (injector.getInstance(ServerConfig.class).isCoordinator()) {
            HeuristicIndexerManager heuristicIndexerManager = injector.getInstance(HeuristicIndexerManager.class);
            heuristicIndexerManager.preloadIndex();
            heuristicIndexerManager.initCache();
        }
        // register dynamic filter listener
        registerStateStoreListeners(injector.getInstance(StateStoreListenerManager.class), injector.getInstance(DynamicFilterCacheManager.class), injector.getInstance(ServerConfig.class), injector.getInstance(NodeSchedulerConfig.class));
        // Initialize snapshot Manager
        injector.getInstance(SnapshotUtils.class).initialize();
        injector.getInstance(Announcer.class).start();
        injector.getInstance(ServerInfoResource.class).startupComplete();
        log.info("======== SERVER STARTED ========");
    } catch (Throwable e) {
        log.error(e);
        System.exit(1);
    }
}

Example 2 with HeuristicIndexerManager

use of io.prestosql.heuristicindex.HeuristicIndexerManager in project hetu-core by openlookeng.

the class TestSourcePartitionedScheduler method getSourcePartitionedScheduler.

private static StageScheduler getSourcePartitionedScheduler(StageExecutionPlan plan, SqlStageExecution stage, InternalNodeManager nodeManager, NodeTaskMap nodeTaskMap, int splitBatchSize) {
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setIncludeCoordinator(false).setMaxSplitsPerNode(20).setMaxPendingSplitsPerTask(0);
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, nodeSchedulerConfig, nodeTaskMap);
    PlanNodeId sourceNode = Iterables.getOnlyElement(plan.getSplitSources().keySet());
    SplitSource splitSource = Iterables.getOnlyElement(plan.getSplitSources().values());
    SplitPlacementPolicy placementPolicy = new DynamicSplitPlacementPolicy(nodeScheduler.createNodeSelector(splitSource.getCatalogName(), false, null), stage::getAllTasks);
    return newSourcePartitionedSchedulerAsStageScheduler(stage, sourceNode, splitSource, placementPolicy, splitBatchSize, session, new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager()));
}

Example 3 with HeuristicIndexerManager

use of io.prestosql.heuristicindex.HeuristicIndexerManager in project hetu-core by openlookeng.

the class TaskTestUtils method createTestingPlanner.

public static LocalExecutionPlanner createTestingPlanner() {
    Metadata metadata = createTestMetadataManager();
    PageSourceManager pageSourceManager = new PageSourceManager();
    HetuMetaStoreManager hetuMetaStoreManager = new HetuMetaStoreManager();
    FeaturesConfig featuresConfig = new FeaturesConfig();
    CubeManager cubeManager = new CubeManager(featuresConfig, hetuMetaStoreManager);
    pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
    // we don't start the finalizer so nothing will be collected, which is ok for a test
    FinalizerService finalizerService = new FinalizerService();
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService));
    NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler);
    PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0);
    NodeInfo nodeInfo = new NodeInfo("test");
    FileSystemClientManager fileSystemClientManager = new FileSystemClientManager();
    SeedStoreManager seedStoreManager = new SeedStoreManager(fileSystemClientManager);
    StateStoreProvider stateStoreProvider = new LocalStateStoreProvider(seedStoreManager);
    HeuristicIndexerManager heuristicIndexerManager = new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager());
    return new LocalExecutionPlanner(metadata, new TypeAnalyzer(new SqlParser(), metadata), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockExchangeClientSupplier(), new ExpressionCompiler(metadata, pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }), (types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, (types, partitionFunction, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, new PagesIndex.TestingFactory(false), new JoinCompiler(metadata), new LookupJoinOperators(), new OrderingCompiler(), nodeInfo, stateStoreProvider, new StateStoreListenerManager(stateStoreProvider), new DynamicFilterCacheManager(), heuristicIndexerManager, cubeManager);
}

Example 4 with HeuristicIndexerManager

use of io.prestosql.heuristicindex.HeuristicIndexerManager in project hetu-core by openlookeng.

the class TestCommitTask method testCommit.

@Test
public void testCommit() {
    TransactionManager transactionManager = createTestTransactionManager();
    Session session = sessionBuilder().setTransactionId(transactionManager.beginTransaction(false)).build();
    QueryStateMachine stateMachine = createQueryStateMachine("COMMIT", session, transactionManager);
    assertTrue(stateMachine.getSession().getTransactionId().isPresent());
    assertEquals(transactionManager.getAllTransactionInfos().size(), 1);
    getFutureValue(new CommitTask().execute(new Commit(), transactionManager, metadata, new AllowAllAccessControl(), stateMachine, emptyList(), new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager())));
    assertTrue(stateMachine.getQueryInfo(Optional.empty()).isClearTransactionId());
    assertFalse(stateMachine.getQueryInfo(Optional.empty()).getStartedTransactionId().isPresent());
    assertTrue(transactionManager.getAllTransactionInfos().isEmpty());
}

Example 5 with HeuristicIndexerManager

use of io.prestosql.heuristicindex.HeuristicIndexerManager in project hetu-core by openlookeng.

the class TestCommitTask method testNoTransactionCommit.

@Test
public void testNoTransactionCommit() {
    TransactionManager transactionManager = createTestTransactionManager();
    Session session = sessionBuilder().build();
    QueryStateMachine stateMachine = createQueryStateMachine("COMMIT", session, transactionManager);
    assertPrestoExceptionThrownBy(() -> getFutureValue(new CommitTask().execute(new Commit(), transactionManager, metadata, new AllowAllAccessControl(), stateMachine, emptyList(), new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager())))).hasErrorCode(NOT_IN_TRANSACTION);
    assertFalse(stateMachine.getQueryInfo(Optional.empty()).isClearTransactionId());
    assertFalse(stateMachine.getQueryInfo(Optional.empty()).getStartedTransactionId().isPresent());
    assertTrue(transactionManager.getAllTransactionInfos().isEmpty());
}