Examples with HetuMetaStoreManager io.prestosql.metastore.HetuMetaStoreManager used on opensource projects

Example 1 with HetuMetaStoreManager

use of io.prestosql.metastore.HetuMetaStoreManager in project hetu-core by openlookeng.

the class TestSplitFiltering method testGetFilteredSplit.

/**
 * This test will not actually filter any splits since the indexes will not be found,
 * instead it's just testing the flow. The actual filtering is tested in other classes.
 */
@Test
public void testGetFilteredSplit() {
    PropertyService.setProperty(HetuConstant.FILTER_ENABLED, true);
    PropertyService.setProperty(HetuConstant.INDEXSTORE_URI, "/tmp/hetu/indices");
    PropertyService.setProperty(HetuConstant.INDEXSTORE_FILESYSTEM_PROFILE, "local-config-default");
    PropertyService.setProperty(HetuConstant.FILTER_CACHE_TTL, new Duration(10, TimeUnit.MINUTES));
    PropertyService.setProperty(HetuConstant.FILTER_CACHE_LOADING_DELAY, new Duration(5000, TimeUnit.MILLISECONDS));
    PropertyService.setProperty(HetuConstant.FILTER_CACHE_LOADING_THREADS, 2L);
    RowExpression expression = PlanBuilder.comparison(OperatorType.EQUAL, new VariableReferenceExpression("a", VarcharType.VARCHAR), new ConstantExpression(utf8Slice("test_value"), VarcharType.VARCHAR));
    SqlStageExecution stage = TestUtil.getTestStage(expression);
    List<Split> mockSplits = new ArrayList<>();
    MockSplit mock = new MockSplit("hdfs://hacluster/AppData/BIProd/DWD/EVT/bogus_table/000000_0", 0, 10, 0);
    MockSplit mock1 = new MockSplit("hdfs://hacluster/AppData/BIProd/DWD/EVT/bogus_table/000000_1", 0, 10, 0);
    MockSplit mock2 = new MockSplit("hdfs://hacluster/AppData/BIProd/DWD/EVT/bogus_table/000001_0", 0, 10, 0);
    MockSplit mock3 = new MockSplit("hdfs://hacluster/AppData/BIProd/DWD/EVT/bogus_table/000000_4", 0, 10, 0);
    mockSplits.add(new Split(new CatalogName("bogus_catalog"), mock, Lifespan.taskWide()));
    mockSplits.add(new Split(new CatalogName("bogus_catalog"), mock1, Lifespan.taskWide()));
    mockSplits.add(new Split(new CatalogName("bogus_catalog"), mock2, Lifespan.taskWide()));
    mockSplits.add(new Split(new CatalogName("bogus_catalog"), mock3, Lifespan.taskWide()));
    SplitSource.SplitBatch nextSplits = new SplitSource.SplitBatch(mockSplits, true);
    HeuristicIndexerManager indexerManager = new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager());
    Pair<Optional<RowExpression>, Map<Symbol, ColumnHandle>> pair = SplitFiltering.getExpression(stage);
    List<Split> filteredSplits = SplitFiltering.getFilteredSplit(pair.getFirst(), SplitFiltering.getFullyQualifiedName(stage), pair.getSecond(), nextSplits, indexerManager);
    assertNotNull(filteredSplits);
    assertEquals(filteredSplits.size(), 4);
}

Example 2 with HetuMetaStoreManager

use of io.prestosql.metastore.HetuMetaStoreManager in project hetu-core by openlookeng.

the class TestSourcePartitionedScheduler method getSourcePartitionedScheduler.

private static StageScheduler getSourcePartitionedScheduler(StageExecutionPlan plan, SqlStageExecution stage, InternalNodeManager nodeManager, NodeTaskMap nodeTaskMap, int splitBatchSize) {
    NodeSchedulerConfig nodeSchedulerConfig = new NodeSchedulerConfig().setIncludeCoordinator(false).setMaxSplitsPerNode(20).setMaxPendingSplitsPerTask(0);
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), nodeManager, nodeSchedulerConfig, nodeTaskMap);
    PlanNodeId sourceNode = Iterables.getOnlyElement(plan.getSplitSources().keySet());
    SplitSource splitSource = Iterables.getOnlyElement(plan.getSplitSources().values());
    SplitPlacementPolicy placementPolicy = new DynamicSplitPlacementPolicy(nodeScheduler.createNodeSelector(splitSource.getCatalogName(), false, null), stage::getAllTasks);
    return newSourcePartitionedSchedulerAsStageScheduler(stage, sourceNode, splitSource, placementPolicy, splitBatchSize, session, new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager()));
}

Example 3 with HetuMetaStoreManager

use of io.prestosql.metastore.HetuMetaStoreManager in project hetu-core by openlookeng.

the class TaskTestUtils method createTestingPlanner.

public static LocalExecutionPlanner createTestingPlanner() {
    Metadata metadata = createTestMetadataManager();
    PageSourceManager pageSourceManager = new PageSourceManager();
    HetuMetaStoreManager hetuMetaStoreManager = new HetuMetaStoreManager();
    FeaturesConfig featuresConfig = new FeaturesConfig();
    CubeManager cubeManager = new CubeManager(featuresConfig, hetuMetaStoreManager);
    pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
    // we don't start the finalizer so nothing will be collected, which is ok for a test
    FinalizerService finalizerService = new FinalizerService();
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService));
    NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler);
    PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0);
    NodeInfo nodeInfo = new NodeInfo("test");
    FileSystemClientManager fileSystemClientManager = new FileSystemClientManager();
    SeedStoreManager seedStoreManager = new SeedStoreManager(fileSystemClientManager);
    StateStoreProvider stateStoreProvider = new LocalStateStoreProvider(seedStoreManager);
    HeuristicIndexerManager heuristicIndexerManager = new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager());
    return new LocalExecutionPlanner(metadata, new TypeAnalyzer(new SqlParser(), metadata), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockExchangeClientSupplier(), new ExpressionCompiler(metadata, pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }), (types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, (types, partitionFunction, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, new PagesIndex.TestingFactory(false), new JoinCompiler(metadata), new LookupJoinOperators(), new OrderingCompiler(), nodeInfo, stateStoreProvider, new StateStoreListenerManager(stateStoreProvider), new DynamicFilterCacheManager(), heuristicIndexerManager, cubeManager);
}

Example 4 with HetuMetaStoreManager

use of io.prestosql.metastore.HetuMetaStoreManager in project hetu-core by openlookeng.

the class TestCommitTask method testCommit.

@Test
public void testCommit() {
    TransactionManager transactionManager = createTestTransactionManager();
    Session session = sessionBuilder().setTransactionId(transactionManager.beginTransaction(false)).build();
    QueryStateMachine stateMachine = createQueryStateMachine("COMMIT", session, transactionManager);
    assertTrue(stateMachine.getSession().getTransactionId().isPresent());
    assertEquals(transactionManager.getAllTransactionInfos().size(), 1);
    getFutureValue(new CommitTask().execute(new Commit(), transactionManager, metadata, new AllowAllAccessControl(), stateMachine, emptyList(), new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager())));
    assertTrue(stateMachine.getQueryInfo(Optional.empty()).isClearTransactionId());
    assertFalse(stateMachine.getQueryInfo(Optional.empty()).getStartedTransactionId().isPresent());
    assertTrue(transactionManager.getAllTransactionInfos().isEmpty());
}

Example 5 with HetuMetaStoreManager

use of io.prestosql.metastore.HetuMetaStoreManager in project hetu-core by openlookeng.

the class TestCommitTask method testNoTransactionCommit.

@Test
public void testNoTransactionCommit() {
    TransactionManager transactionManager = createTestTransactionManager();
    Session session = sessionBuilder().build();
    QueryStateMachine stateMachine = createQueryStateMachine("COMMIT", session, transactionManager);
    assertPrestoExceptionThrownBy(() -> getFutureValue(new CommitTask().execute(new Commit(), transactionManager, metadata, new AllowAllAccessControl(), stateMachine, emptyList(), new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager())))).hasErrorCode(NOT_IN_TRANSACTION);
    assertFalse(stateMachine.getQueryInfo(Optional.empty()).isClearTransactionId());
    assertFalse(stateMachine.getQueryInfo(Optional.empty()).getStartedTransactionId().isPresent());
    assertTrue(transactionManager.getAllTransactionInfos().isEmpty());
}