Search in sources :

Example 1 with LocalStateStoreProvider

use of io.prestosql.statestore.LocalStateStoreProvider in project hetu-core by openlookeng.

the class TestDynamicFilterSourceOperator method prepareConfigFiles.

@BeforeTest
private void prepareConfigFiles() throws Exception {
    File launcherConfigFile = new File(STATE_STORE_CONFIGURATION_PATH);
    if (launcherConfigFile.exists()) {
        launcherConfigFile.delete();
    }
    launcherConfigFile.createNewFile();
    FileWriter configWriter = new FileWriter(STATE_STORE_CONFIGURATION_PATH);
    configWriter.write("state-store.type=hazelcast\n" + "state-store.name=test\n" + "state-store.cluster=test-cluster\n" + "hazelcast.discovery.mode=tcp-ip\n" + "hazelcast.discovery.port=7980\n");
    configWriter.close();
    Set<Seed> seeds = new HashSet<>();
    SeedStore mockSeedStore = mock(SeedStore.class);
    Seed mockSeed = mock(Seed.class);
    seeds.add(mockSeed);
    SeedStoreManager mockSeedStoreManager = mock(SeedStoreManager.class);
    when(mockSeedStoreManager.getSeedStore(SeedStoreSubType.HAZELCAST)).thenReturn(mockSeedStore);
    when(mockSeed.getLocation()).thenReturn("127.0.0.1:6991");
    when(mockSeedStore.get()).thenReturn(seeds);
    StateStoreFactory factory = new HazelcastStateStoreFactory();
    stateStoreProvider = new LocalStateStoreProvider(mockSeedStoreManager);
    stateStoreProvider.addStateStoreFactory(factory);
    createStateStoreCluster("6991");
    stateStoreProvider.loadStateStore();
}
Also used : LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) Seed(io.prestosql.spi.seedstore.Seed) FileWriter(java.io.FileWriter) SeedStore(io.prestosql.spi.seedstore.SeedStore) HazelcastStateStoreFactory(io.hetu.core.statestore.hazelcast.HazelcastStateStoreFactory) File(java.io.File) HashSet(java.util.HashSet) HazelcastStateStoreFactory(io.hetu.core.statestore.hazelcast.HazelcastStateStoreFactory) StateStoreFactory(io.prestosql.spi.statestore.StateStoreFactory) BeforeTest(org.testng.annotations.BeforeTest)

Example 2 with LocalStateStoreProvider

use of io.prestosql.statestore.LocalStateStoreProvider in project hetu-core by openlookeng.

the class TestSourcePartitionedScheduler method createSqlStageExecution.

private SqlStageExecution createSqlStageExecution(StageExecutionPlan tableScanPlan, NodeTaskMap nodeTaskMap) {
    StageId stageId = new StageId(new QueryId("query"), 0);
    SqlStageExecution stage = SqlStageExecution.createSqlStageExecution(stageId, locationFactory.createStageLocation(stageId), tableScanPlan.getFragment(), tableScanPlan.getTables(), new MockRemoteTaskFactory(queryExecutor, scheduledExecutor), TEST_SESSION, true, nodeTaskMap, queryExecutor, new NoOpFailureDetector(), new SplitSchedulerStats(), new DynamicFilterService(new LocalStateStoreProvider(seedStoreManager)), new QuerySnapshotManager(stageId.getQueryId(), NOOP_SNAPSHOT_UTILS, TEST_SESSION));
    stage.setOutputBuffers(createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds());
    return stage;
}
Also used : NoOpFailureDetector(io.prestosql.failuredetector.NoOpFailureDetector) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) StageId(io.prestosql.execution.StageId) QueryId(io.prestosql.spi.QueryId) DynamicFilterService(io.prestosql.dynamicfilter.DynamicFilterService) SqlStageExecution(io.prestosql.execution.SqlStageExecution) MockRemoteTaskFactory(io.prestosql.execution.MockRemoteTaskFactory) QuerySnapshotManager(io.prestosql.snapshot.QuerySnapshotManager)

Example 3 with LocalStateStoreProvider

use of io.prestosql.statestore.LocalStateStoreProvider in project hetu-core by openlookeng.

the class TaskTestUtils method createTestingPlanner.

public static LocalExecutionPlanner createTestingPlanner() {
    Metadata metadata = createTestMetadataManager();
    PageSourceManager pageSourceManager = new PageSourceManager();
    HetuMetaStoreManager hetuMetaStoreManager = new HetuMetaStoreManager();
    FeaturesConfig featuresConfig = new FeaturesConfig();
    CubeManager cubeManager = new CubeManager(featuresConfig, hetuMetaStoreManager);
    pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
    // we don't start the finalizer so nothing will be collected, which is ok for a test
    FinalizerService finalizerService = new FinalizerService();
    NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService));
    NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler);
    PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0);
    NodeInfo nodeInfo = new NodeInfo("test");
    FileSystemClientManager fileSystemClientManager = new FileSystemClientManager();
    SeedStoreManager seedStoreManager = new SeedStoreManager(fileSystemClientManager);
    StateStoreProvider stateStoreProvider = new LocalStateStoreProvider(seedStoreManager);
    HeuristicIndexerManager heuristicIndexerManager = new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager());
    return new LocalExecutionPlanner(metadata, new TypeAnalyzer(new SqlParser(), metadata), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockExchangeClientSupplier(), new ExpressionCompiler(metadata, pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }), (types, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, (types, partitionFunction, spillContext, memoryContext) -> {
        throw new UnsupportedOperationException();
    }, new PagesIndex.TestingFactory(false), new JoinCompiler(metadata), new LookupJoinOperators(), new OrderingCompiler(), nodeInfo, stateStoreProvider, new StateStoreListenerManager(stateStoreProvider), new DynamicFilterCacheManager(), heuristicIndexerManager, cubeManager);
}
Also used : NodeInfo(io.airlift.node.NodeInfo) SqlParser(io.prestosql.sql.parser.SqlParser) PlanFragmentId(io.prestosql.sql.planner.plan.PlanFragmentId) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) TEST_TABLE_HANDLE(io.prestosql.testing.TestingHandles.TEST_TABLE_HANDLE) PartitioningScheme(io.prestosql.sql.planner.PartitioningScheme) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) ObjectMapperProvider(io.airlift.json.ObjectMapperProvider) StateStoreListenerManager(io.prestosql.statestore.listener.StateStoreListenerManager) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) Partitioning(io.prestosql.sql.planner.Partitioning) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) MockExchangeClientSupplier(io.prestosql.execution.TestSqlTaskManager.MockExchangeClientSupplier) EventListenerManager(io.prestosql.eventlistener.EventListenerManager) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) ImmutableMap(com.google.common.collect.ImmutableMap) MetadataManager.createTestMetadataManager(io.prestosql.metadata.MetadataManager.createTestMetadataManager) CatalogName(io.prestosql.spi.connector.CatalogName) TableScanNode(io.prestosql.spi.plan.TableScanNode) UUID(java.util.UUID) NodeSchedulerConfig(io.prestosql.execution.scheduler.NodeSchedulerConfig) Metadata(io.prestosql.metadata.Metadata) ReuseExchangeOperator(io.prestosql.spi.operator.ReuseExchangeOperator) List(java.util.List) SOURCE_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SOURCE_DISTRIBUTION) PageSinkManager(io.prestosql.split.PageSinkManager) Optional(java.util.Optional) TEST_SESSION(io.prestosql.SessionTestUtils.TEST_SESSION) NodePartitioningManager(io.prestosql.sql.planner.NodePartitioningManager) LocalExecutionPlanner(io.prestosql.sql.planner.LocalExecutionPlanner) LegacyNetworkTopology(io.prestosql.execution.scheduler.LegacyNetworkTopology) SINGLE_DISTRIBUTION(io.prestosql.sql.planner.SystemPartitioningHandle.SINGLE_DISTRIBUTION) ExpressionCompiler(io.prestosql.sql.gen.ExpressionCompiler) StatsAndCosts(io.prestosql.cost.StatsAndCosts) NodeScheduler(io.prestosql.execution.scheduler.NodeScheduler) Split(io.prestosql.metadata.Split) OutputBuffers(io.prestosql.execution.buffer.OutputBuffers) IndexJoinLookupStats(io.prestosql.operator.index.IndexJoinLookupStats) OptionalInt(java.util.OptionalInt) DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) LookupJoinOperators(io.prestosql.operator.LookupJoinOperators) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) VARCHAR(io.prestosql.spi.type.VarcharType.VARCHAR) ImmutableList(com.google.common.collect.ImmutableList) TestingSplit(io.prestosql.testing.TestingSplit) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) InMemoryNodeManager(io.prestosql.metadata.InMemoryNodeManager) PageSourceManager(io.prestosql.split.PageSourceManager) TestingColumnHandle(io.prestosql.testing.TestingMetadata.TestingColumnHandle) PagesIndex(io.prestosql.operator.PagesIndex) Symbol(io.prestosql.spi.plan.Symbol) PlanFragment(io.prestosql.sql.planner.PlanFragment) StageExecutionDescriptor.ungroupedExecution(io.prestosql.operator.StageExecutionDescriptor.ungroupedExecution) IndexManager(io.prestosql.index.IndexManager) SplitMonitor(io.prestosql.event.SplitMonitor) FinalizerService(io.prestosql.util.FinalizerService) CubeManager(io.prestosql.cube.CubeManager) JoinFilterFunctionCompiler(io.prestosql.sql.gen.JoinFilterFunctionCompiler) HetuMetaStoreManager(io.prestosql.metastore.HetuMetaStoreManager) StateStoreProvider(io.prestosql.statestore.StateStoreProvider) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) PageFunctionCompiler(io.prestosql.sql.gen.PageFunctionCompiler) FeaturesConfig(io.prestosql.sql.analyzer.FeaturesConfig) Metadata(io.prestosql.metadata.Metadata) NodeSchedulerConfig(io.prestosql.execution.scheduler.NodeSchedulerConfig) StateStoreListenerManager(io.prestosql.statestore.listener.StateStoreListenerManager) PagesIndex(io.prestosql.operator.PagesIndex) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) StateStoreProvider(io.prestosql.statestore.StateStoreProvider) PageSourceManager(io.prestosql.split.PageSourceManager) NodePartitioningManager(io.prestosql.sql.planner.NodePartitioningManager) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) OrderingCompiler(io.prestosql.sql.gen.OrderingCompiler) NodeScheduler(io.prestosql.execution.scheduler.NodeScheduler) HetuMetaStoreManager(io.prestosql.metastore.HetuMetaStoreManager) GenericSpillerFactory(io.prestosql.spiller.GenericSpillerFactory) PageSinkManager(io.prestosql.split.PageSinkManager) LookupJoinOperators(io.prestosql.operator.LookupJoinOperators) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) DynamicFilterCacheManager(io.prestosql.dynamicfilter.DynamicFilterCacheManager) LocalExecutionPlanner(io.prestosql.sql.planner.LocalExecutionPlanner) MockExchangeClientSupplier(io.prestosql.execution.TestSqlTaskManager.MockExchangeClientSupplier) IndexJoinLookupStats(io.prestosql.operator.index.IndexJoinLookupStats) JoinFilterFunctionCompiler(io.prestosql.sql.gen.JoinFilterFunctionCompiler) HeuristicIndexerManager(io.prestosql.heuristicindex.HeuristicIndexerManager) SqlParser(io.prestosql.sql.parser.SqlParser) CubeManager(io.prestosql.cube.CubeManager) TypeAnalyzer(io.prestosql.sql.planner.TypeAnalyzer) InMemoryNodeManager(io.prestosql.metadata.InMemoryNodeManager) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) IndexManager(io.prestosql.index.IndexManager) FinalizerService(io.prestosql.util.FinalizerService) NodeInfo(io.airlift.node.NodeInfo) LegacyNetworkTopology(io.prestosql.execution.scheduler.LegacyNetworkTopology) ExpressionCompiler(io.prestosql.sql.gen.ExpressionCompiler)

Example 4 with LocalStateStoreProvider

use of io.prestosql.statestore.LocalStateStoreProvider in project hetu-core by openlookeng.

the class TestHiveIntegrationSmokeTest method testRuseExchangeSplitsGroupNotMatchingBetweenProducerConsumer.

@Test
public void testRuseExchangeSplitsGroupNotMatchingBetweenProducerConsumer() {
    setUpNodes();
    NodeTaskMap nodeTasks = new NodeTaskMap(new FinalizerService());
    StageId stageId = new StageId(new QueryId("query"), 0);
    UUID uuid = UUID.randomUUID();
    PlanFragment testFragmentProducer = createTableScanPlanFragment("build", ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_PRODUCER, uuid, 1);
    PlanNodeId tableScanNodeId = new PlanNodeId("plan_id");
    StageExecutionPlan producerStageExecutionPlan = new StageExecutionPlan(testFragmentProducer, ImmutableMap.of(tableScanNodeId, new ConnectorAwareSplitSource(CONNECTOR_ID, createFixedSplitSource(0, TestingSplit::createRemoteSplit))), ImmutableList.of(), ImmutableMap.of(tableScanNodeId, new TableInfo(new QualifiedObjectName("test", TEST_SCHEMA, "test"), TupleDomain.all())));
    SqlStageExecution producerStage = createSqlStageExecution(stageId, new TestSqlTaskManager.MockLocationFactory().createStageLocation(stageId), producerStageExecutionPlan.getFragment(), producerStageExecutionPlan.getTables(), new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor), TEST_SESSION_REUSE, true, nodeTasks, remoteTaskExecutor, new NoOpFailureDetector(), new SplitSchedulerStats(), new DynamicFilterService(new LocalStateStoreProvider(new SeedStoreManager(new FileSystemClientManager()))), new QuerySnapshotManager(stageId.getQueryId(), NOOP_SNAPSHOT_UTILS, TEST_SESSION));
    Set<Split> producerSplits = createAndGetSplits(10);
    Multimap<InternalNode, Split> producerAssignment = nodeSelector.computeAssignments(producerSplits, ImmutableList.copyOf(taskMap.values()), Optional.of(producerStage)).getAssignments();
    PlanFragment testFragmentConsumer = createTableScanPlanFragment("build", ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_CONSUMER, uuid, 1);
    StageExecutionPlan consumerStageExecutionPlan = new StageExecutionPlan(testFragmentConsumer, ImmutableMap.of(tableScanNodeId, new ConnectorAwareSplitSource(CONNECTOR_ID, createFixedSplitSource(0, TestingSplit::createRemoteSplit))), ImmutableList.of(), ImmutableMap.of(tableScanNodeId, new TableInfo(new QualifiedObjectName("test", TEST_SCHEMA, "test"), TupleDomain.all())));
    SqlStageExecution stage = createSqlStageExecution(stageId, new TestSqlTaskManager.MockLocationFactory().createStageLocation(stageId), consumerStageExecutionPlan.getFragment(), consumerStageExecutionPlan.getTables(), new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor), TEST_SESSION_REUSE, true, nodeTasks, remoteTaskExecutor, new NoOpFailureDetector(), new SplitSchedulerStats(), new DynamicFilterService(new LocalStateStoreProvider(new SeedStoreManager(new FileSystemClientManager()))), new QuerySnapshotManager(stageId.getQueryId(), NOOP_SNAPSHOT_UTILS, TEST_SESSION));
    Set<Split> consumerSplits = createAndGetSplits(50);
    try {
        Multimap<InternalNode, Split> consumerAssignment = nodeSelector.computeAssignments(consumerSplits, ImmutableList.copyOf(taskMap.values()), Optional.of(stage)).getAssignments();
    } catch (PrestoException e) {
        assertEquals("Producer & consumer splits are not same", e.getMessage());
        return;
    }
    assertEquals(false, true);
}
Also used : NoOpFailureDetector(io.prestosql.failuredetector.NoOpFailureDetector) StageExecutionPlan(io.prestosql.sql.planner.StageExecutionPlan) StageId(io.prestosql.execution.StageId) PrestoException(io.prestosql.spi.PrestoException) TestPhasedExecutionSchedule.createTableScanPlanFragment(io.prestosql.execution.scheduler.TestPhasedExecutionSchedule.createTableScanPlanFragment) PlanFragment(io.prestosql.sql.planner.PlanFragment) ConnectorAwareSplitSource(io.prestosql.split.ConnectorAwareSplitSource) SqlStageExecution.createSqlStageExecution(io.prestosql.execution.SqlStageExecution.createSqlStageExecution) SqlStageExecution(io.prestosql.execution.SqlStageExecution) QuerySnapshotManager(io.prestosql.snapshot.QuerySnapshotManager) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) TableInfo(io.prestosql.execution.TableInfo) DynamicFilterService(io.prestosql.dynamicfilter.DynamicFilterService) UUID(java.util.UUID) NodeTaskMap(io.prestosql.execution.NodeTaskMap) QueryId(io.prestosql.spi.QueryId) QualifiedObjectName(io.prestosql.spi.connector.QualifiedObjectName) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) SplitSchedulerStats(io.prestosql.execution.scheduler.SplitSchedulerStats) FinalizerService(io.prestosql.util.FinalizerService) InternalNode(io.prestosql.metadata.InternalNode) ConnectorSplit(io.prestosql.spi.connector.ConnectorSplit) Split(io.prestosql.metadata.Split) TestingSplit(io.prestosql.testing.TestingSplit) MockRemoteTaskFactory(io.prestosql.execution.MockRemoteTaskFactory) Test(org.testng.annotations.Test) AbstractTestIntegrationSmokeTest(io.prestosql.tests.AbstractTestIntegrationSmokeTest)

Example 5 with LocalStateStoreProvider

use of io.prestosql.statestore.LocalStateStoreProvider in project hetu-core by openlookeng.

the class TestHetuMetastoreGlobalCache method setUpHazelcast.

@BeforeTest
private void setUpHazelcast() throws IOException {
    Set<Seed> seeds = new HashSet<>();
    SeedStore mockSeedStore = mock(SeedStore.class);
    Seed mockSeed = mock(Seed.class);
    seeds.add(mockSeed);
    SeedStoreManager mockSeedStoreManager = mock(SeedStoreManager.class);
    when(mockSeedStoreManager.getSeedStore(any(SeedStoreSubType.class))).thenReturn(mockSeedStore);
    when(mockSeed.getLocation()).thenReturn(LOCALHOST + ":" + PORT3);
    when(mockSeedStore.get()).thenReturn(seeds);
    factory = new HazelcastStateStoreFactory();
    stateStoreProvider = new LocalStateStoreProvider(mockSeedStoreManager);
    stateStoreProvider.addStateStoreFactory(factory);
}
Also used : LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) SeedStoreManager(io.prestosql.seedstore.SeedStoreManager) Seed(io.prestosql.spi.seedstore.Seed) SeedStore(io.prestosql.spi.seedstore.SeedStore) HazelcastStateStoreFactory(io.hetu.core.statestore.hazelcast.HazelcastStateStoreFactory) HashSet(java.util.HashSet) SeedStoreSubType(io.prestosql.spi.seedstore.SeedStoreSubType) BeforeTest(org.testng.annotations.BeforeTest)

Aggregations

LocalStateStoreProvider (io.prestosql.statestore.LocalStateStoreProvider)12 SeedStoreManager (io.prestosql.seedstore.SeedStoreManager)10 FileSystemClientManager (io.prestosql.filesystem.FileSystemClientManager)8 DynamicFilterService (io.prestosql.dynamicfilter.DynamicFilterService)7 NoOpFailureDetector (io.prestosql.failuredetector.NoOpFailureDetector)7 QuerySnapshotManager (io.prestosql.snapshot.QuerySnapshotManager)7 QueryId (io.prestosql.spi.QueryId)7 MockRemoteTaskFactory (io.prestosql.execution.MockRemoteTaskFactory)6 SqlStageExecution (io.prestosql.execution.SqlStageExecution)6 SqlStageExecution.createSqlStageExecution (io.prestosql.execution.SqlStageExecution.createSqlStageExecution)6 StageId (io.prestosql.execution.StageId)6 Split (io.prestosql.metadata.Split)6 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)6 FinalizerService (io.prestosql.util.FinalizerService)5 NodeTaskMap (io.prestosql.execution.NodeTaskMap)4 TableInfo (io.prestosql.execution.TableInfo)4 SplitSchedulerStats (io.prestosql.execution.scheduler.SplitSchedulerStats)4 TestPhasedExecutionSchedule.createTableScanPlanFragment (io.prestosql.execution.scheduler.TestPhasedExecutionSchedule.createTableScanPlanFragment)4 InternalNode (io.prestosql.metadata.InternalNode)4 ConnectorSplit (io.prestosql.spi.connector.ConnectorSplit)4