use of io.prestosql.statestore.LocalStateStoreProvider in project hetu-core by openlookeng.
the class TestDynamicFilterSourceOperator method prepareConfigFiles.
@BeforeTest
private void prepareConfigFiles() throws Exception {
File launcherConfigFile = new File(STATE_STORE_CONFIGURATION_PATH);
if (launcherConfigFile.exists()) {
launcherConfigFile.delete();
}
launcherConfigFile.createNewFile();
FileWriter configWriter = new FileWriter(STATE_STORE_CONFIGURATION_PATH);
configWriter.write("state-store.type=hazelcast\n" + "state-store.name=test\n" + "state-store.cluster=test-cluster\n" + "hazelcast.discovery.mode=tcp-ip\n" + "hazelcast.discovery.port=7980\n");
configWriter.close();
Set<Seed> seeds = new HashSet<>();
SeedStore mockSeedStore = mock(SeedStore.class);
Seed mockSeed = mock(Seed.class);
seeds.add(mockSeed);
SeedStoreManager mockSeedStoreManager = mock(SeedStoreManager.class);
when(mockSeedStoreManager.getSeedStore(SeedStoreSubType.HAZELCAST)).thenReturn(mockSeedStore);
when(mockSeed.getLocation()).thenReturn("127.0.0.1:6991");
when(mockSeedStore.get()).thenReturn(seeds);
StateStoreFactory factory = new HazelcastStateStoreFactory();
stateStoreProvider = new LocalStateStoreProvider(mockSeedStoreManager);
stateStoreProvider.addStateStoreFactory(factory);
createStateStoreCluster("6991");
stateStoreProvider.loadStateStore();
}
use of io.prestosql.statestore.LocalStateStoreProvider in project hetu-core by openlookeng.
the class TestSourcePartitionedScheduler method createSqlStageExecution.
private SqlStageExecution createSqlStageExecution(StageExecutionPlan tableScanPlan, NodeTaskMap nodeTaskMap) {
StageId stageId = new StageId(new QueryId("query"), 0);
SqlStageExecution stage = SqlStageExecution.createSqlStageExecution(stageId, locationFactory.createStageLocation(stageId), tableScanPlan.getFragment(), tableScanPlan.getTables(), new MockRemoteTaskFactory(queryExecutor, scheduledExecutor), TEST_SESSION, true, nodeTaskMap, queryExecutor, new NoOpFailureDetector(), new SplitSchedulerStats(), new DynamicFilterService(new LocalStateStoreProvider(seedStoreManager)), new QuerySnapshotManager(stageId.getQueryId(), NOOP_SNAPSHOT_UTILS, TEST_SESSION));
stage.setOutputBuffers(createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds());
return stage;
}
use of io.prestosql.statestore.LocalStateStoreProvider in project hetu-core by openlookeng.
the class TaskTestUtils method createTestingPlanner.
public static LocalExecutionPlanner createTestingPlanner() {
Metadata metadata = createTestMetadataManager();
PageSourceManager pageSourceManager = new PageSourceManager();
HetuMetaStoreManager hetuMetaStoreManager = new HetuMetaStoreManager();
FeaturesConfig featuresConfig = new FeaturesConfig();
CubeManager cubeManager = new CubeManager(featuresConfig, hetuMetaStoreManager);
pageSourceManager.addConnectorPageSourceProvider(CONNECTOR_ID, new TestingPageSourceProvider());
// we don't start the finalizer so nothing will be collected, which is ok for a test
FinalizerService finalizerService = new FinalizerService();
NodeScheduler nodeScheduler = new NodeScheduler(new LegacyNetworkTopology(), new InMemoryNodeManager(), new NodeSchedulerConfig().setIncludeCoordinator(true), new NodeTaskMap(finalizerService));
NodePartitioningManager nodePartitioningManager = new NodePartitioningManager(nodeScheduler);
PageFunctionCompiler pageFunctionCompiler = new PageFunctionCompiler(metadata, 0);
NodeInfo nodeInfo = new NodeInfo("test");
FileSystemClientManager fileSystemClientManager = new FileSystemClientManager();
SeedStoreManager seedStoreManager = new SeedStoreManager(fileSystemClientManager);
StateStoreProvider stateStoreProvider = new LocalStateStoreProvider(seedStoreManager);
HeuristicIndexerManager heuristicIndexerManager = new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager());
return new LocalExecutionPlanner(metadata, new TypeAnalyzer(new SqlParser(), metadata), Optional.empty(), pageSourceManager, new IndexManager(), nodePartitioningManager, new PageSinkManager(), new MockExchangeClientSupplier(), new ExpressionCompiler(metadata, pageFunctionCompiler), pageFunctionCompiler, new JoinFilterFunctionCompiler(metadata), new IndexJoinLookupStats(), new TaskManagerConfig(), new GenericSpillerFactory((types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}), (types, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, (types, partitionFunction, spillContext, memoryContext) -> {
throw new UnsupportedOperationException();
}, new PagesIndex.TestingFactory(false), new JoinCompiler(metadata), new LookupJoinOperators(), new OrderingCompiler(), nodeInfo, stateStoreProvider, new StateStoreListenerManager(stateStoreProvider), new DynamicFilterCacheManager(), heuristicIndexerManager, cubeManager);
}
use of io.prestosql.statestore.LocalStateStoreProvider in project hetu-core by openlookeng.
the class TestHiveIntegrationSmokeTest method testRuseExchangeSplitsGroupNotMatchingBetweenProducerConsumer.
@Test
public void testRuseExchangeSplitsGroupNotMatchingBetweenProducerConsumer() {
setUpNodes();
NodeTaskMap nodeTasks = new NodeTaskMap(new FinalizerService());
StageId stageId = new StageId(new QueryId("query"), 0);
UUID uuid = UUID.randomUUID();
PlanFragment testFragmentProducer = createTableScanPlanFragment("build", ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_PRODUCER, uuid, 1);
PlanNodeId tableScanNodeId = new PlanNodeId("plan_id");
StageExecutionPlan producerStageExecutionPlan = new StageExecutionPlan(testFragmentProducer, ImmutableMap.of(tableScanNodeId, new ConnectorAwareSplitSource(CONNECTOR_ID, createFixedSplitSource(0, TestingSplit::createRemoteSplit))), ImmutableList.of(), ImmutableMap.of(tableScanNodeId, new TableInfo(new QualifiedObjectName("test", TEST_SCHEMA, "test"), TupleDomain.all())));
SqlStageExecution producerStage = createSqlStageExecution(stageId, new TestSqlTaskManager.MockLocationFactory().createStageLocation(stageId), producerStageExecutionPlan.getFragment(), producerStageExecutionPlan.getTables(), new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor), TEST_SESSION_REUSE, true, nodeTasks, remoteTaskExecutor, new NoOpFailureDetector(), new SplitSchedulerStats(), new DynamicFilterService(new LocalStateStoreProvider(new SeedStoreManager(new FileSystemClientManager()))), new QuerySnapshotManager(stageId.getQueryId(), NOOP_SNAPSHOT_UTILS, TEST_SESSION));
Set<Split> producerSplits = createAndGetSplits(10);
Multimap<InternalNode, Split> producerAssignment = nodeSelector.computeAssignments(producerSplits, ImmutableList.copyOf(taskMap.values()), Optional.of(producerStage)).getAssignments();
PlanFragment testFragmentConsumer = createTableScanPlanFragment("build", ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_CONSUMER, uuid, 1);
StageExecutionPlan consumerStageExecutionPlan = new StageExecutionPlan(testFragmentConsumer, ImmutableMap.of(tableScanNodeId, new ConnectorAwareSplitSource(CONNECTOR_ID, createFixedSplitSource(0, TestingSplit::createRemoteSplit))), ImmutableList.of(), ImmutableMap.of(tableScanNodeId, new TableInfo(new QualifiedObjectName("test", TEST_SCHEMA, "test"), TupleDomain.all())));
SqlStageExecution stage = createSqlStageExecution(stageId, new TestSqlTaskManager.MockLocationFactory().createStageLocation(stageId), consumerStageExecutionPlan.getFragment(), consumerStageExecutionPlan.getTables(), new MockRemoteTaskFactory(remoteTaskExecutor, remoteTaskScheduledExecutor), TEST_SESSION_REUSE, true, nodeTasks, remoteTaskExecutor, new NoOpFailureDetector(), new SplitSchedulerStats(), new DynamicFilterService(new LocalStateStoreProvider(new SeedStoreManager(new FileSystemClientManager()))), new QuerySnapshotManager(stageId.getQueryId(), NOOP_SNAPSHOT_UTILS, TEST_SESSION));
Set<Split> consumerSplits = createAndGetSplits(50);
try {
Multimap<InternalNode, Split> consumerAssignment = nodeSelector.computeAssignments(consumerSplits, ImmutableList.copyOf(taskMap.values()), Optional.of(stage)).getAssignments();
} catch (PrestoException e) {
assertEquals("Producer & consumer splits are not same", e.getMessage());
return;
}
assertEquals(false, true);
}
use of io.prestosql.statestore.LocalStateStoreProvider in project hetu-core by openlookeng.
the class TestHetuMetastoreGlobalCache method setUpHazelcast.
@BeforeTest
private void setUpHazelcast() throws IOException {
Set<Seed> seeds = new HashSet<>();
SeedStore mockSeedStore = mock(SeedStore.class);
Seed mockSeed = mock(Seed.class);
seeds.add(mockSeed);
SeedStoreManager mockSeedStoreManager = mock(SeedStoreManager.class);
when(mockSeedStoreManager.getSeedStore(any(SeedStoreSubType.class))).thenReturn(mockSeedStore);
when(mockSeed.getLocation()).thenReturn(LOCALHOST + ":" + PORT3);
when(mockSeedStore.get()).thenReturn(seeds);
factory = new HazelcastStateStoreFactory();
stateStoreProvider = new LocalStateStoreProvider(mockSeedStoreManager);
stateStoreProvider.addStateStoreFactory(factory);
}
Aggregations