use of io.prestosql.execution.SqlStageExecution in project hetu-core by openlookeng.
the class TestSplitFiltering method testGetFilteredSplit.
/**
* This test will not actually filter any splits since the indexes will not be found,
* instead it's just testing the flow. The actual filtering is tested in other classes.
*/
@Test
public void testGetFilteredSplit() {
PropertyService.setProperty(HetuConstant.FILTER_ENABLED, true);
PropertyService.setProperty(HetuConstant.INDEXSTORE_URI, "/tmp/hetu/indices");
PropertyService.setProperty(HetuConstant.INDEXSTORE_FILESYSTEM_PROFILE, "local-config-default");
PropertyService.setProperty(HetuConstant.FILTER_CACHE_TTL, new Duration(10, TimeUnit.MINUTES));
PropertyService.setProperty(HetuConstant.FILTER_CACHE_LOADING_DELAY, new Duration(5000, TimeUnit.MILLISECONDS));
PropertyService.setProperty(HetuConstant.FILTER_CACHE_LOADING_THREADS, 2L);
RowExpression expression = PlanBuilder.comparison(OperatorType.EQUAL, new VariableReferenceExpression("a", VarcharType.VARCHAR), new ConstantExpression(utf8Slice("test_value"), VarcharType.VARCHAR));
SqlStageExecution stage = TestUtil.getTestStage(expression);
List<Split> mockSplits = new ArrayList<>();
MockSplit mock = new MockSplit("hdfs://hacluster/AppData/BIProd/DWD/EVT/bogus_table/000000_0", 0, 10, 0);
MockSplit mock1 = new MockSplit("hdfs://hacluster/AppData/BIProd/DWD/EVT/bogus_table/000000_1", 0, 10, 0);
MockSplit mock2 = new MockSplit("hdfs://hacluster/AppData/BIProd/DWD/EVT/bogus_table/000001_0", 0, 10, 0);
MockSplit mock3 = new MockSplit("hdfs://hacluster/AppData/BIProd/DWD/EVT/bogus_table/000000_4", 0, 10, 0);
mockSplits.add(new Split(new CatalogName("bogus_catalog"), mock, Lifespan.taskWide()));
mockSplits.add(new Split(new CatalogName("bogus_catalog"), mock1, Lifespan.taskWide()));
mockSplits.add(new Split(new CatalogName("bogus_catalog"), mock2, Lifespan.taskWide()));
mockSplits.add(new Split(new CatalogName("bogus_catalog"), mock3, Lifespan.taskWide()));
SplitSource.SplitBatch nextSplits = new SplitSource.SplitBatch(mockSplits, true);
HeuristicIndexerManager indexerManager = new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager());
Pair<Optional<RowExpression>, Map<Symbol, ColumnHandle>> pair = SplitFiltering.getExpression(stage);
List<Split> filteredSplits = SplitFiltering.getFilteredSplit(pair.getFirst(), SplitFiltering.getFullyQualifiedName(stage), pair.getSecond(), nextSplits, indexerManager);
assertNotNull(filteredSplits);
assertEquals(filteredSplits.size(), 4);
}
use of io.prestosql.execution.SqlStageExecution in project hetu-core by openlookeng.
the class TestSourcePartitionedScheduler method testScheduleSplitsBlock.
@Test
public void testScheduleSplitsBlock() {
StageExecutionPlan plan = createPlan(createFixedSplitSource(80, TestingSplit::createRemoteSplit));
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap);
StageScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 1);
// schedule first 60 splits, which will cause the scheduler to block
for (int i = 0; i <= 60; i++) {
ScheduleResult scheduleResult = scheduler.schedule();
assertFalse(scheduleResult.isFinished());
// blocks at 20 per node
assertEquals(scheduleResult.getBlocked().isDone(), i != 60);
// first three splits create new tasks
assertEquals(scheduleResult.getNewTasks().size(), i < 3 ? 1 : 0);
assertEquals(stage.getAllTasks().size(), i < 3 ? i + 1 : 3);
assertPartitionedSplitCount(stage, min(i + 1, 60));
}
for (RemoteTask remoteTask : stage.getAllTasks()) {
assertEquals(remoteTask.getPartitionedSplitCount(), 20);
}
// todo rewrite MockRemoteTask to fire a tate transition when splits are cleared, and then validate blocked future completes
// drop the 20 splits from one node
((MockRemoteTask) stage.getAllTasks().get(0)).clearSplits();
// schedule remaining 20 splits
for (int i = 0; i < 20; i++) {
ScheduleResult scheduleResult = scheduler.schedule();
// finishes when last split is fetched
if (i == 19) {
assertEffectivelyFinished(scheduleResult, scheduler);
} else {
assertFalse(scheduleResult.isFinished());
}
// does not block again
assertTrue(scheduleResult.getBlocked().isDone());
// no additional tasks will be created
assertEquals(scheduleResult.getNewTasks().size(), 0);
assertEquals(stage.getAllTasks().size(), 3);
// we dropped 20 splits so start at 40 and count to 60
assertPartitionedSplitCount(stage, min(i + 41, 60));
}
for (RemoteTask remoteTask : stage.getAllTasks()) {
assertEquals(remoteTask.getPartitionedSplitCount(), 20);
}
stage.abort();
}
use of io.prestosql.execution.SqlStageExecution in project hetu-core by openlookeng.
the class TestSourcePartitionedScheduler method createSqlStageExecution.
private SqlStageExecution createSqlStageExecution(StageExecutionPlan tableScanPlan, NodeTaskMap nodeTaskMap) {
StageId stageId = new StageId(new QueryId("query"), 0);
SqlStageExecution stage = SqlStageExecution.createSqlStageExecution(stageId, locationFactory.createStageLocation(stageId), tableScanPlan.getFragment(), tableScanPlan.getTables(), new MockRemoteTaskFactory(queryExecutor, scheduledExecutor), TEST_SESSION, true, nodeTaskMap, queryExecutor, new NoOpFailureDetector(), new SplitSchedulerStats(), new DynamicFilterService(new LocalStateStoreProvider(seedStoreManager)), new QuerySnapshotManager(stageId.getQueryId(), NOOP_SNAPSHOT_UTILS, TEST_SESSION));
stage.setOutputBuffers(createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds());
return stage;
}
use of io.prestosql.execution.SqlStageExecution in project hetu-core by openlookeng.
the class TestSourcePartitionedScheduler method testScheduleSplitsOneAtATime.
@Test
public void testScheduleSplitsOneAtATime() {
StageExecutionPlan plan = createPlan(createFixedSplitSource(60, TestingSplit::createRemoteSplit));
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap);
try (StageScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 1)) {
for (int i = 0; i < 60; i++) {
ScheduleResult scheduleResult = scheduler.schedule();
// only finishes when last split is fetched
if (i == 59) {
assertEffectivelyFinished(scheduleResult, scheduler);
} else {
assertFalse(scheduleResult.isFinished());
}
// never blocks
assertTrue(scheduleResult.getBlocked().isDone());
// first three splits create new tasks
assertEquals(scheduleResult.getNewTasks().size(), i < 3 ? 1 : 0);
assertEquals(stage.getAllTasks().size(), i < 3 ? i + 1 : 3);
assertPartitionedSplitCount(stage, min(i + 1, 60));
}
for (RemoteTask remoteTask : stage.getAllTasks()) {
assertEquals(remoteTask.getPartitionedSplitCount(), 20);
}
stage.abort();
}
}
use of io.prestosql.execution.SqlStageExecution in project hetu-core by openlookeng.
the class TestSourcePartitionedScheduler method testScheduleNoSplits.
@Test
public void testScheduleNoSplits() {
StageExecutionPlan plan = createPlan(createFixedSplitSource(0, TestingSplit::createRemoteSplit));
NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap);
StageScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 1);
ScheduleResult scheduleResult = scheduler.schedule();
assertEquals(scheduleResult.getNewTasks().size(), 1);
assertEffectivelyFinished(scheduleResult, scheduler);
stage.abort();
}
Aggregations