Search in sources :

Example 1 with SqlStageExecution

use of io.prestosql.execution.SqlStageExecution in project hetu-core by openlookeng.

the class TestSplitFiltering method testGetFilteredSplit.

/**
 * This test will not actually filter any splits since the indexes will not be found,
 * instead it's just testing the flow. The actual filtering is tested in other classes.
 */
@Test
public void testGetFilteredSplit() {
    PropertyService.setProperty(HetuConstant.FILTER_ENABLED, true);
    PropertyService.setProperty(HetuConstant.INDEXSTORE_URI, "/tmp/hetu/indices");
    PropertyService.setProperty(HetuConstant.INDEXSTORE_FILESYSTEM_PROFILE, "local-config-default");
    PropertyService.setProperty(HetuConstant.FILTER_CACHE_TTL, new Duration(10, TimeUnit.MINUTES));
    PropertyService.setProperty(HetuConstant.FILTER_CACHE_LOADING_DELAY, new Duration(5000, TimeUnit.MILLISECONDS));
    PropertyService.setProperty(HetuConstant.FILTER_CACHE_LOADING_THREADS, 2L);
    RowExpression expression = PlanBuilder.comparison(OperatorType.EQUAL, new VariableReferenceExpression("a", VarcharType.VARCHAR), new ConstantExpression(utf8Slice("test_value"), VarcharType.VARCHAR));
    SqlStageExecution stage = TestUtil.getTestStage(expression);
    List<Split> mockSplits = new ArrayList<>();
    MockSplit mock = new MockSplit("hdfs://hacluster/AppData/BIProd/DWD/EVT/bogus_table/000000_0", 0, 10, 0);
    MockSplit mock1 = new MockSplit("hdfs://hacluster/AppData/BIProd/DWD/EVT/bogus_table/000000_1", 0, 10, 0);
    MockSplit mock2 = new MockSplit("hdfs://hacluster/AppData/BIProd/DWD/EVT/bogus_table/000001_0", 0, 10, 0);
    MockSplit mock3 = new MockSplit("hdfs://hacluster/AppData/BIProd/DWD/EVT/bogus_table/000000_4", 0, 10, 0);
    mockSplits.add(new Split(new CatalogName("bogus_catalog"), mock, Lifespan.taskWide()));
    mockSplits.add(new Split(new CatalogName("bogus_catalog"), mock1, Lifespan.taskWide()));
    mockSplits.add(new Split(new CatalogName("bogus_catalog"), mock2, Lifespan.taskWide()));
    mockSplits.add(new Split(new CatalogName("bogus_catalog"), mock3, Lifespan.taskWide()));
    SplitSource.SplitBatch nextSplits = new SplitSource.SplitBatch(mockSplits, true);
    HeuristicIndexerManager indexerManager = new HeuristicIndexerManager(new FileSystemClientManager(), new HetuMetaStoreManager());
    Pair<Optional<RowExpression>, Map<Symbol, ColumnHandle>> pair = SplitFiltering.getExpression(stage);
    List<Split> filteredSplits = SplitFiltering.getFilteredSplit(pair.getFirst(), SplitFiltering.getFullyQualifiedName(stage), pair.getSecond(), nextSplits, indexerManager);
    assertNotNull(filteredSplits);
    assertEquals(filteredSplits.size(), 4);
}
Also used : Optional(java.util.Optional) ConstantExpression(io.prestosql.spi.relation.ConstantExpression) ArrayList(java.util.ArrayList) RowExpression(io.prestosql.spi.relation.RowExpression) Duration(io.airlift.units.Duration) MockSplit(io.prestosql.utils.MockSplit) SqlStageExecution(io.prestosql.execution.SqlStageExecution) FileSystemClientManager(io.prestosql.filesystem.FileSystemClientManager) VariableReferenceExpression(io.prestosql.spi.relation.VariableReferenceExpression) CatalogName(io.prestosql.spi.connector.CatalogName) Split(io.prestosql.metadata.Split) MockSplit(io.prestosql.utils.MockSplit) SplitSource(io.prestosql.split.SplitSource) HetuMetaStoreManager(io.prestosql.metastore.HetuMetaStoreManager) HashMap(java.util.HashMap) Map(java.util.Map) Test(org.testng.annotations.Test)

Example 2 with SqlStageExecution

use of io.prestosql.execution.SqlStageExecution in project hetu-core by openlookeng.

the class TestSourcePartitionedScheduler method testScheduleSplitsBlock.

@Test
public void testScheduleSplitsBlock() {
    StageExecutionPlan plan = createPlan(createFixedSplitSource(80, TestingSplit::createRemoteSplit));
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap);
    StageScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 1);
    // schedule first 60 splits, which will cause the scheduler to block
    for (int i = 0; i <= 60; i++) {
        ScheduleResult scheduleResult = scheduler.schedule();
        assertFalse(scheduleResult.isFinished());
        // blocks at 20 per node
        assertEquals(scheduleResult.getBlocked().isDone(), i != 60);
        // first three splits create new tasks
        assertEquals(scheduleResult.getNewTasks().size(), i < 3 ? 1 : 0);
        assertEquals(stage.getAllTasks().size(), i < 3 ? i + 1 : 3);
        assertPartitionedSplitCount(stage, min(i + 1, 60));
    }
    for (RemoteTask remoteTask : stage.getAllTasks()) {
        assertEquals(remoteTask.getPartitionedSplitCount(), 20);
    }
    // todo rewrite MockRemoteTask to fire a tate transition when splits are cleared, and then validate blocked future completes
    // drop the 20 splits from one node
    ((MockRemoteTask) stage.getAllTasks().get(0)).clearSplits();
    // schedule remaining 20 splits
    for (int i = 0; i < 20; i++) {
        ScheduleResult scheduleResult = scheduler.schedule();
        // finishes when last split is fetched
        if (i == 19) {
            assertEffectivelyFinished(scheduleResult, scheduler);
        } else {
            assertFalse(scheduleResult.isFinished());
        }
        // does not block again
        assertTrue(scheduleResult.getBlocked().isDone());
        // no additional tasks will be created
        assertEquals(scheduleResult.getNewTasks().size(), 0);
        assertEquals(stage.getAllTasks().size(), 3);
        // we dropped 20 splits so start at 40 and count to 60
        assertPartitionedSplitCount(stage, min(i + 41, 60));
    }
    for (RemoteTask remoteTask : stage.getAllTasks()) {
        assertEquals(remoteTask.getPartitionedSplitCount(), 20);
    }
    stage.abort();
}
Also used : NodeTaskMap(io.prestosql.execution.NodeTaskMap) StageExecutionPlan(io.prestosql.sql.planner.StageExecutionPlan) MockRemoteTask(io.prestosql.execution.MockRemoteTaskFactory.MockRemoteTask) MockRemoteTask(io.prestosql.execution.MockRemoteTaskFactory.MockRemoteTask) RemoteTask(io.prestosql.execution.RemoteTask) SqlStageExecution(io.prestosql.execution.SqlStageExecution) SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler(io.prestosql.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler) Test(org.testng.annotations.Test)

Example 3 with SqlStageExecution

use of io.prestosql.execution.SqlStageExecution in project hetu-core by openlookeng.

the class TestSourcePartitionedScheduler method createSqlStageExecution.

private SqlStageExecution createSqlStageExecution(StageExecutionPlan tableScanPlan, NodeTaskMap nodeTaskMap) {
    StageId stageId = new StageId(new QueryId("query"), 0);
    SqlStageExecution stage = SqlStageExecution.createSqlStageExecution(stageId, locationFactory.createStageLocation(stageId), tableScanPlan.getFragment(), tableScanPlan.getTables(), new MockRemoteTaskFactory(queryExecutor, scheduledExecutor), TEST_SESSION, true, nodeTaskMap, queryExecutor, new NoOpFailureDetector(), new SplitSchedulerStats(), new DynamicFilterService(new LocalStateStoreProvider(seedStoreManager)), new QuerySnapshotManager(stageId.getQueryId(), NOOP_SNAPSHOT_UTILS, TEST_SESSION));
    stage.setOutputBuffers(createInitialEmptyOutputBuffers(PARTITIONED).withBuffer(OUT, 0).withNoMoreBufferIds());
    return stage;
}
Also used : NoOpFailureDetector(io.prestosql.failuredetector.NoOpFailureDetector) LocalStateStoreProvider(io.prestosql.statestore.LocalStateStoreProvider) StageId(io.prestosql.execution.StageId) QueryId(io.prestosql.spi.QueryId) DynamicFilterService(io.prestosql.dynamicfilter.DynamicFilterService) SqlStageExecution(io.prestosql.execution.SqlStageExecution) MockRemoteTaskFactory(io.prestosql.execution.MockRemoteTaskFactory) QuerySnapshotManager(io.prestosql.snapshot.QuerySnapshotManager)

Example 4 with SqlStageExecution

use of io.prestosql.execution.SqlStageExecution in project hetu-core by openlookeng.

the class TestSourcePartitionedScheduler method testScheduleSplitsOneAtATime.

@Test
public void testScheduleSplitsOneAtATime() {
    StageExecutionPlan plan = createPlan(createFixedSplitSource(60, TestingSplit::createRemoteSplit));
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap);
    try (StageScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 1)) {
        for (int i = 0; i < 60; i++) {
            ScheduleResult scheduleResult = scheduler.schedule();
            // only finishes when last split is fetched
            if (i == 59) {
                assertEffectivelyFinished(scheduleResult, scheduler);
            } else {
                assertFalse(scheduleResult.isFinished());
            }
            // never blocks
            assertTrue(scheduleResult.getBlocked().isDone());
            // first three splits create new tasks
            assertEquals(scheduleResult.getNewTasks().size(), i < 3 ? 1 : 0);
            assertEquals(stage.getAllTasks().size(), i < 3 ? i + 1 : 3);
            assertPartitionedSplitCount(stage, min(i + 1, 60));
        }
        for (RemoteTask remoteTask : stage.getAllTasks()) {
            assertEquals(remoteTask.getPartitionedSplitCount(), 20);
        }
        stage.abort();
    }
}
Also used : NodeTaskMap(io.prestosql.execution.NodeTaskMap) StageExecutionPlan(io.prestosql.sql.planner.StageExecutionPlan) MockRemoteTask(io.prestosql.execution.MockRemoteTaskFactory.MockRemoteTask) RemoteTask(io.prestosql.execution.RemoteTask) SqlStageExecution(io.prestosql.execution.SqlStageExecution) SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler(io.prestosql.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler) Test(org.testng.annotations.Test)

Example 5 with SqlStageExecution

use of io.prestosql.execution.SqlStageExecution in project hetu-core by openlookeng.

the class TestSourcePartitionedScheduler method testScheduleNoSplits.

@Test
public void testScheduleNoSplits() {
    StageExecutionPlan plan = createPlan(createFixedSplitSource(0, TestingSplit::createRemoteSplit));
    NodeTaskMap nodeTaskMap = new NodeTaskMap(finalizerService);
    SqlStageExecution stage = createSqlStageExecution(plan, nodeTaskMap);
    StageScheduler scheduler = getSourcePartitionedScheduler(plan, stage, nodeManager, nodeTaskMap, 1);
    ScheduleResult scheduleResult = scheduler.schedule();
    assertEquals(scheduleResult.getNewTasks().size(), 1);
    assertEffectivelyFinished(scheduleResult, scheduler);
    stage.abort();
}
Also used : NodeTaskMap(io.prestosql.execution.NodeTaskMap) StageExecutionPlan(io.prestosql.sql.planner.StageExecutionPlan) SqlStageExecution(io.prestosql.execution.SqlStageExecution) SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler(io.prestosql.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler) Test(org.testng.annotations.Test)

Aggregations

SqlStageExecution (io.prestosql.execution.SqlStageExecution)22 NodeTaskMap (io.prestosql.execution.NodeTaskMap)14 StageExecutionPlan (io.prestosql.sql.planner.StageExecutionPlan)12 Test (org.testng.annotations.Test)12 SqlStageExecution.createSqlStageExecution (io.prestosql.execution.SqlStageExecution.createSqlStageExecution)10 StageId (io.prestosql.execution.StageId)9 SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler (io.prestosql.execution.scheduler.SourcePartitionedScheduler.newSourcePartitionedSchedulerAsStageScheduler)9 RemoteTask (io.prestosql.execution.RemoteTask)8 DynamicFilterService (io.prestosql.dynamicfilter.DynamicFilterService)7 InternalNode (io.prestosql.metadata.InternalNode)7 Split (io.prestosql.metadata.Split)7 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)7 MockRemoteTaskFactory (io.prestosql.execution.MockRemoteTaskFactory)6 QuerySnapshotManager (io.prestosql.snapshot.QuerySnapshotManager)6 MockRemoteTask (io.prestosql.execution.MockRemoteTaskFactory.MockRemoteTask)5 NoOpFailureDetector (io.prestosql.failuredetector.NoOpFailureDetector)5 FileSystemClientManager (io.prestosql.filesystem.FileSystemClientManager)5 SeedStoreManager (io.prestosql.seedstore.SeedStoreManager)5 PrestoException (io.prestosql.spi.PrestoException)5 QueryId (io.prestosql.spi.QueryId)5