Search in sources :

Example 1 with FixedPageSource

use of io.prestosql.spi.connector.FixedPageSource in project hetu-core by openlookeng.

the class MemoryPageSourceProvider method createPageSource.

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, ConnectorTableHandle table, List<ColumnHandle> columns, Optional<DynamicFilterSupplier> dynamicFilterSupplier) {
    MemorySplit memorySplit = (MemorySplit) split;
    long tableId = memorySplit.getTable();
    int logicalPartNumber = memorySplit.getLogicalPartNum();
    long expectedRows = memorySplit.getExpectedRows();
    MemoryTableHandle memoryTable = (MemoryTableHandle) table;
    OptionalDouble sampleRatio = memoryTable.getSampleRatio();
    TupleDomain<ColumnHandle> predicate = memoryTable.getPredicate();
    // Commenting for Dynamic filter changes
    List<Integer> columnIndexes = columns.stream().map(MemoryColumnHandle.class::cast).map(MemoryColumnHandle::getColumnIndex).collect(toList());
    List<Page> pages = pagesStore.getPages(tableId, logicalPartNumber, columnIndexes, expectedRows, memorySplit.getLimit(), sampleRatio, predicate);
    if (dynamicFilterSupplier.isPresent()) {
        return new FixedPageSource(pages.stream().map(page -> applyFilter(page, dynamicFilterSupplier, columns)).collect(toList()));
    } else {
        return new FixedPageSource(pages);
    }
}
Also used : ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Page(io.prestosql.spi.Page) FixedPageSource(io.prestosql.spi.connector.FixedPageSource) OptionalDouble(java.util.OptionalDouble)

Example 2 with FixedPageSource

use of io.prestosql.spi.connector.FixedPageSource in project hetu-core by openlookeng.

the class TestSystemMemoryBlocking method testTableScanSystemMemoryBlocking.

@Test
public void testTableScanSystemMemoryBlocking() {
    PlanNodeId sourceId = new PlanNodeId("source");
    final List<Type> types = ImmutableList.of(VARCHAR);
    TableScanOperator source = new TableScanOperator(driverContext.addOperatorContext(1, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).addSequencePage(10, 1).build()), TEST_TABLE_HANDLE, ImmutableList.of(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), types, false, Optional.empty(), 0, 0);
    PageConsumerOperator sink = createSinkOperator(types);
    Driver driver = Driver.createDriver(driverContext, source, sink);
    assertSame(driver.getDriverContext(), driverContext);
    assertFalse(driver.isFinished());
    Split testSplit = new Split(new CatalogName("test"), new TestSplit(), Lifespan.taskWide());
    driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, testSplit)), true));
    ListenableFuture<?> blocked = driver.processFor(new Duration(1, NANOSECONDS));
    // the driver shouldn't block in the first call as it will be able to move a page between source and the sink operator
    // but the operator should be blocked
    assertTrue(blocked.isDone());
    assertFalse(source.getOperatorContext().isWaitingForMemory().isDone());
    // and they should stay blocked until more memory becomes available
    for (int i = 0; i < 10; i++) {
        blocked = driver.processFor(new Duration(1, NANOSECONDS));
        assertFalse(blocked.isDone());
        assertFalse(source.getOperatorContext().isWaitingForMemory().isDone());
    }
    // free up some memory
    memoryPool.free(QUERY_ID, "test", memoryPool.getReservedBytes());
    // the operator should be unblocked
    assertTrue(source.getOperatorContext().isWaitingForMemory().isDone());
    // the driver shouldn't be blocked
    blocked = driver.processFor(new Duration(1, NANOSECONDS));
    assertTrue(blocked.isDone());
}
Also used : TableScanOperator(io.prestosql.operator.TableScanOperator) ScheduledSplit(io.prestosql.execution.ScheduledSplit) Driver(io.prestosql.operator.Driver) Duration(io.airlift.units.Duration) FixedPageSource(io.prestosql.spi.connector.FixedPageSource) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) PageConsumerOperator(io.prestosql.testing.PageConsumerOperator) Type(io.prestosql.spi.type.Type) CatalogName(io.prestosql.spi.connector.CatalogName) UUID(java.util.UUID) ConnectorSplit(io.prestosql.spi.connector.ConnectorSplit) Split(io.prestosql.metadata.Split) ScheduledSplit(io.prestosql.execution.ScheduledSplit) TaskSource(io.prestosql.execution.TaskSource) Test(org.testng.annotations.Test)

Example 3 with FixedPageSource

use of io.prestosql.spi.connector.FixedPageSource in project hetu-core by openlookeng.

the class TestDriver method testMemoryRevocationRace.

@Test
public void testMemoryRevocationRace() {
    List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
    TableScanOperator source = new AlwaysBlockedMemoryRevokingTableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "scan"), new PlanNodeId("source"), (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()), TEST_TABLE_HANDLE, ImmutableList.of());
    Driver driver = Driver.createDriver(driverContext, source, createSinkOperator(types));
    // the table scan operator will request memory revocation with requestMemoryRevoking()
    // while the driver is still not done with the processFor() method and before it moves to
    // updateDriverBlockedFuture() method.
    assertTrue(driver.processFor(new Duration(100, TimeUnit.MILLISECONDS)).isDone());
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Type(io.prestosql.spi.type.Type) Duration(io.airlift.units.Duration) FixedPageSource(io.prestosql.spi.connector.FixedPageSource) Test(org.testng.annotations.Test)

Example 4 with FixedPageSource

use of io.prestosql.spi.connector.FixedPageSource in project hetu-core by openlookeng.

the class TestDriver method testBrokenOperatorAddSource.

@Test
public void testBrokenOperatorAddSource() throws Exception {
    PlanNodeId sourceId = new PlanNodeId("source");
    final List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
    // create a table scan operator that does not block, which will cause the driver loop to busy wait
    TableScanOperator source = new NotBlockedTableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()), TEST_TABLE_HANDLE, ImmutableList.of());
    BrokenOperator brokenOperator = new BrokenOperator(driverContext.addOperatorContext(0, new PlanNodeId("test"), "source"));
    final Driver driver = Driver.createDriver(driverContext, source, brokenOperator);
    // block thread in operator processing
    Future<Boolean> driverProcessFor = executor.submit(new Callable<Boolean>() {

        @Override
        public Boolean call() {
            return driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone();
        }
    });
    brokenOperator.waitForLocked();
    assertSame(driver.getDriverContext(), driverContext);
    assertFalse(driver.isFinished());
    // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
    assertTrue(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());
    driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));
    assertFalse(driver.isFinished());
    // processFor always returns NOT_BLOCKED, because DriveLockResult was not acquired
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertFalse(driver.isFinished());
    driver.close();
    assertTrue(driver.isFinished());
    try {
        driverProcessFor.get(1, TimeUnit.SECONDS);
        fail("Expected InterruptedException");
    } catch (ExecutionException e) {
        assertDriverInterrupted(e.getCause());
    }
}
Also used : ScheduledSplit(io.prestosql.execution.ScheduledSplit) Duration(io.airlift.units.Duration) FixedPageSource(io.prestosql.spi.connector.FixedPageSource) PlanNodeId(io.prestosql.spi.plan.PlanNodeId) Type(io.prestosql.spi.type.Type) ExecutionException(java.util.concurrent.ExecutionException) TaskSource(io.prestosql.execution.TaskSource) Test(org.testng.annotations.Test)

Example 5 with FixedPageSource

use of io.prestosql.spi.connector.FixedPageSource in project hetu-core by openlookeng.

the class TestHiveDistributedJoinQueriesWithDynamicFiltering method testIsPartitionFiltered.

@Test
public void testIsPartitionFiltered() throws IOException {
    Properties schema = new Properties();
    ImmutableList<HivePartitionKey> partitionKeys = ImmutableList.of(new HivePartitionKey("p1", "100"), new HivePartitionKey("p2", "101"), new HivePartitionKey("p3", "__HIVE_DEFAULT_PARTITION__"));
    HiveSplitWrapper split = HiveSplitWrapper.wrap(new HiveSplit("db", "table", "partitionId", "path", 0, 50, 50, 0, schema, partitionKeys, ImmutableList.of(), OptionalInt.empty(), false, ImmutableMap.of(), Optional.empty(), false, Optional.empty(), Optional.empty(), false, ImmutableMap.of()));
    List<Long> filterValues = ImmutableList.of(1L, 50L, 100L);
    HiveColumnHandle testColumnHandle = new HiveColumnHandle("p1", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), 0, PARTITION_KEY, Optional.empty());
    Supplier<List<Map<ColumnHandle, DynamicFilter>>> dynamicFilter = createDynamicFilterSupplier(filterValues, testColumnHandle, "filter1");
    Optional<DynamicFilterSupplier> dynamicFilterSupplier = Optional.of(new DynamicFilterSupplier(dynamicFilter, System.currentTimeMillis(), 10000));
    HiveColumnHandle testColumnHandle2 = new HiveColumnHandle("p2", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), 0, PARTITION_KEY, Optional.empty());
    Supplier<List<Map<ColumnHandle, DynamicFilter>>> dynamicFilter2 = createDynamicFilterSupplier(filterValues, testColumnHandle2, "filter2");
    Optional<DynamicFilterSupplier> dynamicFilterSupplier2 = Optional.of(new DynamicFilterSupplier(dynamicFilter2, System.currentTimeMillis(), 10000));
    HiveColumnHandle testColumnHandle3 = new HiveColumnHandle("p3", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), 0, PARTITION_KEY, Optional.empty());
    Supplier<List<Map<ColumnHandle, DynamicFilter>>> dynamicFilter3 = createDynamicFilterSupplier(filterValues, testColumnHandle3, "filter3");
    Optional<DynamicFilterSupplier> dynamicFilterSupplier3 = Optional.of(new DynamicFilterSupplier(dynamicFilter3, System.currentTimeMillis(), 10000));
    HiveColumnHandle testColumnHandle4 = new HiveColumnHandle("p4", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), 0, PARTITION_KEY, Optional.empty());
    Supplier<List<Map<ColumnHandle, DynamicFilter>>> dynamicFilter4 = createDynamicFilterSupplier(filterValues, testColumnHandle4, "filter3");
    Optional<DynamicFilterSupplier> dynamicFilterSupplier4 = Optional.of(new DynamicFilterSupplier(dynamicFilter4, System.currentTimeMillis(), 0));
    HiveConfig config = new HiveConfig();
    HivePageSourceProvider provider = new HivePageSourceProvider(config, createTestHdfsEnvironment(config), getDefaultHiveRecordCursorProvider(config), getDefaultHiveDataStreamFactories(config), TYPE_MANAGER, getNoOpIndexCache(), getDefaultHiveSelectiveFactories(config));
    TestingConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(config, new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties());
    ConnectorTableHandle table = new HiveTableHandle("db", "table", ImmutableMap.of(), ImmutableList.of(), Optional.empty());
    HiveTransactionHandle transaction = new HiveTransactionHandle();
    try {
        ConnectorPageSource result = provider.createPageSource(transaction, session, split, table, ImmutableList.of(testColumnHandle), dynamicFilterSupplier);
        assertFalse(result instanceof FixedPageSource);
    } catch (Exception e) {
        assertTrue(e instanceof PrestoException);
    }
    try {
        ConnectorPageSource result = provider.createPageSource(transaction, session, split, table, ImmutableList.of(testColumnHandle2), dynamicFilterSupplier2);
        assertTrue(result instanceof FixedPageSource);
    } catch (Exception e) {
        fail("A FixedPageSource object should have been created");
    }
    try {
        ConnectorPageSource result = provider.createPageSource(transaction, session, split, table, ImmutableList.of(testColumnHandle3), dynamicFilterSupplier3);
        assertFalse(result instanceof FixedPageSource);
    } catch (Exception e) {
        assertTrue(e instanceof PrestoException);
    }
    try {
        ConnectorPageSource result = provider.createPageSource(transaction, session, split, table, ImmutableList.of(testColumnHandle4), dynamicFilterSupplier4);
        assertFalse(result instanceof FixedPageSource);
    } catch (Exception e) {
        assertTrue(e instanceof PrestoException);
    }
}
Also used : PrestoException(io.prestosql.spi.PrestoException) Properties(java.util.Properties) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) DynamicFilterSupplier(io.prestosql.spi.dynamicfilter.DynamicFilterSupplier) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) DynamicFilter(io.prestosql.spi.dynamicfilter.DynamicFilter) TestingConnectorSession(io.prestosql.testing.TestingConnectorSession) FixedPageSource(io.prestosql.spi.connector.FixedPageSource) PrestoException(io.prestosql.spi.PrestoException) IOException(java.io.IOException) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) Test(org.testng.annotations.Test)

Aggregations

FixedPageSource (io.prestosql.spi.connector.FixedPageSource)12 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)8 Test (org.testng.annotations.Test)8 Split (io.prestosql.metadata.Split)5 Page (io.prestosql.spi.Page)5 ImmutableList (com.google.common.collect.ImmutableList)4 DataSize (io.airlift.units.DataSize)4 Duration (io.airlift.units.Duration)4 CursorProcessor (io.prestosql.operator.project.CursorProcessor)4 PageProcessor (io.prestosql.operator.project.PageProcessor)4 CatalogName (io.prestosql.spi.connector.CatalogName)4 RowExpression (io.prestosql.spi.relation.RowExpression)4 Type (io.prestosql.spi.type.Type)4 TestingSplit (io.prestosql.testing.TestingSplit)4 UUID (java.util.UUID)4 ScheduledSplit (io.prestosql.execution.ScheduledSplit)3 TaskSource (io.prestosql.execution.TaskSource)3 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)3 OperatorAssertion.toMaterializedResult (io.prestosql.operator.OperatorAssertion.toMaterializedResult)2 Block (io.prestosql.spi.block.Block)2