Search in sources :

Example 11 with FixedPageSource

use of io.prestosql.spi.connector.FixedPageSource in project hetu-core by openlookeng.

the class InformationSchemaPageSourceProvider method createPageSource.

@Override
public ConnectorPageSource createPageSource(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorSplit split, ConnectorTableHandle tableHandle, List<ColumnHandle> columns) {
    InternalTable table = getInternalTable(session, tableHandle);
    List<Integer> channels = new ArrayList<>();
    for (ColumnHandle column : columns) {
        String columnName = ((InformationSchemaColumnHandle) column).getColumnName();
        int columnIndex = table.getColumnIndex(columnName);
        channels.add(columnIndex);
    }
    ImmutableList.Builder<Page> pages = ImmutableList.builder();
    for (Page page : table.getPages()) {
        Block[] blocks = new Block[channels.size()];
        for (int index = 0; index < blocks.length; index++) {
            blocks[index] = page.getBlock(channels.get(index));
        }
        pages.add(new Page(page.getPositionCount(), blocks));
    }
    return new FixedPageSource(pages.build());
}
Also used : ColumnHandle(io.prestosql.spi.connector.ColumnHandle) ImmutableList(com.google.common.collect.ImmutableList) ArrayList(java.util.ArrayList) Page(io.prestosql.spi.Page) InternalTable(io.prestosql.metadata.InternalTable) FixedPageSource(io.prestosql.spi.connector.FixedPageSource) Block(io.prestosql.spi.block.Block)

Example 12 with FixedPageSource

use of io.prestosql.spi.connector.FixedPageSource in project hetu-core by openlookeng.

the class TestDriver method testAddSourceFinish.

@Test
public void testAddSourceFinish() {
    PlanNodeId sourceId = new PlanNodeId("source");
    final List<Type> types = ImmutableList.of(VARCHAR, BIGINT, BIGINT);
    TableScanOperator source = new TableScanOperator(driverContext.addOperatorContext(99, new PlanNodeId("test"), "values"), sourceId, (session, split, table, columns, dynamicFilter) -> new FixedPageSource(rowPagesBuilder(types).addSequencePage(10, 20, 30, 40).build()), TEST_TABLE_HANDLE, ImmutableList.of(), ReuseExchangeOperator.STRATEGY.REUSE_STRATEGY_DEFAULT, new UUID(0, 0), types, false, Optional.empty(), 0, 0);
    PageConsumerOperator sink = createSinkOperator(types);
    Driver driver = Driver.createDriver(driverContext, source, sink);
    assertSame(driver.getDriverContext(), driverContext);
    assertFalse(driver.isFinished());
    assertFalse(driver.processFor(new Duration(1, TimeUnit.MILLISECONDS)).isDone());
    assertFalse(driver.isFinished());
    driver.updateSource(new TaskSource(sourceId, ImmutableSet.of(new ScheduledSplit(0, sourceId, newMockSplit())), true));
    assertFalse(driver.isFinished());
    assertTrue(driver.processFor(new Duration(1, TimeUnit.SECONDS)).isDone());
    assertTrue(driver.isFinished());
    assertTrue(sink.isFinished());
    assertTrue(source.isFinished());
}
Also used : PlanNodeId(io.prestosql.spi.plan.PlanNodeId) PageConsumerOperator(io.prestosql.testing.PageConsumerOperator) Type(io.prestosql.spi.type.Type) ScheduledSplit(io.prestosql.execution.ScheduledSplit) Duration(io.airlift.units.Duration) UUID(java.util.UUID) FixedPageSource(io.prestosql.spi.connector.FixedPageSource) TaskSource(io.prestosql.execution.TaskSource) Test(org.testng.annotations.Test)

Example 13 with FixedPageSource

use of io.prestosql.spi.connector.FixedPageSource in project boostkit-bigdata by kunpengcompute.

the class TestHiveDistributedJoinQueriesWithDynamicFiltering method testIsPartitionFiltered.

@Test
public void testIsPartitionFiltered() throws IOException {
    Properties schema = new Properties();
    ImmutableList<HivePartitionKey> partitionKeys = ImmutableList.of(new HivePartitionKey("p1", "100"), new HivePartitionKey("p2", "101"), new HivePartitionKey("p3", "__HIVE_DEFAULT_PARTITION__"));
    HiveSplitWrapper split = HiveSplitWrapper.wrap(new HiveSplit("db", "table", "partitionId", "path", 0, 50, 50, 0, schema, partitionKeys, ImmutableList.of(), OptionalInt.empty(), false, ImmutableMap.of(), Optional.empty(), false, Optional.empty(), Optional.empty(), false, ImmutableMap.of()));
    List<Long> filterValues = ImmutableList.of(1L, 50L, 100L);
    HiveColumnHandle testColumnHandle = new HiveColumnHandle("p1", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), 0, PARTITION_KEY, Optional.empty());
    Supplier<List<Map<ColumnHandle, DynamicFilter>>> dynamicFilter = createDynamicFilterSupplier(filterValues, testColumnHandle, "filter1");
    Optional<DynamicFilterSupplier> dynamicFilterSupplier = Optional.of(new DynamicFilterSupplier(dynamicFilter, System.currentTimeMillis(), 10000));
    HiveColumnHandle testColumnHandle2 = new HiveColumnHandle("p2", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), 0, PARTITION_KEY, Optional.empty());
    Supplier<List<Map<ColumnHandle, DynamicFilter>>> dynamicFilter2 = createDynamicFilterSupplier(filterValues, testColumnHandle2, "filter2");
    Optional<DynamicFilterSupplier> dynamicFilterSupplier2 = Optional.of(new DynamicFilterSupplier(dynamicFilter2, System.currentTimeMillis(), 10000));
    HiveColumnHandle testColumnHandle3 = new HiveColumnHandle("p3", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), 0, PARTITION_KEY, Optional.empty());
    Supplier<List<Map<ColumnHandle, DynamicFilter>>> dynamicFilter3 = createDynamicFilterSupplier(filterValues, testColumnHandle3, "filter3");
    Optional<DynamicFilterSupplier> dynamicFilterSupplier3 = Optional.of(new DynamicFilterSupplier(dynamicFilter3, System.currentTimeMillis(), 10000));
    HiveColumnHandle testColumnHandle4 = new HiveColumnHandle("p4", HIVE_INT, parseTypeSignature(StandardTypes.INTEGER), 0, PARTITION_KEY, Optional.empty());
    Supplier<List<Map<ColumnHandle, DynamicFilter>>> dynamicFilter4 = createDynamicFilterSupplier(filterValues, testColumnHandle4, "filter3");
    Optional<DynamicFilterSupplier> dynamicFilterSupplier4 = Optional.of(new DynamicFilterSupplier(dynamicFilter4, System.currentTimeMillis(), 0));
    HiveConfig config = new HiveConfig();
    HivePageSourceProvider provider = new HivePageSourceProvider(config, createTestHdfsEnvironment(config), getDefaultHiveRecordCursorProvider(config), getDefaultHiveDataStreamFactories(config), TYPE_MANAGER, getNoOpIndexCache(), getDefaultHiveSelectiveFactories(config));
    TestingConnectorSession session = new TestingConnectorSession(new HiveSessionProperties(config, new OrcFileWriterConfig(), new ParquetFileWriterConfig()).getSessionProperties());
    ConnectorTableHandle table = new HiveTableHandle("db", "table", ImmutableMap.of(), ImmutableList.of(), Optional.empty());
    HiveTransactionHandle transaction = new HiveTransactionHandle();
    try {
        ConnectorPageSource result = provider.createPageSource(transaction, session, split, table, ImmutableList.of(testColumnHandle), dynamicFilterSupplier);
        assertFalse(result instanceof FixedPageSource);
    } catch (Exception e) {
        assertTrue(e instanceof PrestoException);
    }
    try {
        ConnectorPageSource result = provider.createPageSource(transaction, session, split, table, ImmutableList.of(testColumnHandle2), dynamicFilterSupplier2);
        assertTrue(result instanceof FixedPageSource);
    } catch (Exception e) {
        fail("A FixedPageSource object should have been created");
    }
    try {
        ConnectorPageSource result = provider.createPageSource(transaction, session, split, table, ImmutableList.of(testColumnHandle3), dynamicFilterSupplier3);
        assertFalse(result instanceof FixedPageSource);
    } catch (Exception e) {
        assertTrue(e instanceof PrestoException);
    }
    try {
        ConnectorPageSource result = provider.createPageSource(transaction, session, split, table, ImmutableList.of(testColumnHandle4), dynamicFilterSupplier4);
        assertFalse(result instanceof FixedPageSource);
    } catch (Exception e) {
        assertTrue(e instanceof PrestoException);
    }
}
Also used : PrestoException(io.prestosql.spi.PrestoException) Properties(java.util.Properties) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) DynamicFilterSupplier(io.prestosql.spi.dynamicfilter.DynamicFilterSupplier) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) DynamicFilter(io.prestosql.spi.dynamicfilter.DynamicFilter) TestingConnectorSession(io.prestosql.testing.TestingConnectorSession) FixedPageSource(io.prestosql.spi.connector.FixedPageSource) PrestoException(io.prestosql.spi.PrestoException) IOException(java.io.IOException) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) Test(org.testng.annotations.Test)

Aggregations

FixedPageSource (io.prestosql.spi.connector.FixedPageSource)13 Test (org.testng.annotations.Test)9 PlanNodeId (io.prestosql.spi.plan.PlanNodeId)8 ImmutableList (com.google.common.collect.ImmutableList)5 Split (io.prestosql.metadata.Split)5 Page (io.prestosql.spi.Page)5 ColumnHandle (io.prestosql.spi.connector.ColumnHandle)5 DataSize (io.airlift.units.DataSize)4 Duration (io.airlift.units.Duration)4 CursorProcessor (io.prestosql.operator.project.CursorProcessor)4 PageProcessor (io.prestosql.operator.project.PageProcessor)4 CatalogName (io.prestosql.spi.connector.CatalogName)4 ConnectorPageSource (io.prestosql.spi.connector.ConnectorPageSource)4 RowExpression (io.prestosql.spi.relation.RowExpression)4 Type (io.prestosql.spi.type.Type)4 TestingSplit (io.prestosql.testing.TestingSplit)4 List (java.util.List)4 UUID (java.util.UUID)4 ScheduledSplit (io.prestosql.execution.ScheduledSplit)3 TaskSource (io.prestosql.execution.TaskSource)3