Examples with TableScanRedirectApplicationResult io.trino.spi.connector.TableScanRedirectApplicationResult used on opensource projects

Example 1 with TableScanRedirectApplicationResult

use of io.trino.spi.connector.TableScanRedirectApplicationResult in project trino by trinodb.

the class AbstractTestHive method setup.

protected final void setup(String databaseName, HiveConfig hiveConfig, HiveMetastore hiveMetastore, HdfsEnvironment hdfsConfiguration) {
    setupHive(databaseName);
    metastoreClient = hiveMetastore;
    hdfsEnvironment = hdfsConfiguration;
    HivePartitionManager partitionManager = new HivePartitionManager(hiveConfig);
    locationService = new HiveLocationService(hdfsEnvironment);
    JsonCodec<PartitionUpdate> partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class);
    metadataFactory = new HiveMetadataFactory(new CatalogName("hive"), HiveMetastoreFactory.ofInstance(metastoreClient), hdfsEnvironment, partitionManager, 10, 10, 10, false, false, false, true, true, false, 1000, Optional.empty(), true, TESTING_TYPE_MANAGER, NOOP_METADATA_PROVIDER, locationService, partitionUpdateCodec, newFixedThreadPool(2), heartbeatService, TEST_SERVER_VERSION, (session, tableHandle) -> {
        if (!tableHandle.getTableName().contains("apply_redirection_tester")) {
            return Optional.empty();
        }
        return Optional.of(new TableScanRedirectApplicationResult(new CatalogSchemaTableName("hive", databaseName, "mock_redirection_target"), ImmutableMap.of(), TupleDomain.all()));
    }, ImmutableSet.of(new PartitionsSystemTableProvider(partitionManager, TESTING_TYPE_MANAGER), new PropertiesSystemTableProvider()), metastore -> new NoneHiveMaterializedViewMetadata() {

        @Override
        public Optional<ConnectorMaterializedViewDefinition> getMaterializedView(ConnectorSession session, SchemaTableName viewName) {
            if (!viewName.getTableName().contains("materialized_view_tester")) {
                return Optional.empty();
            }
            return Optional.of(new ConnectorMaterializedViewDefinition("dummy_view_sql", Optional.empty(), Optional.empty(), Optional.empty(), ImmutableList.of(new ConnectorMaterializedViewDefinition.Column("abc", TypeId.of("type"))), Optional.empty(), Optional.of("alice"), ImmutableMap.of()));
        }
    }, SqlStandardAccessControlMetadata::new, NO_REDIRECTIONS, TableInvalidationCallback.NOOP);
    transactionManager = new HiveTransactionManager(metadataFactory);
    splitManager = new HiveSplitManager(transactionManager, partitionManager, new NamenodeStats(), hdfsEnvironment, new CachingDirectoryLister(hiveConfig), directExecutor(), new CounterStat(), 100, hiveConfig.getMaxOutstandingSplitsSize(), hiveConfig.getMinPartitionBatchSize(), hiveConfig.getMaxPartitionBatchSize(), hiveConfig.getMaxInitialSplits(), hiveConfig.getSplitLoaderConcurrency(), hiveConfig.getMaxSplitsPerSecond(), false, TESTING_TYPE_MANAGER);
    pageSinkProvider = new HivePageSinkProvider(getDefaultHiveFileWriterFactories(hiveConfig, hdfsEnvironment), hdfsEnvironment, PAGE_SORTER, HiveMetastoreFactory.ofInstance(metastoreClient), new GroupByHashPageIndexerFactory(JOIN_COMPILER, BLOCK_TYPE_OPERATORS), TESTING_TYPE_MANAGER, getHiveConfig(), locationService, partitionUpdateCodec, new TestingNodeManager("fake-environment"), new HiveEventClient(), getHiveSessionProperties(hiveConfig), new HiveWriterStats());
    pageSourceProvider = new HivePageSourceProvider(TESTING_TYPE_MANAGER, hdfsEnvironment, hiveConfig, getDefaultHivePageSourceFactories(hdfsEnvironment, hiveConfig), getDefaultHiveRecordCursorProviders(hiveConfig, hdfsEnvironment), new GenericHiveRecordCursorProvider(hdfsEnvironment, hiveConfig), Optional.empty());
    nodePartitioningProvider = new HiveNodePartitioningProvider(new TestingNodeManager("fake-environment"), TESTING_TYPE_MANAGER);
}

Example 2 with TableScanRedirectApplicationResult

use of io.trino.spi.connector.TableScanRedirectApplicationResult in project trino by trinodb.

the class AbstractTestHive method testApplyRedirection.

@Test
public void testApplyRedirection() throws Exception {
    SchemaTableName sourceTableName = temporaryTable("apply_redirection_tester");
    doCreateEmptyTable(sourceTableName, ORC, CREATE_TABLE_COLUMNS);
    SchemaTableName tableName = temporaryTable("apply_no_redirection_tester");
    doCreateEmptyTable(tableName, ORC, CREATE_TABLE_COLUMNS);
    try (Transaction transaction = newTransaction()) {
        ConnectorSession session = newSession();
        ConnectorMetadata metadata = transaction.getMetadata();
        assertThat(metadata.applyTableScanRedirect(session, getTableHandle(metadata, tableName))).isEmpty();
        Optional<TableScanRedirectApplicationResult> result = metadata.applyTableScanRedirect(session, getTableHandle(metadata, sourceTableName));
        assertThat(result).isPresent();
        assertThat(result.get().getDestinationTable()).isEqualTo(new CatalogSchemaTableName("hive", database, "mock_redirection_target"));
    } finally {
        dropTable(sourceTableName);
        dropTable(tableName);
    }
}

Example 3 with TableScanRedirectApplicationResult

use of io.trino.spi.connector.TableScanRedirectApplicationResult in project trino by trinodb.

the class ApplyTableScanRedirection method apply.

@Override
public Result apply(TableScanNode scanNode, Captures captures, Context context) {
    Optional<TableScanRedirectApplicationResult> tableScanRedirectApplicationResult = plannerContext.getMetadata().applyTableScanRedirect(context.getSession(), scanNode.getTable());
    if (tableScanRedirectApplicationResult.isEmpty()) {
        return Result.empty();
    }
    CatalogSchemaTableName destinationTable = tableScanRedirectApplicationResult.get().getDestinationTable();
    QualifiedObjectName destinationObjectName = convertFromSchemaTableName(destinationTable.getCatalogName()).apply(destinationTable.getSchemaTableName());
    Optional<QualifiedObjectName> redirectedObjectName = plannerContext.getMetadata().getRedirectionAwareTableHandle(context.getSession(), destinationObjectName).getRedirectedTableName();
    redirectedObjectName.ifPresent(name -> {
        throw new TrinoException(NOT_SUPPORTED, format("Further redirection of destination table '%s' to '%s' is not supported", destinationObjectName, name));
    });
    TableMetadata tableMetadata = plannerContext.getMetadata().getTableMetadata(context.getSession(), scanNode.getTable());
    CatalogSchemaTableName sourceTable = new CatalogSchemaTableName(tableMetadata.getCatalogName().getCatalogName(), tableMetadata.getTable());
    if (destinationTable.equals(sourceTable)) {
        return Result.empty();
    }
    Optional<TableHandle> destinationTableHandle = plannerContext.getMetadata().getTableHandle(context.getSession(), convertFromSchemaTableName(destinationTable.getCatalogName()).apply(destinationTable.getSchemaTableName()));
    if (destinationTableHandle.isEmpty()) {
        throw new TrinoException(TABLE_NOT_FOUND, format("Destination table %s from table scan redirection not found", destinationTable));
    }
    Map<ColumnHandle, String> columnMapping = tableScanRedirectApplicationResult.get().getDestinationColumns();
    Map<String, ColumnHandle> destinationColumnHandles = plannerContext.getMetadata().getColumnHandles(context.getSession(), destinationTableHandle.get());
    ImmutableMap.Builder<Symbol, Cast> casts = ImmutableMap.builder();
    ImmutableMap.Builder<Symbol, ColumnHandle> newAssignmentsBuilder = ImmutableMap.builder();
    for (Map.Entry<Symbol, ColumnHandle> assignment : scanNode.getAssignments().entrySet()) {
        String destinationColumn = columnMapping.get(assignment.getValue());
        if (destinationColumn == null) {
            throw new TrinoException(COLUMN_NOT_FOUND, format("Did not find mapping for source column %s in table scan redirection", assignment.getValue()));
        }
        ColumnHandle destinationColumnHandle = destinationColumnHandles.get(destinationColumn);
        if (destinationColumnHandle == null) {
            throw new TrinoException(COLUMN_NOT_FOUND, format("Did not find handle for column %s in destination table %s", destinationColumn, destinationTable));
        }
        // insert ts if redirected types don't match source types
        Type sourceType = context.getSymbolAllocator().getTypes().get(assignment.getKey());
        Type redirectedType = plannerContext.getMetadata().getColumnMetadata(context.getSession(), destinationTableHandle.get(), destinationColumnHandle).getType();
        if (!sourceType.equals(redirectedType)) {
            Symbol redirectedSymbol = context.getSymbolAllocator().newSymbol(destinationColumn, redirectedType);
            Cast cast = getCast(context.getSession(), destinationTable, destinationColumn, redirectedType, redirectedSymbol, sourceTable, assignment.getValue(), sourceType);
            casts.put(assignment.getKey(), cast);
            newAssignmentsBuilder.put(redirectedSymbol, destinationColumnHandle);
        } else {
            newAssignmentsBuilder.put(assignment.getKey(), destinationColumnHandle);
        }
    }
    TupleDomain<String> requiredFilter = tableScanRedirectApplicationResult.get().getFilter();
    if (requiredFilter.isAll()) {
        ImmutableMap<Symbol, ColumnHandle> newAssignments = newAssignmentsBuilder.buildOrThrow();
        return Result.ofPlanNode(applyProjection(context.getIdAllocator(), ImmutableSet.copyOf(scanNode.getOutputSymbols()), casts.buildOrThrow(), new TableScanNode(scanNode.getId(), destinationTableHandle.get(), ImmutableList.copyOf(newAssignments.keySet()), newAssignments, TupleDomain.all(), // Use table statistics from destination table
        Optional.empty(), scanNode.isUpdateTarget(), Optional.empty())));
    }
    Map<ColumnHandle, Symbol> inverseAssignments = ImmutableBiMap.copyOf(scanNode.getAssignments()).inverse();
    Map<String, ColumnHandle> inverseColumnsMapping = ImmutableBiMap.copyOf(columnMapping).inverse();
    TupleDomain<Symbol> transformedConstraint = requiredFilter.transformKeys(destinationColumn -> {
        ColumnHandle sourceColumnHandle = inverseColumnsMapping.get(destinationColumn);
        if (sourceColumnHandle == null) {
            throw new TrinoException(COLUMN_NOT_FOUND, format("Did not find mapping for destination column %s in table scan redirection", destinationColumn));
        }
        Symbol symbol = inverseAssignments.get(sourceColumnHandle);
        if (symbol != null) {
            // domain symbol should already be mapped in redirected table scan
            return symbol;
        }
        // Column pruning after predicate is pushed into table scan can remove assignments for filter columns from the scan node
        Type domainType = requiredFilter.getDomains().get().get(destinationColumn).getType();
        symbol = context.getSymbolAllocator().newSymbol(destinationColumn, domainType);
        ColumnHandle destinationColumnHandle = destinationColumnHandles.get(destinationColumn);
        if (destinationColumnHandle == null) {
            throw new TrinoException(COLUMN_NOT_FOUND, format("Did not find handle for column %s in destination table %s", destinationColumn, destinationTable));
        }
        // insert casts if redirected types don't match domain types
        Type redirectedType = plannerContext.getMetadata().getColumnMetadata(context.getSession(), destinationTableHandle.get(), destinationColumnHandle).getType();
        if (!domainType.equals(redirectedType)) {
            Symbol redirectedSymbol = context.getSymbolAllocator().newSymbol(destinationColumn, redirectedType);
            Cast cast = getCast(context.getSession(), destinationTable, destinationColumn, redirectedType, redirectedSymbol, sourceTable, sourceColumnHandle, domainType);
            casts.put(symbol, cast);
            newAssignmentsBuilder.put(redirectedSymbol, destinationColumnHandle);
        } else {
            newAssignmentsBuilder.put(symbol, destinationColumnHandle);
        }
        return symbol;
    });
    Map<Symbol, ColumnHandle> newAssignments = newAssignmentsBuilder.buildOrThrow();
    TableScanNode newScanNode = new TableScanNode(scanNode.getId(), destinationTableHandle.get(), ImmutableList.copyOf(newAssignments.keySet()), newAssignments, TupleDomain.all(), // Use table statistics from destination table
    Optional.empty(), scanNode.isUpdateTarget(), Optional.empty());
    DomainTranslator domainTranslator = new DomainTranslator(plannerContext);
    FilterNode filterNode = new FilterNode(context.getIdAllocator().getNextId(), applyProjection(context.getIdAllocator(), newAssignments.keySet(), casts.buildOrThrow(), newScanNode), domainTranslator.toPredicate(context.getSession(), transformedConstraint));
    return Result.ofPlanNode(applyProjection(context.getIdAllocator(), ImmutableSet.copyOf(scanNode.getOutputSymbols()), ImmutableMap.of(), filterNode));
}

Example 4 with TableScanRedirectApplicationResult

use of io.trino.spi.connector.TableScanRedirectApplicationResult in project trino by trinodb.

the class TpchMetadata method applyTableScanRedirect.

@Override
public Optional<TableScanRedirectApplicationResult> applyTableScanRedirect(ConnectorSession session, ConnectorTableHandle table) {
    TpchTableHandle handle = (TpchTableHandle) table;
    if (destinationCatalog.isEmpty()) {
        return Optional.empty();
    }
    CatalogSchemaTableName destinationTable = new CatalogSchemaTableName(destinationCatalog.get(), destinationSchema.orElse(handle.getSchemaName()), handle.getTableName());
    return Optional.of(new TableScanRedirectApplicationResult(destinationTable, ImmutableBiMap.copyOf(getColumnHandles(session, table)).inverse(), handle.getConstraint().transformKeys(TpchColumnHandle.class::cast).transformKeys(TpchColumnHandle::getColumnName)));
}