Examples with ConnectorTableHandle io.trino.spi.connector.ConnectorTableHandle used on opensource projects

Example 41 with ConnectorTableHandle

use of io.trino.spi.connector.ConnectorTableHandle in project trino by trinodb.

the class TestDefaultJdbcMetadata method tesMultiGroupKeyPredicatePushdown.

@Test
public void tesMultiGroupKeyPredicatePushdown() {
    ConnectorSession session = TestingConnectorSession.builder().setPropertyMetadata(new JdbcMetadataSessionProperties(new JdbcMetadataConfig().setAggregationPushdownEnabled(true), Optional.empty()).getSessionProperties()).build();
    Map<String, ColumnHandle> columnHandles = metadata.getColumnHandles(session, tableHandle);
    ColumnHandle textColumn = columnHandles.get("text");
    ColumnHandle valueColumn = columnHandles.get("value");
    ConnectorTableHandle baseTableHandle = metadata.getTableHandle(session, new SchemaTableName("example", "numbers"));
    ConnectorTableHandle aggregatedTable = applyCountAggregation(session, baseTableHandle, ImmutableList.of(ImmutableList.of(textColumn, valueColumn), ImmutableList.of(textColumn)));
    Domain domain = Domain.singleValue(BIGINT, 123L);
    JdbcTableHandle tableHandleWithFilter = applyFilter(session, aggregatedTable, new Constraint(TupleDomain.withColumnDomains(ImmutableMap.of(valueColumn, domain))));
    assertEquals(tableHandleWithFilter.getConstraint().getDomains(), Optional.of(ImmutableMap.of(valueColumn, domain)));
    assertEquals(((JdbcQueryRelationHandle) tableHandleWithFilter.getRelationHandle()).getPreparedQuery().getQuery(), "SELECT \"TEXT\", \"VALUE\", count(*) AS \"_pfgnrtd_0\" " + "FROM \"" + database.getDatabaseName() + "\".\"EXAMPLE\".\"NUMBERS\" " + "GROUP BY GROUPING SETS ((\"TEXT\", \"VALUE\"), (\"TEXT\"))");
}

Example 42 with ConnectorTableHandle

use of io.trino.spi.connector.ConnectorTableHandle in project trino by trinodb.

the class TestPruneTableScanColumns method mockApplyProjection.

private Optional<ProjectionApplicationResult<ConnectorTableHandle>> mockApplyProjection(ConnectorSession session, ConnectorTableHandle tableHandle, List<ConnectorExpression> projections, Map<String, ColumnHandle> assignments) {
    MockConnectorTableHandle handle = (MockConnectorTableHandle) tableHandle;
    List<Variable> variables = projections.stream().map(Variable.class::cast).collect(toImmutableList());
    List<ColumnHandle> newColumns = variables.stream().map(variable -> assignments.get(variable.getName())).collect(toImmutableList());
    if (handle.getColumns().isPresent() && newColumns.equals(handle.getColumns().get())) {
        return Optional.empty();
    }
    return Optional.of(new ProjectionApplicationResult<>(new MockConnectorTableHandle(handle.getTableName(), handle.getConstraint(), Optional.of(newColumns)), projections, variables.stream().map(variable -> new Assignment(variable.getName(), assignments.get(variable.getName()), ((MockConnectorColumnHandle) assignments.get(variable.getName())).getType())).collect(toImmutableList()), false));
}

Example 43 with ConnectorTableHandle

use of io.trino.spi.connector.ConnectorTableHandle in project trino by trinodb.

the class DeltaLakeMetadata method finishStatisticsCollection.

@Override
public void finishStatisticsCollection(ConnectorSession session, ConnectorTableHandle table, Collection<ComputedStatistics> computedStatistics) {
    DeltaLakeTableHandle tableHandle = (DeltaLakeTableHandle) table;
    AnalyzeHandle analyzeHandle = tableHandle.getAnalyzeHandle().orElseThrow(() -> new IllegalArgumentException("analyzeHandle not set"));
    String location = metastore.getTableLocation(tableHandle.getSchemaTableName(), session);
    Optional<DeltaLakeStatistics> oldStatistics = statisticsAccess.readDeltaLakeStatistics(session, location);
    // more elaborate logic for handling statistics model evaluation may need to be introduced in the future
    // for now let's have a simple check rejecting update
    oldStatistics.ifPresent(statistics -> checkArgument(statistics.getModelVersion() == DeltaLakeStatistics.CURRENT_MODEL_VERSION, "Existing table statistics are incompatible, run the drop statistics procedure on this table before re-analyzing"));
    Map<String, DeltaLakeColumnStatistics> oldColumnStatistics = oldStatistics.map(DeltaLakeStatistics::getColumnStatistics).orElseGet(ImmutableMap::of);
    Map<String, DeltaLakeColumnStatistics> newColumnStatistics = toDeltaLakeColumnStatistics(computedStatistics);
    Map<String, DeltaLakeColumnStatistics> mergedColumnStatistics = new HashMap<>();
    // only keep stats for existing columns
    Set<String> newColumns = newColumnStatistics.keySet();
    oldColumnStatistics.entrySet().stream().filter(entry -> newColumns.contains(entry.getKey())).forEach(entry -> mergedColumnStatistics.put(entry.getKey(), entry.getValue()));
    newColumnStatistics.forEach((columnName, columnStatistics) -> {
        mergedColumnStatistics.merge(columnName, columnStatistics, DeltaLakeColumnStatistics::update);
    });
    Optional<Instant> maxFileModificationTime = getMaxFileModificationTime(computedStatistics);
    // We do not want to hinder our future calls to ANALYZE if one of the files we analyzed have modification time far in the future.
    // Therefore we cap the value we store in extended_stats.json to current_time as observed on Trino coordinator.
    Instant finalAlreadyAnalyzedModifiedTimeMax = Instant.now();
    if (maxFileModificationTime.isPresent()) {
        finalAlreadyAnalyzedModifiedTimeMax = Comparators.min(maxFileModificationTime.get(), finalAlreadyAnalyzedModifiedTimeMax);
    }
    // also ensure that we are not traveling back in time
    if (oldStatistics.isPresent()) {
        finalAlreadyAnalyzedModifiedTimeMax = Comparators.max(oldStatistics.get().getAlreadyAnalyzedModifiedTimeMax(), finalAlreadyAnalyzedModifiedTimeMax);
    }
    if (analyzeHandle.getColumns().isPresent() && !mergedColumnStatistics.keySet().equals(analyzeHandle.getColumns().get())) {
        // sanity validation
        throw new IllegalStateException(format("Unexpected columns in in mergedColumnStatistics %s; expected %s", mergedColumnStatistics.keySet(), analyzeHandle.getColumns().get()));
    }
    DeltaLakeStatistics mergedDeltaLakeStatistics = new DeltaLakeStatistics(finalAlreadyAnalyzedModifiedTimeMax, mergedColumnStatistics, analyzeHandle.getColumns());
    statisticsAccess.updateDeltaLakeStatistics(session, location, mergedDeltaLakeStatistics);
}

Example 44 with ConnectorTableHandle

use of io.trino.spi.connector.ConnectorTableHandle in project trino by trinodb.

the class KafkaSplitManager method getSplits.

@Override
public ConnectorSplitSource getSplits(ConnectorTransactionHandle transaction, ConnectorSession session, ConnectorTableHandle table, SplitSchedulingStrategy splitSchedulingStrategy, DynamicFilter dynamicFilter) {
    KafkaTableHandle kafkaTableHandle = (KafkaTableHandle) table;
    try (KafkaConsumer<byte[], byte[]> kafkaConsumer = consumerFactory.create(session)) {
        List<PartitionInfo> partitionInfos = kafkaConsumer.partitionsFor(kafkaTableHandle.getTopicName());
        List<TopicPartition> topicPartitions = partitionInfos.stream().map(KafkaSplitManager::toTopicPartition).collect(toImmutableList());
        Map<TopicPartition, Long> partitionBeginOffsets = kafkaConsumer.beginningOffsets(topicPartitions);
        Map<TopicPartition, Long> partitionEndOffsets = kafkaConsumer.endOffsets(topicPartitions);
        KafkaFilteringResult kafkaFilteringResult = kafkaFilterManager.getKafkaFilterResult(session, kafkaTableHandle, partitionInfos, partitionBeginOffsets, partitionEndOffsets);
        partitionInfos = kafkaFilteringResult.getPartitionInfos();
        partitionBeginOffsets = kafkaFilteringResult.getPartitionBeginOffsets();
        partitionEndOffsets = kafkaFilteringResult.getPartitionEndOffsets();
        ImmutableList.Builder<KafkaSplit> splits = ImmutableList.builder();
        Optional<String> keyDataSchemaContents = contentSchemaReader.readKeyContentSchema(kafkaTableHandle);
        Optional<String> messageDataSchemaContents = contentSchemaReader.readValueContentSchema(kafkaTableHandle);
        for (PartitionInfo partitionInfo : partitionInfos) {
            TopicPartition topicPartition = toTopicPartition(partitionInfo);
            HostAddress leader = HostAddress.fromParts(partitionInfo.leader().host(), partitionInfo.leader().port());
            new Range(partitionBeginOffsets.get(topicPartition), partitionEndOffsets.get(topicPartition)).partition(messagesPerSplit).stream().map(range -> new KafkaSplit(kafkaTableHandle.getTopicName(), kafkaTableHandle.getKeyDataFormat(), kafkaTableHandle.getMessageDataFormat(), keyDataSchemaContents, messageDataSchemaContents, partitionInfo.partition(), range, leader)).forEach(splits::add);
        }
        return new FixedSplitSource(splits.build());
    } catch (Exception e) {
        // Catch all exceptions because Kafka library is written in scala and checked exceptions are not declared in method signature.
        if (e instanceof TrinoException) {
            throw e;
        }
        throw new TrinoException(KAFKA_SPLIT_ERROR, format("Cannot list splits for table '%s' reading topic '%s'", kafkaTableHandle.getTableName(), kafkaTableHandle.getTopicName()), e);
    }
}

Example 45 with ConnectorTableHandle

use of io.trino.spi.connector.ConnectorTableHandle in project trino by trinodb.

the class TestPrometheusRecordSetProvider method testGetRecordSet.

@Test
public void testGetRecordSet() {
    ConnectorTableHandle tableHandle = new PrometheusTableHandle("schema", "table");
    PrometheusRecordSetProvider recordSetProvider = new PrometheusRecordSetProvider(client);
    RecordSet recordSet = recordSetProvider.getRecordSet(PrometheusTransactionHandle.INSTANCE, SESSION, new PrometheusSplit(dataUri), tableHandle, ImmutableList.of(new PrometheusColumnHandle("labels", varcharMapType, 0), new PrometheusColumnHandle("timestamp", TIMESTAMP_COLUMN_TYPE, 1), new PrometheusColumnHandle("value", DoubleType.DOUBLE, 2)));
    assertNotNull(recordSet, "recordSet is null");
    RecordCursor cursor = recordSet.cursor();
    assertNotNull(cursor, "cursor is null");
    Map<Instant, Map<?, ?>> actual = new LinkedHashMap<>();
    while (cursor.advanceNextPosition()) {
        actual.put((Instant) cursor.getObject(1), getMapFromBlock(varcharMapType, (Block) cursor.getObject(0)));
    }
    Map<Instant, Map<String, String>> expected = ImmutableMap.<Instant, Map<String, String>>builder().put(ofEpochMilli(1565962969044L), ImmutableMap.of("instance", "localhost:9090", "__name__", "up", "job", "prometheus")).put(ofEpochMilli(1565962984045L), ImmutableMap.of("instance", "localhost:9090", "__name__", "up", "job", "prometheus")).put(ofEpochMilli(1565962999044L), ImmutableMap.of("instance", "localhost:9090", "__name__", "up", "job", "prometheus")).put(ofEpochMilli(1565963014044L), ImmutableMap.of("instance", "localhost:9090", "__name__", "up", "job", "prometheus")).buildOrThrow();
    assertEquals(actual, expected);
}