Examples with ConnectorTableMetadata io.trino.spi.connector.ConnectorTableMetadata used on opensource projects

Example 6 with ConnectorTableMetadata

use of io.trino.spi.connector.ConnectorTableMetadata in project trino by trinodb.

the class AbstractTestHiveFileSystemAbfs method ensureTableExists.

private void ensureTableExists(SchemaTableName table, String tableDirectoryName, Map<String, Object> tableProperties) {
    try (Transaction transaction = newTransaction()) {
        ConnectorTableMetadata tableMetadata = new ConnectorTableMetadata(table, ImmutableList.of(new ColumnMetadata("t_bigint", BIGINT)), ImmutableMap.<String, Object>builder().putAll(tableProperties).put(STORAGE_FORMAT_PROPERTY, HiveStorageFormat.TEXTFILE).put(EXTERNAL_LOCATION_PROPERTY, getBasePath().toString() + "/" + tableDirectoryName).put(BUCKET_COUNT_PROPERTY, 0).put(BUCKETED_BY_PROPERTY, ImmutableList.of()).put(SORTED_BY_PROPERTY, ImmutableList.of()).buildOrThrow());
        if (!transaction.getMetadata().listTables(newSession(), Optional.of(table.getSchemaName())).contains(table)) {
            transaction.getMetadata().createTable(newSession(), tableMetadata, false);
        }
        transaction.commit();
    }
}

Example 7 with ConnectorTableMetadata

use of io.trino.spi.connector.ConnectorTableMetadata in project trino by trinodb.

the class HiveMetadata method getTableHandleForStatisticsCollection.

@Override
public ConnectorTableHandle getTableHandleForStatisticsCollection(ConnectorSession session, SchemaTableName tableName, Map<String, Object> analyzeProperties) {
    HiveTableHandle handle = getTableHandle(session, tableName);
    if (handle == null) {
        return null;
    }
    Optional<List<List<String>>> partitionValuesList = getPartitionList(analyzeProperties);
    Optional<Set<String>> analyzeColumnNames = getColumnNames(analyzeProperties);
    ConnectorTableMetadata tableMetadata = getTableMetadata(session, handle.getSchemaTableName());
    List<String> partitionedBy = getPartitionedBy(tableMetadata.getProperties());
    if (partitionValuesList.isPresent()) {
        List<List<String>> list = partitionValuesList.get();
        if (partitionedBy.isEmpty()) {
            throw new TrinoException(INVALID_ANALYZE_PROPERTY, "Partition list provided but table is not partitioned");
        }
        for (List<String> values : list) {
            if (values.size() != partitionedBy.size()) {
                throw new TrinoException(INVALID_ANALYZE_PROPERTY, "Partition value count does not match partition column count");
            }
        }
        handle = handle.withAnalyzePartitionValues(list);
        HivePartitionResult partitions = partitionManager.getPartitions(handle, list);
        handle = partitionManager.applyPartitionResult(handle, partitions, alwaysTrue());
    }
    if (analyzeColumnNames.isPresent()) {
        Set<String> columnNames = analyzeColumnNames.get();
        Set<String> allColumnNames = tableMetadata.getColumns().stream().map(ColumnMetadata::getName).collect(toImmutableSet());
        if (!allColumnNames.containsAll(columnNames)) {
            throw new TrinoException(INVALID_ANALYZE_PROPERTY, format("Invalid columns specified for analysis: %s", Sets.difference(columnNames, allColumnNames)));
        }
        handle = handle.withAnalyzeColumnNames(columnNames);
    }
    return handle;
}

Example 8 with ConnectorTableMetadata

use of io.trino.spi.connector.ConnectorTableMetadata in project trino by trinodb.

the class HiveMetadata method doGetTableMetadata.

private ConnectorTableMetadata doGetTableMetadata(ConnectorSession session, SchemaTableName tableName) {
    Table table = metastore.getTable(tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    if (isIcebergTable(table) || isDeltaLakeTable(table)) {
        throw new TrinoException(HIVE_UNSUPPORTED_FORMAT, format("Not a Hive table '%s'", tableName));
    }
    if (!translateHiveViews && isHiveOrPrestoView(table)) {
        throw new TableNotFoundException(tableName);
    }
    Function<HiveColumnHandle, ColumnMetadata> metadataGetter = columnMetadataGetter(table);
    ImmutableList.Builder<ColumnMetadata> columns = ImmutableList.builder();
    for (HiveColumnHandle columnHandle : hiveColumnHandles(table, typeManager, getTimestampPrecision(session))) {
        columns.add(metadataGetter.apply(columnHandle));
    }
    // External location property
    ImmutableMap.Builder<String, Object> properties = ImmutableMap.builder();
    if (table.getTableType().equals(EXTERNAL_TABLE.name())) {
        properties.put(EXTERNAL_LOCATION_PROPERTY, table.getStorage().getLocation());
    }
    // Storage format property
    try {
        HiveStorageFormat format = extractHiveStorageFormat(table);
        properties.put(STORAGE_FORMAT_PROPERTY, format);
    } catch (TrinoException ignored) {
    // todo fail if format is not known
    }
    // Partitioning property
    List<String> partitionedBy = table.getPartitionColumns().stream().map(Column::getName).collect(toImmutableList());
    if (!partitionedBy.isEmpty()) {
        properties.put(PARTITIONED_BY_PROPERTY, partitionedBy);
    }
    // Bucket properties
    table.getStorage().getBucketProperty().ifPresent(property -> {
        properties.put(BUCKETING_VERSION, property.getBucketingVersion().getVersion());
        properties.put(BUCKET_COUNT_PROPERTY, property.getBucketCount());
        properties.put(BUCKETED_BY_PROPERTY, property.getBucketedBy());
        properties.put(SORTED_BY_PROPERTY, property.getSortedBy());
    });
    // Transactional properties
    String transactionalProperty = table.getParameters().get(HiveMetadata.TRANSACTIONAL);
    if (parseBoolean(transactionalProperty)) {
        properties.put(HiveTableProperties.TRANSACTIONAL, true);
    }
    // ORC format specific properties
    String orcBloomFilterColumns = table.getParameters().get(ORC_BLOOM_FILTER_COLUMNS_KEY);
    if (orcBloomFilterColumns != null) {
        properties.put(ORC_BLOOM_FILTER_COLUMNS, Splitter.on(',').trimResults().omitEmptyStrings().splitToList(orcBloomFilterColumns));
    }
    String orcBloomFilterFfp = table.getParameters().get(ORC_BLOOM_FILTER_FPP_KEY);
    if (orcBloomFilterFfp != null) {
        properties.put(ORC_BLOOM_FILTER_FPP, Double.parseDouble(orcBloomFilterFfp));
    }
    // Avro specific property
    String avroSchemaUrl = table.getParameters().get(AVRO_SCHEMA_URL_KEY);
    if (avroSchemaUrl != null) {
        properties.put(AVRO_SCHEMA_URL, avroSchemaUrl);
    }
    // Textfile and CSV specific properties
    getSerdeProperty(table, SKIP_HEADER_COUNT_KEY).ifPresent(skipHeaderCount -> properties.put(SKIP_HEADER_LINE_COUNT, Integer.valueOf(skipHeaderCount)));
    getSerdeProperty(table, SKIP_FOOTER_COUNT_KEY).ifPresent(skipFooterCount -> properties.put(SKIP_FOOTER_LINE_COUNT, Integer.valueOf(skipFooterCount)));
    // Multi-format property
    getSerdeProperty(table, NULL_FORMAT_KEY).ifPresent(nullFormat -> properties.put(NULL_FORMAT_PROPERTY, nullFormat));
    // Textfile specific properties
    getSerdeProperty(table, TEXT_FIELD_SEPARATOR_KEY).ifPresent(fieldSeparator -> properties.put(TEXTFILE_FIELD_SEPARATOR, fieldSeparator));
    getSerdeProperty(table, TEXT_FIELD_SEPARATOR_ESCAPE_KEY).ifPresent(fieldEscape -> properties.put(TEXTFILE_FIELD_SEPARATOR_ESCAPE, fieldEscape));
    // CSV specific properties
    getCsvSerdeProperty(table, CSV_SEPARATOR_KEY).ifPresent(csvSeparator -> properties.put(CSV_SEPARATOR, csvSeparator));
    getCsvSerdeProperty(table, CSV_QUOTE_KEY).ifPresent(csvQuote -> properties.put(CSV_QUOTE, csvQuote));
    getCsvSerdeProperty(table, CSV_ESCAPE_KEY).ifPresent(csvEscape -> properties.put(CSV_ESCAPE, csvEscape));
    Optional<String> comment = Optional.ofNullable(table.getParameters().get(TABLE_COMMENT));
    String autoPurgeProperty = table.getParameters().get(AUTO_PURGE_KEY);
    if (parseBoolean(autoPurgeProperty)) {
        properties.put(AUTO_PURGE, true);
    }
    return new ConnectorTableMetadata(tableName, columns.build(), properties.buildOrThrow(), comment);
}

Example 9 with ConnectorTableMetadata

use of io.trino.spi.connector.ConnectorTableMetadata in project trino by trinodb.

the class HiveMetadata method getNewTableLayout.

@Override
public Optional<ConnectorTableLayout> getNewTableLayout(ConnectorSession session, ConnectorTableMetadata tableMetadata) {
    validateTimestampColumns(tableMetadata.getColumns(), getTimestampPrecision(session));
    validatePartitionColumns(tableMetadata);
    validateBucketColumns(tableMetadata);
    validateColumns(tableMetadata);
    Optional<HiveBucketProperty> bucketProperty = getBucketProperty(tableMetadata.getProperties());
    List<String> partitionedBy = getPartitionedBy(tableMetadata.getProperties());
    if (bucketProperty.isEmpty()) {
        // return preferred layout which is partitioned by partition columns
        if (partitionedBy.isEmpty()) {
            return Optional.empty();
        }
        return Optional.of(new ConnectorTableLayout(partitionedBy));
    }
    if (!bucketProperty.get().getSortedBy().isEmpty() && !isSortedWritingEnabled(session)) {
        throw new TrinoException(NOT_SUPPORTED, "Writing to bucketed sorted Hive tables is disabled");
    }
    List<String> bucketedBy = bucketProperty.get().getBucketedBy();
    Map<String, HiveType> hiveTypeMap = tableMetadata.getColumns().stream().collect(toMap(ColumnMetadata::getName, column -> toHiveType(column.getType())));
    return Optional.of(new ConnectorTableLayout(new HivePartitioningHandle(bucketProperty.get().getBucketingVersion(), bucketProperty.get().getBucketCount(), bucketedBy.stream().map(hiveTypeMap::get).collect(toImmutableList()), OptionalInt.of(bucketProperty.get().getBucketCount()), !partitionedBy.isEmpty() && isParallelPartitionedBucketedWrites(session)), ImmutableList.<String>builder().addAll(bucketedBy).addAll(partitionedBy).build()));
}

Example 10 with ConnectorTableMetadata

use of io.trino.spi.connector.ConnectorTableMetadata in project trino by trinodb.

the class PropertiesSystemTableProvider method getSystemTable.

@Override
public Optional<SystemTable> getSystemTable(HiveMetadata metadata, ConnectorSession session, SchemaTableName tableName) {
    if (!PROPERTIES.matches(tableName)) {
        return Optional.empty();
    }
    SchemaTableName sourceTableName = PROPERTIES.getSourceTableName(tableName);
    Table table = metadata.getMetastore().getTable(sourceTableName.getSchemaName(), sourceTableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
    if (isDeltaLakeTable(table) || isIcebergTable(table)) {
        return Optional.empty();
    }
    Map<String, String> sortedTableParameters = ImmutableSortedMap.copyOf(table.getParameters());
    List<ColumnMetadata> columns = sortedTableParameters.keySet().stream().map(key -> new ColumnMetadata(key, VarcharType.VARCHAR)).collect(toImmutableList());
    List<Type> types = columns.stream().map(ColumnMetadata::getType).collect(toImmutableList());
    Iterable<List<Object>> propertyValues = ImmutableList.of(ImmutableList.copyOf(sortedTableParameters.values()));
    return Optional.of(createSystemTable(new ConnectorTableMetadata(sourceTableName, columns), constraint -> new InMemoryRecordSet(types, propertyValues).cursor()));
}