Search in sources :

Example 76 with BIGINT

use of io.trino.spi.type.BigintType.BIGINT in project trino by trinodb.

the class DeltaLakeMetadata method getUpdateRowIdColumnHandle.

// The rowId is a RowType where the first field is a BigInt containing the row number. The second field contains the values of any columns that were unmodified.
// They are needed to write the complete row after modifications in DeltaLakeUpdatablePageSource. If there are no unmodified columns, the rowId only has one field.
@Override
public ColumnHandle getUpdateRowIdColumnHandle(ConnectorSession session, ConnectorTableHandle tableHandle, List<ColumnHandle> updatedColumns) {
    DeltaLakeTableHandle handle = (DeltaLakeTableHandle) tableHandle;
    List<DeltaLakeColumnHandle> unmodifiedColumns = getUnmodifiedColumns(handle, updatedColumns);
    Type rowIdType;
    if (unmodifiedColumns.isEmpty()) {
        rowIdType = RowType.rowType(RowType.field(BIGINT));
    } else {
        List<RowType.Field> unmodifiedColumnFields = unmodifiedColumns.stream().map(columnMetadata -> RowType.field(columnMetadata.getName(), columnMetadata.getType())).collect(toImmutableList());
        rowIdType = RowType.rowType(RowType.field(BIGINT), RowType.field(RowType.from(unmodifiedColumnFields)));
    }
    return new DeltaLakeColumnHandle(ROW_ID_COLUMN_NAME, rowIdType, SYNTHESIZED);
}
Also used : TransactionLogUtil.getTransactionLogDir(io.trino.plugin.deltalake.transactionlog.TransactionLogUtil.getTransactionLogDir) FileSystem(org.apache.hadoop.fs.FileSystem) TableSnapshot(io.trino.plugin.deltalake.transactionlog.TableSnapshot) ColumnStatisticMetadata(io.trino.spi.statistics.ColumnStatisticMetadata) FileStatus(org.apache.hadoop.fs.FileStatus) DeltaLakeSchemaSupport.validateType(io.trino.plugin.deltalake.transactionlog.DeltaLakeSchemaSupport.validateType) TypeUtils.isFloatingPointNaN(io.trino.spi.type.TypeUtils.isFloatingPointNaN) RemoveFileEntry(io.trino.plugin.deltalake.transactionlog.RemoveFileEntry) ConnectorTableExecuteHandle(io.trino.spi.connector.ConnectorTableExecuteHandle) Collections.singletonList(java.util.Collections.singletonList) NOT_SUPPORTED(io.trino.spi.StandardErrorCode.NOT_SUPPORTED) TransactionLogWriterFactory(io.trino.plugin.deltalake.transactionlog.writer.TransactionLogWriterFactory) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) TimestampWithTimeZoneType(io.trino.spi.type.TimestampWithTimeZoneType) ValueSet.ofRanges(io.trino.spi.predicate.ValueSet.ofRanges) Column(io.trino.plugin.hive.metastore.Column) ConnectorOutputTableHandle(io.trino.spi.connector.ConnectorOutputTableHandle) ConnectorTableHandle(io.trino.spi.connector.ConnectorTableHandle) Map(java.util.Map) PARTITIONED_BY_PROPERTY(io.trino.plugin.deltalake.DeltaLakeTableProperties.PARTITIONED_BY_PROPERTY) ProjectionApplicationResult(io.trino.spi.connector.ProjectionApplicationResult) PRESTO_QUERY_ID_NAME(io.trino.plugin.hive.HiveMetadata.PRESTO_QUERY_ID_NAME) ENGLISH(java.util.Locale.ENGLISH) SMALLINT(io.trino.spi.type.SmallintType.SMALLINT) HdfsEnvironment(io.trino.plugin.hive.HdfsEnvironment) Table(io.trino.plugin.hive.metastore.Table) Domain(io.trino.spi.predicate.Domain) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) TABLE_PROVIDER_PROPERTY(io.trino.plugin.deltalake.metastore.HiveMetastoreBackedDeltaLakeMetastore.TABLE_PROVIDER_PROPERTY) HiveWriteUtils.pathExists(io.trino.plugin.hive.util.HiveWriteUtils.pathExists) MANAGED_TABLE(org.apache.hadoop.hive.metastore.TableType.MANAGED_TABLE) SchemaTableName(io.trino.spi.connector.SchemaTableName) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) Stream(java.util.stream.Stream) TrinoPrincipal(io.trino.spi.security.TrinoPrincipal) CatalogSchemaTableName(io.trino.spi.connector.CatalogSchemaTableName) SchemaTablePrefix(io.trino.spi.connector.SchemaTablePrefix) HyperLogLog(io.airlift.stats.cardinality.HyperLogLog) DateTimeEncoding.unpackMillisUtc(io.trino.spi.type.DateTimeEncoding.unpackMillisUtc) FILE_MODIFIED_TIME_COLUMN_NAME(io.trino.plugin.deltalake.DeltaLakeColumnHandle.FILE_MODIFIED_TIME_COLUMN_NAME) Predicate.not(java.util.function.Predicate.not) TableColumnsMetadata(io.trino.spi.connector.TableColumnsMetadata) RemoteIterator(org.apache.hadoop.fs.RemoteIterator) ANALYZE_COLUMNS_PROPERTY(io.trino.plugin.deltalake.DeltaLakeTableProperties.ANALYZE_COLUMNS_PROPERTY) REGULAR(io.trino.plugin.deltalake.DeltaLakeColumnType.REGULAR) TransactionLogParser.getMandatoryCurrentVersion(io.trino.plugin.deltalake.transactionlog.TransactionLogParser.getMandatoryCurrentVersion) DATE(io.trino.spi.type.DateType.DATE) REAL(io.trino.spi.type.RealType.REAL) Iterables(com.google.common.collect.Iterables) ConnectorTableLayout(io.trino.spi.connector.ConnectorTableLayout) ConnectorInsertTableHandle(io.trino.spi.connector.ConnectorInsertTableHandle) DeltaLakeColumnHandle.fileSizeColumnHandle(io.trino.plugin.deltalake.DeltaLakeColumnHandle.fileSizeColumnHandle) Slice(io.airlift.slice.Slice) ColumnMetadata(io.trino.spi.connector.ColumnMetadata) DeltaLakeTableProcedureId(io.trino.plugin.deltalake.procedure.DeltaLakeTableProcedureId) INVALID_ANALYZE_PROPERTY(io.trino.spi.StandardErrorCode.INVALID_ANALYZE_PROPERTY) BOOLEAN(io.trino.spi.type.BooleanType.BOOLEAN) ConnectorTableMetadata(io.trino.spi.connector.ConnectorTableMetadata) Variable(io.trino.spi.expression.Variable) DeltaLakeTableProperties.getLocation(io.trino.plugin.deltalake.DeltaLakeTableProperties.getLocation) Range.greaterThanOrEqual(io.trino.spi.predicate.Range.greaterThanOrEqual) TransactionConflictException(io.trino.plugin.deltalake.transactionlog.writer.TransactionConflictException) HiveType(io.trino.plugin.hive.HiveType) VARCHAR(io.trino.spi.type.VarcharType.VARCHAR) DeltaLakeStatisticsAccess(io.trino.plugin.deltalake.statistics.DeltaLakeStatisticsAccess) DeltaLakeSchemaSupport.extractPartitionColumns(io.trino.plugin.deltalake.transactionlog.DeltaLakeSchemaSupport.extractPartitionColumns) ColumnHandle(io.trino.spi.connector.ColumnHandle) ImmutableSet.toImmutableSet(com.google.common.collect.ImmutableSet.toImmutableSet) INVALID_TABLE_PROPERTY(io.trino.spi.StandardErrorCode.INVALID_TABLE_PROPERTY) DeltaLakeSchemaSupport.serializeStatsAsJson(io.trino.plugin.deltalake.transactionlog.DeltaLakeSchemaSupport.serializeStatsAsJson) Nullable(javax.annotation.Nullable) ConstraintApplicationResult(io.trino.spi.connector.ConstraintApplicationResult) MapType(io.trino.spi.type.MapType) PARTITION_KEY(io.trino.plugin.deltalake.DeltaLakeColumnType.PARTITION_KEY) IOException(java.io.IOException) ConnectorSession(io.trino.spi.connector.ConnectorSession) DELTA_LAKE_INVALID_SCHEMA(io.trino.plugin.deltalake.DeltaLakeErrorCode.DELTA_LAKE_INVALID_SCHEMA) CheckpointWriterManager(io.trino.plugin.deltalake.transactionlog.checkpoint.CheckpointWriterManager) ROW_ID_COLUMN_TYPE(io.trino.plugin.deltalake.DeltaLakeColumnHandle.ROW_ID_COLUMN_TYPE) DOUBLE(io.trino.spi.type.DoubleType.DOUBLE) HiveUtil.isHiveSystemSchema(io.trino.plugin.hive.util.HiveUtil.isHiveSystemSchema) ConnectorTableProperties(io.trino.spi.connector.ConnectorTableProperties) ConnectorExpression(io.trino.spi.expression.ConnectorExpression) MAX_VALUE(io.trino.spi.statistics.ColumnStatisticType.MAX_VALUE) DeltaLakeSessionProperties.isTableStatisticsEnabled(io.trino.plugin.deltalake.DeltaLakeSessionProperties.isTableStatisticsEnabled) LOCATION_PROPERTY(io.trino.plugin.deltalake.DeltaLakeTableProperties.LOCATION_PROPERTY) TableStatisticsMetadata(io.trino.spi.statistics.TableStatisticsMetadata) TINYINT(io.trino.spi.type.TinyintType.TINYINT) NotADeltaLakeTableException(io.trino.plugin.deltalake.metastore.NotADeltaLakeTableException) DeltaLakeStatistics(io.trino.plugin.deltalake.statistics.DeltaLakeStatistics) HiveUtil.isDeltaLakeTable(io.trino.plugin.hive.util.HiveUtil.isDeltaLakeTable) NodeManager(io.trino.spi.NodeManager) EXTERNAL_TABLE(org.apache.hadoop.hive.metastore.TableType.EXTERNAL_TABLE) Database(io.trino.plugin.hive.metastore.Database) DeltaLakeSchemaSupport.extractSchema(io.trino.plugin.deltalake.transactionlog.DeltaLakeSchemaSupport.extractSchema) SYNTHESIZED(io.trino.plugin.deltalake.DeltaLakeColumnType.SYNTHESIZED) TABLE_PROVIDER_VALUE(io.trino.plugin.deltalake.metastore.HiveMetastoreBackedDeltaLakeMetastore.TABLE_PROVIDER_VALUE) SchemaNotFoundException(io.trino.spi.connector.SchemaNotFoundException) AddFileEntry(io.trino.plugin.deltalake.transactionlog.AddFileEntry) DeltaLakeMetastore(io.trino.plugin.deltalake.metastore.DeltaLakeMetastore) Format(io.trino.plugin.deltalake.transactionlog.MetadataEntry.Format) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Locale(java.util.Locale) CatalogSchemaName(io.trino.spi.connector.CatalogSchemaName) Path(org.apache.hadoop.fs.Path) HyperLogLogType(io.trino.spi.type.HyperLogLogType) INTEGER(io.trino.spi.type.IntegerType.INTEGER) StorageFormat(io.trino.plugin.hive.metastore.StorageFormat) RowType(io.trino.spi.type.RowType) Range.range(io.trino.spi.predicate.Range.range) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) HiveWriteUtils.isS3FileSystem(io.trino.plugin.hive.util.HiveWriteUtils.isS3FileSystem) TransactionLogWriter(io.trino.plugin.deltalake.transactionlog.writer.TransactionLogWriter) Collection(java.util.Collection) DeltaLakeTableExecuteHandle(io.trino.plugin.deltalake.procedure.DeltaLakeTableExecuteHandle) MetadataEntry(io.trino.plugin.deltalake.transactionlog.MetadataEntry) ComputedStatistics(io.trino.spi.statistics.ComputedStatistics) TrinoException(io.trino.spi.TrinoException) ArrayType(io.trino.spi.type.ArrayType) Instant(java.time.Instant) ConnectorOutputMetadata(io.trino.spi.connector.ConnectorOutputMetadata) Sets(com.google.common.collect.Sets) FileNotFoundException(java.io.FileNotFoundException) String.format(java.lang.String.format) Preconditions.checkState(com.google.common.base.Preconditions.checkState) ROW_ID_COLUMN_NAME(io.trino.plugin.deltalake.DeltaLakeColumnHandle.ROW_ID_COLUMN_NAME) INVALID_SCHEMA_PROPERTY(io.trino.spi.StandardErrorCode.INVALID_SCHEMA_PROPERTY) DataSize(io.airlift.units.DataSize) HdfsContext(io.trino.plugin.hive.HdfsEnvironment.HdfsContext) List(java.util.List) BIGINT(io.trino.spi.type.BigintType.BIGINT) MetastoreUtil.buildInitialPrivilegeSet(io.trino.plugin.hive.metastore.MetastoreUtil.buildInitialPrivilegeSet) Assignment(io.trino.spi.connector.Assignment) BeginTableExecuteResult(io.trino.spi.connector.BeginTableExecuteResult) Function.identity(java.util.function.Function.identity) Optional(java.util.Optional) ConnectorMetadata(io.trino.spi.connector.ConnectorMetadata) DecimalType(io.trino.spi.type.DecimalType) OPTIMIZE(io.trino.plugin.deltalake.procedure.DeltaLakeTableProcedureId.OPTIMIZE) JsonCodec(io.airlift.json.JsonCodec) Comparators(com.google.common.collect.Comparators) Constraint(io.trino.spi.connector.Constraint) Range.lessThanOrEqual(io.trino.spi.predicate.Range.lessThanOrEqual) DeltaLakeFileStatistics(io.trino.plugin.deltalake.transactionlog.statistics.DeltaLakeFileStatistics) Logger(io.airlift.log.Logger) DeltaLakeSchemaSupport.serializeSchemaAsJson(io.trino.plugin.deltalake.transactionlog.DeltaLakeSchemaSupport.serializeSchemaAsJson) DeltaLakeColumnStatistics(io.trino.plugin.deltalake.statistics.DeltaLakeColumnStatistics) Type(io.trino.spi.type.Type) HashMap(java.util.HashMap) DeltaLakeColumnHandle.pathColumnHandle(io.trino.plugin.deltalake.DeltaLakeColumnHandle.pathColumnHandle) DeltaLakeColumnHandle.fileModifiedTimeColumnHandle(io.trino.plugin.deltalake.DeltaLakeColumnHandle.fileModifiedTimeColumnHandle) AtomicReference(java.util.concurrent.atomic.AtomicReference) VarcharType(io.trino.spi.type.VarcharType) ImmutableList(com.google.common.collect.ImmutableList) Verify.verify(com.google.common.base.Verify.verify) Objects.requireNonNull(java.util.Objects.requireNonNull) TableStatistics(io.trino.spi.statistics.TableStatistics) DeltaLakeSessionProperties.isExtendedStatisticsEnabled(io.trino.plugin.deltalake.DeltaLakeSessionProperties.isExtendedStatisticsEnabled) VIRTUAL_VIEW(org.apache.hadoop.hive.metastore.TableType.VIRTUAL_VIEW) CHECKPOINT_INTERVAL_PROPERTY(io.trino.plugin.deltalake.DeltaLakeTableProperties.CHECKPOINT_INTERVAL_PROPERTY) StorageFormat.create(io.trino.plugin.hive.metastore.StorageFormat.create) MetadataEntry.buildDeltaMetadataConfiguration(io.trino.plugin.deltalake.transactionlog.MetadataEntry.buildDeltaMetadataConfiguration) TupleDomain.withColumnDomains(io.trino.spi.predicate.TupleDomain.withColumnDomains) DELTA_LAKE_BAD_WRITE(io.trino.plugin.deltalake.DeltaLakeErrorCode.DELTA_LAKE_BAD_WRITE) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) TupleDomain(io.trino.spi.predicate.TupleDomain) DeltaLakeTableProperties.getPartitionedBy(io.trino.plugin.deltalake.DeltaLakeTableProperties.getPartitionedBy) HiveWriteUtils.createDirectory(io.trino.plugin.hive.util.HiveWriteUtils.createDirectory) GENERIC_INTERNAL_ERROR(io.trino.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR) SchemaTableName.schemaTableName(io.trino.spi.connector.SchemaTableName.schemaTableName) UUID.randomUUID(java.util.UUID.randomUUID) ProtocolEntry(io.trino.plugin.deltalake.transactionlog.ProtocolEntry) DeltaTableOptimizeHandle(io.trino.plugin.deltalake.procedure.DeltaTableOptimizeHandle) Collections.unmodifiableMap(java.util.Collections.unmodifiableMap) CommitInfoEntry(io.trino.plugin.deltalake.transactionlog.CommitInfoEntry) PrincipalPrivileges(io.trino.plugin.hive.metastore.PrincipalPrivileges) TypeManager(io.trino.spi.type.TypeManager) Collections(java.util.Collections) NUMBER_OF_DISTINCT_VALUES_SUMMARY(io.trino.spi.statistics.ColumnStatisticType.NUMBER_OF_DISTINCT_VALUES_SUMMARY) DeltaLakeSchemaSupport.validateType(io.trino.plugin.deltalake.transactionlog.DeltaLakeSchemaSupport.validateType) TimestampWithTimeZoneType(io.trino.spi.type.TimestampWithTimeZoneType) HiveType(io.trino.plugin.hive.HiveType) MapType(io.trino.spi.type.MapType) HyperLogLogType(io.trino.spi.type.HyperLogLogType) RowType(io.trino.spi.type.RowType) ArrayType(io.trino.spi.type.ArrayType) DecimalType(io.trino.spi.type.DecimalType) Type(io.trino.spi.type.Type) VarcharType(io.trino.spi.type.VarcharType)

Example 77 with BIGINT

use of io.trino.spi.type.BigintType.BIGINT in project trino by trinodb.

the class TestHivePartitionedBucketFunction method testMultiplePartitions.

@Test(dataProvider = "hiveBucketingVersion")
public void testMultiplePartitions(BucketingVersion hiveBucketingVersion) {
    int numValues = 1024;
    int numBuckets = 10;
    Block bucketColumn = createLongSequenceBlockWithNull(numValues);
    Page bucketedColumnPage = new Page(bucketColumn);
    BucketFunction hiveBucketFunction = bucketFunction(hiveBucketingVersion, numBuckets, ImmutableList.of(HIVE_LONG));
    int numPartitions = 8;
    List<Long> partitionValues = new ArrayList<>();
    for (int i = 0; i < numPartitions - 1; i++) {
        partitionValues.addAll(Collections.nCopies(numValues / numPartitions, i * 348349L));
    }
    partitionValues.addAll(Collections.nCopies(numValues / numPartitions, null));
    Block partitionColumn = createLongsBlock(partitionValues);
    Page page = new Page(bucketColumn, partitionColumn);
    Map<Long, HashMultimap<Integer, Integer>> partitionedBucketPositions = new HashMap<>();
    for (int i = 0; i < numValues; i++) {
        int hiveBucket = hiveBucketFunction.getBucket(bucketedColumnPage, i);
        Long hivePartition = partitionValues.get(i);
        // record list of positions for each combination of hive partition and bucket
        partitionedBucketPositions.computeIfAbsent(hivePartition, ignored -> HashMultimap.create()).put(hiveBucket, i);
    }
    BucketFunction hivePartitionedBucketFunction = partitionedBucketFunction(hiveBucketingVersion, numBuckets, ImmutableList.of(HIVE_LONG), ImmutableList.of(BIGINT), 4000);
    // All positions of a hive partition and bucket should hash to the same partitioned bucket
    for (Map.Entry<Long, HashMultimap<Integer, Integer>> partitionEntry : partitionedBucketPositions.entrySet()) {
        for (Map.Entry<Integer, Collection<Integer>> entry : partitionEntry.getValue().asMap().entrySet()) {
            assertBucketCount(hivePartitionedBucketFunction, page, entry.getValue(), 1);
        }
    }
    assertBucketCount(hivePartitionedBucketFunction, page, IntStream.range(0, numValues).boxed().collect(toImmutableList()), numBuckets * numPartitions);
}
Also used : IntStream(java.util.stream.IntStream) BucketFunction(io.trino.spi.connector.BucketFunction) DataProvider(org.testng.annotations.DataProvider) Collections.min(java.util.Collections.min) Type(io.trino.spi.type.Type) BUCKETING_V2(io.trino.plugin.hive.util.HiveBucketing.BucketingVersion.BUCKETING_V2) Page(io.trino.spi.Page) Assert.assertEquals(org.testng.Assert.assertEquals) Test(org.testng.annotations.Test) HashMap(java.util.HashMap) TypeOperators(io.trino.spi.type.TypeOperators) Multimap(com.google.common.collect.Multimap) ArrayList(java.util.ArrayList) BUCKETING_V1(io.trino.plugin.hive.util.HiveBucketing.BucketingVersion.BUCKETING_V1) HashMultimap(com.google.common.collect.HashMultimap) ImmutableList(com.google.common.collect.ImmutableList) Block(io.trino.spi.block.Block) Map(java.util.Map) HIVE_LONG(io.trino.plugin.hive.HiveType.HIVE_LONG) Collection(java.util.Collection) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) List(java.util.List) BucketingVersion(io.trino.plugin.hive.util.HiveBucketing.BucketingVersion) BIGINT(io.trino.spi.type.BigintType.BIGINT) Collections.max(java.util.Collections.max) BlockAssertions.createLongsBlock(io.trino.block.BlockAssertions.createLongsBlock) BlockAssertions.createLongRepeatBlock(io.trino.block.BlockAssertions.createLongRepeatBlock) BlockBuilder(io.trino.spi.block.BlockBuilder) Collections(java.util.Collections) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Page(io.trino.spi.Page) HashMultimap(com.google.common.collect.HashMultimap) Block(io.trino.spi.block.Block) BlockAssertions.createLongsBlock(io.trino.block.BlockAssertions.createLongsBlock) BlockAssertions.createLongRepeatBlock(io.trino.block.BlockAssertions.createLongRepeatBlock) Collection(java.util.Collection) BucketFunction(io.trino.spi.connector.BucketFunction) HashMap(java.util.HashMap) Map(java.util.Map) Test(org.testng.annotations.Test)

Example 78 with BIGINT

use of io.trino.spi.type.BigintType.BIGINT in project trino by trinodb.

the class BaseIcebergConnectorTest method testStatisticsConstraints.

@Test
public void testStatisticsConstraints() {
    String tableName = "iceberg.tpch.test_simple_partitioned_table_statistics";
    assertUpdate("CREATE TABLE iceberg.tpch.test_simple_partitioned_table_statistics (col1 BIGINT, col2 BIGINT) WITH (partitioning = ARRAY['col1'])");
    String insertStart = "INSERT INTO iceberg.tpch.test_simple_partitioned_table_statistics";
    assertUpdate(insertStart + " VALUES (1, 101), (2, 102), (3, 103), (4, 104)", 4);
    TableStatistics tableStatistics = getTableStatistics(tableName, new Constraint(TupleDomain.all()));
    IcebergColumnHandle col1Handle = getColumnHandleFromStatistics(tableStatistics, "col1");
    IcebergColumnHandle col2Handle = getColumnHandleFromStatistics(tableStatistics, "col2");
    // Constraint.predicate is currently not supported, because it's never provided by the engine.
    // TODO add (restore) test coverage when this changes.
    // predicate on a partition column
    assertThatThrownBy(() -> getTableStatistics(tableName, new Constraint(TupleDomain.all(), new TestRelationalNumberPredicate("col1", 3, i1 -> i1 >= 0), Set.of(col1Handle)))).isInstanceOf(VerifyException.class).hasMessage("Unexpected Constraint predicate");
    // predicate on a non-partition column
    assertThatThrownBy(() -> getTableStatistics(tableName, new Constraint(TupleDomain.all(), new TestRelationalNumberPredicate("col2", 102, i -> i >= 0), Set.of(col2Handle)))).isInstanceOf(VerifyException.class).hasMessage("Unexpected Constraint predicate");
    dropTable(tableName);
}
Also used : SkipException(org.testng.SkipException) FileSystem(org.apache.hadoop.fs.FileSystem) Test(org.testng.annotations.Test) TestTable(io.trino.testing.sql.TestTable) BROADCAST(io.trino.sql.planner.OptimizerConfig.JoinDistributionType.BROADCAST) TableNotFoundException(io.trino.spi.connector.TableNotFoundException) Matcher(java.util.regex.Matcher) Map(java.util.Map) MaterializedRow(io.trino.testing.MaterializedRow) Path(java.nio.file.Path) Assert.assertFalse(org.testng.Assert.assertFalse) Assert.assertEquals(io.trino.testing.assertions.Assert.assertEquals) GenericDatumWriter(org.apache.avro.generic.GenericDatumWriter) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) HdfsEnvironment(io.trino.plugin.hive.HdfsEnvironment) Schema(org.apache.avro.Schema) DataProviders(io.trino.testing.DataProviders) Domain(io.trino.spi.predicate.Domain) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) Set(java.util.Set) DataFileWriter(org.apache.avro.file.DataFileWriter) HDFS_ENVIRONMENT(io.trino.plugin.hive.HiveTestUtils.HDFS_ENVIRONMENT) Collectors.joining(java.util.stream.Collectors.joining) ImmutableMap.toImmutableMap(com.google.common.collect.ImmutableMap.toImmutableMap) TestingSession.testSessionBuilder(io.trino.testing.TestingSession.testSessionBuilder) Stream(java.util.stream.Stream) ORC(io.trino.plugin.iceberg.IcebergFileFormat.ORC) Session(io.trino.Session) NullableValue(io.trino.spi.predicate.NullableValue) GenericData(org.apache.avro.generic.GenericData) VARCHAR(io.trino.spi.type.VarcharType.VARCHAR) String.join(java.lang.String.join) ColumnHandle(io.trino.spi.connector.ColumnHandle) ConstraintApplicationResult(io.trino.spi.connector.ConstraintApplicationResult) LongStream(java.util.stream.LongStream) OperatorStats(io.trino.operator.OperatorStats) Language(org.intellij.lang.annotations.Language) Files(java.nio.file.Files) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) File(java.io.File) BaseConnectorTest(io.trino.testing.BaseConnectorTest) DOUBLE(io.trino.spi.type.DoubleType.DOUBLE) TableHandle(io.trino.metadata.TableHandle) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) Paths(java.nio.file.Paths) QueryRunner(io.trino.testing.QueryRunner) Domain.multipleValues(io.trino.spi.predicate.Domain.multipleValues) QueryId(io.trino.spi.QueryId) TransactionBuilder.transaction(io.trino.transaction.TransactionBuilder.transaction) IntStream.range(java.util.stream.IntStream.range) MaterializedResult(io.trino.testing.MaterializedResult) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) PREFERRED_WRITE_PARTITIONING_MIN_NUMBER_OF_PARTITIONS(io.trino.SystemSessionProperties.PREFERRED_WRITE_PARTITIONING_MIN_NUMBER_OF_PARTITIONS) MoreCollectors.onlyElement(com.google.common.collect.MoreCollectors.onlyElement) ICEBERG_DOMAIN_COMPACTION_THRESHOLD(io.trino.plugin.iceberg.IcebergSplitManager.ICEBERG_DOMAIN_COMPACTION_THRESHOLD) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) QueryAssertions.assertEqualsIgnoreOrder(io.trino.testing.QueryAssertions.assertEqualsIgnoreOrder) Locale(java.util.Locale) Iterables.concat(com.google.common.collect.Iterables.concat) TestingConnectorBehavior(io.trino.testing.TestingConnectorBehavior) TestTable.randomTableSuffix(io.trino.testing.sql.TestTable.randomTableSuffix) TpchTable(io.trino.tpch.TpchTable) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) Collections.nCopies(java.util.Collections.nCopies) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) ICEBERG_CATALOG(io.trino.plugin.iceberg.IcebergQueryRunner.ICEBERG_CATALOG) DataSize(io.airlift.units.DataSize) List(java.util.List) HdfsContext(io.trino.plugin.hive.HdfsEnvironment.HdfsContext) BIGINT(io.trino.spi.type.BigintType.BIGINT) Domain.singleValue(io.trino.spi.predicate.Domain.singleValue) Optional(java.util.Optional) Pattern(java.util.regex.Pattern) DataFileReader(org.apache.avro.file.DataFileReader) GenericDatumReader(org.apache.avro.generic.GenericDatumReader) IcebergQueryRunner.createIcebergQueryRunner(io.trino.plugin.iceberg.IcebergQueryRunner.createIcebergQueryRunner) LINE_ITEM(io.trino.tpch.TpchTable.LINE_ITEM) IntStream(java.util.stream.IntStream) Constraint(io.trino.spi.connector.Constraint) DataProvider(org.testng.annotations.DataProvider) PARQUET(io.trino.plugin.iceberg.IcebergFileFormat.PARQUET) ImmutableList(com.google.common.collect.ImmutableList) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) Objects.requireNonNull(java.util.Objects.requireNonNull) TableStatistics(io.trino.spi.statistics.TableStatistics) NoSuchElementException(java.util.NoSuchElementException) VerifyException(com.google.common.base.VerifyException) OutputStream(java.io.OutputStream) ResultWithQueryId(io.trino.testing.ResultWithQueryId) TupleDomain(io.trino.spi.predicate.TupleDomain) Consumer(java.util.function.Consumer) Assert.assertEventually(io.trino.testing.assertions.Assert.assertEventually) JOIN_DISTRIBUTION_TYPE(io.trino.SystemSessionProperties.JOIN_DISTRIBUTION_TYPE) Metadata(io.trino.metadata.Metadata) Assert.assertTrue(org.testng.Assert.assertTrue) MaterializedResult.resultBuilder(io.trino.testing.MaterializedResult.resultBuilder) Constraint(io.trino.spi.connector.Constraint) VerifyException(com.google.common.base.VerifyException) TableStatistics(io.trino.spi.statistics.TableStatistics) Test(org.testng.annotations.Test) BaseConnectorTest(io.trino.testing.BaseConnectorTest)

Example 79 with BIGINT

use of io.trino.spi.type.BigintType.BIGINT in project trino by trinodb.

the class TestCreateTableTask method testCreateTableNotExistsTrue.

@Test
public void testCreateTableNotExistsTrue() {
    CreateTable statement = new CreateTable(QualifiedName.of("test_table"), ImmutableList.of(new ColumnDefinition(identifier("a"), toSqlType(BIGINT), true, emptyList(), Optional.empty())), true, ImmutableList.of(), Optional.empty());
    CreateTableTask createTableTask = new CreateTableTask(plannerContext, new AllowAllAccessControl(), columnPropertyManager, tablePropertyManager);
    getFutureValue(createTableTask.internalExecute(statement, testSession, emptyList(), output -> {
    }));
    assertEquals(metadata.getCreateTableCallCount(), 1);
}
Also used : TransactionManager(io.trino.transaction.TransactionManager) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) Test(org.testng.annotations.Test) TestingTableHandle(io.trino.testing.TestingMetadata.TestingTableHandle) AfterMethod(org.testng.annotations.AfterMethod) SHOW_CREATE_TABLE(io.trino.testing.TestingAccessControlManager.TestingPrivilegeType.SHOW_CREATE_TABLE) TrinoExceptionAssert.assertTrinoExceptionThrownBy(io.trino.testing.assertions.TrinoExceptionAssert.assertTrinoExceptionThrownBy) TestingConnectorTransactionHandle(io.trino.sql.planner.TestingConnectorTransactionHandle) NOT_SUPPORTED(io.trino.spi.StandardErrorCode.NOT_SUPPORTED) CatalogName(io.trino.connector.CatalogName) MockConnectorFactory(io.trino.connector.MockConnectorFactory) Map(java.util.Map) TEST_SESSION(io.trino.SessionTestUtils.TEST_SESSION) ALREADY_EXISTS(io.trino.spi.StandardErrorCode.ALREADY_EXISTS) TestingAccessControlManager(io.trino.testing.TestingAccessControlManager) Property(io.trino.sql.tree.Property) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) Identifier(io.trino.sql.tree.Identifier) SMALLINT(io.trino.spi.type.SmallintType.SMALLINT) TableElement(io.trino.sql.tree.TableElement) ImmutableMap(com.google.common.collect.ImmutableMap) INCLUDING(io.trino.sql.tree.LikeClause.PropertiesOption.INCLUDING) Collections.emptyList(java.util.Collections.emptyList) TypeSignatureTranslator.toSqlType(io.trino.sql.analyzer.TypeSignatureTranslator.toSqlType) BeforeMethod(org.testng.annotations.BeforeMethod) Set(java.util.Set) TrinoException(io.trino.spi.TrinoException) SchemaTableName(io.trino.spi.connector.SchemaTableName) TableMetadata(io.trino.metadata.TableMetadata) List(java.util.List) TestingSession.testSessionBuilder(io.trino.testing.TestingSession.testSessionBuilder) CreateTable(io.trino.sql.tree.CreateTable) BIGINT(io.trino.spi.type.BigintType.BIGINT) Optional(java.util.Optional) ColumnPropertyManager(io.trino.metadata.ColumnPropertyManager) DATE(io.trino.spi.type.DateType.DATE) Session(io.trino.Session) PlannerContext(io.trino.sql.PlannerContext) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) AccessDeniedException(io.trino.spi.security.AccessDeniedException) SELECT_COLUMN(io.trino.testing.TestingAccessControlManager.TestingPrivilegeType.SELECT_COLUMN) AllowAllAccessControl(io.trino.security.AllowAllAccessControl) ColumnMetadata(io.trino.spi.connector.ColumnMetadata) Assert.assertEquals(org.testng.Assert.assertEquals) ConnectorTableMetadata(io.trino.spi.connector.ConnectorTableMetadata) VARCHAR(io.trino.spi.type.VarcharType.VARCHAR) ImmutableList(com.google.common.collect.ImmutableList) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) TestingPlannerContext.plannerContextBuilder(io.trino.sql.planner.TestingPlannerContext.plannerContextBuilder) LocalQueryRunner(io.trino.testing.LocalQueryRunner) Objects.requireNonNull(java.util.Objects.requireNonNull) ColumnHandle(io.trino.spi.connector.ColumnHandle) INVALID_TABLE_PROPERTY(io.trino.spi.StandardErrorCode.INVALID_TABLE_PROPERTY) VARBINARY(io.trino.spi.type.VarbinaryType.VARBINARY) EventListenerConfig(io.trino.eventlistener.EventListenerConfig) PropertyMetadata.stringProperty(io.trino.spi.session.PropertyMetadata.stringProperty) QueryUtil.identifier(io.trino.sql.QueryUtil.identifier) Sets.immutableEnumSet(com.google.common.collect.Sets.immutableEnumSet) NOT_NULL_COLUMN_CONSTRAINT(io.trino.spi.connector.ConnectorCapabilities.NOT_NULL_COLUMN_CONSTRAINT) StringLiteral(io.trino.sql.tree.StringLiteral) Collections.emptySet(java.util.Collections.emptySet) ConnectorCapabilities(io.trino.spi.connector.ConnectorCapabilities) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) QualifiedName(io.trino.sql.tree.QualifiedName) LikeClause(io.trino.sql.tree.LikeClause) AbstractMockMetadata(io.trino.metadata.AbstractMockMetadata) TableHandle(io.trino.metadata.TableHandle) TestingAccessControlManager.privilege(io.trino.testing.TestingAccessControlManager.privilege) QualifiedObjectName(io.trino.metadata.QualifiedObjectName) EventListenerManager(io.trino.eventlistener.EventListenerManager) TablePropertyManager(io.trino.metadata.TablePropertyManager) Assert.assertTrue(org.testng.Assert.assertTrue) ColumnDefinition(io.trino.sql.tree.ColumnDefinition) AllowAllAccessControl(io.trino.security.AllowAllAccessControl) CreateTable(io.trino.sql.tree.CreateTable) ColumnDefinition(io.trino.sql.tree.ColumnDefinition) Test(org.testng.annotations.Test)

Example 80 with BIGINT

use of io.trino.spi.type.BigintType.BIGINT in project trino by trinodb.

the class TestValuesNodeStats method testStatsForValuesNode.

@Test
public void testStatsForValuesNode() {
    tester().assertStatsFor(pb -> pb.values(ImmutableList.of(pb.symbol("a", BIGINT), pb.symbol("b", DOUBLE)), ImmutableList.of(ImmutableList.of(expression("3+3"), expression("13.5e0")), ImmutableList.of(expression("55"), expression("null")), ImmutableList.of(expression("6"), expression("13.5e0"))))).check(outputStats -> outputStats.equalTo(PlanNodeStatsEstimate.builder().setOutputRowCount(3).addSymbolStatistics(new Symbol("a"), SymbolStatsEstimate.builder().setNullsFraction(0).setLowValue(6).setHighValue(55).setDistinctValuesCount(2).build()).addSymbolStatistics(new Symbol("b"), SymbolStatsEstimate.builder().setNullsFraction(0.33333333333333333).setLowValue(13.5).setHighValue(13.5).setDistinctValuesCount(1).build()).build()));
    tester().assertStatsFor(pb -> pb.values(ImmutableList.of(pb.symbol("v", createVarcharType(30))), ImmutableList.of(ImmutableList.of(expression("'Alice'")), ImmutableList.of(expression("'has'")), ImmutableList.of(expression("'a cat'")), ImmutableList.of(expression("null"))))).check(outputStats -> outputStats.equalTo(PlanNodeStatsEstimate.builder().setOutputRowCount(4).addSymbolStatistics(new Symbol("v"), SymbolStatsEstimate.builder().setNullsFraction(0.25).setDistinctValuesCount(3).build()).build()));
}
Also used : DOUBLE(io.trino.spi.type.DoubleType.DOUBLE) Symbol(io.trino.sql.planner.Symbol) ImmutableList(com.google.common.collect.ImmutableList) BIGINT(io.trino.spi.type.BigintType.BIGINT) UNKNOWN(io.trino.type.UnknownType.UNKNOWN) Test(org.testng.annotations.Test) VarcharType.createVarcharType(io.trino.spi.type.VarcharType.createVarcharType) PlanBuilder.expression(io.trino.sql.planner.iterative.rule.test.PlanBuilder.expression) Symbol(io.trino.sql.planner.Symbol) Test(org.testng.annotations.Test)

Aggregations

BIGINT (io.trino.spi.type.BigintType.BIGINT)106 ImmutableList (com.google.common.collect.ImmutableList)99 Optional (java.util.Optional)87 Test (org.testng.annotations.Test)86 ImmutableMap (com.google.common.collect.ImmutableMap)84 VARCHAR (io.trino.spi.type.VarcharType.VARCHAR)44 List (java.util.List)44 Map (java.util.Map)44 ColumnHandle (io.trino.spi.connector.ColumnHandle)38 Type (io.trino.spi.type.Type)38 Symbol (io.trino.sql.planner.Symbol)38 QualifiedName (io.trino.sql.tree.QualifiedName)38 ImmutableSet (com.google.common.collect.ImmutableSet)37 TupleDomain (io.trino.spi.predicate.TupleDomain)36 PlanMatchPattern.values (io.trino.sql.planner.assertions.PlanMatchPattern.values)36 TableHandle (io.trino.metadata.TableHandle)35 BaseRuleTest (io.trino.sql.planner.iterative.rule.test.BaseRuleTest)35 Session (io.trino.Session)34 PlanMatchPattern.filter (io.trino.sql.planner.assertions.PlanMatchPattern.filter)33 PlanBuilder.expression (io.trino.sql.planner.iterative.rule.test.PlanBuilder.expression)32