Examples with Column io.trino.plugin.hive.metastore.Column used on opensource projects

Example 16 with Column

use of io.trino.plugin.hive.metastore.Column in project trino by trinodb.

the class AbstractTestHive method testDisallowQueryingOfIcebergTables.

@Test
public void testDisallowQueryingOfIcebergTables() {
    ConnectorSession session = newSession();
    SchemaTableName tableName = temporaryTable("trino_iceberg_table");
    Table.Builder table = Table.builder().setDatabaseName(tableName.getSchemaName()).setTableName(tableName.getTableName()).setOwner(Optional.of(session.getUser())).setTableType(MANAGED_TABLE.name()).setPartitionColumns(List.of(new Column("a_partition_column", HIVE_INT, Optional.empty()))).setDataColumns(List.of(new Column("a_column", HIVE_STRING, Optional.empty()))).setParameter(ICEBERG_TABLE_TYPE_NAME, ICEBERG_TABLE_TYPE_VALUE);
    table.getStorageBuilder().setStorageFormat(fromHiveStorageFormat(PARQUET)).setLocation(getTableDefaultLocation(metastoreClient.getDatabase(tableName.getSchemaName()).orElseThrow(), new HdfsContext(session.getIdentity()), hdfsEnvironment, tableName.getSchemaName(), tableName.getTableName()).toString());
    metastoreClient.createTable(table.build(), NO_PRIVILEGES);
    try {
        // Verify that the table was created as a Iceberg table can't be queried in hive
        try (Transaction transaction = newTransaction()) {
            ConnectorMetadata metadata = transaction.getMetadata();
            metadata.beginQuery(session);
            assertThatThrownBy(() -> getTableHandle(metadata, tableName)).hasMessage(format("Cannot query Iceberg table '%s'", tableName));
        }
        // Verify the hidden `$properties` and `$partitions` hive system tables table handle can't be obtained for the Iceberg tables
        try (Transaction transaction = newTransaction()) {
            ConnectorMetadata metadata = transaction.getMetadata();
            metadata.beginQuery(session);
            SchemaTableName propertiesTableName = new SchemaTableName(tableName.getSchemaName(), format("%s$properties", tableName.getTableName()));
            assertThat(metadata.getSystemTable(newSession(), propertiesTableName)).isEmpty();
            SchemaTableName partitionsTableName = new SchemaTableName(tableName.getSchemaName(), format("%s$partitions", tableName.getTableName()));
            assertThat(metadata.getSystemTable(newSession(), partitionsTableName)).isEmpty();
        }
    } finally {
        // Clean up
        metastoreClient.dropTable(tableName.getSchemaName(), tableName.getTableName(), true);
    }
}

Example 17 with Column

use of io.trino.plugin.hive.metastore.Column in project trino by trinodb.

the class AbstractTestHive method doTestBucketedTableEvolution.

private void doTestBucketedTableEvolution(HiveStorageFormat storageFormat, SchemaTableName tableName) throws Exception {
    int rowCount = 100;
    // 
    // Produce a table with 8 buckets.
    // The table has 3 partitions of 3 different bucket count (4, 8, 16).
    createEmptyTable(tableName, storageFormat, ImmutableList.of(new Column("id", HIVE_LONG, Optional.empty()), new Column("name", HIVE_STRING, Optional.empty())), ImmutableList.of(new Column("pk", HIVE_STRING, Optional.empty())), Optional.of(new HiveBucketProperty(ImmutableList.of("id"), BUCKETING_V1, 4, ImmutableList.of())));
    // write a 4-bucket partition
    MaterializedResult.Builder bucket4Builder = MaterializedResult.resultBuilder(SESSION, BIGINT, VARCHAR, VARCHAR);
    IntStream.range(0, rowCount).forEach(i -> bucket4Builder.row((long) i, String.valueOf(i), "four"));
    insertData(tableName, bucket4Builder.build());
    // write a 16-bucket partition
    alterBucketProperty(tableName, Optional.of(new HiveBucketProperty(ImmutableList.of("id"), BUCKETING_V1, 16, ImmutableList.of())));
    MaterializedResult.Builder bucket16Builder = MaterializedResult.resultBuilder(SESSION, BIGINT, VARCHAR, VARCHAR);
    IntStream.range(0, rowCount).forEach(i -> bucket16Builder.row((long) i, String.valueOf(i), "sixteen"));
    insertData(tableName, bucket16Builder.build());
    // write an 8-bucket partition
    alterBucketProperty(tableName, Optional.of(new HiveBucketProperty(ImmutableList.of("id"), BUCKETING_V1, 8, ImmutableList.of())));
    MaterializedResult.Builder bucket8Builder = MaterializedResult.resultBuilder(SESSION, BIGINT, VARCHAR, VARCHAR);
    IntStream.range(0, rowCount).forEach(i -> bucket8Builder.row((long) i, String.valueOf(i), "eight"));
    insertData(tableName, bucket8Builder.build());
    try (Transaction transaction = newTransaction()) {
        ConnectorMetadata metadata = transaction.getMetadata();
        ConnectorSession session = newSession();
        metadata.beginQuery(session);
        ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
        // read entire table
        List<ColumnHandle> columnHandles = ImmutableList.<ColumnHandle>builder().addAll(metadata.getColumnHandles(session, tableHandle).values()).build();
        MaterializedResult result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.all(), OptionalInt.empty(), Optional.empty());
        assertBucketTableEvolutionResult(result, columnHandles, ImmutableSet.of(0, 1, 2, 3, 4, 5, 6, 7), rowCount);
        // read single bucket (table/logical bucket)
        result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.fromFixedValues(ImmutableMap.of(bucketColumnHandle(), NullableValue.of(INTEGER, 6L))), OptionalInt.empty(), Optional.empty());
        assertBucketTableEvolutionResult(result, columnHandles, ImmutableSet.of(6), rowCount);
        // read single bucket, without selecting the bucketing column (i.e. id column)
        columnHandles = ImmutableList.<ColumnHandle>builder().addAll(metadata.getColumnHandles(session, tableHandle).values().stream().filter(columnHandle -> !"id".equals(((HiveColumnHandle) columnHandle).getName())).collect(toImmutableList())).build();
        result = readTable(transaction, tableHandle, columnHandles, session, TupleDomain.fromFixedValues(ImmutableMap.of(bucketColumnHandle(), NullableValue.of(INTEGER, 6L))), OptionalInt.empty(), Optional.empty());
        assertBucketTableEvolutionResult(result, columnHandles, ImmutableSet.of(6), rowCount);
    }
}

Example 18 with Column

use of io.trino.plugin.hive.metastore.Column in project trino by trinodb.

the class AbstractTestHiveLocal method doTestSparkBucketedTableValidation.

private void doTestSparkBucketedTableValidation(SchemaTableName tableName) throws Exception {
    java.nio.file.Path externalLocation = copyResourceDirToTemporaryDirectory("spark_bucketed_nation");
    try {
        createExternalTable(tableName, ORC, ImmutableList.of(new Column("nationkey", HIVE_INT, Optional.empty()), new Column("name", HIVE_STRING, Optional.empty()), new Column("regionkey", HIVE_INT, Optional.empty()), new Column("comment", HIVE_STRING, Optional.empty())), ImmutableList.of(), Optional.of(new HiveBucketProperty(ImmutableList.of("nationkey"), BUCKETING_V1, 3, ImmutableList.of(new SortingColumn("name", SortingColumn.Order.ASCENDING)))), new Path(URI.create("file://" + externalLocation.toString())));
        assertReadFailsWithMessageMatching(ORC, tableName, "Hive table is corrupt\\. File '.*/.*' is for bucket [0-2], but contains a row for bucket [0-2].");
        markTableAsCreatedBySpark(tableName, "orc");
        assertReadReturnsRowCount(ORC, tableName, 25);
    } finally {
        deleteRecursively(externalLocation, RecursiveDeleteOption.ALLOW_INSECURE);
    }
}

Example 19 with Column

use of io.trino.plugin.hive.metastore.Column in project trino by trinodb.

the class AbstractTestHive method doTestTransactionDeleteInsert.

protected void doTestTransactionDeleteInsert(HiveStorageFormat storageFormat, boolean allowInsertExisting, List<TransactionDeleteInsertTestCase> testCases) throws Exception {
    // There are 4 types of operations on a partition: add, drop, alter (drop then add), insert existing.
    // There are 12 partitions in this test, 3 for each type.
    // 3 is chosen to verify that cleanups, commit aborts, rollbacks are always as complete as possible regardless of failure.
    MaterializedResult beforeData = MaterializedResult.resultBuilder(SESSION, BIGINT, createUnboundedVarcharType(), createUnboundedVarcharType()).row(110L, "a", "alter1").row(120L, "a", "insert1").row(140L, "a", "drop1").row(210L, "b", "drop2").row(310L, "c", "alter2").row(320L, "c", "alter3").row(510L, "e", "drop3").row(610L, "f", "insert2").row(620L, "f", "insert3").build();
    Domain domainToDrop = Domain.create(ValueSet.of(createUnboundedVarcharType(), utf8Slice("alter1"), utf8Slice("alter2"), utf8Slice("alter3"), utf8Slice("drop1"), utf8Slice("drop2"), utf8Slice("drop3")), false);
    List<MaterializedRow> extraRowsForInsertExisting = ImmutableList.of();
    if (allowInsertExisting) {
        extraRowsForInsertExisting = MaterializedResult.resultBuilder(SESSION, BIGINT, createUnboundedVarcharType(), createUnboundedVarcharType()).row(121L, "a", "insert1").row(611L, "f", "insert2").row(621L, "f", "insert3").build().getMaterializedRows();
    }
    MaterializedResult insertData = MaterializedResult.resultBuilder(SESSION, BIGINT, createUnboundedVarcharType(), createUnboundedVarcharType()).row(111L, "a", "alter1").row(131L, "a", "add1").row(221L, "b", "add2").row(311L, "c", "alter2").row(321L, "c", "alter3").row(411L, "d", "add3").rows(extraRowsForInsertExisting).build();
    MaterializedResult afterData = MaterializedResult.resultBuilder(SESSION, BIGINT, createUnboundedVarcharType(), createUnboundedVarcharType()).row(120L, "a", "insert1").row(610L, "f", "insert2").row(620L, "f", "insert3").rows(insertData.getMaterializedRows()).build();
    for (TransactionDeleteInsertTestCase testCase : testCases) {
        SchemaTableName temporaryDeleteInsert = temporaryTable("delete_insert");
        try {
            createEmptyTable(temporaryDeleteInsert, storageFormat, ImmutableList.of(new Column("col1", HIVE_LONG, Optional.empty())), ImmutableList.of(new Column("pk1", HIVE_STRING, Optional.empty()), new Column("pk2", HIVE_STRING, Optional.empty())));
            insertData(temporaryDeleteInsert, beforeData);
            try {
                doTestTransactionDeleteInsert(storageFormat, temporaryDeleteInsert, domainToDrop, insertData, testCase.isExpectCommitedData() ? afterData : beforeData, testCase.getTag(), testCase.isExpectQuerySucceed(), testCase.getConflictTrigger());
            } catch (AssertionError e) {
                throw new AssertionError(format("Test case: %s", testCase), e);
            }
        } finally {
            dropTable(temporaryDeleteInsert);
        }
    }
}

Example 20 with Column

use of io.trino.plugin.hive.metastore.Column in project trino by trinodb.

the class AbstractTestHive method testPreferredInsertLayout.

@Test
public void testPreferredInsertLayout() throws Exception {
    SchemaTableName tableName = temporaryTable("empty_partitioned_table");
    try {
        Column partitioningColumn = new Column("column2", HIVE_STRING, Optional.empty());
        List<Column> columns = ImmutableList.of(new Column("column1", HIVE_STRING, Optional.empty()), partitioningColumn);
        createEmptyTable(tableName, ORC, columns, ImmutableList.of(partitioningColumn));
        try (Transaction transaction = newTransaction()) {
            ConnectorMetadata metadata = transaction.getMetadata();
            ConnectorSession session = newSession();
            ConnectorTableHandle tableHandle = getTableHandle(metadata, tableName);
            Optional<ConnectorTableLayout> insertLayout = metadata.getInsertLayout(session, tableHandle);
            assertTrue(insertLayout.isPresent());
            assertFalse(insertLayout.get().getPartitioning().isPresent());
            assertEquals(insertLayout.get().getPartitionColumns(), ImmutableList.of(partitioningColumn.getName()));
        }
    } finally {
        dropTable(tableName);
    }
}