Examples with TEXTFILE io.trino.plugin.hive.HiveStorageFormat.TEXTFILE used on opensource projects

Example 1 with TEXTFILE

use of io.trino.plugin.hive.HiveStorageFormat.TEXTFILE in project trino by trinodb.

the class AbstractTestHive method testStorePartitionWithStatistics.

protected void testStorePartitionWithStatistics(List<ColumnMetadata> columns, PartitionStatistics statsForAllColumns1, PartitionStatistics statsForAllColumns2, PartitionStatistics statsForSubsetOfColumns, PartitionStatistics emptyStatistics) throws Exception {
    SchemaTableName tableName = temporaryTable("store_partition_with_statistics");
    try {
        doCreateEmptyTable(tableName, ORC, columns);
        HiveMetastoreClosure metastoreClient = new HiveMetastoreClosure(getMetastoreClient());
        Table table = metastoreClient.getTable(tableName.getSchemaName(), tableName.getTableName()).orElseThrow(() -> new TableNotFoundException(tableName));
        List<String> partitionValues = ImmutableList.of("2016-01-01");
        String partitionName = makePartName(ImmutableList.of("ds"), partitionValues);
        Partition partition = createDummyPartition(table, partitionName);
        // create partition with stats for all columns
        metastoreClient.addPartitions(tableName.getSchemaName(), tableName.getTableName(), ImmutableList.of(new PartitionWithStatistics(partition, partitionName, statsForAllColumns1)));
        assertEquals(metastoreClient.getPartition(tableName.getSchemaName(), tableName.getTableName(), partitionValues).get().getStorage().getStorageFormat(), fromHiveStorageFormat(ORC));
        assertThat(metastoreClient.getPartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), ImmutableSet.of(partitionName))).isEqualTo(ImmutableMap.of(partitionName, statsForAllColumns1));
        // alter the partition into one with other stats
        Partition modifiedPartition = Partition.builder(partition).withStorage(storage -> storage.setStorageFormat(fromHiveStorageFormat(RCBINARY)).setLocation(partitionTargetPath(tableName, partitionName))).build();
        metastoreClient.alterPartition(tableName.getSchemaName(), tableName.getTableName(), new PartitionWithStatistics(modifiedPartition, partitionName, statsForAllColumns2));
        assertEquals(metastoreClient.getPartition(tableName.getSchemaName(), tableName.getTableName(), partitionValues).get().getStorage().getStorageFormat(), fromHiveStorageFormat(RCBINARY));
        assertThat(metastoreClient.getPartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), ImmutableSet.of(partitionName))).isEqualTo(ImmutableMap.of(partitionName, statsForAllColumns2));
        // alter the partition into one with stats for only subset of columns
        modifiedPartition = Partition.builder(partition).withStorage(storage -> storage.setStorageFormat(fromHiveStorageFormat(TEXTFILE)).setLocation(partitionTargetPath(tableName, partitionName))).build();
        metastoreClient.alterPartition(tableName.getSchemaName(), tableName.getTableName(), new PartitionWithStatistics(modifiedPartition, partitionName, statsForSubsetOfColumns));
        assertThat(metastoreClient.getPartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), ImmutableSet.of(partitionName))).isEqualTo(ImmutableMap.of(partitionName, statsForSubsetOfColumns));
        // alter the partition into one without stats
        modifiedPartition = Partition.builder(partition).withStorage(storage -> storage.setStorageFormat(fromHiveStorageFormat(TEXTFILE)).setLocation(partitionTargetPath(tableName, partitionName))).build();
        metastoreClient.alterPartition(tableName.getSchemaName(), tableName.getTableName(), new PartitionWithStatistics(modifiedPartition, partitionName, emptyStatistics));
        assertThat(metastoreClient.getPartitionStatistics(tableName.getSchemaName(), tableName.getTableName(), ImmutableSet.of(partitionName))).isEqualTo(ImmutableMap.of(partitionName, emptyStatistics));
    } finally {
        dropTable(tableName);
    }
}

Example 2 with TEXTFILE

use of io.trino.plugin.hive.HiveStorageFormat.TEXTFILE in project trino by trinodb.

the class ViewReaderUtil method coralTableRedirectionResolver.

private static CoralTableRedirectionResolver coralTableRedirectionResolver(ConnectorSession session, BiFunction<ConnectorSession, SchemaTableName, Optional<CatalogSchemaTableName>> tableRedirectionResolver, MetadataProvider metadataProvider) {
    return schemaTableName -> tableRedirectionResolver.apply(session, schemaTableName).map(target -> {
        ConnectorTableSchema tableSchema = metadataProvider.getRelationMetadata(session, target).orElseThrow(() -> new TableNotFoundException(target.getSchemaTableName(), format("%s is redirected to %s, but that relation cannot be found", schemaTableName, target)));
        List<Column> columns = tableSchema.getColumns().stream().filter(columnSchema -> !columnSchema.isHidden()).map(columnSchema -> new Column(columnSchema.getName(), toHiveType(columnSchema.getType()), Optional.empty())).collect(toImmutableList());
        Table table = Table.builder().setDatabaseName(schemaTableName.getSchemaName()).setTableName(schemaTableName.getTableName()).setTableType(EXTERNAL_TABLE.name()).setDataColumns(columns).withStorage(storage -> storage.setStorageFormat(fromHiveStorageFormat(TEXTFILE))).setOwner(Optional.empty()).build();
        return toMetastoreApiTable(table);
    });
}

Example 3 with TEXTFILE

use of io.trino.plugin.hive.HiveStorageFormat.TEXTFILE in project trino by trinodb.

the class TestCachingHiveMetastore method testLoadAfterInvalidate.

@Test(timeOut = 60_000, dataProviderClass = DataProviders.class, dataProvider = "trueFalse")
public void testLoadAfterInvalidate(boolean invalidateAll) throws Exception {
    // State
    CopyOnWriteArrayList<Column> tableColumns = new CopyOnWriteArrayList<>();
    ConcurrentMap<String, Partition> tablePartitionsByName = new ConcurrentHashMap<>();
    Map<String, String> tableParameters = new ConcurrentHashMap<>();
    tableParameters.put("frequent-changing-table-parameter", "parameter initial value");
    // Initialize data
    String databaseName = "my_database";
    String tableName = "my_table_name";
    tableColumns.add(new Column("value", toHiveType(VARCHAR), Optional.empty()));
    tableColumns.add(new Column("pk", toHiveType(VARCHAR), Optional.empty()));
    List<String> partitionNames = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String partitionName = "pk=" + i;
        tablePartitionsByName.put(partitionName, Partition.builder().setDatabaseName(databaseName).setTableName(tableName).setColumns(ImmutableList.copyOf(tableColumns)).setValues(List.of(Integer.toString(i))).withStorage(storage -> storage.setStorageFormat(fromHiveStorageFormat(TEXTFILE))).setParameters(Map.of("frequent-changing-partition-parameter", "parameter initial value")).build());
        partitionNames.add(partitionName);
    }
    // Mock metastore
    CountDownLatch getTableEnteredLatch = new CountDownLatch(1);
    CountDownLatch getTableReturnLatch = new CountDownLatch(1);
    CountDownLatch getTableFinishedLatch = new CountDownLatch(1);
    CountDownLatch getPartitionsByNamesEnteredLatch = new CountDownLatch(1);
    CountDownLatch getPartitionsByNamesReturnLatch = new CountDownLatch(1);
    CountDownLatch getPartitionsByNamesFinishedLatch = new CountDownLatch(1);
    HiveMetastore mockMetastore = new UnimplementedHiveMetastore() {

        @Override
        public Optional<Table> getTable(String databaseName, String tableName) {
            Optional<Table> table = Optional.of(Table.builder().setDatabaseName(databaseName).setTableName(tableName).setTableType(EXTERNAL_TABLE.name()).setDataColumns(tableColumns).setParameters(ImmutableMap.copyOf(tableParameters)).withStorage(storage -> storage.setStorageFormat(fromHiveStorageFormat(TEXTFILE))).setOwner(Optional.empty()).build());
            // 1
            getTableEnteredLatch.countDown();
            // 2
            await(getTableReturnLatch, 10, SECONDS);
            return table;
        }

        @Override
        public Map<String, Optional<Partition>> getPartitionsByNames(Table table, List<String> partitionNames) {
            Map<String, Optional<Partition>> result = new HashMap<>();
            for (String partitionName : partitionNames) {
                result.put(partitionName, Optional.ofNullable(tablePartitionsByName.get(partitionName)));
            }
            // loader#1
            getPartitionsByNamesEnteredLatch.countDown();
            // loader#2
            await(getPartitionsByNamesReturnLatch, 10, SECONDS);
            return result;
        }
    };
    // Caching metastore
    metastore = cachingHiveMetastore(mockMetastore, executor, new Duration(5, TimeUnit.MINUTES), Optional.of(new Duration(1, TimeUnit.MINUTES)), 1000);
    // The test. Main thread does modifications and verifies subsequent load sees them. Background thread loads the state into the cache.
    ExecutorService executor = Executors.newFixedThreadPool(1);
    try {
        Future<Void> future = executor.submit(() -> {
            try {
                Table table;
                table = metastore.getTable(databaseName, tableName).orElseThrow();
                // 3
                getTableFinishedLatch.countDown();
                metastore.getPartitionsByNames(table, partitionNames);
                // 6
                getPartitionsByNamesFinishedLatch.countDown();
                return (Void) null;
            } catch (Throwable e) {
                log.error(e);
                throw e;
            }
        });
        // 21
        await(getTableEnteredLatch, 10, SECONDS);
        tableParameters.put("frequent-changing-table-parameter", "main-thread-put-xyz");
        if (invalidateAll) {
            metastore.flushCache();
        } else {
            metastore.invalidateTable(databaseName, tableName);
        }
        // 2
        getTableReturnLatch.countDown();
        // 3
        await(getTableFinishedLatch, 10, SECONDS);
        Table table = metastore.getTable(databaseName, tableName).orElseThrow();
        assertThat(table.getParameters()).isEqualTo(Map.of("frequent-changing-table-parameter", "main-thread-put-xyz"));
        // 4
        await(getPartitionsByNamesEnteredLatch, 10, SECONDS);
        String partitionName = partitionNames.get(2);
        Map<String, String> newPartitionParameters = Map.of("frequent-changing-partition-parameter", "main-thread-put-alice");
        tablePartitionsByName.put(partitionName, Partition.builder(tablePartitionsByName.get(partitionName)).setParameters(newPartitionParameters).build());
        if (invalidateAll) {
            metastore.flushCache();
        } else {
            metastore.invalidateTable(databaseName, tableName);
        }
        // 5
        getPartitionsByNamesReturnLatch.countDown();
        // 6
        await(getPartitionsByNamesFinishedLatch, 10, SECONDS);
        Map<String, Optional<Partition>> loadedPartitions = metastore.getPartitionsByNames(table, partitionNames);
        assertThat(loadedPartitions.get(partitionName)).isNotNull().isPresent().hasValueSatisfying(partition -> assertThat(partition.getParameters()).isEqualTo(newPartitionParameters));
        // verify no failure in the background thread
        future.get(10, SECONDS);
    } finally {
        getTableEnteredLatch.countDown();
        getTableReturnLatch.countDown();
        getTableFinishedLatch.countDown();
        getPartitionsByNamesEnteredLatch.countDown();
        getPartitionsByNamesReturnLatch.countDown();
        getPartitionsByNamesFinishedLatch.countDown();
        executor.shutdownNow();
        executor.awaitTermination(10, SECONDS);
    }
}