Examples with TpchTable io.trino.tpch.TpchTable used on opensource projects

Example 1 with TpchTable

use of io.trino.tpch.TpchTable in project trino by trinodb.

the class BenchmarkFileFormatsUtils method createTpchDataSet.

public static <E extends TpchEntity> TestData createTpchDataSet(FileFormat format, TpchTable<E> tpchTable, List<TpchColumn<E>> columns) {
    List<String> columnNames = columns.stream().map(TpchColumn::getColumnName).collect(toList());
    List<Type> columnTypes = columns.stream().map(BenchmarkFileFormatsUtils::getColumnType).map(type -> format.supportsDate() || !DATE.equals(type) ? type : createUnboundedVarcharType()).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    ImmutableList.Builder<Page> pages = ImmutableList.builder();
    long dataSize = 0;
    for (E row : tpchTable.createGenerator(10, 1, 1)) {
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            TpchColumn<E> column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(row));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(row));
                    break;
                case DATE:
                    if (format.supportsDate()) {
                        DATE.writeLong(blockBuilder, column.getDate(row));
                    } else {
                        createUnboundedVarcharType().writeString(blockBuilder, column.getString(row));
                    }
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(row));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(row)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
        if (pageBuilder.isFull()) {
            Page page = pageBuilder.build();
            pages.add(page);
            pageBuilder.reset();
            dataSize += page.getSizeInBytes();
            if (dataSize >= MIN_DATA_SIZE) {
                break;
            }
        }
    }
    if (!pageBuilder.isEmpty()) {
        pages.add(pageBuilder.build());
    }
    return new TestData(columnNames, columnTypes, pages.build());
}

Example 2 with TpchTable

use of io.trino.tpch.TpchTable in project trino by trinodb.

the class TestIcebergQueryFailureRecoveryTest method createQueryRunner.

@Override
protected QueryRunner createQueryRunner(List<TpchTable<?>> requiredTpchTables, Map<String, String> configProperties, Map<String, String> coordinatorProperties) throws Exception {
    this.minioStorage = new MinioStorage("test-exchange-spooling-" + randomTableSuffix());
    minioStorage.start();
    return IcebergQueryRunner.builder().setInitialTables(requiredTpchTables).setCoordinatorProperties(coordinatorProperties).setExtraProperties(configProperties).setAdditionalSetup(runner -> {
        runner.installPlugin(new FileSystemExchangePlugin());
        runner.loadExchangeManager("filesystem", getExchangeManagerProperties(minioStorage));
    }).build();
}

Example 3 with TpchTable

use of io.trino.tpch.TpchTable in project trino by trinodb.

the class ThriftTpchService method getTableMetadata.

@Override
public final TrinoThriftNullableTableMetadata getTableMetadata(TrinoThriftSchemaTableName schemaTableName) {
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();
    if (!SCHEMAS.contains(schemaName) || TpchTable.getTables().stream().noneMatch(table -> table.getTableName().equals(tableName))) {
        return new TrinoThriftNullableTableMetadata(null);
    }
    TpchTable<?> tpchTable = TpchTable.getTable(schemaTableName.getTableName());
    List<TrinoThriftColumnMetadata> columns = new ArrayList<>();
    for (TpchColumn<? extends TpchEntity> column : tpchTable.getColumns()) {
        columns.add(new TrinoThriftColumnMetadata(column.getSimplifiedColumnName(), getTypeString(column), null, false));
    }
    List<Set<String>> indexableKeys = getIndexableKeys(schemaName, tableName);
    return new TrinoThriftNullableTableMetadata(new TrinoThriftTableMetadata(schemaTableName, columns, null, !indexableKeys.isEmpty() ? indexableKeys : null));
}

Example 4 with TpchTable

use of io.trino.tpch.TpchTable in project trino by trinodb.

the class TestIcebergTaskFailureRecoveryTest method createQueryRunner.

@Override
protected QueryRunner createQueryRunner(List<TpchTable<?>> requiredTpchTables, Map<String, String> configProperties, Map<String, String> coordinatorProperties) throws Exception {
    this.minioStorage = new MinioStorage("test-exchange-spooling-" + randomTableSuffix());
    minioStorage.start();
    return IcebergQueryRunner.builder().setInitialTables(requiredTpchTables).setCoordinatorProperties(coordinatorProperties).setExtraProperties(ImmutableMap.<String, String>builder().putAll(configProperties).put("enable-dynamic-filtering", "false").buildOrThrow()).setAdditionalSetup(runner -> {
        runner.installPlugin(new FileSystemExchangePlugin());
        runner.loadExchangeManager("filesystem", getExchangeManagerProperties(minioStorage));
    }).build();
}

Example 5 with TpchTable

use of io.trino.tpch.TpchTable in project trino by trinodb.

the class TestDeltaLakePerTransactionMetastoreCache method createQueryRunner.

private DistributedQueryRunner createQueryRunner(boolean enablePerTransactionHiveMetastoreCaching) throws Exception {
    boolean createdDeltaLake = false;
    if (dockerizedMinioDataLake == null) {
        // share environment between testcases to speed things up
        dockerizedMinioDataLake = createDockerizedMinioDataLakeForDeltaLake(BUCKET_NAME);
        createdDeltaLake = true;
    }
    Session session = testSessionBuilder().setCatalog(DELTA_CATALOG).setSchema("default").build();
    DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(session).build();
    queryRunner.installPlugin(new Plugin() {

        @Override
        public Iterable<ConnectorFactory> getConnectorFactories() {
            return ImmutableList.of(new ConnectorFactory() {

                @Override
                public String getName() {
                    return TEST_DELTA_CONNECTOR_NAME;
                }

                @Override
                public Connector create(String catalogName, Map<String, String> config, ConnectorContext context) {
                    return InternalDeltaLakeConnectorFactory.createConnector(catalogName, config, context, new AbstractConfigurationAwareModule() {

                        @Override
                        protected void setup(Binder binder) {
                            newOptionalBinder(binder, ThriftMetastoreClientFactory.class).setDefault().to(DefaultThriftMetastoreClientFactory.class).in(Scopes.SINGLETON);
                            binder.bind(MetastoreLocator.class).to(StaticMetastoreLocator.class).in(Scopes.SINGLETON);
                            configBinder(binder).bindConfig(StaticMetastoreConfig.class);
                            configBinder(binder).bindConfig(ThriftMetastoreConfig.class);
                            binder.bind(ThriftMetastore.class).to(ThriftHiveMetastore.class).in(Scopes.SINGLETON);
                            newExporter(binder).export(ThriftMetastore.class).as((generator) -> generator.generatedNameOf(ThriftHiveMetastore.class));
                            install(new ThriftMetastoreAuthenticationModule());
                            binder.bind(Boolean.class).annotatedWith(HideNonDeltaLakeTables.class).toInstance(false);
                            binder.bind(BridgingHiveMetastoreFactory.class).in(Scopes.SINGLETON);
                        }

                        @Provides
                        @Singleton
                        @RawHiveMetastoreFactory
                        public HiveMetastoreFactory getCountingHiveMetastoreFactory(BridgingHiveMetastoreFactory bridgingHiveMetastoreFactory) {
                            return new HiveMetastoreFactory() {

                                @Override
                                public boolean isImpersonationEnabled() {
                                    return false;
                                }

                                @Override
                                public HiveMetastore createMetastore(Optional<ConnectorIdentity> identity) {
                                    HiveMetastore bridgingHiveMetastore = bridgingHiveMetastoreFactory.createMetastore(identity);
                                    // bind HiveMetastore which counts method executions
                                    return Reflection.newProxy(HiveMetastore.class, (proxy, method, args) -> {
                                        String methodName = method.getName();
                                        long count = hiveMetastoreInvocationCounts.getOrDefault(methodName, 0L);
                                        hiveMetastoreInvocationCounts.put(methodName, count + 1);
                                        return method.invoke(bridgingHiveMetastore, args);
                                    });
                                }
                            };
                        }
                    });
                }
            });
        }
    });
    ImmutableMap.Builder<String, String> deltaLakeProperties = ImmutableMap.builder();
    deltaLakeProperties.put("hive.metastore.uri", dockerizedMinioDataLake.getTestingHadoop().getMetastoreAddress());
    deltaLakeProperties.put("hive.s3.aws-access-key", MINIO_ACCESS_KEY);
    deltaLakeProperties.put("hive.s3.aws-secret-key", MINIO_SECRET_KEY);
    deltaLakeProperties.put("hive.s3.endpoint", dockerizedMinioDataLake.getMinioAddress());
    deltaLakeProperties.put("hive.s3.path-style-access", "true");
    // use test value so we do not get clash with default bindings)
    deltaLakeProperties.put("hive.metastore", "test");
    if (!enablePerTransactionHiveMetastoreCaching) {
        // almost disable the cache; 0 is not allowed as config property value
        deltaLakeProperties.put("hive.per-transaction-metastore-cache-maximum-size", "1");
    }
    queryRunner.createCatalog(DELTA_CATALOG, TEST_DELTA_CONNECTOR_NAME, deltaLakeProperties.buildOrThrow());
    if (createdDeltaLake) {
        List<TpchTable<? extends TpchEntity>> tpchTables = List.of(TpchTable.NATION, TpchTable.REGION);
        tpchTables.forEach(table -> {
            String tableName = table.getTableName();
            dockerizedMinioDataLake.copyResources("io/trino/plugin/deltalake/testing/resources/databricks/" + tableName, tableName);
            queryRunner.execute(format("CREATE TABLE %s.%s.%s (dummy int) WITH (location = 's3://%s/%3$s')", DELTA_CATALOG, "default", tableName, BUCKET_NAME));
        });
    }
    return queryRunner;
}