Search in sources :

Example 1 with TpchColumn

use of io.trino.tpch.TpchColumn in project trino by trinodb.

the class BenchmarkFileFormatsUtils method createTpchDataSet.

public static <E extends TpchEntity> TestData createTpchDataSet(FileFormat format, TpchTable<E> tpchTable, List<TpchColumn<E>> columns) {
    List<String> columnNames = columns.stream().map(TpchColumn::getColumnName).collect(toList());
    List<Type> columnTypes = columns.stream().map(BenchmarkFileFormatsUtils::getColumnType).map(type -> format.supportsDate() || !DATE.equals(type) ? type : createUnboundedVarcharType()).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    ImmutableList.Builder<Page> pages = ImmutableList.builder();
    long dataSize = 0;
    for (E row : tpchTable.createGenerator(10, 1, 1)) {
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            TpchColumn<E> column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(row));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(row));
                    break;
                case DATE:
                    if (format.supportsDate()) {
                        DATE.writeLong(blockBuilder, column.getDate(row));
                    } else {
                        createUnboundedVarcharType().writeString(blockBuilder, column.getString(row));
                    }
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(row));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(row)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
        if (pageBuilder.isFull()) {
            Page page = pageBuilder.build();
            pages.add(page);
            pageBuilder.reset();
            dataSize += page.getSizeInBytes();
            if (dataSize >= MIN_DATA_SIZE) {
                break;
            }
        }
    }
    if (!pageBuilder.isEmpty()) {
        pages.add(pageBuilder.build());
    }
    return new TestData(columnNames, columnTypes, pages.build());
}
Also used : PageBuilder(io.trino.spi.PageBuilder) MEGABYTE(io.airlift.units.DataSize.Unit.MEGABYTE) Type(io.trino.spi.type.Type) RunResult(org.openjdk.jmh.results.RunResult) Page(io.trino.spi.Page) VarcharType.createUnboundedVarcharType(io.trino.spi.type.VarcharType.createUnboundedVarcharType) Random(java.util.Random) Statistics(org.openjdk.jmh.util.Statistics) ImmutableList(com.google.common.collect.ImmutableList) Files.createTempDirectory(java.nio.file.Files.createTempDirectory) Slices(io.airlift.slice.Slices) INTEGER(io.trino.spi.type.IntegerType.INTEGER) TpchEntity(io.trino.tpch.TpchEntity) TpchTable(io.trino.tpch.TpchTable) Collection(java.util.Collection) IOException(java.io.IOException) File(java.io.File) String.format(java.lang.String.format) UncheckedIOException(java.io.UncheckedIOException) DOUBLE(io.trino.spi.type.DoubleType.DOUBLE) DataSize(io.airlift.units.DataSize) TpchColumn(io.trino.tpch.TpchColumn) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) BIGINT(io.trino.spi.type.BigintType.BIGINT) BlockBuilder(io.trino.spi.block.BlockBuilder) DATE(io.trino.spi.type.DateType.DATE) MEGABYTE(io.airlift.units.DataSize.Unit.MEGABYTE) DOUBLE(io.trino.spi.type.DoubleType.DOUBLE) DATE(io.trino.spi.type.DateType.DATE) ImmutableList(com.google.common.collect.ImmutableList) Page(io.trino.spi.Page) PageBuilder(io.trino.spi.PageBuilder) Type(io.trino.spi.type.Type) VarcharType.createUnboundedVarcharType(io.trino.spi.type.VarcharType.createUnboundedVarcharType) BlockBuilder(io.trino.spi.block.BlockBuilder)

Example 2 with TpchColumn

use of io.trino.tpch.TpchColumn in project trino by trinodb.

the class ThriftTpchService method getTableMetadata.

@Override
public final TrinoThriftNullableTableMetadata getTableMetadata(TrinoThriftSchemaTableName schemaTableName) {
    String schemaName = schemaTableName.getSchemaName();
    String tableName = schemaTableName.getTableName();
    if (!SCHEMAS.contains(schemaName) || TpchTable.getTables().stream().noneMatch(table -> table.getTableName().equals(tableName))) {
        return new TrinoThriftNullableTableMetadata(null);
    }
    TpchTable<?> tpchTable = TpchTable.getTable(schemaTableName.getTableName());
    List<TrinoThriftColumnMetadata> columns = new ArrayList<>();
    for (TpchColumn<? extends TpchEntity> column : tpchTable.getColumns()) {
        columns.add(new TrinoThriftColumnMetadata(column.getSimplifiedColumnName(), getTypeString(column), null, false));
    }
    List<Set<String>> indexableKeys = getIndexableKeys(schemaName, tableName);
    return new TrinoThriftNullableTableMetadata(new TrinoThriftTableMetadata(schemaTableName, columns, null, !indexableKeys.isEmpty() ? indexableKeys : null));
}
Also used : MoreExecutors.listeningDecorator(com.google.common.util.concurrent.MoreExecutors.listeningDecorator) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) TrinoThriftTupleDomain(io.trino.plugin.thrift.api.TrinoThriftTupleDomain) SplitInfo.normalSplit(io.trino.plugin.thrift.server.SplitInfo.normalSplit) Type(io.trino.spi.type.Type) Page(io.trino.spi.Page) TpchRecordSet.createTpchRecordSet(io.trino.plugin.tpch.TpchRecordSet.createTpchRecordSet) DEFAULT_MAX_PAGE_SIZE_IN_BYTES(io.trino.spi.block.PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES) DecimalTypeMapping(io.trino.plugin.tpch.DecimalTypeMapping) TrinoThriftServiceException(io.trino.plugin.thrift.api.TrinoThriftServiceException) ArrayList(java.util.ArrayList) PreDestroy(javax.annotation.PreDestroy) TrinoThriftBlock.fromBlock(io.trino.plugin.thrift.api.TrinoThriftBlock.fromBlock) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ImmutableList(com.google.common.collect.ImmutableList) TrinoThriftId(io.trino.plugin.thrift.api.TrinoThriftId) RecordPageSource(io.trino.spi.connector.RecordPageSource) TrinoThriftService(io.trino.plugin.thrift.api.TrinoThriftService) TrinoThriftNullableToken(io.trino.plugin.thrift.api.TrinoThriftNullableToken) ConnectorPageSource(io.trino.spi.connector.ConnectorPageSource) TrinoThriftSplit(io.trino.plugin.thrift.api.TrinoThriftSplit) Nullable(javax.annotation.Nullable) TrinoThriftBlock(io.trino.plugin.thrift.api.TrinoThriftBlock) TpchEntity(io.trino.tpch.TpchEntity) TpchTable(io.trino.tpch.TpchTable) TrinoThriftPageResult(io.trino.plugin.thrift.api.TrinoThriftPageResult) Set(java.util.Set) TrinoThriftSplitBatch(io.trino.plugin.thrift.api.TrinoThriftSplitBatch) TrinoThriftNullableTableMetadata(io.trino.plugin.thrift.api.TrinoThriftNullableTableMetadata) Math.min(java.lang.Math.min) TupleDomain(io.trino.spi.predicate.TupleDomain) Threads.threadsNamed(io.airlift.concurrent.Threads.threadsNamed) Ints(com.google.common.primitives.Ints) TpchMetadata.getTrinoType(io.trino.plugin.tpch.TpchMetadata.getTrinoType) Executors.newFixedThreadPool(java.util.concurrent.Executors.newFixedThreadPool) Preconditions.checkState(com.google.common.base.Preconditions.checkState) JsonCodec.jsonCodec(io.airlift.json.JsonCodec.jsonCodec) TpchColumn(io.trino.tpch.TpchColumn) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) TrinoThriftSchemaTableName(io.trino.plugin.thrift.api.TrinoThriftSchemaTableName) Closeable(java.io.Closeable) TrinoThriftNullableColumnSet(io.trino.plugin.thrift.api.TrinoThriftNullableColumnSet) TrinoThriftNullableSchemaName(io.trino.plugin.thrift.api.TrinoThriftNullableSchemaName) TrinoThriftColumnMetadata(io.trino.plugin.thrift.api.TrinoThriftColumnMetadata) TrinoThriftTableMetadata(io.trino.plugin.thrift.api.TrinoThriftTableMetadata) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) JsonCodec(io.airlift.json.JsonCodec) TpchRecordSet.createTpchRecordSet(io.trino.plugin.tpch.TpchRecordSet.createTpchRecordSet) Set(java.util.Set) TrinoThriftNullableColumnSet(io.trino.plugin.thrift.api.TrinoThriftNullableColumnSet) TrinoThriftColumnMetadata(io.trino.plugin.thrift.api.TrinoThriftColumnMetadata) ArrayList(java.util.ArrayList) TrinoThriftNullableTableMetadata(io.trino.plugin.thrift.api.TrinoThriftNullableTableMetadata) TrinoThriftTableMetadata(io.trino.plugin.thrift.api.TrinoThriftTableMetadata)

Aggregations

ImmutableList (com.google.common.collect.ImmutableList)2 Page (io.trino.spi.Page)2 Type (io.trino.spi.type.Type)2 TpchColumn (io.trino.tpch.TpchColumn)2 TpchEntity (io.trino.tpch.TpchEntity)2 TpchTable (io.trino.tpch.TpchTable)2 List (java.util.List)2 Collectors.toList (java.util.stream.Collectors.toList)2 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)1 Preconditions.checkState (com.google.common.base.Preconditions.checkState)1 Ints (com.google.common.primitives.Ints)1 ListenableFuture (com.google.common.util.concurrent.ListenableFuture)1 ListeningExecutorService (com.google.common.util.concurrent.ListeningExecutorService)1 MoreExecutors.listeningDecorator (com.google.common.util.concurrent.MoreExecutors.listeningDecorator)1 Threads.threadsNamed (io.airlift.concurrent.Threads.threadsNamed)1 JsonCodec (io.airlift.json.JsonCodec)1 JsonCodec.jsonCodec (io.airlift.json.JsonCodec.jsonCodec)1 Slices (io.airlift.slice.Slices)1 DataSize (io.airlift.units.DataSize)1 MEGABYTE (io.airlift.units.DataSize.Unit.MEGABYTE)1