Search in sources :

Example 1 with LineItemGenerator

use of io.airlift.tpch.LineItemGenerator in project presto by prestodb.

the class BenchmarkPageProcessor method createInputPage.

public static Page createInputPage() {
    PageBuilder pageBuilder = new PageBuilder(ImmutableList.of(DOUBLE, DOUBLE, VARCHAR, DOUBLE));
    LineItemGenerator lineItemGenerator = new LineItemGenerator(1, 1, 1);
    Iterator<LineItem> iterator = lineItemGenerator.iterator();
    for (int i = 0; i < 10_000; i++) {
        pageBuilder.declarePosition();
        LineItem lineItem = iterator.next();
        DOUBLE.writeDouble(pageBuilder.getBlockBuilder(EXTENDED_PRICE), lineItem.getExtendedPrice());
        DOUBLE.writeDouble(pageBuilder.getBlockBuilder(DISCOUNT), lineItem.getDiscount());
        DATE.writeLong(pageBuilder.getBlockBuilder(SHIP_DATE), lineItem.getShipDate());
        DOUBLE.writeDouble(pageBuilder.getBlockBuilder(QUANTITY), lineItem.getQuantity());
    }
    return pageBuilder.build();
}
Also used : LineItem(io.airlift.tpch.LineItem) PageBuilder(com.facebook.presto.spi.PageBuilder) LineItemGenerator(io.airlift.tpch.LineItemGenerator)

Example 2 with LineItemGenerator

use of io.airlift.tpch.LineItemGenerator in project presto by prestodb.

the class TestHivePageSink method writeTestFile.

private static long writeTestFile(HiveClientConfig config, ExtendedHiveMetastore metastore, String outputPath) {
    HiveTransactionHandle transaction = new HiveTransactionHandle();
    ConnectorPageSink pageSink = createPageSink(transaction, config, metastore, new Path("file:///" + outputPath));
    List<LineItemColumn> columns = getTestColumns();
    List<Type> columnTypes = columns.stream().map(LineItemColumn::getType).map(TestHivePageSink::getHiveType).map(hiveType -> hiveType.getType(TYPE_MANAGER)).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    int rows = 0;
    for (LineItem lineItem : new LineItemGenerator(0.01, 1, 1)) {
        rows++;
        if (rows >= NUM_ROWS) {
            break;
        }
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            LineItemColumn column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(lineItem));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(lineItem));
                    break;
                case DATE:
                    DATE.writeLong(blockBuilder, column.getDate(lineItem));
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(lineItem));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(lineItem)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
    }
    Page page = pageBuilder.build();
    pageSink.appendPage(page);
    getFutureValue(pageSink.finish());
    File outputDir = new File(outputPath);
    List<File> files = ImmutableList.copyOf(outputDir.listFiles((dir, name) -> !name.endsWith(".crc")));
    File outputFile = getOnlyElement(files);
    long length = outputFile.length();
    ConnectorPageSource pageSource = createPageSource(transaction, config, outputFile);
    List<Page> pages = new ArrayList<>();
    while (!pageSource.isFinished()) {
        Page nextPage = pageSource.getNextPage();
        if (nextPage != null) {
            nextPage.assureLoaded();
            pages.add(nextPage);
        }
    }
    MaterializedResult expectedResults = toMaterializedResult(getSession(config), columnTypes, ImmutableList.of(page));
    MaterializedResult results = toMaterializedResult(getSession(config), columnTypes, pages);
    assertEquals(results, expectedResults);
    return length;
}
Also used : Path(org.apache.hadoop.fs.Path) Page(com.facebook.presto.spi.Page) HiveTestUtils.getDefaultHiveDataStreamFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveDataStreamFactories) HiveTestUtils.getDefaultHiveFileWriterFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) Assertions.assertGreaterThan(io.airlift.testing.Assertions.assertGreaterThan) Test(org.testng.annotations.Test) TpchColumnTypes(io.airlift.tpch.TpchColumnTypes) BIGINT(com.facebook.presto.spi.type.BigintType.BIGINT) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) Slices(io.airlift.slice.Slices) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) Path(org.apache.hadoop.fs.Path) HIVE_LONG(com.facebook.presto.hive.HiveType.HIVE_LONG) SERIALIZATION_LIB(org.apache.hadoop.hive.serde.serdeConstants.SERIALIZATION_LIB) TpchColumnType(io.airlift.tpch.TpchColumnType) FileUtils(io.airlift.testing.FileUtils) ImmutableMap(com.google.common.collect.ImmutableMap) BlockBuilder(com.facebook.presto.spi.block.BlockBuilder) String.format(java.lang.String.format) ConnectorSession(com.facebook.presto.spi.ConnectorSession) LineItemGenerator(io.airlift.tpch.LineItemGenerator) TupleDomain(com.facebook.presto.spi.predicate.TupleDomain) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) List(java.util.List) Stream(java.util.stream.Stream) TYPE_MANAGER(com.facebook.presto.hive.HiveTestUtils.TYPE_MANAGER) Optional(java.util.Optional) INTEGER(com.facebook.presto.spi.type.IntegerType.INTEGER) Joiner(com.google.common.base.Joiner) JsonCodec(io.airlift.json.JsonCodec) DOUBLE(com.facebook.presto.spi.type.DoubleType.DOUBLE) LineItem(io.airlift.tpch.LineItem) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) HivePageSinkMetadata(com.facebook.presto.hive.metastore.HivePageSinkMetadata) Assert.assertEquals(org.testng.Assert.assertEquals) OptionalInt(java.util.OptionalInt) ArrayList(java.util.ArrayList) HIVE_DATE(com.facebook.presto.hive.HiveType.HIVE_DATE) HIVE_INT(com.facebook.presto.hive.HiveType.HIVE_INT) HiveTestUtils.createTestHdfsEnvironment(com.facebook.presto.hive.HiveTestUtils.createTestHdfsEnvironment) TestingHiveMetastore(com.facebook.presto.hive.metastore.TestingHiveMetastore) ImmutableList(com.google.common.collect.ImmutableList) HIVE_DOUBLE(com.facebook.presto.hive.HiveType.HIVE_DOUBLE) Files(com.google.common.io.Files) Type(com.facebook.presto.spi.type.Type) LineItemColumn(io.airlift.tpch.LineItemColumn) Properties(java.util.Properties) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) File(java.io.File) VarcharType.createUnboundedVarcharType(com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType) MaterializedResult(com.facebook.presto.testing.MaterializedResult) NONE(com.facebook.presto.hive.HiveCompressionCodec.NONE) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) PageBuilder(com.facebook.presto.spi.PageBuilder) DATE(com.facebook.presto.spi.type.DateType.DATE) FILE_INPUT_FORMAT(org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.FILE_INPUT_FORMAT) Assert.assertTrue(org.testng.Assert.assertTrue) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) HiveTestUtils.getDefaultHiveRecordCursorProvider(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) LineItemColumn(io.airlift.tpch.LineItemColumn) ArrayList(java.util.ArrayList) LineItem(io.airlift.tpch.LineItem) Page(com.facebook.presto.spi.Page) PageBuilder(com.facebook.presto.spi.PageBuilder) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) TpchColumnType(io.airlift.tpch.TpchColumnType) Type(com.facebook.presto.spi.type.Type) VarcharType.createUnboundedVarcharType(com.facebook.presto.spi.type.VarcharType.createUnboundedVarcharType) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) MaterializedResult(com.facebook.presto.testing.MaterializedResult) File(java.io.File) LineItemGenerator(io.airlift.tpch.LineItemGenerator) BlockBuilder(com.facebook.presto.spi.block.BlockBuilder)

Example 3 with LineItemGenerator

use of io.airlift.tpch.LineItemGenerator in project presto by prestodb.

the class BenchmarkGroupedTopNBuilder method createInputPages.

private static List<Page> createInputPages(int positions, List<Type> types, int positionsPerPage, int groupCount, int seed) {
    Random random = new Random(seed);
    List<Page> pages = new ArrayList<>();
    PageBuilder pageBuilder = new PageBuilder(types);
    LineItemGenerator lineItemGenerator = new LineItemGenerator(1, 1, 1);
    Iterator<LineItem> iterator = lineItemGenerator.iterator();
    for (int i = 0; i < positions; i++) {
        pageBuilder.declarePosition();
        LineItem lineItem = iterator.next();
        BIGINT.writeLong(pageBuilder.getBlockBuilder(HASH_GROUP), groupCount > 1 ? random.nextInt(groupCount) : 1);
        DOUBLE.writeDouble(pageBuilder.getBlockBuilder(EXTENDED_PRICE), lineItem.getExtendedPrice());
        DOUBLE.writeDouble(pageBuilder.getBlockBuilder(DISCOUNT), lineItem.getDiscount());
        DATE.writeLong(pageBuilder.getBlockBuilder(SHIP_DATE), lineItem.getShipDate());
        DOUBLE.writeDouble(pageBuilder.getBlockBuilder(QUANTITY), lineItem.getQuantity());
        if (pageBuilder.getPositionCount() >= positionsPerPage) {
            pages.add(pageBuilder.build());
            pageBuilder.reset();
        }
    }
    if (!pageBuilder.isEmpty()) {
        pages.add(pageBuilder.build());
    }
    return pages;
}
Also used : Random(java.util.Random) ArrayList(java.util.ArrayList) LineItem(io.airlift.tpch.LineItem) Page(com.facebook.presto.common.Page) PageBuilder(com.facebook.presto.common.PageBuilder) LineItemGenerator(io.airlift.tpch.LineItemGenerator)

Example 4 with LineItemGenerator

use of io.airlift.tpch.LineItemGenerator in project presto by prestodb.

the class TestHivePageSink method writeTestFile.

private static long writeTestFile(HiveClientConfig config, MetastoreClientConfig metastoreClientConfig, ExtendedHiveMetastore metastore, String outputPath) {
    HiveTransactionHandle transaction = new HiveTransactionHandle();
    HiveWriterStats stats = new HiveWriterStats();
    ConnectorPageSink pageSink = createPageSink(transaction, config, metastoreClientConfig, metastore, new Path("file:///" + outputPath), stats);
    List<LineItemColumn> columns = getTestColumns();
    List<Type> columnTypes = columns.stream().map(LineItemColumn::getType).map(TestHivePageSink::getHiveType).map(hiveType -> hiveType.getType(FUNCTION_AND_TYPE_MANAGER)).collect(toList());
    PageBuilder pageBuilder = new PageBuilder(columnTypes);
    int rows = 0;
    for (LineItem lineItem : new LineItemGenerator(0.01, 1, 1)) {
        rows++;
        if (rows >= NUM_ROWS) {
            break;
        }
        pageBuilder.declarePosition();
        for (int i = 0; i < columns.size(); i++) {
            LineItemColumn column = columns.get(i);
            BlockBuilder blockBuilder = pageBuilder.getBlockBuilder(i);
            switch(column.getType().getBase()) {
                case IDENTIFIER:
                    BIGINT.writeLong(blockBuilder, column.getIdentifier(lineItem));
                    break;
                case INTEGER:
                    INTEGER.writeLong(blockBuilder, column.getInteger(lineItem));
                    break;
                case DATE:
                    DATE.writeLong(blockBuilder, column.getDate(lineItem));
                    break;
                case DOUBLE:
                    DOUBLE.writeDouble(blockBuilder, column.getDouble(lineItem));
                    break;
                case VARCHAR:
                    createUnboundedVarcharType().writeSlice(blockBuilder, Slices.utf8Slice(column.getString(lineItem)));
                    break;
                default:
                    throw new IllegalArgumentException("Unsupported type " + column.getType());
            }
        }
    }
    Page page = pageBuilder.build();
    pageSink.appendPage(page);
    getFutureValue(pageSink.finish());
    File outputDir = new File(outputPath);
    List<File> files = ImmutableList.copyOf(outputDir.listFiles((dir, name) -> !name.endsWith(".crc")));
    File outputFile = getOnlyElement(files);
    long length = outputFile.length();
    ConnectorPageSource pageSource = createPageSource(transaction, config, metastoreClientConfig, outputFile);
    List<Page> pages = new ArrayList<>();
    while (!pageSource.isFinished()) {
        Page nextPage = pageSource.getNextPage();
        if (nextPage != null) {
            pages.add(nextPage.getLoadedPage());
        }
    }
    MaterializedResult expectedResults = toMaterializedResult(getSession(config), columnTypes, ImmutableList.of(page));
    MaterializedResult results = toMaterializedResult(getSession(config), columnTypes, pages);
    assertEquals(results, expectedResults);
    assertEquals(stats.getInputPageSizeInBytes().getAllTime().getMax(), page.getRetainedSizeInBytes());
    return length;
}
Also used : Path(org.apache.hadoop.fs.Path) Page(com.facebook.presto.common.Page) MetadataManager(com.facebook.presto.metadata.MetadataManager) VarcharType.createUnboundedVarcharType(com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType) HiveTestUtils.getDefaultHiveFileWriterFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) NO_PREFERENCE(com.facebook.presto.spi.schedule.NodeSelectionStrategy.NO_PREFERENCE) MoreFiles.deleteRecursively(com.google.common.io.MoreFiles.deleteRecursively) Test(org.testng.annotations.Test) NO_CACHE_REQUIREMENT(com.facebook.presto.hive.CacheQuotaRequirement.NO_CACHE_REQUIREMENT) TpchColumnTypes(io.airlift.tpch.TpchColumnTypes) CacheConfig(com.facebook.presto.cache.CacheConfig) HiveTestUtils.getDefaultOrcFileWriterFactory(com.facebook.presto.hive.HiveTestUtils.getDefaultOrcFileWriterFactory) MoreFutures.getFutureValue(com.facebook.airlift.concurrent.MoreFutures.getFutureValue) DIRECT_TO_TARGET_NEW_DIRECTORY(com.facebook.presto.hive.LocationHandle.WriteMode.DIRECT_TO_TARGET_NEW_DIRECTORY) PageBuilder(com.facebook.presto.common.PageBuilder) SchemaTableName(com.facebook.presto.spi.SchemaTableName) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) Slices(io.airlift.slice.Slices) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) Path(org.apache.hadoop.fs.Path) HIVE_LONG(com.facebook.presto.hive.HiveType.HIVE_LONG) TpchColumnType(io.airlift.tpch.TpchColumnType) METASTORE_CONTEXT(com.facebook.presto.hive.HiveQueryRunner.METASTORE_CONTEXT) BlockBuilder(com.facebook.presto.common.block.BlockBuilder) ImmutableSet(com.google.common.collect.ImmutableSet) TestHiveUtil.createTestingFileHiveMetastore(com.facebook.presto.hive.TestHiveUtil.createTestingFileHiveMetastore) StorageFormat(com.facebook.presto.hive.metastore.StorageFormat) ImmutableMap(com.google.common.collect.ImmutableMap) DOUBLE(com.facebook.presto.common.type.DoubleType.DOUBLE) ImmutableList.toImmutableList(com.google.common.collect.ImmutableList.toImmutableList) TestingNodeManager(com.facebook.presto.testing.TestingNodeManager) TRUE_CONSTANT(com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT) String.format(java.lang.String.format) ConnectorSession(com.facebook.presto.spi.ConnectorSession) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) LineItemGenerator(io.airlift.tpch.LineItemGenerator) PAGE_SORTER(com.facebook.presto.hive.HiveTestUtils.PAGE_SORTER) HIVE_STRING(com.facebook.presto.hive.HiveType.HIVE_STRING) List(java.util.List) Stream(java.util.stream.Stream) INTEGER(com.facebook.presto.common.type.IntegerType.INTEGER) Assertions.assertGreaterThan(com.facebook.airlift.testing.Assertions.assertGreaterThan) Optional(java.util.Optional) ConnectorId(com.facebook.presto.spi.ConnectorId) HiveTestUtils.getDefaultHiveBatchPageSourceFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveBatchPageSourceFactories) LineItem(io.airlift.tpch.LineItem) Assert.assertEquals(com.facebook.presto.testing.assertions.Assert.assertEquals) Column(com.facebook.presto.hive.metastore.Column) REGULAR(com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR) HivePageSinkMetadata(com.facebook.presto.hive.metastore.HivePageSinkMetadata) OptionalInt(java.util.OptionalInt) DATE(com.facebook.presto.common.type.DateType.DATE) ArrayList(java.util.ArrayList) HIVE_DATE(com.facebook.presto.hive.HiveType.HIVE_DATE) ROW_EXPRESSION_SERVICE(com.facebook.presto.hive.HiveTestUtils.ROW_EXPRESSION_SERVICE) HIVE_INT(com.facebook.presto.hive.HiveType.HIVE_INT) HiveTestUtils.createTestHdfsEnvironment(com.facebook.presto.hive.HiveTestUtils.createTestHdfsEnvironment) ALLOW_INSECURE(com.google.common.io.RecursiveDeleteOption.ALLOW_INSECURE) ImmutableList(com.google.common.collect.ImmutableList) HIVE_DOUBLE(com.facebook.presto.hive.HiveType.HIVE_DOUBLE) NON_CACHEABLE(com.facebook.presto.spi.SplitContext.NON_CACHEABLE) Files(com.google.common.io.Files) LineItemColumn(io.airlift.tpch.LineItemColumn) TableHandle(com.facebook.presto.spi.TableHandle) Type(com.facebook.presto.common.type.Type) BIGINT(com.facebook.presto.common.type.BigintType.BIGINT) Storage(com.facebook.presto.hive.metastore.Storage) HiveTestUtils.getDefaultHiveSelectivePageSourceFactories(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveSelectivePageSourceFactories) HIVE_CATALOG(com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG) TEST_HIVE_PAGE_SINK_CONTEXT(com.facebook.presto.hive.AbstractTestHiveClient.TEST_HIVE_PAGE_SINK_CONTEXT) NEW(com.facebook.presto.hive.LocationHandle.TableType.NEW) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) TestingConnectorSession(com.facebook.presto.testing.TestingConnectorSession) TupleDomain(com.facebook.presto.common.predicate.TupleDomain) File(java.io.File) FUNCTION_AND_TYPE_MANAGER(com.facebook.presto.hive.HiveTestUtils.FUNCTION_AND_TYPE_MANAGER) MaterializedResult(com.facebook.presto.testing.MaterializedResult) NONE(com.facebook.presto.hive.HiveCompressionCodec.NONE) Collectors.toList(java.util.stream.Collectors.toList) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) Assert.assertTrue(org.testng.Assert.assertTrue) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) SplitWeight(com.facebook.presto.spi.SplitWeight) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) HiveTestUtils.getDefaultHiveRecordCursorProvider(com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) LineItemColumn(io.airlift.tpch.LineItemColumn) ArrayList(java.util.ArrayList) LineItem(io.airlift.tpch.LineItem) Page(com.facebook.presto.common.Page) PageBuilder(com.facebook.presto.common.PageBuilder) ConnectorPageSource(com.facebook.presto.spi.ConnectorPageSource) VarcharType.createUnboundedVarcharType(com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType) TpchColumnType(io.airlift.tpch.TpchColumnType) Type(com.facebook.presto.common.type.Type) ConnectorPageSink(com.facebook.presto.spi.ConnectorPageSink) MaterializedResult(com.facebook.presto.testing.MaterializedResult) File(java.io.File) LineItemGenerator(io.airlift.tpch.LineItemGenerator) BlockBuilder(com.facebook.presto.common.block.BlockBuilder)

Aggregations

LineItem (io.airlift.tpch.LineItem)3 LineItemGenerator (io.airlift.tpch.LineItemGenerator)3 GroupByHashPageIndexerFactory (com.facebook.presto.GroupByHashPageIndexerFactory)2 Page (com.facebook.presto.common.Page)2 PageBuilder (com.facebook.presto.common.PageBuilder)2 REGULAR (com.facebook.presto.hive.HiveColumnHandle.ColumnType.REGULAR)2 NONE (com.facebook.presto.hive.HiveCompressionCodec.NONE)2 HiveTestUtils.createTestHdfsEnvironment (com.facebook.presto.hive.HiveTestUtils.createTestHdfsEnvironment)2 HiveTestUtils.getDefaultHiveFileWriterFactories (com.facebook.presto.hive.HiveTestUtils.getDefaultHiveFileWriterFactories)2 HiveTestUtils.getDefaultHiveRecordCursorProvider (com.facebook.presto.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider)2 HIVE_DATE (com.facebook.presto.hive.HiveType.HIVE_DATE)2 HIVE_DOUBLE (com.facebook.presto.hive.HiveType.HIVE_DOUBLE)2 HIVE_INT (com.facebook.presto.hive.HiveType.HIVE_INT)2 HIVE_LONG (com.facebook.presto.hive.HiveType.HIVE_LONG)2 HIVE_STRING (com.facebook.presto.hive.HiveType.HIVE_STRING)2 ExtendedHiveMetastore (com.facebook.presto.hive.metastore.ExtendedHiveMetastore)2 PageBuilder (com.facebook.presto.spi.PageBuilder)2 ArrayList (java.util.ArrayList)2 MoreFutures.getFutureValue (com.facebook.airlift.concurrent.MoreFutures.getFutureValue)1 Assertions.assertGreaterThan (com.facebook.airlift.testing.Assertions.assertGreaterThan)1