Search in sources :

Example 36 with FeaturesConfig

use of com.facebook.presto.sql.analyzer.FeaturesConfig in project presto by prestodb.

the class TestAggregationOperator method testDistinctMaskWithNull.

@Test
public void testDistinctMaskWithNull() {
    AccumulatorFactory distinctFactory = COUNT.bind(ImmutableList.of(0), Optional.of(1), ImmutableList.of(BIGINT, BOOLEAN), ImmutableList.of(), ImmutableList.of(), null, // distinct
    true, new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig()), ImmutableList.of(), false, TEST_SESSION, new TempStorageStandaloneSpillerFactory(new TestingTempStorageManager(), new BlockEncodingManager(), new NodeSpillConfig(), new FeaturesConfig(), new SpillerStats()));
    OperatorFactory operatorFactory = new AggregationOperatorFactory(0, new PlanNodeId("test"), Step.SINGLE, ImmutableList.of(distinctFactory), false);
    DriverContext driverContext = createTaskContext(executor, scheduledExecutor, TEST_SESSION).addPipelineContext(0, true, true, false).addDriverContext();
    ByteArrayBlock trueMaskAllNull = new ByteArrayBlock(4, Optional.of(new boolean[] { true, true, true, true }), /* all positions are null */
    new byte[] { 1, 1, 1, 1 });
    /* non-zero value is true, all masks are true */
    Block trueNullRleMask = new RunLengthEncodedBlock(trueMaskAllNull.getSingleValueBlock(0), 4);
    List<Page> input = ImmutableList.of(new Page(4, createLongsBlock(1, 2, 3, 4), trueMaskAllNull), new Page(4, createLongsBlock(5, 6, 7, 8), trueNullRleMask));
    MaterializedResult expected = resultBuilder(driverContext.getSession(), BIGINT).row(// all rows should be filtered by nulls
    0L).build();
    assertOperatorEquals(operatorFactory, driverContext, input, expected);
}
Also used : JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) AggregationOperatorFactory(com.facebook.presto.operator.AggregationOperator.AggregationOperatorFactory) TempStorageStandaloneSpillerFactory(com.facebook.presto.spiller.TempStorageStandaloneSpillerFactory) NodeSpillConfig(com.facebook.presto.spiller.NodeSpillConfig) Page(com.facebook.presto.common.Page) SpillerStats(com.facebook.presto.spiller.SpillerStats) PlanNodeId(com.facebook.presto.spi.plan.PlanNodeId) BlockEncodingManager(com.facebook.presto.common.block.BlockEncodingManager) TestingTempStorageManager(com.facebook.presto.testing.TestingTempStorageManager) AggregationOperatorFactory(com.facebook.presto.operator.AggregationOperator.AggregationOperatorFactory) AccumulatorFactory(com.facebook.presto.operator.aggregation.AccumulatorFactory) RunLengthEncodedBlock(com.facebook.presto.common.block.RunLengthEncodedBlock) ByteArrayBlock(com.facebook.presto.common.block.ByteArrayBlock) BlockAssertions.createLongsBlock(com.facebook.presto.block.BlockAssertions.createLongsBlock) Block(com.facebook.presto.common.block.Block) ByteArrayBlock(com.facebook.presto.common.block.ByteArrayBlock) RunLengthEncodedBlock(com.facebook.presto.common.block.RunLengthEncodedBlock) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Test(org.testng.annotations.Test)

Example 37 with FeaturesConfig

use of com.facebook.presto.sql.analyzer.FeaturesConfig in project presto by prestodb.

the class TestRowExpressionDomainTranslator method testLegacyCharComparedToVarcharExpression.

@Test
public void testLegacyCharComparedToVarcharExpression() {
    metadata = createTestMetadataManager(new FeaturesConfig().setLegacyCharToVarcharCoercion(true));
    domainTranslator = new RowExpressionDomainTranslator(metadata);
    String maxCodePoint = new String(Character.toChars(Character.MAX_CODE_POINT));
    // greater than or equal
    testSimpleComparison(greaterThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Range.greaterThan(createCharType(10), utf8Slice("123456788" + maxCodePoint)));
    testSimpleComparison(greaterThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.greaterThanOrEqual(createCharType(10), Slices.utf8Slice("1234567890")));
    testSimpleComparison(greaterThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890")));
    // greater than
    testSimpleComparison(greaterThan(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Range.greaterThan(createCharType(10), utf8Slice("123456788" + maxCodePoint)));
    testSimpleComparison(greaterThan(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890")));
    testSimpleComparison(greaterThan(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890")));
    // less than or equal
    testSimpleComparison(lessThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Range.lessThanOrEqual(createCharType(10), utf8Slice("123456788" + maxCodePoint)));
    testSimpleComparison(lessThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.lessThanOrEqual(createCharType(10), Slices.utf8Slice("1234567890")));
    testSimpleComparison(lessThanOrEqual(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Range.lessThanOrEqual(createCharType(10), Slices.utf8Slice("1234567890")));
    // less than
    testSimpleComparison(lessThan(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Range.lessThanOrEqual(createCharType(10), utf8Slice("123456788" + maxCodePoint)));
    testSimpleComparison(lessThan(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.lessThan(createCharType(10), Slices.utf8Slice("1234567890")));
    testSimpleComparison(lessThan(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Range.lessThanOrEqual(createCharType(10), Slices.utf8Slice("1234567890")));
    // equal
    testSimpleComparison(equal(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Domain.none(createCharType(10)));
    testSimpleComparison(equal(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Range.equal(createCharType(10), Slices.utf8Slice("1234567890")));
    testSimpleComparison(equal(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Domain.none(createCharType(10)));
    // not equal
    testSimpleComparison(notEqual(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Domain.notNull(createCharType(10)));
    testSimpleComparison(notEqual(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Domain.create(ValueSet.ofRanges(Range.lessThan(createCharType(10), Slices.utf8Slice("1234567890")), Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890"))), false));
    testSimpleComparison(notEqual(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Domain.notNull(createCharType(10)));
    // is distinct from
    testSimpleComparison(isDistinctFrom(cast(C_CHAR, VARCHAR), stringLiteral("123456789")), C_CHAR, Domain.all(createCharType(10)));
    testSimpleComparison(isDistinctFrom(cast(C_CHAR, VARCHAR), stringLiteral("1234567890")), C_CHAR, Domain.create(ValueSet.ofRanges(Range.lessThan(createCharType(10), Slices.utf8Slice("1234567890")), Range.greaterThan(createCharType(10), Slices.utf8Slice("1234567890"))), true));
    testSimpleComparison(isDistinctFrom(cast(C_CHAR, VARCHAR), stringLiteral("12345678901")), C_CHAR, Domain.all(createCharType(10)));
}
Also used : FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) RowExpressionDomainTranslator(com.facebook.presto.sql.relational.RowExpressionDomainTranslator) Test(org.testng.annotations.Test)

Example 38 with FeaturesConfig

use of com.facebook.presto.sql.analyzer.FeaturesConfig in project presto by prestodb.

the class AbstractTestHiveFileSystem method setup.

protected void setup(String host, int port, String databaseName, BiFunction<HiveClientConfig, MetastoreClientConfig, HdfsConfiguration> hdfsConfigurationProvider, boolean s3SelectPushdownEnabled) {
    database = databaseName;
    table = new SchemaTableName(database, "presto_test_external_fs");
    String random = UUID.randomUUID().toString().toLowerCase(ENGLISH).replace("-", "");
    temporaryCreateTable = new SchemaTableName(database, "tmp_presto_test_create_" + random);
    config = new HiveClientConfig().setS3SelectPushdownEnabled(s3SelectPushdownEnabled);
    cacheConfig = new CacheConfig();
    metastoreClientConfig = new MetastoreClientConfig();
    String proxy = System.getProperty("hive.metastore.thrift.client.socks-proxy");
    if (proxy != null) {
        metastoreClientConfig.setMetastoreSocksProxy(HostAndPort.fromString(proxy));
    }
    HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, host, port);
    ExecutorService executor = newCachedThreadPool(daemonThreadsNamed("hive-%s"));
    HivePartitionManager hivePartitionManager = new HivePartitionManager(FUNCTION_AND_TYPE_MANAGER, config);
    HdfsConfiguration hdfsConfiguration = hdfsConfigurationProvider.apply(config, metastoreClientConfig);
    hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication());
    ColumnConverterProvider columnConverterProvider = HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER;
    metastoreClient = new TestingHiveMetastore(new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster, metastoreClientConfig), new HivePartitionMutator()), executor, metastoreClientConfig, getBasePath(), hdfsEnvironment);
    locationService = new HiveLocationService(hdfsEnvironment);
    metadataFactory = new HiveMetadataFactory(config, metastoreClientConfig, metastoreClient, hdfsEnvironment, hivePartitionManager, newDirectExecutorService(), FUNCTION_AND_TYPE_MANAGER, locationService, FUNCTION_RESOLUTION, ROW_EXPRESSION_SERVICE, FILTER_STATS_CALCULATOR_SERVICE, new TableParameterCodec(), HiveTestUtils.PARTITION_UPDATE_CODEC, HiveTestUtils.PARTITION_UPDATE_SMILE_CODEC, new HiveTypeTranslator(), new HiveStagingFileCommitter(hdfsEnvironment, listeningDecorator(executor)), new HiveZeroRowFileCreator(hdfsEnvironment, new OutputStreamDataSinkFactory(), listeningDecorator(executor)), new NodeVersion("test_version"), new HivePartitionObjectBuilder(), new HiveEncryptionInformationProvider(ImmutableList.of()), new HivePartitionStats(), new HiveFileRenamer(), columnConverterProvider);
    transactionManager = new HiveTransactionManager();
    splitManager = new HiveSplitManager(transactionManager, new NamenodeStats(), hdfsEnvironment, new CachingDirectoryLister(new HadoopDirectoryLister(), new HiveClientConfig()), new BoundedExecutor(executor, config.getMaxSplitIteratorThreads()), new HiveCoercionPolicy(FUNCTION_AND_TYPE_MANAGER), new CounterStat(), config.getMaxOutstandingSplits(), config.getMaxOutstandingSplitsSize(), config.getMinPartitionBatchSize(), config.getMaxPartitionBatchSize(), config.getSplitLoaderConcurrency(), config.getRecursiveDirWalkerEnabled(), new ConfigBasedCacheQuotaRequirementProvider(cacheConfig), new HiveEncryptionInformationProvider(ImmutableSet.of()));
    pageSinkProvider = new HivePageSinkProvider(getDefaultHiveFileWriterFactories(config, metastoreClientConfig), hdfsEnvironment, PAGE_SORTER, metastoreClient, new GroupByHashPageIndexerFactory(new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig())), FUNCTION_AND_TYPE_MANAGER, config, metastoreClientConfig, locationService, HiveTestUtils.PARTITION_UPDATE_CODEC, HiveTestUtils.PARTITION_UPDATE_SMILE_CODEC, new TestingNodeManager("fake-environment"), new HiveEventClient(), new HiveSessionProperties(config, new OrcFileWriterConfig(), new ParquetFileWriterConfig(), new CacheConfig()), new HiveWriterStats(), getDefaultOrcFileWriterFactory(config, metastoreClientConfig), columnConverterProvider);
    pageSourceProvider = new HivePageSourceProvider(config, hdfsEnvironment, getDefaultHiveRecordCursorProvider(config, metastoreClientConfig), getDefaultHiveBatchPageSourceFactories(config, metastoreClientConfig), getDefaultHiveSelectivePageSourceFactories(config, metastoreClientConfig), FUNCTION_AND_TYPE_MANAGER, ROW_EXPRESSION_SERVICE);
}
Also used : CounterStat(com.facebook.airlift.stats.CounterStat) HivePartitionMutator(com.facebook.presto.hive.metastore.HivePartitionMutator) TestingNodeManager(com.facebook.presto.testing.TestingNodeManager) BridgingHiveMetastore(com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) TestingHiveCluster(com.facebook.presto.hive.metastore.thrift.TestingHiveCluster) OutputStreamDataSinkFactory(com.facebook.presto.hive.datasink.OutputStreamDataSinkFactory) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) TestingHiveCluster(com.facebook.presto.hive.metastore.thrift.TestingHiveCluster) HiveCluster(com.facebook.presto.hive.metastore.thrift.HiveCluster) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) CacheConfig(com.facebook.presto.cache.CacheConfig) ThriftHiveMetastore(com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore) SchemaTableName(com.facebook.presto.spi.SchemaTableName) BoundedExecutor(com.facebook.airlift.concurrent.BoundedExecutor) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) ExecutorService(java.util.concurrent.ExecutorService)

Example 39 with FeaturesConfig

use of com.facebook.presto.sql.analyzer.FeaturesConfig in project presto by prestodb.

the class AbstractTestQueryFramework method getQueryExplainer.

private QueryExplainer getQueryExplainer() {
    Metadata metadata = queryRunner.getMetadata();
    FeaturesConfig featuresConfig = new FeaturesConfig().setOptimizeHashGeneration(true);
    boolean forceSingleNode = queryRunner.getNodeCount() == 1;
    TaskCountEstimator taskCountEstimator = new TaskCountEstimator(queryRunner::getNodeCount);
    CostCalculator costCalculator = new CostCalculatorUsingExchanges(taskCountEstimator);
    List<PlanOptimizer> optimizers = new PlanOptimizers(metadata, sqlParser, forceSingleNode, new MBeanExporter(new TestingMBeanServer()), queryRunner.getSplitManager(), queryRunner.getPlanOptimizerManager(), queryRunner.getPageSourceManager(), queryRunner.getStatsCalculator(), costCalculator, new CostCalculatorWithEstimatedExchanges(costCalculator, taskCountEstimator), new CostComparator(featuresConfig), taskCountEstimator, new PartitioningProviderManager()).getPlanningTimeOptimizers();
    return new QueryExplainer(optimizers, new PlanFragmenter(metadata, queryRunner.getNodePartitioningManager(), new QueryManagerConfig(), sqlParser, new FeaturesConfig()), metadata, queryRunner.getAccessControl(), sqlParser, queryRunner.getStatsCalculator(), costCalculator, ImmutableMap.of(), new PlanChecker(new FeaturesConfig(), false));
}
Also used : TaskCountEstimator(com.facebook.presto.cost.TaskCountEstimator) CostComparator(com.facebook.presto.cost.CostComparator) PlanOptimizer(com.facebook.presto.sql.planner.optimizations.PlanOptimizer) TestingMBeanServer(org.weakref.jmx.testing.TestingMBeanServer) QueryExplainer(com.facebook.presto.sql.analyzer.QueryExplainer) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) MBeanExporter(org.weakref.jmx.MBeanExporter) PartitioningProviderManager(com.facebook.presto.sql.planner.PartitioningProviderManager) Metadata(com.facebook.presto.metadata.Metadata) PlanFragmenter(com.facebook.presto.sql.planner.PlanFragmenter) CostCalculator(com.facebook.presto.cost.CostCalculator) PlanChecker(com.facebook.presto.sql.planner.sanity.PlanChecker) PlanOptimizers(com.facebook.presto.sql.planner.PlanOptimizers) QueryManagerConfig(com.facebook.presto.execution.QueryManagerConfig) CostCalculatorUsingExchanges(com.facebook.presto.cost.CostCalculatorUsingExchanges) CostCalculatorWithEstimatedExchanges(com.facebook.presto.cost.CostCalculatorWithEstimatedExchanges)

Aggregations

FeaturesConfig (com.facebook.presto.sql.analyzer.FeaturesConfig)39 Test (org.testng.annotations.Test)20 JoinCompiler (com.facebook.presto.sql.gen.JoinCompiler)14 BlockEncodingManager (com.facebook.presto.block.BlockEncodingManager)10 PlanNodeId (com.facebook.presto.spi.plan.PlanNodeId)9 TypeRegistry (com.facebook.presto.type.TypeRegistry)9 Page (com.facebook.presto.common.Page)7 Type (com.facebook.presto.common.type.Type)7 SetBuilderOperatorFactory (com.facebook.presto.operator.SetBuilderOperator.SetBuilderOperatorFactory)6 RowPagesBuilder (com.facebook.presto.RowPagesBuilder)5 TypeSignature.parseTypeSignature (com.facebook.presto.spi.type.TypeSignature.parseTypeSignature)5 BlockEncodingManager (com.facebook.presto.common.block.BlockEncodingManager)4 CatalogManager (com.facebook.presto.metadata.CatalogManager)4 FunctionRegistry (com.facebook.presto.metadata.FunctionRegistry)4 FunctionRegistry.getMagicLiteralFunctionSignature (com.facebook.presto.metadata.FunctionRegistry.getMagicLiteralFunctionSignature)4 HashSemiJoinOperatorFactory (com.facebook.presto.operator.HashSemiJoinOperator.HashSemiJoinOperatorFactory)4 ImmutableList (com.google.common.collect.ImmutableList)4 List (java.util.List)4 BeforeClass (org.testng.annotations.BeforeClass)4 CounterStat (com.facebook.airlift.stats.CounterStat)3