Search in sources :

Example 6 with OutputStreamDataSinkFactory

use of com.facebook.presto.hive.datasink.OutputStreamDataSinkFactory in project presto by prestodb.

the class AbstractTestHiveClient method setup.

protected final void setup(String databaseName, HiveClientConfig hiveClientConfig, CacheConfig cacheConfig, MetastoreClientConfig metastoreClientConfig, ExtendedHiveMetastore hiveMetastore) {
    HiveConnectorId connectorId = new HiveConnectorId("hive-test");
    setupHive(connectorId.toString(), databaseName, hiveClientConfig.getTimeZone());
    hivePartitionManager = new HivePartitionManager(FUNCTION_AND_TYPE_MANAGER, hiveClientConfig);
    metastoreClient = hiveMetastore;
    HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(hiveClientConfig, metastoreClientConfig), ImmutableSet.of());
    hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication());
    locationService = new HiveLocationService(hdfsEnvironment);
    metadataFactory = new HiveMetadataFactory(metastoreClient, hdfsEnvironment, hivePartitionManager, timeZone, true, false, false, false, true, true, getHiveClientConfig().getMaxPartitionBatchSize(), getHiveClientConfig().getMaxPartitionsPerScan(), false, FUNCTION_AND_TYPE_MANAGER, locationService, FUNCTION_RESOLUTION, ROW_EXPRESSION_SERVICE, FILTER_STATS_CALCULATOR_SERVICE, new TableParameterCodec(), HiveTestUtils.PARTITION_UPDATE_CODEC, HiveTestUtils.PARTITION_UPDATE_SMILE_CODEC, listeningDecorator(executor), new HiveTypeTranslator(), new HiveStagingFileCommitter(hdfsEnvironment, listeningDecorator(executor)), new HiveZeroRowFileCreator(hdfsEnvironment, new OutputStreamDataSinkFactory(), listeningDecorator(executor)), TEST_SERVER_VERSION, new HivePartitionObjectBuilder(), new HiveEncryptionInformationProvider(ImmutableList.of()), new HivePartitionStats(), new HiveFileRenamer(), DEFAULT_COLUMN_CONVERTER_PROVIDER);
    transactionManager = new HiveTransactionManager();
    encryptionInformationProvider = new HiveEncryptionInformationProvider(ImmutableList.of());
    splitManager = new HiveSplitManager(transactionManager, new NamenodeStats(), hdfsEnvironment, new CachingDirectoryLister(new HadoopDirectoryLister(), new HiveClientConfig()), directExecutor(), new HiveCoercionPolicy(FUNCTION_AND_TYPE_MANAGER), new CounterStat(), 100, hiveClientConfig.getMaxOutstandingSplitsSize(), hiveClientConfig.getMinPartitionBatchSize(), hiveClientConfig.getMaxPartitionBatchSize(), hiveClientConfig.getSplitLoaderConcurrency(), false, new ConfigBasedCacheQuotaRequirementProvider(cacheConfig), encryptionInformationProvider);
    pageSinkProvider = new HivePageSinkProvider(getDefaultHiveFileWriterFactories(hiveClientConfig, metastoreClientConfig), hdfsEnvironment, PAGE_SORTER, metastoreClient, new GroupByHashPageIndexerFactory(JOIN_COMPILER), FUNCTION_AND_TYPE_MANAGER, getHiveClientConfig(), getMetastoreClientConfig(), locationService, HiveTestUtils.PARTITION_UPDATE_CODEC, HiveTestUtils.PARTITION_UPDATE_SMILE_CODEC, new TestingNodeManager("fake-environment"), new HiveEventClient(), new HiveSessionProperties(hiveClientConfig, new OrcFileWriterConfig(), new ParquetFileWriterConfig(), new CacheConfig()), new HiveWriterStats(), getDefaultOrcFileWriterFactory(hiveClientConfig, metastoreClientConfig), DEFAULT_COLUMN_CONVERTER_PROVIDER);
    pageSourceProvider = new HivePageSourceProvider(hiveClientConfig, hdfsEnvironment, getDefaultHiveRecordCursorProvider(hiveClientConfig, metastoreClientConfig), getDefaultHiveBatchPageSourceFactories(hiveClientConfig, metastoreClientConfig), getDefaultHiveSelectivePageSourceFactories(hiveClientConfig, metastoreClientConfig), FUNCTION_AND_TYPE_MANAGER, ROW_EXPRESSION_SERVICE);
}
Also used : OutputStreamDataSinkFactory(com.facebook.presto.hive.datasink.OutputStreamDataSinkFactory) CounterStat(com.facebook.airlift.stats.CounterStat) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) TestingNodeManager(com.facebook.presto.testing.TestingNodeManager) CacheConfig(com.facebook.presto.cache.CacheConfig) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory)

Aggregations

OutputStreamDataSinkFactory (com.facebook.presto.hive.datasink.OutputStreamDataSinkFactory)6 CacheConfig (com.facebook.presto.cache.CacheConfig)5 CounterStat (com.facebook.airlift.stats.CounterStat)3 NoHdfsAuthentication (com.facebook.presto.hive.authentication.NoHdfsAuthentication)3 Storage (com.facebook.presto.hive.metastore.Storage)3 File (java.io.File)3 BeforeClass (org.testng.annotations.BeforeClass)3 GroupByHashPageIndexerFactory (com.facebook.presto.GroupByHashPageIndexerFactory)2 TupleDomain (com.facebook.presto.common.predicate.TupleDomain)2 ArrayType (com.facebook.presto.common.type.ArrayType)2 BIGINT (com.facebook.presto.common.type.BigintType.BIGINT)2 BOOLEAN (com.facebook.presto.common.type.BooleanType.BOOLEAN)2 DOUBLE (com.facebook.presto.common.type.DoubleType.DOUBLE)2 INTEGER (com.facebook.presto.common.type.IntegerType.INTEGER)2 RowType (com.facebook.presto.common.type.RowType)2 VarcharType.createUnboundedVarcharType (com.facebook.presto.common.type.VarcharType.createUnboundedVarcharType)2 TRUE_CONSTANT (com.facebook.presto.expressions.LogicalRowExpressions.TRUE_CONSTANT)2 NO_ENCRYPTION (com.facebook.presto.hive.HiveDwrfEncryptionProvider.NO_ENCRYPTION)2 HIVE_INVALID_PARTITION_VALUE (com.facebook.presto.hive.HiveErrorCode.HIVE_INVALID_PARTITION_VALUE)2 HIVE_PARTITION_SCHEMA_MISMATCH (com.facebook.presto.hive.HiveErrorCode.HIVE_PARTITION_SCHEMA_MISMATCH)2