Search in sources :

Example 1 with TestingHiveCluster

use of com.facebook.presto.hive.metastore.thrift.TestingHiveCluster in project presto by prestodb.

the class AbstractTestHiveClient method setup.

protected final void setup(String host, int port, String databaseName, String timeZone) {
    HiveClientConfig hiveClientConfig = getHiveClientConfig();
    CacheConfig cacheConfig = getCacheConfig();
    MetastoreClientConfig metastoreClientConfig = getMetastoreClientConfig();
    hiveClientConfig.setTimeZone(timeZone);
    String proxy = System.getProperty("hive.metastore.thrift.client.socks-proxy");
    if (proxy != null) {
        metastoreClientConfig.setMetastoreSocksProxy(HostAndPort.fromString(proxy));
    }
    HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, host, port);
    ExtendedHiveMetastore metastore = new CachingHiveMetastore(new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster, metastoreClientConfig), new HivePartitionMutator()), executor, false, Duration.valueOf("1m"), Duration.valueOf("15s"), 10000, false, MetastoreCacheScope.ALL, 0.0);
    setup(databaseName, hiveClientConfig, cacheConfig, metastoreClientConfig, metastore);
}
Also used : TestingHiveCluster(com.facebook.presto.hive.metastore.thrift.TestingHiveCluster) CachingHiveMetastore(com.facebook.presto.hive.metastore.CachingHiveMetastore) ThriftHiveMetastore(com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) CacheConfig(com.facebook.presto.cache.CacheConfig) HiveCluster(com.facebook.presto.hive.metastore.thrift.HiveCluster) TestingHiveCluster(com.facebook.presto.hive.metastore.thrift.TestingHiveCluster) HivePartitionMutator(com.facebook.presto.hive.metastore.HivePartitionMutator) BridgingHiveMetastore(com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore)

Example 2 with TestingHiveCluster

use of com.facebook.presto.hive.metastore.thrift.TestingHiveCluster in project presto by prestodb.

the class TestingSemiTransactionalHiveMetastore method create.

public static TestingSemiTransactionalHiveMetastore create() {
    // none of these values matter, as we never use them
    HiveClientConfig config = new HiveClientConfig();
    MetastoreClientConfig metastoreClientConfig = new MetastoreClientConfig();
    HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(config, metastoreClientConfig), ImmutableSet.of());
    HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication());
    HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, HOST, PORT);
    ColumnConverterProvider columnConverterProvider = HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER;
    ExtendedHiveMetastore delegate = new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster, metastoreClientConfig), new HivePartitionMutator());
    ExecutorService executor = newCachedThreadPool(daemonThreadsNamed("hive-%s"));
    ListeningExecutorService renameExecutor = listeningDecorator(executor);
    return new TestingSemiTransactionalHiveMetastore(hdfsEnvironment, delegate, renameExecutor, false, false, true, columnConverterProvider);
}
Also used : TestingHiveCluster(com.facebook.presto.hive.metastore.thrift.TestingHiveCluster) ThriftHiveMetastore(com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) HiveCluster(com.facebook.presto.hive.metastore.thrift.HiveCluster) TestingHiveCluster(com.facebook.presto.hive.metastore.thrift.TestingHiveCluster) HivePartitionMutator(com.facebook.presto.hive.metastore.HivePartitionMutator) ExecutorService(java.util.concurrent.ExecutorService) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) ExtendedHiveMetastore(com.facebook.presto.hive.metastore.ExtendedHiveMetastore) BridgingHiveMetastore(com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore)

Example 3 with TestingHiveCluster

use of com.facebook.presto.hive.metastore.thrift.TestingHiveCluster in project presto by prestodb.

the class AbstractTestHiveFileSystem method setup.

protected void setup(String host, int port, String databaseName, BiFunction<HiveClientConfig, MetastoreClientConfig, HdfsConfiguration> hdfsConfigurationProvider, boolean s3SelectPushdownEnabled) {
    database = databaseName;
    table = new SchemaTableName(database, "presto_test_external_fs");
    String random = UUID.randomUUID().toString().toLowerCase(ENGLISH).replace("-", "");
    temporaryCreateTable = new SchemaTableName(database, "tmp_presto_test_create_" + random);
    config = new HiveClientConfig().setS3SelectPushdownEnabled(s3SelectPushdownEnabled);
    cacheConfig = new CacheConfig();
    metastoreClientConfig = new MetastoreClientConfig();
    String proxy = System.getProperty("hive.metastore.thrift.client.socks-proxy");
    if (proxy != null) {
        metastoreClientConfig.setMetastoreSocksProxy(HostAndPort.fromString(proxy));
    }
    HiveCluster hiveCluster = new TestingHiveCluster(metastoreClientConfig, host, port);
    ExecutorService executor = newCachedThreadPool(daemonThreadsNamed("hive-%s"));
    HivePartitionManager hivePartitionManager = new HivePartitionManager(FUNCTION_AND_TYPE_MANAGER, config);
    HdfsConfiguration hdfsConfiguration = hdfsConfigurationProvider.apply(config, metastoreClientConfig);
    hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, metastoreClientConfig, new NoHdfsAuthentication());
    ColumnConverterProvider columnConverterProvider = HiveColumnConverterProvider.DEFAULT_COLUMN_CONVERTER_PROVIDER;
    metastoreClient = new TestingHiveMetastore(new BridgingHiveMetastore(new ThriftHiveMetastore(hiveCluster, metastoreClientConfig), new HivePartitionMutator()), executor, metastoreClientConfig, getBasePath(), hdfsEnvironment);
    locationService = new HiveLocationService(hdfsEnvironment);
    metadataFactory = new HiveMetadataFactory(config, metastoreClientConfig, metastoreClient, hdfsEnvironment, hivePartitionManager, newDirectExecutorService(), FUNCTION_AND_TYPE_MANAGER, locationService, FUNCTION_RESOLUTION, ROW_EXPRESSION_SERVICE, FILTER_STATS_CALCULATOR_SERVICE, new TableParameterCodec(), HiveTestUtils.PARTITION_UPDATE_CODEC, HiveTestUtils.PARTITION_UPDATE_SMILE_CODEC, new HiveTypeTranslator(), new HiveStagingFileCommitter(hdfsEnvironment, listeningDecorator(executor)), new HiveZeroRowFileCreator(hdfsEnvironment, new OutputStreamDataSinkFactory(), listeningDecorator(executor)), new NodeVersion("test_version"), new HivePartitionObjectBuilder(), new HiveEncryptionInformationProvider(ImmutableList.of()), new HivePartitionStats(), new HiveFileRenamer(), columnConverterProvider);
    transactionManager = new HiveTransactionManager();
    splitManager = new HiveSplitManager(transactionManager, new NamenodeStats(), hdfsEnvironment, new CachingDirectoryLister(new HadoopDirectoryLister(), new HiveClientConfig()), new BoundedExecutor(executor, config.getMaxSplitIteratorThreads()), new HiveCoercionPolicy(FUNCTION_AND_TYPE_MANAGER), new CounterStat(), config.getMaxOutstandingSplits(), config.getMaxOutstandingSplitsSize(), config.getMinPartitionBatchSize(), config.getMaxPartitionBatchSize(), config.getSplitLoaderConcurrency(), config.getRecursiveDirWalkerEnabled(), new ConfigBasedCacheQuotaRequirementProvider(cacheConfig), new HiveEncryptionInformationProvider(ImmutableSet.of()));
    pageSinkProvider = new HivePageSinkProvider(getDefaultHiveFileWriterFactories(config, metastoreClientConfig), hdfsEnvironment, PAGE_SORTER, metastoreClient, new GroupByHashPageIndexerFactory(new JoinCompiler(MetadataManager.createTestMetadataManager(), new FeaturesConfig())), FUNCTION_AND_TYPE_MANAGER, config, metastoreClientConfig, locationService, HiveTestUtils.PARTITION_UPDATE_CODEC, HiveTestUtils.PARTITION_UPDATE_SMILE_CODEC, new TestingNodeManager("fake-environment"), new HiveEventClient(), new HiveSessionProperties(config, new OrcFileWriterConfig(), new ParquetFileWriterConfig(), new CacheConfig()), new HiveWriterStats(), getDefaultOrcFileWriterFactory(config, metastoreClientConfig), columnConverterProvider);
    pageSourceProvider = new HivePageSourceProvider(config, hdfsEnvironment, getDefaultHiveRecordCursorProvider(config, metastoreClientConfig), getDefaultHiveBatchPageSourceFactories(config, metastoreClientConfig), getDefaultHiveSelectivePageSourceFactories(config, metastoreClientConfig), FUNCTION_AND_TYPE_MANAGER, ROW_EXPRESSION_SERVICE);
}
Also used : CounterStat(com.facebook.airlift.stats.CounterStat) HivePartitionMutator(com.facebook.presto.hive.metastore.HivePartitionMutator) TestingNodeManager(com.facebook.presto.testing.TestingNodeManager) BridgingHiveMetastore(com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore) JoinCompiler(com.facebook.presto.sql.gen.JoinCompiler) GroupByHashPageIndexerFactory(com.facebook.presto.GroupByHashPageIndexerFactory) TestingHiveCluster(com.facebook.presto.hive.metastore.thrift.TestingHiveCluster) OutputStreamDataSinkFactory(com.facebook.presto.hive.datasink.OutputStreamDataSinkFactory) FeaturesConfig(com.facebook.presto.sql.analyzer.FeaturesConfig) TestingHiveCluster(com.facebook.presto.hive.metastore.thrift.TestingHiveCluster) HiveCluster(com.facebook.presto.hive.metastore.thrift.HiveCluster) NoHdfsAuthentication(com.facebook.presto.hive.authentication.NoHdfsAuthentication) CacheConfig(com.facebook.presto.cache.CacheConfig) ThriftHiveMetastore(com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore) SchemaTableName(com.facebook.presto.spi.SchemaTableName) BoundedExecutor(com.facebook.airlift.concurrent.BoundedExecutor) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) ExecutorService(java.util.concurrent.ExecutorService)

Aggregations

HivePartitionMutator (com.facebook.presto.hive.metastore.HivePartitionMutator)3 BridgingHiveMetastore (com.facebook.presto.hive.metastore.thrift.BridgingHiveMetastore)3 HiveCluster (com.facebook.presto.hive.metastore.thrift.HiveCluster)3 TestingHiveCluster (com.facebook.presto.hive.metastore.thrift.TestingHiveCluster)3 ThriftHiveMetastore (com.facebook.presto.hive.metastore.thrift.ThriftHiveMetastore)3 CacheConfig (com.facebook.presto.cache.CacheConfig)2 NoHdfsAuthentication (com.facebook.presto.hive.authentication.NoHdfsAuthentication)2 ExtendedHiveMetastore (com.facebook.presto.hive.metastore.ExtendedHiveMetastore)2 ExecutorService (java.util.concurrent.ExecutorService)2 BoundedExecutor (com.facebook.airlift.concurrent.BoundedExecutor)1 CounterStat (com.facebook.airlift.stats.CounterStat)1 GroupByHashPageIndexerFactory (com.facebook.presto.GroupByHashPageIndexerFactory)1 OutputStreamDataSinkFactory (com.facebook.presto.hive.datasink.OutputStreamDataSinkFactory)1 CachingHiveMetastore (com.facebook.presto.hive.metastore.CachingHiveMetastore)1 SchemaTableName (com.facebook.presto.spi.SchemaTableName)1 FeaturesConfig (com.facebook.presto.sql.analyzer.FeaturesConfig)1 JoinCompiler (com.facebook.presto.sql.gen.JoinCompiler)1 TestingNodeManager (com.facebook.presto.testing.TestingNodeManager)1 ListeningExecutorService (com.google.common.util.concurrent.ListeningExecutorService)1 MoreExecutors.newDirectExecutorService (com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService)1