Examples with MetastoreLocator io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator used on opensource projects

Example 1 with MetastoreLocator

use of io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator in project hetu-core by openlookeng.

the class TestCachingHiveMetastore method setUp.

@BeforeMethod
public void setUp() {
    mockClient = new MockThriftMetastoreClient();
    MetastoreLocator metastoreLocator = new MockMetastoreLocator(mockClient);
    ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s")));
    ListeningExecutorService executorRefresh = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s")));
    ThriftHiveMetastore thriftHiveMetastore = new ThriftHiveMetastore(metastoreLocator, new ThriftHiveMetastoreConfig());
    metastore = new CachingHiveMetastore(new BridgingHiveMetastore(thriftHiveMetastore), executor, executorRefresh, new Duration(6, TimeUnit.MINUTES), new Duration(6, TimeUnit.MINUTES), new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), 1000, false);
    stats = thriftHiveMetastore.getStats();
}

Example 2 with MetastoreLocator

use of io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator in project hetu-core by openlookeng.

the class TestSemiTransactionalHiveMetastore method setUp.

@BeforeMethod
public void setUp() {
    mockClient = new MockThriftMetastoreClient();
    MetastoreLocator metastoreLocator = new MockMetastoreLocator(mockClient);
    thriftHiveMetastore = new ThriftHiveMetastore(metastoreLocator, new ThriftHiveMetastoreConfig());
    stats = thriftHiveMetastore.getStats();
}

Example 3 with MetastoreLocator

use of io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator in project boostkit-bigdata by kunpengcompute.

the class AbstractTestHiveFileSystem method setup.

protected void setup(String host, int port, String databaseName, Function<HiveConfig, HdfsConfiguration> hdfsConfigurationProvider, boolean s3SelectPushdownEnabled) {
    database = databaseName;
    table = new SchemaTableName(database, "presto_test_external_fs");
    String random = UUID.randomUUID().toString().toLowerCase(ENGLISH).replace("-", "");
    temporaryCreateTable = new SchemaTableName(database, "tmp_presto_test_create_" + random);
    config = new HiveConfig().setS3SelectPushdownEnabled(s3SelectPushdownEnabled);
    String proxy = System.getProperty("hive.metastore.thrift.client.socks-proxy");
    if (proxy != null) {
        config.setMetastoreSocksProxy(HostAndPort.fromString(proxy));
    }
    MetastoreLocator metastoreLocator = new TestingMetastoreLocator(config, host, port);
    ExecutorService executors = newCachedThreadPool(daemonThreadsNamed("hive-%s"));
    ExecutorService executorRefresh = newCachedThreadPool(daemonThreadsNamed("hive-refresh-%s"));
    HivePartitionManager hivePartitionManager = new HivePartitionManager(TYPE_MANAGER, config);
    HdfsConfiguration hdfsConfiguration = hdfsConfigurationProvider.apply(config);
    hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, config, new NoHdfsAuthentication());
    metastoreClient = new TestingHiveMetastore(new BridgingHiveMetastore(new ThriftHiveMetastore(metastoreLocator, new ThriftHiveMetastoreConfig())), executors, executorRefresh, config, getBasePath(), hdfsEnvironment);
    locationService = new HiveLocationService(hdfsEnvironment);
    JsonCodec<PartitionUpdate> partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class);
    metadataFactory = new HiveMetadataFactory(config, metastoreClient, hdfsEnvironment, hivePartitionManager, newDirectExecutorService(), vacuumExecutorService, heartbeatService, vacuumExecutorService, TYPE_MANAGER, locationService, partitionUpdateCodec, new HiveTypeTranslator(), new NodeVersion("test_version"), SqlStandardAccessControlMetadata::new);
    transactionManager = new HiveTransactionManager();
    splitManager = new HiveSplitManager(transactionHandle -> ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore(), hivePartitionManager, new NamenodeStats(), hdfsEnvironment, new CachingDirectoryLister(new HiveConfig()), new BoundedExecutor(executors, config.getMaxSplitIteratorThreads()), new HiveCoercionPolicy(TYPE_MANAGER), new CounterStat(), config.getMaxOutstandingSplits(), config.getMaxOutstandingSplitsSize(), config.getMinPartitionBatchSize(), config.getMaxPartitionBatchSize(), config.getMaxInitialSplits(), config.getSplitLoaderConcurrency(), config.getMaxSplitsPerSecond(), config.getRecursiveDirWalkerEnabled(), null, config);
    pageSinkProvider = new HivePageSinkProvider(getDefaultHiveFileWriterFactories(config), hdfsEnvironment, PAGE_SORTER, metastoreClient, new GroupByHashPageIndexerFactory(new JoinCompiler(createTestMetadataManager())), TYPE_MANAGER, config, locationService, partitionUpdateCodec, new TestingNodeManager("fake-environment"), new HiveEventClient(), new HiveSessionProperties(config, new OrcFileWriterConfig(), new ParquetFileWriterConfig()), new HiveWriterStats(), getDefaultOrcFileWriterFactory(config));
    pageSourceProvider = new HivePageSourceProvider(config, hdfsEnvironment, getDefaultHiveRecordCursorProvider(config), getDefaultHiveDataStreamFactories(config), TYPE_MANAGER, getNoOpIndexCache(), getDefaultHiveSelectiveFactories(config));
}

Example 4 with MetastoreLocator

use of io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator in project boostkit-bigdata by kunpengcompute.

the class AbstractTestHive method setup.

protected final void setup(String host, int port, String databaseName, String timeZone) {
    HiveConfig hiveConfig = getHiveConfig().setParquetTimeZone(timeZone).setRcfileTimeZone(timeZone);
    String proxy = System.getProperty("hive.metastore.thrift.client.socks-proxy");
    if (proxy != null) {
        hiveConfig.setMetastoreSocksProxy(HostAndPort.fromString(proxy));
    }
    MetastoreLocator metastoreLocator = new TestingMetastoreLocator(hiveConfig, host, port);
    HiveMetastore metastore = new CachingHiveMetastore(new BridgingHiveMetastore(new ThriftHiveMetastore(metastoreLocator, new ThriftHiveMetastoreConfig())), executor, executorRefresh, Duration.valueOf("1m"), Duration.valueOf("15s"), Duration.valueOf("1m"), Duration.valueOf("15s"), 10000, false);
    setup(databaseName, hiveConfig, metastore);
}

Example 5 with MetastoreLocator

use of io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator in project boostkit-bigdata by kunpengcompute.

the class TestHiveWriterFactory method setUp.

private void setUp() {
    mockClient = new MockThriftMetastoreClient();
    executor = newCachedThreadPool(daemonThreadsNamed("hive-%s"));
    executorRefresh = newCachedThreadPool(daemonThreadsNamed("hive-refresh-%s"));
    MetastoreLocator metastoreLocator = new MockMetastoreLocator(mockClient);
    metastore = new CachingHiveMetastore(new BridgingHiveMetastore(new ThriftHiveMetastore(metastoreLocator, new ThriftHiveMetastoreConfig())), executor, executorRefresh, Duration.valueOf("1m"), Duration.valueOf("15s"), Duration.valueOf("1m"), Duration.valueOf("15s"), 10000, false);
}