Search in sources :

Example 1 with ThriftHiveMetastoreConfig

use of io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig in project hetu-core by openlookeng.

the class TestCachingHiveMetastore method setUp.

@BeforeMethod
public void setUp() {
    mockClient = new MockThriftMetastoreClient();
    MetastoreLocator metastoreLocator = new MockMetastoreLocator(mockClient);
    ListeningExecutorService executor = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s")));
    ListeningExecutorService executorRefresh = listeningDecorator(newCachedThreadPool(daemonThreadsNamed("test-%s")));
    ThriftHiveMetastore thriftHiveMetastore = new ThriftHiveMetastore(metastoreLocator, new ThriftHiveMetastoreConfig());
    metastore = new CachingHiveMetastore(new BridgingHiveMetastore(thriftHiveMetastore), executor, executorRefresh, new Duration(6, TimeUnit.MINUTES), new Duration(6, TimeUnit.MINUTES), new Duration(5, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), 1000, false);
    stats = thriftHiveMetastore.getStats();
}
Also used : MockThriftMetastoreClient(io.prestosql.plugin.hive.metastore.thrift.MockThriftMetastoreClient) ThriftHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastore) MetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) Duration(io.airlift.units.Duration) ThriftHiveMetastoreConfig(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig) BridgingHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.BridgingHiveMetastore) BeforeMethod(org.testng.annotations.BeforeMethod)

Example 2 with ThriftHiveMetastoreConfig

use of io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig in project hetu-core by openlookeng.

the class TestSemiTransactionalHiveMetastore method setUp.

@BeforeMethod
public void setUp() {
    mockClient = new MockThriftMetastoreClient();
    MetastoreLocator metastoreLocator = new MockMetastoreLocator(mockClient);
    thriftHiveMetastore = new ThriftHiveMetastore(metastoreLocator, new ThriftHiveMetastoreConfig());
    stats = thriftHiveMetastore.getStats();
}
Also used : MockThriftMetastoreClient(io.prestosql.plugin.hive.metastore.thrift.MockThriftMetastoreClient) ThriftHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastore) MetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator) ThriftHiveMetastoreConfig(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig) BeforeMethod(org.testng.annotations.BeforeMethod)

Example 3 with ThriftHiveMetastoreConfig

use of io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig in project boostkit-bigdata by kunpengcompute.

the class AbstractTestHiveFileSystem method setup.

protected void setup(String host, int port, String databaseName, Function<HiveConfig, HdfsConfiguration> hdfsConfigurationProvider, boolean s3SelectPushdownEnabled) {
    database = databaseName;
    table = new SchemaTableName(database, "presto_test_external_fs");
    String random = UUID.randomUUID().toString().toLowerCase(ENGLISH).replace("-", "");
    temporaryCreateTable = new SchemaTableName(database, "tmp_presto_test_create_" + random);
    config = new HiveConfig().setS3SelectPushdownEnabled(s3SelectPushdownEnabled);
    String proxy = System.getProperty("hive.metastore.thrift.client.socks-proxy");
    if (proxy != null) {
        config.setMetastoreSocksProxy(HostAndPort.fromString(proxy));
    }
    MetastoreLocator metastoreLocator = new TestingMetastoreLocator(config, host, port);
    ExecutorService executors = newCachedThreadPool(daemonThreadsNamed("hive-%s"));
    ExecutorService executorRefresh = newCachedThreadPool(daemonThreadsNamed("hive-refresh-%s"));
    HivePartitionManager hivePartitionManager = new HivePartitionManager(TYPE_MANAGER, config);
    HdfsConfiguration hdfsConfiguration = hdfsConfigurationProvider.apply(config);
    hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, config, new NoHdfsAuthentication());
    metastoreClient = new TestingHiveMetastore(new BridgingHiveMetastore(new ThriftHiveMetastore(metastoreLocator, new ThriftHiveMetastoreConfig())), executors, executorRefresh, config, getBasePath(), hdfsEnvironment);
    locationService = new HiveLocationService(hdfsEnvironment);
    JsonCodec<PartitionUpdate> partitionUpdateCodec = JsonCodec.jsonCodec(PartitionUpdate.class);
    metadataFactory = new HiveMetadataFactory(config, metastoreClient, hdfsEnvironment, hivePartitionManager, newDirectExecutorService(), vacuumExecutorService, heartbeatService, vacuumExecutorService, TYPE_MANAGER, locationService, partitionUpdateCodec, new HiveTypeTranslator(), new NodeVersion("test_version"), SqlStandardAccessControlMetadata::new);
    transactionManager = new HiveTransactionManager();
    splitManager = new HiveSplitManager(transactionHandle -> ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore(), hivePartitionManager, new NamenodeStats(), hdfsEnvironment, new CachingDirectoryLister(new HiveConfig()), new BoundedExecutor(executors, config.getMaxSplitIteratorThreads()), new HiveCoercionPolicy(TYPE_MANAGER), new CounterStat(), config.getMaxOutstandingSplits(), config.getMaxOutstandingSplitsSize(), config.getMinPartitionBatchSize(), config.getMaxPartitionBatchSize(), config.getMaxInitialSplits(), config.getSplitLoaderConcurrency(), config.getMaxSplitsPerSecond(), config.getRecursiveDirWalkerEnabled(), null, config);
    pageSinkProvider = new HivePageSinkProvider(getDefaultHiveFileWriterFactories(config), hdfsEnvironment, PAGE_SORTER, metastoreClient, new GroupByHashPageIndexerFactory(new JoinCompiler(createTestMetadataManager())), TYPE_MANAGER, config, locationService, partitionUpdateCodec, new TestingNodeManager("fake-environment"), new HiveEventClient(), new HiveSessionProperties(config, new OrcFileWriterConfig(), new ParquetFileWriterConfig()), new HiveWriterStats(), getDefaultOrcFileWriterFactory(config));
    pageSourceProvider = new HivePageSourceProvider(config, hdfsEnvironment, getDefaultHiveRecordCursorProvider(config), getDefaultHiveDataStreamFactories(config), TYPE_MANAGER, getNoOpIndexCache(), getDefaultHiveSelectiveFactories(config));
}
Also used : ConnectorMetadata(io.prestosql.spi.connector.ConnectorMetadata) HiveTestUtils.getDefaultHiveSelectiveFactories(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveSelectiveFactories) FileSystem(org.apache.hadoop.fs.FileSystem) HdfsContext(io.prestosql.plugin.hive.HdfsEnvironment.HdfsContext) ConnectorSplitManager(io.prestosql.spi.connector.ConnectorSplitManager) NoHdfsAuthentication(io.prestosql.plugin.hive.authentication.NoHdfsAuthentication) ConnectorPageSink(io.prestosql.spi.connector.ConnectorPageSink) Test(org.testng.annotations.Test) MaterializedResult.materializeSourceDataStream(io.prestosql.testing.MaterializedResult.materializeSourceDataStream) MaterializedResult(io.prestosql.testing.MaterializedResult) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) ConnectorSession(io.prestosql.spi.connector.ConnectorSession) TableNotFoundException(io.prestosql.spi.connector.TableNotFoundException) BoundedExecutor(io.airlift.concurrent.BoundedExecutor) Executors.newScheduledThreadPool(java.util.concurrent.Executors.newScheduledThreadPool) Map(java.util.Map) Path(org.apache.hadoop.fs.Path) ConnectorPageSinkProvider(io.prestosql.spi.connector.ConnectorPageSinkProvider) BIGINT(io.prestosql.spi.type.BigintType.BIGINT) ENGLISH(java.util.Locale.ENGLISH) Assert.assertFalse(org.testng.Assert.assertFalse) Function(com.google.common.base.Function) CachingHiveMetastore(io.prestosql.plugin.hive.metastore.CachingHiveMetastore) UNGROUPED_SCHEDULING(io.prestosql.spi.connector.ConnectorSplitManager.SplitSchedulingStrategy.UNGROUPED_SCHEDULING) ImmutableMap(com.google.common.collect.ImmutableMap) MetadataManager.createTestMetadataManager(io.prestosql.metadata.MetadataManager.createTestMetadataManager) BeforeClass(org.testng.annotations.BeforeClass) Collection(java.util.Collection) SqlStandardAccessControlMetadata(io.prestosql.plugin.hive.security.SqlStandardAccessControlMetadata) ConnectorSplitSource(io.prestosql.spi.connector.ConnectorSplitSource) UUID(java.util.UUID) HiveTestUtils.getDefaultHiveFileWriterFactories(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveFileWriterFactories) UncheckedIOException(java.io.UncheckedIOException) List(java.util.List) ConnectorPageSource(io.prestosql.spi.connector.ConnectorPageSource) Table(io.prestosql.plugin.hive.metastore.Table) HiveTestUtils.getTypes(io.prestosql.plugin.hive.HiveTestUtils.getTypes) Optional(java.util.Optional) AbstractTestHive.getAllSplits(io.prestosql.plugin.hive.AbstractTestHive.getAllSplits) TestingNodeManager(io.prestosql.testing.TestingNodeManager) AbstractTestHive.filterNonHiddenColumnHandles(io.prestosql.plugin.hive.AbstractTestHive.filterNonHiddenColumnHandles) JsonCodec(io.airlift.json.JsonCodec) Database(io.prestosql.plugin.hive.metastore.Database) Slice(io.airlift.slice.Slice) TYPE_MANAGER(io.prestosql.plugin.hive.HiveTestUtils.TYPE_MANAGER) ConnectorSplit(io.prestosql.spi.connector.ConnectorSplit) MetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) HiveTransaction(io.prestosql.plugin.hive.AbstractTestHive.HiveTransaction) ThriftHiveMetastoreConfig(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig) AbstractTestHive.createTableProperties(io.prestosql.plugin.hive.AbstractTestHive.createTableProperties) Assert.assertEquals(org.testng.Assert.assertEquals) CounterStat(io.airlift.stats.CounterStat) AbstractTestHive.filterNonHiddenColumnMetadata(io.prestosql.plugin.hive.AbstractTestHive.filterNonHiddenColumnMetadata) GroupByHashPageIndexerFactory(io.prestosql.GroupByHashPageIndexerFactory) BridgingHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.BridgingHiveMetastore) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) ImmutableList(com.google.common.collect.ImmutableList) TestingMetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.TestingMetastoreLocator) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ThriftHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastore) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) HiveMetastore(io.prestosql.plugin.hive.metastore.HiveMetastore) ExecutorService(java.util.concurrent.ExecutorService) ConnectorOutputTableHandle(io.prestosql.spi.connector.ConnectorOutputTableHandle) AfterClass(org.testng.annotations.AfterClass) HiveTestUtils.getNoOpIndexCache(io.prestosql.plugin.hive.HiveTestUtils.getNoOpIndexCache) HiveIdentity(io.prestosql.plugin.hive.authentication.HiveIdentity) Executor(java.util.concurrent.Executor) ColumnMetadata(io.prestosql.spi.connector.ColumnMetadata) ConnectorTableHandle(io.prestosql.spi.connector.ConnectorTableHandle) HiveTestUtils.getDefaultHiveRecordCursorProvider(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveRecordCursorProvider) ConnectorIdentity(io.prestosql.spi.security.ConnectorIdentity) IOException(java.io.IOException) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) HiveTestUtils.getDefaultHiveDataStreamFactories(io.prestosql.plugin.hive.HiveTestUtils.getDefaultHiveDataStreamFactories) MoreFutures.getFutureValue(io.airlift.concurrent.MoreFutures.getFutureValue) PAGE_SORTER(io.prestosql.plugin.hive.HiveTestUtils.PAGE_SORTER) HostAndPort(com.google.common.net.HostAndPort) MaterializedRow(io.prestosql.testing.MaterializedRow) PrincipalPrivileges(io.prestosql.plugin.hive.metastore.PrincipalPrivileges) ConnectorTableMetadata(io.prestosql.spi.connector.ConnectorTableMetadata) Assertions.assertEqualsIgnoreOrder(io.airlift.testing.Assertions.assertEqualsIgnoreOrder) Transaction(io.prestosql.plugin.hive.AbstractTestHive.Transaction) ColumnHandle(io.prestosql.spi.connector.ColumnHandle) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) Assert.assertTrue(org.testng.Assert.assertTrue) TestingConnectorSession(io.prestosql.testing.TestingConnectorSession) HiveTestUtils.getDefaultOrcFileWriterFactory(io.prestosql.plugin.hive.HiveTestUtils.getDefaultOrcFileWriterFactory) ConnectorPageSourceProvider(io.prestosql.spi.connector.ConnectorPageSourceProvider) TestingMetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.TestingMetastoreLocator) CounterStat(io.airlift.stats.CounterStat) MetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator) TestingMetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.TestingMetastoreLocator) NoHdfsAuthentication(io.prestosql.plugin.hive.authentication.NoHdfsAuthentication) TestingNodeManager(io.prestosql.testing.TestingNodeManager) ThriftHiveMetastoreConfig(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig) BridgingHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.BridgingHiveMetastore) JoinCompiler(io.prestosql.sql.gen.JoinCompiler) ThriftHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastore) SchemaTableName(io.prestosql.spi.connector.SchemaTableName) BoundedExecutor(io.airlift.concurrent.BoundedExecutor) MoreExecutors.newDirectExecutorService(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) ExecutorService(java.util.concurrent.ExecutorService) GroupByHashPageIndexerFactory(io.prestosql.GroupByHashPageIndexerFactory)

Example 4 with ThriftHiveMetastoreConfig

use of io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig in project boostkit-bigdata by kunpengcompute.

the class AbstractTestHive method setup.

protected final void setup(String host, int port, String databaseName, String timeZone) {
    HiveConfig hiveConfig = getHiveConfig().setParquetTimeZone(timeZone).setRcfileTimeZone(timeZone);
    String proxy = System.getProperty("hive.metastore.thrift.client.socks-proxy");
    if (proxy != null) {
        hiveConfig.setMetastoreSocksProxy(HostAndPort.fromString(proxy));
    }
    MetastoreLocator metastoreLocator = new TestingMetastoreLocator(hiveConfig, host, port);
    HiveMetastore metastore = new CachingHiveMetastore(new BridgingHiveMetastore(new ThriftHiveMetastore(metastoreLocator, new ThriftHiveMetastoreConfig())), executor, executorRefresh, Duration.valueOf("1m"), Duration.valueOf("15s"), Duration.valueOf("1m"), Duration.valueOf("15s"), 10000, false);
    setup(databaseName, hiveConfig, metastore);
}
Also used : TestingMetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.TestingMetastoreLocator) CachingHiveMetastore(io.prestosql.plugin.hive.metastore.CachingHiveMetastore) ThriftHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastore) CachingHiveMetastore(io.prestosql.plugin.hive.metastore.CachingHiveMetastore) SemiTransactionalHiveMetastore(io.prestosql.plugin.hive.metastore.SemiTransactionalHiveMetastore) HiveMetastore(io.prestosql.plugin.hive.metastore.HiveMetastore) BridgingHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.BridgingHiveMetastore) ThriftHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastore) MetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator) TestingMetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.TestingMetastoreLocator) ThriftHiveMetastoreConfig(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig) BridgingHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.BridgingHiveMetastore)

Example 5 with ThriftHiveMetastoreConfig

use of io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig in project boostkit-bigdata by kunpengcompute.

the class TestHiveWriterFactory method setUp.

private void setUp() {
    mockClient = new MockThriftMetastoreClient();
    executor = newCachedThreadPool(daemonThreadsNamed("hive-%s"));
    executorRefresh = newCachedThreadPool(daemonThreadsNamed("hive-refresh-%s"));
    MetastoreLocator metastoreLocator = new MockMetastoreLocator(mockClient);
    metastore = new CachingHiveMetastore(new BridgingHiveMetastore(new ThriftHiveMetastore(metastoreLocator, new ThriftHiveMetastoreConfig())), executor, executorRefresh, Duration.valueOf("1m"), Duration.valueOf("15s"), Duration.valueOf("1m"), Duration.valueOf("15s"), 10000, false);
}
Also used : MockThriftMetastoreClient(io.prestosql.plugin.hive.metastore.thrift.MockThriftMetastoreClient) CachingHiveMetastore(io.prestosql.plugin.hive.metastore.CachingHiveMetastore) ThriftHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastore) MetastoreLocator(io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator) ThriftHiveMetastoreConfig(io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig) BridgingHiveMetastore(io.prestosql.plugin.hive.metastore.thrift.BridgingHiveMetastore)

Aggregations

MetastoreLocator (io.prestosql.plugin.hive.metastore.thrift.MetastoreLocator)10 ThriftHiveMetastore (io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastore)10 ThriftHiveMetastoreConfig (io.prestosql.plugin.hive.metastore.thrift.ThriftHiveMetastoreConfig)10 BridgingHiveMetastore (io.prestosql.plugin.hive.metastore.thrift.BridgingHiveMetastore)8 CachingHiveMetastore (io.prestosql.plugin.hive.metastore.CachingHiveMetastore)6 MockThriftMetastoreClient (io.prestosql.plugin.hive.metastore.thrift.MockThriftMetastoreClient)6 HiveMetastore (io.prestosql.plugin.hive.metastore.HiveMetastore)4 TestingMetastoreLocator (io.prestosql.plugin.hive.metastore.thrift.TestingMetastoreLocator)4 BeforeMethod (org.testng.annotations.BeforeMethod)3 Function (com.google.common.base.Function)2 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)2 ImmutableList (com.google.common.collect.ImmutableList)2 ImmutableMap (com.google.common.collect.ImmutableMap)2 ImmutableMultimap (com.google.common.collect.ImmutableMultimap)2 Iterables.getOnlyElement (com.google.common.collect.Iterables.getOnlyElement)2 HostAndPort (com.google.common.net.HostAndPort)2 ListeningExecutorService (com.google.common.util.concurrent.ListeningExecutorService)2 MoreExecutors.newDirectExecutorService (com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService)2 BoundedExecutor (io.airlift.concurrent.BoundedExecutor)2 MoreFutures.getFutureValue (io.airlift.concurrent.MoreFutures.getFutureValue)2