Search in sources :

Example 1 with HiveMetastoreFactory

use of io.trino.plugin.hive.metastore.HiveMetastoreFactory in project trino by trinodb.

the class TestDeltaLakePerTransactionMetastoreCache method createQueryRunner.

private DistributedQueryRunner createQueryRunner(boolean enablePerTransactionHiveMetastoreCaching) throws Exception {
    boolean createdDeltaLake = false;
    if (dockerizedMinioDataLake == null) {
        // share environment between testcases to speed things up
        dockerizedMinioDataLake = createDockerizedMinioDataLakeForDeltaLake(BUCKET_NAME);
        createdDeltaLake = true;
    }
    Session session = testSessionBuilder().setCatalog(DELTA_CATALOG).setSchema("default").build();
    DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(session).build();
    queryRunner.installPlugin(new Plugin() {

        @Override
        public Iterable<ConnectorFactory> getConnectorFactories() {
            return ImmutableList.of(new ConnectorFactory() {

                @Override
                public String getName() {
                    return TEST_DELTA_CONNECTOR_NAME;
                }

                @Override
                public Connector create(String catalogName, Map<String, String> config, ConnectorContext context) {
                    return InternalDeltaLakeConnectorFactory.createConnector(catalogName, config, context, new AbstractConfigurationAwareModule() {

                        @Override
                        protected void setup(Binder binder) {
                            newOptionalBinder(binder, ThriftMetastoreClientFactory.class).setDefault().to(DefaultThriftMetastoreClientFactory.class).in(Scopes.SINGLETON);
                            binder.bind(MetastoreLocator.class).to(StaticMetastoreLocator.class).in(Scopes.SINGLETON);
                            configBinder(binder).bindConfig(StaticMetastoreConfig.class);
                            configBinder(binder).bindConfig(ThriftMetastoreConfig.class);
                            binder.bind(ThriftMetastore.class).to(ThriftHiveMetastore.class).in(Scopes.SINGLETON);
                            newExporter(binder).export(ThriftMetastore.class).as((generator) -> generator.generatedNameOf(ThriftHiveMetastore.class));
                            install(new ThriftMetastoreAuthenticationModule());
                            binder.bind(Boolean.class).annotatedWith(HideNonDeltaLakeTables.class).toInstance(false);
                            binder.bind(BridgingHiveMetastoreFactory.class).in(Scopes.SINGLETON);
                        }

                        @Provides
                        @Singleton
                        @RawHiveMetastoreFactory
                        public HiveMetastoreFactory getCountingHiveMetastoreFactory(BridgingHiveMetastoreFactory bridgingHiveMetastoreFactory) {
                            return new HiveMetastoreFactory() {

                                @Override
                                public boolean isImpersonationEnabled() {
                                    return false;
                                }

                                @Override
                                public HiveMetastore createMetastore(Optional<ConnectorIdentity> identity) {
                                    HiveMetastore bridgingHiveMetastore = bridgingHiveMetastoreFactory.createMetastore(identity);
                                    // bind HiveMetastore which counts method executions
                                    return Reflection.newProxy(HiveMetastore.class, (proxy, method, args) -> {
                                        String methodName = method.getName();
                                        long count = hiveMetastoreInvocationCounts.getOrDefault(methodName, 0L);
                                        hiveMetastoreInvocationCounts.put(methodName, count + 1);
                                        return method.invoke(bridgingHiveMetastore, args);
                                    });
                                }
                            };
                        }
                    });
                }
            });
        }
    });
    ImmutableMap.Builder<String, String> deltaLakeProperties = ImmutableMap.builder();
    deltaLakeProperties.put("hive.metastore.uri", dockerizedMinioDataLake.getTestingHadoop().getMetastoreAddress());
    deltaLakeProperties.put("hive.s3.aws-access-key", MINIO_ACCESS_KEY);
    deltaLakeProperties.put("hive.s3.aws-secret-key", MINIO_SECRET_KEY);
    deltaLakeProperties.put("hive.s3.endpoint", dockerizedMinioDataLake.getMinioAddress());
    deltaLakeProperties.put("hive.s3.path-style-access", "true");
    // use test value so we do not get clash with default bindings)
    deltaLakeProperties.put("hive.metastore", "test");
    if (!enablePerTransactionHiveMetastoreCaching) {
        // almost disable the cache; 0 is not allowed as config property value
        deltaLakeProperties.put("hive.per-transaction-metastore-cache-maximum-size", "1");
    }
    queryRunner.createCatalog(DELTA_CATALOG, TEST_DELTA_CONNECTOR_NAME, deltaLakeProperties.buildOrThrow());
    if (createdDeltaLake) {
        List<TpchTable<? extends TpchEntity>> tpchTables = List.of(TpchTable.NATION, TpchTable.REGION);
        tpchTables.forEach(table -> {
            String tableName = table.getTableName();
            dockerizedMinioDataLake.copyResources("io/trino/plugin/deltalake/testing/resources/databricks/" + tableName, tableName);
            queryRunner.execute(format("CREATE TABLE %s.%s.%s (dummy int) WITH (location = 's3://%s/%3$s')", DELTA_CATALOG, "default", tableName, BUCKET_NAME));
        });
    }
    return queryRunner;
}
Also used : BridgingHiveMetastoreFactory(io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastoreFactory) DELTA_CATALOG(io.trino.plugin.deltalake.DeltaLakeQueryRunner.DELTA_CATALOG) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) MINIO_SECRET_KEY(io.trino.plugin.deltalake.util.MinioContainer.MINIO_SECRET_KEY) Test(org.testng.annotations.Test) ConnectorContext(io.trino.spi.connector.ConnectorContext) Singleton(javax.inject.Singleton) StaticMetastoreConfig(io.trino.plugin.hive.metastore.thrift.StaticMetastoreConfig) MINIO_ACCESS_KEY(io.trino.plugin.deltalake.util.MinioContainer.MINIO_ACCESS_KEY) ThriftMetastoreClientFactory(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreClientFactory) HiveMetastoreFactory(io.trino.plugin.hive.metastore.HiveMetastoreFactory) AbstractConfigurationAwareModule(io.airlift.configuration.AbstractConfigurationAwareModule) HiveMetastore(io.trino.plugin.hive.metastore.HiveMetastore) DistributedQueryRunner(io.trino.testing.DistributedQueryRunner) ImmutableList(com.google.common.collect.ImmutableList) Binder(com.google.inject.Binder) DockerizedMinioDataLake(io.trino.plugin.deltalake.util.DockerizedMinioDataLake) Map(java.util.Map) ExportBinder.newExporter(org.weakref.jmx.guice.ExportBinder.newExporter) DefaultThriftMetastoreClientFactory(io.trino.plugin.hive.metastore.thrift.DefaultThriftMetastoreClientFactory) ConnectorFactory(io.trino.spi.connector.ConnectorFactory) OptionalBinder.newOptionalBinder(com.google.inject.multibindings.OptionalBinder.newOptionalBinder) TpchEntity(io.trino.tpch.TpchEntity) TpchTable(io.trino.tpch.TpchTable) StaticMetastoreLocator(io.trino.plugin.hive.metastore.thrift.StaticMetastoreLocator) AfterClass(org.testng.annotations.AfterClass) ConnectorIdentity(io.trino.spi.security.ConnectorIdentity) ImmutableMap(com.google.common.collect.ImmutableMap) ThriftHiveMetastore(io.trino.plugin.hive.metastore.thrift.ThriftHiveMetastore) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Reflection(com.google.common.reflect.Reflection) ThriftMetastoreAuthenticationModule(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreAuthenticationModule) Scopes(com.google.inject.Scopes) RawHiveMetastoreFactory(io.trino.plugin.hive.metastore.RawHiveMetastoreFactory) String.format(java.lang.String.format) Plugin(io.trino.spi.Plugin) List(java.util.List) TestingSession.testSessionBuilder(io.trino.testing.TestingSession.testSessionBuilder) Provides(com.google.inject.Provides) DeltaLakeDockerizedMinioDataLake.createDockerizedMinioDataLakeForDeltaLake(io.trino.plugin.deltalake.DeltaLakeDockerizedMinioDataLake.createDockerizedMinioDataLakeForDeltaLake) MetastoreLocator(io.trino.plugin.hive.metastore.thrift.MetastoreLocator) ThriftMetastore(io.trino.plugin.hive.metastore.thrift.ThriftMetastore) ThriftMetastoreConfig(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreConfig) Optional(java.util.Optional) ConfigBinder.configBinder(io.airlift.configuration.ConfigBinder.configBinder) Session(io.trino.Session) Connector(io.trino.spi.connector.Connector) DistributedQueryRunner(io.trino.testing.DistributedQueryRunner) RawHiveMetastoreFactory(io.trino.plugin.hive.metastore.RawHiveMetastoreFactory) StaticMetastoreLocator(io.trino.plugin.hive.metastore.thrift.StaticMetastoreLocator) MetastoreLocator(io.trino.plugin.hive.metastore.thrift.MetastoreLocator) TpchTable(io.trino.tpch.TpchTable) ThriftMetastoreConfig(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreConfig) Binder(com.google.inject.Binder) OptionalBinder.newOptionalBinder(com.google.inject.multibindings.OptionalBinder.newOptionalBinder) ConfigBinder.configBinder(io.airlift.configuration.ConfigBinder.configBinder) ConnectorFactory(io.trino.spi.connector.ConnectorFactory) ConnectorContext(io.trino.spi.connector.ConnectorContext) AbstractConfigurationAwareModule(io.airlift.configuration.AbstractConfigurationAwareModule) ThriftMetastoreAuthenticationModule(io.trino.plugin.hive.metastore.thrift.ThriftMetastoreAuthenticationModule) BridgingHiveMetastoreFactory(io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastoreFactory) HiveMetastoreFactory(io.trino.plugin.hive.metastore.HiveMetastoreFactory) RawHiveMetastoreFactory(io.trino.plugin.hive.metastore.RawHiveMetastoreFactory) HiveMetastore(io.trino.plugin.hive.metastore.HiveMetastore) ThriftHiveMetastore(io.trino.plugin.hive.metastore.thrift.ThriftHiveMetastore) ThriftMetastore(io.trino.plugin.hive.metastore.thrift.ThriftMetastore) ConnectorIdentity(io.trino.spi.security.ConnectorIdentity) Provides(com.google.inject.Provides) TpchEntity(io.trino.tpch.TpchEntity) ImmutableMap(com.google.common.collect.ImmutableMap) BridgingHiveMetastoreFactory(io.trino.plugin.hive.metastore.thrift.BridgingHiveMetastoreFactory) Singleton(javax.inject.Singleton) StaticMetastoreConfig(io.trino.plugin.hive.metastore.thrift.StaticMetastoreConfig) Map(java.util.Map) ImmutableMap(com.google.common.collect.ImmutableMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Session(io.trino.Session) Plugin(io.trino.spi.Plugin)

Example 2 with HiveMetastoreFactory

use of io.trino.plugin.hive.metastore.HiveMetastoreFactory in project trino by trinodb.

the class TestDeltaLakeMetadata method setUp.

@BeforeClass
public void setUp() throws IOException {
    temporaryCatalogDirectory = createTempDirectory("HiveCatalog").toFile();
    Map<String, String> config = ImmutableMap.<String, String>builder().put("hive.metastore", "file").put("hive.metastore.catalog.dir", temporaryCatalogDirectory.getPath()).buildOrThrow();
    Bootstrap app = new Bootstrap(// connector dependencies
    new JsonModule(), binder -> {
        ConnectorContext context = new TestingConnectorContext();
        binder.bind(NodeVersion.class).toInstance(new NodeVersion(context.getNodeManager().getCurrentNode().getVersion()));
        binder.bind(CatalogName.class).toInstance(new CatalogName("test"));
        binder.bind(TypeManager.class).toInstance(context.getTypeManager());
        binder.bind(NodeManager.class).toInstance(context.getNodeManager());
        binder.bind(PageIndexerFactory.class).toInstance(context.getPageIndexerFactory());
    }, // connector modules
    new DeltaLakeMetastoreModule(), new DeltaLakeModule(), // test setup
    binder -> {
        binder.bind(HdfsEnvironment.class).toInstance(HDFS_ENVIRONMENT);
    }, new AbstractModule() {

        @Provides
        public DeltaLakeMetastore getDeltaLakeMetastore(@RawHiveMetastoreFactory HiveMetastoreFactory hiveMetastoreFactory, TransactionLogAccess transactionLogAccess, TypeManager typeManager, CachingDeltaLakeStatisticsAccess statistics) {
            return new HiveMetastoreBackedDeltaLakeMetastore(hiveMetastoreFactory.createMetastore(Optional.empty()), transactionLogAccess, typeManager, statistics);
        }
    });
    Injector injector = app.doNotInitializeLogging().setRequiredConfigurationProperties(config).initialize();
    deltaLakeMetadataFactory = injector.getInstance(DeltaLakeMetadataFactory.class);
    injector.getInstance(DeltaLakeMetastore.class).createDatabase(Database.builder().setDatabaseName(DATABASE_NAME).setOwnerName(Optional.of("test")).setOwnerType(Optional.of(USER)).setLocation(Optional.empty()).build());
}
Also used : HiveMetastoreFactory(io.trino.plugin.hive.metastore.HiveMetastoreFactory) RawHiveMetastoreFactory(io.trino.plugin.hive.metastore.RawHiveMetastoreFactory) TransactionLogAccess(io.trino.plugin.deltalake.transactionlog.TransactionLogAccess) TestingConnectorContext(io.trino.testing.TestingConnectorContext) Provides(com.google.inject.Provides) DeltaLakeMetastore(io.trino.plugin.deltalake.metastore.DeltaLakeMetastore) HiveMetastoreBackedDeltaLakeMetastore(io.trino.plugin.deltalake.metastore.HiveMetastoreBackedDeltaLakeMetastore) JsonModule(io.airlift.json.JsonModule) PageIndexerFactory(io.trino.spi.PageIndexerFactory) HdfsEnvironment(io.trino.plugin.hive.HdfsEnvironment) AbstractModule(com.google.inject.AbstractModule) NodeVersion(io.trino.plugin.hive.NodeVersion) NodeManager(io.trino.spi.NodeManager) HiveMetastoreBackedDeltaLakeMetastore(io.trino.plugin.deltalake.metastore.HiveMetastoreBackedDeltaLakeMetastore) Injector(com.google.inject.Injector) ConnectorContext(io.trino.spi.connector.ConnectorContext) TestingConnectorContext(io.trino.testing.TestingConnectorContext) DeltaLakeMetastoreModule(io.trino.plugin.deltalake.metastore.DeltaLakeMetastoreModule) Bootstrap(io.airlift.bootstrap.Bootstrap) TypeManager(io.trino.spi.type.TypeManager) CatalogName(io.trino.plugin.base.CatalogName) CachingDeltaLakeStatisticsAccess(io.trino.plugin.deltalake.statistics.CachingDeltaLakeStatisticsAccess) BeforeClass(org.testng.annotations.BeforeClass)

Aggregations

Provides (com.google.inject.Provides)2 HiveMetastoreFactory (io.trino.plugin.hive.metastore.HiveMetastoreFactory)2 RawHiveMetastoreFactory (io.trino.plugin.hive.metastore.RawHiveMetastoreFactory)2 ConnectorContext (io.trino.spi.connector.ConnectorContext)2 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 Reflection (com.google.common.reflect.Reflection)1 AbstractModule (com.google.inject.AbstractModule)1 Binder (com.google.inject.Binder)1 Injector (com.google.inject.Injector)1 Scopes (com.google.inject.Scopes)1 OptionalBinder.newOptionalBinder (com.google.inject.multibindings.OptionalBinder.newOptionalBinder)1 Bootstrap (io.airlift.bootstrap.Bootstrap)1 AbstractConfigurationAwareModule (io.airlift.configuration.AbstractConfigurationAwareModule)1 ConfigBinder.configBinder (io.airlift.configuration.ConfigBinder.configBinder)1 JsonModule (io.airlift.json.JsonModule)1 Session (io.trino.Session)1 CatalogName (io.trino.plugin.base.CatalogName)1 DeltaLakeDockerizedMinioDataLake.createDockerizedMinioDataLakeForDeltaLake (io.trino.plugin.deltalake.DeltaLakeDockerizedMinioDataLake.createDockerizedMinioDataLakeForDeltaLake)1 DELTA_CATALOG (io.trino.plugin.deltalake.DeltaLakeQueryRunner.DELTA_CATALOG)1