use of io.trino.plugin.hive.metastore.HiveMetastoreFactory in project trino by trinodb.
the class TestDeltaLakePerTransactionMetastoreCache method createQueryRunner.
private DistributedQueryRunner createQueryRunner(boolean enablePerTransactionHiveMetastoreCaching) throws Exception {
boolean createdDeltaLake = false;
if (dockerizedMinioDataLake == null) {
// share environment between testcases to speed things up
dockerizedMinioDataLake = createDockerizedMinioDataLakeForDeltaLake(BUCKET_NAME);
createdDeltaLake = true;
}
Session session = testSessionBuilder().setCatalog(DELTA_CATALOG).setSchema("default").build();
DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(session).build();
queryRunner.installPlugin(new Plugin() {
@Override
public Iterable<ConnectorFactory> getConnectorFactories() {
return ImmutableList.of(new ConnectorFactory() {
@Override
public String getName() {
return TEST_DELTA_CONNECTOR_NAME;
}
@Override
public Connector create(String catalogName, Map<String, String> config, ConnectorContext context) {
return InternalDeltaLakeConnectorFactory.createConnector(catalogName, config, context, new AbstractConfigurationAwareModule() {
@Override
protected void setup(Binder binder) {
newOptionalBinder(binder, ThriftMetastoreClientFactory.class).setDefault().to(DefaultThriftMetastoreClientFactory.class).in(Scopes.SINGLETON);
binder.bind(MetastoreLocator.class).to(StaticMetastoreLocator.class).in(Scopes.SINGLETON);
configBinder(binder).bindConfig(StaticMetastoreConfig.class);
configBinder(binder).bindConfig(ThriftMetastoreConfig.class);
binder.bind(ThriftMetastore.class).to(ThriftHiveMetastore.class).in(Scopes.SINGLETON);
newExporter(binder).export(ThriftMetastore.class).as((generator) -> generator.generatedNameOf(ThriftHiveMetastore.class));
install(new ThriftMetastoreAuthenticationModule());
binder.bind(Boolean.class).annotatedWith(HideNonDeltaLakeTables.class).toInstance(false);
binder.bind(BridgingHiveMetastoreFactory.class).in(Scopes.SINGLETON);
}
@Provides
@Singleton
@RawHiveMetastoreFactory
public HiveMetastoreFactory getCountingHiveMetastoreFactory(BridgingHiveMetastoreFactory bridgingHiveMetastoreFactory) {
return new HiveMetastoreFactory() {
@Override
public boolean isImpersonationEnabled() {
return false;
}
@Override
public HiveMetastore createMetastore(Optional<ConnectorIdentity> identity) {
HiveMetastore bridgingHiveMetastore = bridgingHiveMetastoreFactory.createMetastore(identity);
// bind HiveMetastore which counts method executions
return Reflection.newProxy(HiveMetastore.class, (proxy, method, args) -> {
String methodName = method.getName();
long count = hiveMetastoreInvocationCounts.getOrDefault(methodName, 0L);
hiveMetastoreInvocationCounts.put(methodName, count + 1);
return method.invoke(bridgingHiveMetastore, args);
});
}
};
}
});
}
});
}
});
ImmutableMap.Builder<String, String> deltaLakeProperties = ImmutableMap.builder();
deltaLakeProperties.put("hive.metastore.uri", dockerizedMinioDataLake.getTestingHadoop().getMetastoreAddress());
deltaLakeProperties.put("hive.s3.aws-access-key", MINIO_ACCESS_KEY);
deltaLakeProperties.put("hive.s3.aws-secret-key", MINIO_SECRET_KEY);
deltaLakeProperties.put("hive.s3.endpoint", dockerizedMinioDataLake.getMinioAddress());
deltaLakeProperties.put("hive.s3.path-style-access", "true");
// use test value so we do not get clash with default bindings)
deltaLakeProperties.put("hive.metastore", "test");
if (!enablePerTransactionHiveMetastoreCaching) {
// almost disable the cache; 0 is not allowed as config property value
deltaLakeProperties.put("hive.per-transaction-metastore-cache-maximum-size", "1");
}
queryRunner.createCatalog(DELTA_CATALOG, TEST_DELTA_CONNECTOR_NAME, deltaLakeProperties.buildOrThrow());
if (createdDeltaLake) {
List<TpchTable<? extends TpchEntity>> tpchTables = List.of(TpchTable.NATION, TpchTable.REGION);
tpchTables.forEach(table -> {
String tableName = table.getTableName();
dockerizedMinioDataLake.copyResources("io/trino/plugin/deltalake/testing/resources/databricks/" + tableName, tableName);
queryRunner.execute(format("CREATE TABLE %s.%s.%s (dummy int) WITH (location = 's3://%s/%3$s')", DELTA_CATALOG, "default", tableName, BUCKET_NAME));
});
}
return queryRunner;
}
use of io.trino.plugin.hive.metastore.HiveMetastoreFactory in project trino by trinodb.
the class TestDeltaLakeMetadata method setUp.
@BeforeClass
public void setUp() throws IOException {
temporaryCatalogDirectory = createTempDirectory("HiveCatalog").toFile();
Map<String, String> config = ImmutableMap.<String, String>builder().put("hive.metastore", "file").put("hive.metastore.catalog.dir", temporaryCatalogDirectory.getPath()).buildOrThrow();
Bootstrap app = new Bootstrap(// connector dependencies
new JsonModule(), binder -> {
ConnectorContext context = new TestingConnectorContext();
binder.bind(NodeVersion.class).toInstance(new NodeVersion(context.getNodeManager().getCurrentNode().getVersion()));
binder.bind(CatalogName.class).toInstance(new CatalogName("test"));
binder.bind(TypeManager.class).toInstance(context.getTypeManager());
binder.bind(NodeManager.class).toInstance(context.getNodeManager());
binder.bind(PageIndexerFactory.class).toInstance(context.getPageIndexerFactory());
}, // connector modules
new DeltaLakeMetastoreModule(), new DeltaLakeModule(), // test setup
binder -> {
binder.bind(HdfsEnvironment.class).toInstance(HDFS_ENVIRONMENT);
}, new AbstractModule() {
@Provides
public DeltaLakeMetastore getDeltaLakeMetastore(@RawHiveMetastoreFactory HiveMetastoreFactory hiveMetastoreFactory, TransactionLogAccess transactionLogAccess, TypeManager typeManager, CachingDeltaLakeStatisticsAccess statistics) {
return new HiveMetastoreBackedDeltaLakeMetastore(hiveMetastoreFactory.createMetastore(Optional.empty()), transactionLogAccess, typeManager, statistics);
}
});
Injector injector = app.doNotInitializeLogging().setRequiredConfigurationProperties(config).initialize();
deltaLakeMetadataFactory = injector.getInstance(DeltaLakeMetadataFactory.class);
injector.getInstance(DeltaLakeMetastore.class).createDatabase(Database.builder().setDatabaseName(DATABASE_NAME).setOwnerName(Optional.of("test")).setOwnerType(Optional.of(USER)).setLocation(Optional.empty()).build());
}
Aggregations