Examples with HdfsConfigurationInitializer io.trino.plugin.hive.HdfsConfigurationInitializer used on opensource projects

Example 6 with HdfsConfigurationInitializer

use of io.trino.plugin.hive.HdfsConfigurationInitializer in project trino by trinodb.

the class TestHiveProjectionPushdownIntoTableScan method createLocalQueryRunner.

@Override
protected LocalQueryRunner createLocalQueryRunner() {
    baseDir = Files.createTempDir();
    HdfsConfig config = new HdfsConfig();
    HdfsConfiguration configuration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(config), ImmutableSet.of());
    HdfsEnvironment environment = new HdfsEnvironment(configuration, config, new NoHdfsAuthentication());
    HiveMetastore metastore = new FileHiveMetastore(new NodeVersion("test_version"), environment, new MetastoreConfig(), new FileHiveMetastoreConfig().setCatalogDirectory(baseDir.toURI().toString()).setMetastoreUser("test"));
    Database database = Database.builder().setDatabaseName(SCHEMA_NAME).setOwnerName(Optional.of("public")).setOwnerType(Optional.of(PrincipalType.ROLE)).build();
    metastore.createDatabase(database);
    LocalQueryRunner queryRunner = LocalQueryRunner.create(HIVE_SESSION);
    queryRunner.createCatalog(HIVE_CATALOG_NAME, new TestingHiveConnectorFactory(metastore), ImmutableMap.of());
    return queryRunner;
}

Example 7 with HdfsConfigurationInitializer

use of io.trino.plugin.hive.HdfsConfigurationInitializer in project trino by trinodb.

the class TestRubixCaching method getNonCachingFileSystem.

private FileSystem getNonCachingFileSystem() throws IOException {
    HdfsConfigurationInitializer configurationInitializer = new HdfsConfigurationInitializer(config);
    HiveHdfsConfiguration configuration = new HiveHdfsConfiguration(configurationInitializer, ImmutableSet.of());
    HdfsEnvironment environment = new HdfsEnvironment(configuration, config, new NoHdfsAuthentication());
    return environment.getFileSystem(context, cacheStoragePath);
}

Example 8 with HdfsConfigurationInitializer

use of io.trino.plugin.hive.HdfsConfigurationInitializer in project trino by trinodb.

the class TestRubixCaching method testCoordinatorNotJoining.

@Test
public void testCoordinatorNotJoining() {
    RubixConfig rubixConfig = new RubixConfig().setCacheLocation("/tmp/not/existing/dir");
    HdfsConfigurationInitializer configurationInitializer = new HdfsConfigurationInitializer(config, ImmutableSet.of());
    InternalNode workerNode = new InternalNode("worker", URI.create("http://127.0.0.2:8080"), UNKNOWN, false);
    RubixInitializer rubixInitializer = new RubixInitializer(retry().maxAttempts(1), rubixConfig.setStartServerOnCoordinator(true), new TestingNodeManager(ImmutableList.of(workerNode)), new CatalogName("catalog"), configurationInitializer, new DefaultRubixHdfsInitializer(new HdfsAuthenticationConfig()));
    assertThatThrownBy(rubixInitializer::initializeRubix).hasMessage("No coordinator node available");
}

Example 9 with HdfsConfigurationInitializer

use of io.trino.plugin.hive.HdfsConfigurationInitializer in project trino by trinodb.

the class TestRubixCaching method initializeRubix.

private void initializeRubix(RubixConfig rubixConfig, List<Node> nodes) throws Exception {
    tempDirectory = createTempDirectory(getClass().getSimpleName());
    // create cache directories
    List<java.nio.file.Path> cacheDirectories = ImmutableList.of(tempDirectory.resolve("cache1"), tempDirectory.resolve("cache2"));
    for (java.nio.file.Path directory : cacheDirectories) {
        createDirectories(directory);
    }
    // initialize rubix in master-only mode
    rubixConfig.setStartServerOnCoordinator(true);
    rubixConfig.setCacheLocation(Joiner.on(",").join(cacheDirectories.stream().map(java.nio.file.Path::toString).collect(toImmutableList())));
    HdfsConfigurationInitializer configurationInitializer = new HdfsConfigurationInitializer(config, ImmutableSet.of(// fetch data immediately in async mode
    config -> setRemoteFetchProcessInterval(config, 0)));
    TestingNodeManager nodeManager = new TestingNodeManager(nodes);
    rubixInitializer = new RubixInitializer(rubixConfig, nodeManager, new CatalogName("catalog"), configurationInitializer, new DefaultRubixHdfsInitializer(new HdfsAuthenticationConfig()));
    rubixConfigInitializer = new RubixConfigurationInitializer(rubixInitializer);
    rubixInitializer.initializeRubix();
    retry().run("wait for rubix to startup", () -> {
        if (!rubixInitializer.isServerUp()) {
            throw new IllegalStateException("Rubix server has not started");
        }
        return null;
    });
}

Example 10 with HdfsConfigurationInitializer

use of io.trino.plugin.hive.HdfsConfigurationInitializer in project trino by trinodb.

the class TestIcebergSplitSource method createQueryRunner.

@Override
protected QueryRunner createQueryRunner() throws Exception {
    HdfsConfig config = new HdfsConfig();
    HdfsConfiguration configuration = new HiveHdfsConfiguration(new HdfsConfigurationInitializer(config), ImmutableSet.of());
    HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(configuration, config, new NoHdfsAuthentication());
    File tempDir = Files.createTempDirectory("test_iceberg_split_source").toFile();
    this.metastoreDir = new File(tempDir, "iceberg_data");
    HiveMetastore metastore = createTestingFileHiveMetastore(metastoreDir);
    IcebergTableOperationsProvider operationsProvider = new FileMetastoreTableOperationsProvider(new HdfsFileIoProvider(hdfsEnvironment));
    this.catalog = new TrinoHiveCatalog(new CatalogName("hive"), memoizeMetastore(metastore, 1000), hdfsEnvironment, new TestingTypeManager(), operationsProvider, "test", false, false, false);
    return createIcebergQueryRunner(ImmutableMap.of(), ImmutableMap.of(), ImmutableList.of(NATION), Optional.of(metastoreDir));
}