Examples with HetuFileSystemClientPlugin io.hetu.core.filesystem.HetuFileSystemClientPlugin used on opensource projects

Example 1 with HetuFileSystemClientPlugin

use of io.hetu.core.filesystem.HetuFileSystemClientPlugin in project hetu-core by openlookeng.

the class TestQuerySpillLimits method createLocalQueryRunner.

private LocalQueryRunner createLocalQueryRunner(NodeSpillConfig nodeSpillConfig) {
    LocalQueryRunner queryRunner = null;
    try {
        String canonicalPath = spillPath.getCanonicalPath();
        queryRunner = new LocalQueryRunner(SESSION, new FeaturesConfig().setSpillerSpillPaths(canonicalPath).setSpillEnabled(true), nodeSpillConfig, false, true);
    } catch (IOException e) {
        throw new IllegalArgumentException(e);
    }
    queryRunner.createCatalog(SESSION.getCatalog().get(), new TpchConnectorFactory(1), ImmutableMap.of());
    queryRunner.installPlugin(new HetuFileSystemClientPlugin());
    return queryRunner;
}

Example 2 with HetuFileSystemClientPlugin

use of io.hetu.core.filesystem.HetuFileSystemClientPlugin in project hetu-core by openlookeng.

the class TestDistributedSpilledQueries method createQueryRunner.

public static DistributedQueryRunner createQueryRunner() throws Exception {
    Session defaultSession = testSessionBuilder().setCatalog("tpch").setSchema(TINY_SCHEMA_NAME).setSystemProperty(SystemSessionProperties.TASK_CONCURRENCY, "2").setSystemProperty(SystemSessionProperties.SPILL_ENABLED, "true").setSystemProperty(SystemSessionProperties.SPILL_ORDER_BY, "true").setSystemProperty(SystemSessionProperties.AGGREGATION_OPERATOR_UNSPILL_MEMORY_LIMIT, "128kB").build();
    Map<String, String> extraProperties = new HashMap<String, String>();
    extraProperties.put("experimental.spiller-spill-path", Paths.get(System.getProperty("java.io.tmpdir"), "presto", "spills").toString());
    extraProperties.put("experimental.spiller-max-used-space-threshold", "1.0");
    // revoke always
    extraProperties.put("experimental.memory-revoking-threshold", "0.0");
    extraProperties.put("experimental.memory-revoking-target", "0.0");
    extraProperties.put("experimental.spiller-spill-to-hdfs", "false");
    DistributedQueryRunner queryRunner = new DistributedQueryRunner(defaultSession, 2, extraProperties);
    try {
        queryRunner.installPlugin(new TpchPlugin());
        queryRunner.installPlugin(new HetuFileSystemClientPlugin());
        queryRunner.createCatalog("tpch", "tpch");
        return queryRunner;
    } catch (Exception e) {
        queryRunner.close();
        throw e;
    }
}

Example 3 with HetuFileSystemClientPlugin

use of io.hetu.core.filesystem.HetuFileSystemClientPlugin in project hetu-core by openlookeng.

the class HindexQueryRunner method createQueryRunner.

public static DistributedQueryRunner createQueryRunner(Map<String, String> extraProperties, Map<String, String> metastoreProperties, Map<String, String> coordinatorProperties) throws Exception {
    Session session = testSessionBuilder().setSource("test").setCatalog("hive").setSchema("test").build();
    DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(session).setNodeCount(1).setExtraProperties(extraProperties).setCoordinatorProperties(coordinatorProperties).build();
    try {
        File tempDir = Files.createTempDirectory("test-hive").toFile();
        File hiveDir = new File(tempDir, "hive_data");
        HiveMetastore metastore = createTestingFileHiveMetastore(hiveDir);
        HiveIdentity identity = new HiveIdentity(SESSION);
        metastore.createDatabase(identity, Database.builder().setDatabaseName("test").setOwnerName("public").setOwnerType(PrincipalType.ROLE).build());
        queryRunner.installPlugin(new HetuFileSystemClientPlugin());
        queryRunner.installPlugin(new HetuMetastorePlugin());
        queryRunner.installPlugin(new HiveHadoop2Plugin());
        queryRunner.installPlugin(new HeuristicIndexPlugin());
        queryRunner.installPlugin(new HivePlugin("Hive", Optional.of(metastore)));
        queryRunner.getServers().forEach(server -> {
            try {
                server.loadMetastore(metastoreProperties);
                server.getHeuristicIndexerManager().buildIndexClient();
                server.getHeuristicIndexerManager().initCache();
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        Map<String, String> hiveProperties = ImmutableMap.<String, String>builder().put("hive.allow-drop-table", "true").build();
        queryRunner.createCatalog("hive", "Hive", hiveProperties);
        return queryRunner;
    } catch (Exception e) {
        queryRunner.close();
        throw e;
    }
}

Example 4 with HetuFileSystemClientPlugin

use of io.hetu.core.filesystem.HetuFileSystemClientPlugin in project hetu-core by openlookeng.

the class TestSpatialJoinPlanning method createQueryRunner.

private static LocalQueryRunner createQueryRunner() throws IOException {
    LocalQueryRunner queryRunner = new LocalQueryRunner(testSessionBuilder().setCatalog("memory").setSchema("default").build());
    queryRunner.installPlugin(new HetuFileSystemClientPlugin());
    queryRunner.installPlugin(new HetuMetastorePlugin());
    queryRunner.installPlugin(new GeoPlugin());
    queryRunner.createCatalog("tpch", new TpchConnectorFactory(1), ImmutableMap.of());
    TempFolder folder = new TempFolder().create();
    Runtime.getRuntime().addShutdownHook(new Thread(folder::close));
    HashMap<String, String> metastoreConfig = new HashMap<>();
    metastoreConfig.put("hetu.metastore.type", "hetufilesystem");
    metastoreConfig.put("hetu.metastore.hetufilesystem.profile-name", "default");
    metastoreConfig.put("hetu.metastore.hetufilesystem.path", folder.newFolder("metastore").getAbsolutePath());
    metastoreConfig.put("hetu.metastore.cache.type", "local");
    queryRunner.loadMetastore(metastoreConfig);
    queryRunner.createCatalog("memory", new MemoryConnectorFactory(), ImmutableMap.of("memory.spill-path", folder.newFolder("memory-connector").getAbsolutePath()));
    queryRunner.execute(format("CREATE TABLE kdb_tree AS SELECT '%s' AS v", KDB_TREE_JSON));
    queryRunner.execute("CREATE TABLE points (lng, lat, name) AS (VALUES (2.1e0, 2.1e0, 'x'))");
    queryRunner.execute("CREATE TABLE polygons (wkt, name) AS (VALUES ('POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))', 'a'))");
    return queryRunner;
}

Example 5 with HetuFileSystemClientPlugin

use of io.hetu.core.filesystem.HetuFileSystemClientPlugin in project hetu-core by openlookeng.

the class TestHetuConnection method setupServer.

@BeforeClass
public void setupServer() throws Exception {
    Logging.initialize();
    TempFolder folder = new TempFolder().create();
    Runtime.getRuntime().addShutdownHook(new Thread(folder::close));
    HashMap<String, String> metastoreConfig = new HashMap<>();
    metastoreConfig.put("hetu.metastore.type", "hetufilesystem");
    metastoreConfig.put("hetu.metastore.hetufilesystem.profile-name", "default");
    metastoreConfig.put("hetu.metastore.hetufilesystem.path", folder.newFolder("metastore").getAbsolutePath());
    server = new TestingPrestoServer();
    server.installPlugin(new HetuFileSystemClientPlugin());
    server.installPlugin(new HetuMetastorePlugin());
    server.installPlugin(new MemoryPlugin());
    server.loadMetastore(metastoreConfig);
    server.createCatalog("memory", "memory", ImmutableMap.of("memory.spill-path", folder.newFolder("memory-connector").getAbsolutePath()));
    try (Connection connection = createConnection();
        Statement statement = connection.createStatement()) {
        statement.execute("CREATE SCHEMA testschema");
    }
}