Examples with HetuMetastorePlugin io.hetu.core.metastore.HetuMetastorePlugin used on opensource projects

Example 6 with HetuMetastorePlugin

use of io.hetu.core.metastore.HetuMetastorePlugin in project hetu-core by openlookeng.

the class TestDynamicHiveScalarFunction method setUpClass.

@BeforeClass
public void setUpClass() throws Exception {
    RecognizedFunctions.addRecognizedFunction("io.hetu.core.hive.dynamicfunctions.examples.udf.ListStringUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.TimeOutUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.BooleanUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.MapDoubleUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.DoubleUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.EvaluateOverloadUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.ShortUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.ArrayListDoubleUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.FloatWrapperUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.LongUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.IntThreeArgsUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.IntUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.IntTwoArgsUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.IntFiveArgsUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.MapIntUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.MapStringUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.EmptyParameterUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.IntWrapperUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.IntSixArgsUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.IntFourArgsUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.BooleanWrappperUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.ListIntUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.LongWrapperUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.ByteWrapperUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.ListDoubleUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.ShortWrapperUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.ByteUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.DoubleWrapperUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.FloatUDF", "io.hetu.core.hive.dynamicfunctions.examples.udf.NullInputUDF");
    try {
        queryRunner = DistributedQueryRunner.builder(testSessionBuilder().setCatalog("memory").setSchema("default").build()).setNodeCount(1).build();
        TempFolder folder = new TempFolder();
        folder.create();
        Runtime.getRuntime().addShutdownHook(new Thread(folder::close));
        // need dep on hetu-filesystem
        queryRunner.installPlugin(new HetuFileSystemClientPlugin());
        // need dep on hetu-metastore
        queryRunner.installPlugin(new HetuMetastorePlugin());
        queryRunner.installPlugin(new MemoryPlugin());
        HashMap<String, String> metastoreConfig = new HashMap<>();
        metastoreConfig.put("hetu.metastore.type", "hetufilesystem");
        metastoreConfig.put("hetu.metastore.hetufilesystem.profile-name", "default");
        metastoreConfig.put("hetu.metastore.hetufilesystem.path", folder.newFolder("metastore").getAbsolutePath());
        queryRunner.getCoordinator().loadMetastore(metastoreConfig);
        queryRunner.createCatalog("memory", "memory", ImmutableMap.of("memory.spill-path", folder.newFolder("memory-connector").getAbsolutePath()));
        createBooleanTable();
        createByteTable();
        createShortTable();
        createIntTable();
        createLongTable();
        createFloatTable();
        createDoubleTable();
        createListTable();
        createMapTable();
        createMultipleParameterTable();
        createEvaluateOverloadTable();
    } catch (Exception e) {
        closeAllSuppress(e, queryRunner);
        throw e;
    }
}

Example 7 with HetuMetastorePlugin

use of io.hetu.core.metastore.HetuMetastorePlugin in project hetu-core by openlookeng.

the class VdmQueryRunner method createVdmQueryRunner.

public static QueryRunner createVdmQueryRunner(String jdbcUrl, Map<String, String> connectorProperties, Iterable<TpchTable<?>> tables) throws Exception {
    DistributedQueryRunner queryRunner = null;
    try {
        File directory = new File("");
        String courseFile = directory.getCanonicalPath();
        System.setProperty("config", courseFile + "/etc/");
        String configDir = System.getProperty("config");
        String hetumetastoreConfig = configDir + "hetu-metastore.properties";
        File file = new File(configDir);
        if (!file.exists()) {
            file.mkdirs();
        }
        File file2 = new File(configDir, "hetu-metastore.properties");
        if (!file2.exists()) {
            try {
                file2.createNewFile();
            } catch (IOException e) {
                LOG.info("Error message: " + e.getStackTrace());
            }
        }
        try (BufferedWriter bufferedWriter = new BufferedWriter(new FileWriter(hetumetastoreConfig))) {
            bufferedWriter.write("hetu.metastore.db.url = " + jdbcUrl);
            bufferedWriter.write("\n");
            bufferedWriter.write("hetu.metastore.type = jdbc\n");
            bufferedWriter.write("hetu.metastore.db.user = user\n");
            bufferedWriter.write("hetu.metastore.db.password = testpass\n");
            bufferedWriter.write("hetu.metastore.cache.ttl = 0s");
        }
        queryRunner = DistributedQueryRunner.builder(createSession()).setNodeCount(1).setExtraProperties(ImmutableMap.of()).setCoordinatorProperties(ImmutableMap.of()).setParserOptions(DEFAULT_SQL_PARSER_OPTIONS).setEnvironment("vdmtest").setBaseDataDir(Optional.empty()).build();
        queryRunner.installPlugin(new TpchPlugin());
        queryRunner.installPlugin(new VdmPlugin());
        queryRunner.installPlugin(new HetuMetastorePlugin());
        queryRunner.getCoordinator().loadMetastore();
        queryRunner.createCatalog("tpch", "tpch");
        queryRunner.createCatalog("vdm", "vdm", ImmutableMap.of("vdm.metadata-cache-enabled", "false"));
        return queryRunner;
    } catch (Throwable e) {
        closeAllSuppress(e, queryRunner);
        throw e;
    }
}

Example 8 with HetuMetastorePlugin

use of io.hetu.core.metastore.HetuMetastorePlugin in project hetu-core by openlookeng.

the class MemoryQueryRunner method createQueryRunner.

public static DistributedQueryRunner createQueryRunner(int nodes, Map<String, String> extraProperties, Map<String, String> memoryProperties, Boolean createTpchTables) throws Exception {
    Session session = testSessionBuilder().setCatalog(CATALOG).setSchema("default").build();
    DistributedQueryRunner queryRunner = new DistributedQueryRunner(session, nodes, extraProperties);
    Runtime.getRuntime().addShutdownHook(new Thread(queryRunner::close));
    try {
        queryRunner.installPlugin(new HetuFileSystemClientPlugin());
        queryRunner.installPlugin(new HetuMetastorePlugin());
        TempFolder metastoreFolder = new TempFolder().create();
        Runtime.getRuntime().addShutdownHook(new Thread(metastoreFolder::close));
        Map<String, String> metastoreConfig = new HashMap<>();
        metastoreConfig.put("hetu.metastore.type", "hetufilesystem");
        metastoreConfig.put("hetu.metastore.hetufilesystem.profile-name", "default");
        metastoreConfig.put("hetu.metastore.hetufilesystem.path", metastoreFolder.newFolder("metastore").getAbsolutePath());
        queryRunner.getServers().forEach(server -> {
            try {
                server.loadMetastore(new HashMap<>(metastoreConfig));
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        queryRunner.installPlugin(new MemoryPlugin());
        queryRunner.getServers().forEach(server -> {
            try {
                TempFolder folder = new TempFolder();
                folder.create();
                Runtime.getRuntime().addShutdownHook(new Thread(folder::close));
                // create memory catalog with custom memory properties
                server.createCatalog(CATALOG, "memory", createConfig(folder, memoryProperties));
            } catch (Exception e) {
                throw new RuntimeException(e);
            }
        });
        queryRunner.installPlugin(new TpchPlugin());
        queryRunner.createCatalog("tpch", "tpch", ImmutableMap.of());
        queryRunner.installPlugin(new TpcdsPlugin());
        queryRunner.createCatalog("tpcds", "tpcds", ImmutableMap.of());
        if (createTpchTables) {
            copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, session, TpchTable.getTables());
        }
        return queryRunner;
    } catch (Exception e) {
        closeAllSuppress(e, queryRunner);
        throw e;
    }
}