Examples with DistributedQueryRunner com.facebook.presto.tests.DistributedQueryRunner used on opensource projects

Example 1 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class HiveQueryRunner method createQueryRunner.

public static DistributedQueryRunner createQueryRunner(Iterable<TpchTable<?>> tables, Map<String, String> extraProperties, String security, Map<String, String> extraHiveProperties) throws Exception {
    assertEquals(DateTimeZone.getDefault(), TIME_ZONE, "Timezone not configured correctly. Add -Duser.timezone=Asia/Katmandu to your JVM arguments");
    DistributedQueryRunner queryRunner = new DistributedQueryRunner(createSession(), 4, extraProperties);
    try {
        queryRunner.installPlugin(new TpchPlugin());
        queryRunner.createCatalog("tpch", "tpch");
        File baseDir = queryRunner.getCoordinator().getBaseDataDir().resolve("hive_data").toFile();
        HiveClientConfig hiveClientConfig = new HiveClientConfig();
        HiveS3Config s3Config = new HiveS3Config();
        HdfsConfiguration hdfsConfiguration = new HiveHdfsConfiguration(new HdfsConfigurationUpdater(hiveClientConfig, s3Config));
        HdfsEnvironment hdfsEnvironment = new HdfsEnvironment(hdfsConfiguration, hiveClientConfig, new NoHdfsAuthentication());
        FileHiveMetastore metastore = new FileHiveMetastore(hdfsEnvironment, baseDir.toURI().toString(), "test");
        metastore.createDatabase(createDatabaseMetastoreObject(TPCH_SCHEMA));
        metastore.createDatabase(createDatabaseMetastoreObject(TPCH_BUCKETED_SCHEMA));
        queryRunner.installPlugin(new HivePlugin(HIVE_CATALOG, metastore));
        Map<String, String> hiveProperties = ImmutableMap.<String, String>builder().putAll(extraHiveProperties).put("hive.metastore.uri", "thrift://localhost:8080").put("hive.time-zone", TIME_ZONE.getID()).put("hive.security", security).build();
        Map<String, String> hiveBucketedProperties = ImmutableMap.<String, String>builder().putAll(hiveProperties).put("hive.max-initial-split-size", // so that each bucket has multiple splits
        "10kB").put("hive.max-split-size", // so that each bucket has multiple splits
        "10kB").put("hive.storage-format", // so that there's no minimum split size for the file
        "TEXTFILE").put("hive.compression-codec", // so that the file is splittable
        "NONE").build();
        queryRunner.createCatalog(HIVE_CATALOG, HIVE_CATALOG, hiveProperties);
        queryRunner.createCatalog(HIVE_BUCKETED_CATALOG, HIVE_CATALOG, hiveBucketedProperties);
        copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), tables);
        copyTpchTablesBucketed(queryRunner, "tpch", TINY_SCHEMA_NAME, createBucketedSession(), tables);
        return queryRunner;
    } catch (Exception e) {
        queryRunner.close();
        throw e;
    }
}

Example 2 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class PostgreSqlQueryRunner method createPostgreSqlQueryRunner.

public static QueryRunner createPostgreSqlQueryRunner(TestingPostgreSqlServer server, Iterable<TpchTable<?>> tables) throws Exception {
    DistributedQueryRunner queryRunner = null;
    try {
        queryRunner = new DistributedQueryRunner(createSession(), 3);
        queryRunner.installPlugin(new TpchPlugin());
        queryRunner.createCatalog("tpch", "tpch");
        Map<String, String> properties = ImmutableMap.<String, String>builder().put("connection-url", server.getJdbcUrl()).put("allow-drop-table", "true").build();
        createSchema(server.getJdbcUrl(), "tpch");
        queryRunner.installPlugin(new PostgreSqlPlugin());
        queryRunner.createCatalog("postgresql", "postgresql", properties);
        copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), tables);
        return queryRunner;
    } catch (Throwable e) {
        closeAllSuppress(e, queryRunner, server);
        throw e;
    }
}

Example 3 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class TestRaptorIntegrationSmokeTestMySql method createRaptorMySqlQueryRunner.

private static DistributedQueryRunner createRaptorMySqlQueryRunner(TestingMySqlServer server) throws Exception {
    DistributedQueryRunner queryRunner = new DistributedQueryRunner(createSession("tpch"), 2);
    queryRunner.installPlugin(new TpchPlugin());
    queryRunner.createCatalog("tpch", "tpch");
    queryRunner.installPlugin(new RaptorPlugin());
    File baseDir = queryRunner.getCoordinator().getBaseDataDir().toFile();
    Map<String, String> raptorProperties = ImmutableMap.<String, String>builder().put("metadata.db.type", "mysql").put("metadata.db.url", jdbcUrl(server)).put("storage.data-directory", new File(baseDir, "data").getAbsolutePath()).put("storage.max-shard-rows", "2000").put("backup.provider", "file").put("backup.directory", new File(baseDir, "backup").getAbsolutePath()).build();
    queryRunner.createCatalog("raptor", "raptor", raptorProperties);
    copyTables(queryRunner, "tpch", createSession(), false);
    return queryRunner;
}

Example 4 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class TestQueues method testBasic.

private void testBasic(boolean resourceGroups) throws Exception {
    ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
    if (resourceGroups) {
        builder.put("experimental.resource-groups-enabled", "true");
    } else {
        builder.put("query.queue-config-file", getResourceFilePath("queue_config_dashboard.json"));
    }
    Map<String, String> properties = builder.build();
    try (DistributedQueryRunner queryRunner = createQueryRunner(properties)) {
        queryRunner.installPlugin(new ResourceGroupManagerPlugin());
        queryRunner.getCoordinator().getResourceGroupManager().get().setConfigurationManager("file", ImmutableMap.of("resource-groups.config-file", getResourceFilePath("resource_groups_config_dashboard.json")));
        QueryManager queryManager = queryRunner.getCoordinator().getQueryManager();
        // submit first "dashboard" query
        QueryId firstDashboardQuery = createQuery(queryRunner, newDashboardSession(), LONG_LASTING_QUERY);
        // wait for the first "dashboard" query to start
        waitForQueryState(queryRunner, firstDashboardQuery, RUNNING);
        // submit second "dashboard" query
        QueryId secondDashboardQuery = createQuery(queryRunner, newDashboardSession(), LONG_LASTING_QUERY);
        // wait for the second "dashboard" query to be queued ("dashboard.${USER}" queue strategy only allows one "dashboard" query to be accepted for execution)
        waitForQueryState(queryRunner, secondDashboardQuery, QUEUED);
        // submit first non "dashboard" query
        QueryId firstNonDashboardQuery = createQuery(queryRunner, newSession(), LONG_LASTING_QUERY);
        // wait for the first non "dashboard" query to start
        waitForQueryState(queryRunner, firstNonDashboardQuery, RUNNING);
        // submit second non "dashboard" query
        QueryId secondNonDashboardQuery = createQuery(queryRunner, newSession(), LONG_LASTING_QUERY);
        // wait for the second non "dashboard" query to start
        waitForQueryState(queryRunner, secondNonDashboardQuery, RUNNING);
        // cancel first "dashboard" query, second "dashboard" query and second non "dashboard" query should start running
        cancelQuery(queryRunner, firstDashboardQuery);
        waitForQueryState(queryRunner, firstDashboardQuery, FAILED);
        waitForQueryState(queryRunner, secondDashboardQuery, RUNNING);
    }
}

Example 5 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class TestQueues method testRejection.

private void testRejection(boolean resourceGroups) throws Exception {
    ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
    if (resourceGroups) {
        builder.put("experimental.resource-groups-enabled", "true");
    } else {
        builder.put("query.queue-config-file", getResourceFilePath("queue_config_dashboard.json"));
    }
    Map<String, String> properties = builder.build();
    try (DistributedQueryRunner queryRunner = createQueryRunner(properties)) {
        queryRunner.installPlugin(new ResourceGroupManagerPlugin());
        queryRunner.getCoordinator().getResourceGroupManager().get().setConfigurationManager("file", ImmutableMap.of("resource-groups.config-file", getResourceFilePath("resource_groups_config_dashboard.json")));
        QueryId queryId = createQuery(queryRunner, newRejectionSession(), LONG_LASTING_QUERY);
        waitForQueryState(queryRunner, queryId, FAILED);
        QueryManager queryManager = queryRunner.getCoordinator().getQueryManager();
        assertEquals(queryManager.getQueryInfo(queryId).getErrorCode(), QUERY_REJECTED.toErrorCode());
    }
}