Search in sources :

Example 76 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class TestIcebergSystemTables method createQueryRunner.

@Override
protected QueryRunner createQueryRunner() throws Exception {
    Session session = testSessionBuilder().setCatalog(ICEBERG_CATALOG).build();
    DistributedQueryRunner queryRunner = DistributedQueryRunner.builder(session).build();
    Path catalogDir = queryRunner.getCoordinator().getBaseDataDir().resolve("iceberg_data").resolve("catalog");
    queryRunner.installPlugin(new IcebergPlugin());
    Map<String, String> icebergProperties = ImmutableMap.<String, String>builder().put("hive.metastore", "file").put("hive.metastore.catalog.dir", catalogDir.toFile().toURI().toString()).build();
    queryRunner.createCatalog(ICEBERG_CATALOG, "iceberg", icebergProperties);
    return queryRunner;
}
Also used : Path(java.nio.file.Path) DistributedQueryRunner(com.facebook.presto.tests.DistributedQueryRunner) Session(com.facebook.presto.Session)

Example 77 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class CassandraQueryRunner method createCassandraQueryRunner.

public static synchronized DistributedQueryRunner createCassandraQueryRunner() throws Exception {
    EmbeddedCassandra.start();
    DistributedQueryRunner queryRunner = new DistributedQueryRunner(createCassandraSession("tpch"), 4);
    queryRunner.installPlugin(new TpchPlugin());
    queryRunner.createCatalog("tpch", "tpch");
    queryRunner.installPlugin(new CassandraPlugin());
    queryRunner.createCatalog("cassandra", "cassandra", ImmutableMap.of("cassandra.contact-points", EmbeddedCassandra.getHost(), "cassandra.native-protocol-port", Integer.toString(EmbeddedCassandra.getPort()), "cassandra.allow-drop-table", "true"));
    if (!tpchLoaded) {
        createKeyspace(EmbeddedCassandra.getSession(), "tpch");
        List<TpchTable<?>> tables = TpchTable.getTables();
        copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createCassandraSession("tpch"), tables);
        for (TpchTable table : tables) {
            EmbeddedCassandra.refreshSizeEstimates("tpch", table.getTableName());
        }
        tpchLoaded = true;
    }
    return queryRunner;
}
Also used : DistributedQueryRunner(com.facebook.presto.tests.DistributedQueryRunner) TpchPlugin(com.facebook.presto.tpch.TpchPlugin) TpchTable(io.airlift.tpch.TpchTable)

Example 78 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class TestHiveRecoverableExecution method testRecoverableGroupedExecution.

private void testRecoverableGroupedExecution(DistributedQueryRunner queryRunner, int writerConcurrency, boolean optimizedPartitionUpdateSerializationEnabled, List<String> preQueries, @Language("SQL") String queryWithoutFailure, @Language("SQL") String queryWithFailure, int expectedUpdateCount, List<String> postQueries) throws Exception {
    waitUntilAllNodesAreHealthy(queryRunner, new Duration(10, SECONDS));
    Session recoverableSession = createRecoverableSession(writerConcurrency, optimizedPartitionUpdateSerializationEnabled);
    for (@Language("SQL") String postQuery : postQueries) {
        queryRunner.execute(recoverableSession, postQuery);
    }
    try {
        for (@Language("SQL") String preQuery : preQueries) {
            queryRunner.execute(recoverableSession, preQuery);
        }
        // test no failure case
        Stopwatch noRecoveryStopwatch = Stopwatch.createStarted();
        assertEquals(queryRunner.execute(recoverableSession, queryWithoutFailure).getUpdateCount(), OptionalLong.of(expectedUpdateCount));
        log.info("Query with no recovery took %sms", noRecoveryStopwatch.elapsed(MILLISECONDS));
        // cancel all queries and tasks to make sure we are dealing only with a single running query
        cancelAllQueries(queryRunner);
        cancelAllTasks(queryRunner);
        // test failure case
        Stopwatch recoveryStopwatch = Stopwatch.createStarted();
        ListenableFuture<MaterializedResult> result = executor.submit(() -> queryRunner.execute(recoverableSession, queryWithFailure));
        List<TestingPrestoServer> workers = queryRunner.getServers().stream().filter(server -> !server.isCoordinator()).collect(toList());
        shuffle(workers);
        TestingPrestoServer worker1 = workers.get(0);
        // kill worker1 right away, to make sure recoverable execution works in cases when the task hasn't been yet submitted
        worker1.stopResponding();
        // kill worker2 only after the task has been scheduled
        TestingPrestoServer worker2 = workers.get(1);
        sleep(1000);
        worker2.stopResponding();
        assertEquals(result.get(1000, SECONDS).getUpdateCount(), OptionalLong.of(expectedUpdateCount));
        log.info("Query with recovery took %sms", recoveryStopwatch.elapsed(MILLISECONDS));
    } finally {
        queryRunner.getServers().forEach(TestingPrestoServer::startResponding);
        cancelAllQueries(queryRunner);
        cancelAllTasks(queryRunner);
        for (@Language("SQL") String postQuery : postQueries) {
            queryRunner.execute(recoverableSession, postQuery);
        }
    }
}
Also used : Collections.shuffle(java.util.Collections.shuffle) TestingPrestoServer(com.facebook.presto.server.testing.TestingPrestoServer) TimeoutException(java.util.concurrent.TimeoutException) Test(org.testng.annotations.Test) DistributedQueryRunner(com.facebook.presto.tests.DistributedQueryRunner) Duration(io.airlift.units.Duration) RECOVERABLE_GROUPED_EXECUTION(com.facebook.presto.SystemSessionProperties.RECOVERABLE_GROUPED_EXECUTION) Thread.sleep(java.lang.Thread.sleep) TASK_WRITER_COUNT(com.facebook.presto.SystemSessionProperties.TASK_WRITER_COUNT) EXCHANGE_MATERIALIZATION_STRATEGY(com.facebook.presto.SystemSessionProperties.EXCHANGE_MATERIALIZATION_STRATEGY) TASK_PARTITIONED_WRITER_COUNT(com.facebook.presto.SystemSessionProperties.TASK_PARTITIONED_WRITER_COUNT) ImmutableMap(com.google.common.collect.ImmutableMap) BeforeClass(org.testng.annotations.BeforeClass) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) String.format(java.lang.String.format) List(java.util.List) HASH_PARTITION_COUNT(com.facebook.presto.SystemSessionProperties.HASH_PARTITION_COUNT) CONCURRENT_LIFESPANS_PER_NODE(com.facebook.presto.SystemSessionProperties.CONCURRENT_LIFESPANS_PER_NODE) Optional(java.util.Optional) VIRTUAL_BUCKET_COUNT(com.facebook.presto.hive.HiveSessionProperties.VIRTUAL_BUCKET_COUNT) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) MoreExecutors.listeningDecorator(com.google.common.util.concurrent.MoreExecutors.listeningDecorator) Logger(com.facebook.airlift.log.Logger) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) DataProvider(org.testng.annotations.DataProvider) TPCH_BUCKETED_SCHEMA(com.facebook.presto.hive.HiveQueryRunner.TPCH_BUCKETED_SCHEMA) Stopwatch(com.google.common.base.Stopwatch) PARTITIONING_PROVIDER_CATALOG(com.facebook.presto.SystemSessionProperties.PARTITIONING_PROVIDER_CATALOG) ALL(com.facebook.presto.spi.security.SelectedRole.Type.ALL) ORDERS(io.airlift.tpch.TpchTable.ORDERS) Assert.assertEquals(org.testng.Assert.assertEquals) ROLE(com.facebook.presto.spi.security.SelectedRole.Type.ROLE) OptionalLong(java.util.OptionalLong) OPTIMIZED_PARTITION_UPDATE_SERIALIZATION_ENABLED(com.facebook.presto.hive.HiveSessionProperties.OPTIMIZED_PARTITION_UPDATE_SERIALIZATION_ENABLED) ImmutableList(com.google.common.collect.ImmutableList) Identity(com.facebook.presto.spi.security.Identity) GROUPED_EXECUTION(com.facebook.presto.SystemSessionProperties.GROUPED_EXECUTION) AfterClass(org.testng.annotations.AfterClass) SelectedRole(com.facebook.presto.spi.security.SelectedRole) HIVE_CATALOG(com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG) Language(org.intellij.lang.annotations.Language) Session(com.facebook.presto.Session) MAX_STAGE_RETRIES(com.facebook.presto.SystemSessionProperties.MAX_STAGE_RETRIES) TestingSession.testSessionBuilder(com.facebook.presto.testing.TestingSession.testSessionBuilder) AllNodes(com.facebook.presto.metadata.AllNodes) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Collectors.toList(java.util.stream.Collectors.toList) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) COLOCATED_JOIN(com.facebook.presto.SystemSessionProperties.COLOCATED_JOIN) REDISTRIBUTE_WRITES(com.facebook.presto.SystemSessionProperties.REDISTRIBUTE_WRITES) SECONDS(java.util.concurrent.TimeUnit.SECONDS) SCALE_WRITERS(com.facebook.presto.SystemSessionProperties.SCALE_WRITERS) Language(org.intellij.lang.annotations.Language) Stopwatch(com.google.common.base.Stopwatch) TestingPrestoServer(com.facebook.presto.server.testing.TestingPrestoServer) Duration(io.airlift.units.Duration) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Session(com.facebook.presto.Session)

Example 79 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class OracleQueryRunner method main.

public static void main(String[] args) throws Exception {
    Logging.initialize();
    DistributedQueryRunner queryRunner = createOracleQueryRunner(new TestingOracleServer(), TpchTable.getTables());
    Logger log = Logger.get(OracleQueryRunner.class);
    log.info("======== SERVER STARTED ========");
    log.info("\n====\n%s\n====", queryRunner.getCoordinator().getBaseUrl());
}
Also used : DistributedQueryRunner(com.facebook.presto.tests.DistributedQueryRunner) Logger(com.facebook.airlift.log.Logger)

Example 80 with DistributedQueryRunner

use of com.facebook.presto.tests.DistributedQueryRunner in project presto by prestodb.

the class MySqlQueryRunner method createMySqlQueryRunner.

public static QueryRunner createMySqlQueryRunner(String jdbcUrl, Map<String, String> connectorProperties, Iterable<TpchTable<?>> tables) throws Exception {
    DistributedQueryRunner queryRunner = null;
    try {
        queryRunner = new DistributedQueryRunner(createSession(), 3);
        queryRunner.installPlugin(new TpchPlugin());
        queryRunner.createCatalog("tpch", "tpch");
        connectorProperties = new HashMap<>(ImmutableMap.copyOf(connectorProperties));
        connectorProperties.putIfAbsent("connection-url", jdbcUrl);
        connectorProperties.putIfAbsent("allow-drop-table", "true");
        queryRunner.installPlugin(new MySqlPlugin());
        queryRunner.createCatalog("mysql", "mysql", connectorProperties);
        copyTpchTables(queryRunner, "tpch", TINY_SCHEMA_NAME, createSession(), tables);
        return queryRunner;
    } catch (Throwable e) {
        closeAllSuppress(e, queryRunner);
        throw e;
    }
}
Also used : DistributedQueryRunner(com.facebook.presto.tests.DistributedQueryRunner) TpchPlugin(com.facebook.presto.tpch.TpchPlugin)

Aggregations

DistributedQueryRunner (com.facebook.presto.tests.DistributedQueryRunner)111 Test (org.testng.annotations.Test)36 TpchPlugin (com.facebook.presto.tpch.TpchPlugin)33 Session (com.facebook.presto.Session)26 QueryId (com.facebook.presto.spi.QueryId)19 Logger (com.facebook.airlift.log.Logger)15 TestingPrestoServer (com.facebook.presto.server.testing.TestingPrestoServer)14 MaterializedResult (com.facebook.presto.testing.MaterializedResult)13 QueryRunner (com.facebook.presto.testing.QueryRunner)13 ImmutableMap (com.google.common.collect.ImmutableMap)10 ResourceGroupManagerPlugin (com.facebook.presto.resourceGroups.ResourceGroupManagerPlugin)9 ArrayList (java.util.ArrayList)8 File (java.io.File)7 BeforeClass (org.testng.annotations.BeforeClass)7 JettyHttpClient (com.facebook.airlift.http.client.jetty.JettyHttpClient)6 FileResourceGroupConfigurationManagerFactory (com.facebook.presto.resourceGroups.FileResourceGroupConfigurationManagerFactory)6 H2ResourceGroupsDao (com.facebook.presto.resourceGroups.db.H2ResourceGroupsDao)6 Path (java.nio.file.Path)6 Future (java.util.concurrent.Future)6 MetastoreContext (com.facebook.presto.hive.metastore.MetastoreContext)5