Search in sources :

Example 1 with ALL

use of com.facebook.presto.spi.security.SelectedRole.Type.ALL in project presto by prestodb.

the class TestHiveRecoverableExecution method testRecoverableGroupedExecution.

private void testRecoverableGroupedExecution(DistributedQueryRunner queryRunner, int writerConcurrency, boolean optimizedPartitionUpdateSerializationEnabled, List<String> preQueries, @Language("SQL") String queryWithoutFailure, @Language("SQL") String queryWithFailure, int expectedUpdateCount, List<String> postQueries) throws Exception {
    waitUntilAllNodesAreHealthy(queryRunner, new Duration(10, SECONDS));
    Session recoverableSession = createRecoverableSession(writerConcurrency, optimizedPartitionUpdateSerializationEnabled);
    for (@Language("SQL") String postQuery : postQueries) {
        queryRunner.execute(recoverableSession, postQuery);
    }
    try {
        for (@Language("SQL") String preQuery : preQueries) {
            queryRunner.execute(recoverableSession, preQuery);
        }
        // test no failure case
        Stopwatch noRecoveryStopwatch = Stopwatch.createStarted();
        assertEquals(queryRunner.execute(recoverableSession, queryWithoutFailure).getUpdateCount(), OptionalLong.of(expectedUpdateCount));
        log.info("Query with no recovery took %sms", noRecoveryStopwatch.elapsed(MILLISECONDS));
        // cancel all queries and tasks to make sure we are dealing only with a single running query
        cancelAllQueries(queryRunner);
        cancelAllTasks(queryRunner);
        // test failure case
        Stopwatch recoveryStopwatch = Stopwatch.createStarted();
        ListenableFuture<MaterializedResult> result = executor.submit(() -> queryRunner.execute(recoverableSession, queryWithFailure));
        List<TestingPrestoServer> workers = queryRunner.getServers().stream().filter(server -> !server.isCoordinator()).collect(toList());
        shuffle(workers);
        TestingPrestoServer worker1 = workers.get(0);
        // kill worker1 right away, to make sure recoverable execution works in cases when the task hasn't been yet submitted
        worker1.stopResponding();
        // kill worker2 only after the task has been scheduled
        TestingPrestoServer worker2 = workers.get(1);
        sleep(1000);
        worker2.stopResponding();
        assertEquals(result.get(1000, SECONDS).getUpdateCount(), OptionalLong.of(expectedUpdateCount));
        log.info("Query with recovery took %sms", recoveryStopwatch.elapsed(MILLISECONDS));
    } finally {
        queryRunner.getServers().forEach(TestingPrestoServer::startResponding);
        cancelAllQueries(queryRunner);
        cancelAllTasks(queryRunner);
        for (@Language("SQL") String postQuery : postQueries) {
            queryRunner.execute(recoverableSession, postQuery);
        }
    }
}
Also used : Collections.shuffle(java.util.Collections.shuffle) TestingPrestoServer(com.facebook.presto.server.testing.TestingPrestoServer) TimeoutException(java.util.concurrent.TimeoutException) Test(org.testng.annotations.Test) DistributedQueryRunner(com.facebook.presto.tests.DistributedQueryRunner) Duration(io.airlift.units.Duration) RECOVERABLE_GROUPED_EXECUTION(com.facebook.presto.SystemSessionProperties.RECOVERABLE_GROUPED_EXECUTION) Thread.sleep(java.lang.Thread.sleep) TASK_WRITER_COUNT(com.facebook.presto.SystemSessionProperties.TASK_WRITER_COUNT) EXCHANGE_MATERIALIZATION_STRATEGY(com.facebook.presto.SystemSessionProperties.EXCHANGE_MATERIALIZATION_STRATEGY) TASK_PARTITIONED_WRITER_COUNT(com.facebook.presto.SystemSessionProperties.TASK_PARTITIONED_WRITER_COUNT) ImmutableMap(com.google.common.collect.ImmutableMap) BeforeClass(org.testng.annotations.BeforeClass) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) String.format(java.lang.String.format) List(java.util.List) HASH_PARTITION_COUNT(com.facebook.presto.SystemSessionProperties.HASH_PARTITION_COUNT) CONCURRENT_LIFESPANS_PER_NODE(com.facebook.presto.SystemSessionProperties.CONCURRENT_LIFESPANS_PER_NODE) Optional(java.util.Optional) VIRTUAL_BUCKET_COUNT(com.facebook.presto.hive.HiveSessionProperties.VIRTUAL_BUCKET_COUNT) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) MoreExecutors.listeningDecorator(com.google.common.util.concurrent.MoreExecutors.listeningDecorator) Logger(com.facebook.airlift.log.Logger) ListenableFuture(com.google.common.util.concurrent.ListenableFuture) DataProvider(org.testng.annotations.DataProvider) TPCH_BUCKETED_SCHEMA(com.facebook.presto.hive.HiveQueryRunner.TPCH_BUCKETED_SCHEMA) Stopwatch(com.google.common.base.Stopwatch) PARTITIONING_PROVIDER_CATALOG(com.facebook.presto.SystemSessionProperties.PARTITIONING_PROVIDER_CATALOG) ALL(com.facebook.presto.spi.security.SelectedRole.Type.ALL) ORDERS(io.airlift.tpch.TpchTable.ORDERS) Assert.assertEquals(org.testng.Assert.assertEquals) ROLE(com.facebook.presto.spi.security.SelectedRole.Type.ROLE) OptionalLong(java.util.OptionalLong) OPTIMIZED_PARTITION_UPDATE_SERIALIZATION_ENABLED(com.facebook.presto.hive.HiveSessionProperties.OPTIMIZED_PARTITION_UPDATE_SERIALIZATION_ENABLED) ImmutableList(com.google.common.collect.ImmutableList) Identity(com.facebook.presto.spi.security.Identity) GROUPED_EXECUTION(com.facebook.presto.SystemSessionProperties.GROUPED_EXECUTION) AfterClass(org.testng.annotations.AfterClass) SelectedRole(com.facebook.presto.spi.security.SelectedRole) HIVE_CATALOG(com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG) Language(org.intellij.lang.annotations.Language) Session(com.facebook.presto.Session) MAX_STAGE_RETRIES(com.facebook.presto.SystemSessionProperties.MAX_STAGE_RETRIES) TestingSession.testSessionBuilder(com.facebook.presto.testing.TestingSession.testSessionBuilder) AllNodes(com.facebook.presto.metadata.AllNodes) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Collectors.toList(java.util.stream.Collectors.toList) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) COLOCATED_JOIN(com.facebook.presto.SystemSessionProperties.COLOCATED_JOIN) REDISTRIBUTE_WRITES(com.facebook.presto.SystemSessionProperties.REDISTRIBUTE_WRITES) SECONDS(java.util.concurrent.TimeUnit.SECONDS) SCALE_WRITERS(com.facebook.presto.SystemSessionProperties.SCALE_WRITERS) Language(org.intellij.lang.annotations.Language) Stopwatch(com.google.common.base.Stopwatch) TestingPrestoServer(com.facebook.presto.server.testing.TestingPrestoServer) Duration(io.airlift.units.Duration) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Session(com.facebook.presto.Session)

Aggregations

Logger (com.facebook.airlift.log.Logger)1 Session (com.facebook.presto.Session)1 COLOCATED_JOIN (com.facebook.presto.SystemSessionProperties.COLOCATED_JOIN)1 CONCURRENT_LIFESPANS_PER_NODE (com.facebook.presto.SystemSessionProperties.CONCURRENT_LIFESPANS_PER_NODE)1 EXCHANGE_MATERIALIZATION_STRATEGY (com.facebook.presto.SystemSessionProperties.EXCHANGE_MATERIALIZATION_STRATEGY)1 GROUPED_EXECUTION (com.facebook.presto.SystemSessionProperties.GROUPED_EXECUTION)1 HASH_PARTITION_COUNT (com.facebook.presto.SystemSessionProperties.HASH_PARTITION_COUNT)1 MAX_STAGE_RETRIES (com.facebook.presto.SystemSessionProperties.MAX_STAGE_RETRIES)1 PARTITIONING_PROVIDER_CATALOG (com.facebook.presto.SystemSessionProperties.PARTITIONING_PROVIDER_CATALOG)1 RECOVERABLE_GROUPED_EXECUTION (com.facebook.presto.SystemSessionProperties.RECOVERABLE_GROUPED_EXECUTION)1 REDISTRIBUTE_WRITES (com.facebook.presto.SystemSessionProperties.REDISTRIBUTE_WRITES)1 SCALE_WRITERS (com.facebook.presto.SystemSessionProperties.SCALE_WRITERS)1 TASK_PARTITIONED_WRITER_COUNT (com.facebook.presto.SystemSessionProperties.TASK_PARTITIONED_WRITER_COUNT)1 TASK_WRITER_COUNT (com.facebook.presto.SystemSessionProperties.TASK_WRITER_COUNT)1 HIVE_CATALOG (com.facebook.presto.hive.HiveQueryRunner.HIVE_CATALOG)1 TPCH_BUCKETED_SCHEMA (com.facebook.presto.hive.HiveQueryRunner.TPCH_BUCKETED_SCHEMA)1 OPTIMIZED_PARTITION_UPDATE_SERIALIZATION_ENABLED (com.facebook.presto.hive.HiveSessionProperties.OPTIMIZED_PARTITION_UPDATE_SERIALIZATION_ENABLED)1 VIRTUAL_BUCKET_COUNT (com.facebook.presto.hive.HiveSessionProperties.VIRTUAL_BUCKET_COUNT)1 AllNodes (com.facebook.presto.metadata.AllNodes)1 TestingPrestoServer (com.facebook.presto.server.testing.TestingPrestoServer)1