Search in sources :

Example 1 with TEST_SESSION

use of com.facebook.presto.SessionTestUtils.TEST_SESSION in project presto by prestodb.

the class TestHttpRemoteTask method testRemoteTaskMismatch.

// This timeout should never be reached because a daemon thread in test should fail the test and do proper cleanup.
@Test(timeOut = 30000)
public void testRemoteTaskMismatch() throws InterruptedException, ExecutionException {
    Duration idleTimeout = new Duration(3, SECONDS);
    Duration failTimeout = new Duration(20, SECONDS);
    JsonCodec<TaskStatus> taskStatusCodec = JsonCodec.jsonCodec(TaskStatus.class);
    JsonCodec<TaskInfo> taskInfoCodec = JsonCodec.jsonCodec(TaskInfo.class);
    TaskManagerConfig taskManagerConfig = new TaskManagerConfig();
    // Shorten status refresh wait and info update interval so that we can have a shorter test timeout
    taskManagerConfig.setStatusRefreshMaxWait(new Duration(idleTimeout.roundTo(MILLISECONDS) / 100, MILLISECONDS));
    taskManagerConfig.setInfoUpdateInterval(new Duration(idleTimeout.roundTo(MILLISECONDS) / 10, MILLISECONDS));
    AtomicLong lastActivityNanos = new AtomicLong(System.nanoTime());
    HttpProcessor httpProcessor = new HttpProcessor(taskStatusCodec, taskInfoCodec, lastActivityNanos);
    TestingHttpClient testingHttpClient = new TestingHttpClient(httpProcessor);
    HttpRemoteTaskFactory httpRemoteTaskFactory = new HttpRemoteTaskFactory(new QueryManagerConfig(), taskManagerConfig, testingHttpClient, new TestSqlTaskManager.MockLocationFactory(), taskStatusCodec, taskInfoCodec, JsonCodec.jsonCodec(TaskUpdateRequest.class), new RemoteTaskStats());
    RemoteTask remoteTask = httpRemoteTaskFactory.createRemoteTask(TEST_SESSION, new TaskId("test", 1, 2), new PrestoNode("node-id", URI.create("http://192.0.1.2"), new NodeVersion("version"), false), TaskTestUtils.PLAN_FRAGMENT, ImmutableMultimap.of(), createInitialEmptyOutputBuffers(OutputBuffers.BufferType.BROADCAST), new NodeTaskMap.PartitionedSplitCountTracker(i -> {
    }), true);
    httpProcessor.setInitialTaskInfo(remoteTask.getTaskInfo());
    remoteTask.start();
    CompletableFuture<Void> testComplete = new CompletableFuture<>();
    asyncRun(idleTimeout.roundTo(MILLISECONDS), failTimeout.roundTo(MILLISECONDS), lastActivityNanos, () -> testComplete.complete(null), (message, cause) -> testComplete.completeExceptionally(new AssertionError(message, cause)));
    testComplete.get();
    httpRemoteTaskFactory.stop();
    assertTrue(remoteTask.getTaskStatus().getState().isDone(), format("TaskStatus is not in a done state: %s", remoteTask.getTaskStatus()));
    assertEquals(getOnlyElement(remoteTask.getTaskStatus().getFailures()).getErrorCode(), REMOTE_TASK_MISMATCH.toErrorCode());
    assertTrue(remoteTask.getTaskInfo().getTaskStatus().getState().isDone(), format("TaskInfo is not in a done state: %s", remoteTask.getTaskInfo()));
}
Also used : OutputBuffers(com.facebook.presto.OutputBuffers) CONTENT_TYPE(javax.ws.rs.core.HttpHeaders.CONTENT_TYPE) NodeTaskMap(com.facebook.presto.execution.NodeTaskMap) TaskStatus(com.facebook.presto.execution.TaskStatus) Assert.assertEquals(org.testng.Assert.assertEquals) TaskState(com.facebook.presto.execution.TaskState) Test(org.testng.annotations.Test) CompletableFuture(java.util.concurrent.CompletableFuture) NodeVersion(com.facebook.presto.client.NodeVersion) TaskManagerConfig(com.facebook.presto.execution.TaskManagerConfig) Duration(io.airlift.units.Duration) PRESTO_TASK_INSTANCE_ID(com.facebook.presto.client.PrestoHeaders.PRESTO_TASK_INSTANCE_ID) TEST_SESSION(com.facebook.presto.SessionTestUtils.TEST_SESSION) Request(io.airlift.http.client.Request) BiConsumer(java.util.function.BiConsumer) TestingResponse(io.airlift.http.client.testing.TestingResponse) QueryManagerConfig(com.facebook.presto.execution.QueryManagerConfig) URI(java.net.URI) ImmutableMultimap(com.google.common.collect.ImmutableMultimap) REMOTE_TASK_MISMATCH(com.facebook.presto.spi.StandardErrorCode.REMOTE_TASK_MISMATCH) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) TestSqlTaskManager(com.facebook.presto.execution.TestSqlTaskManager) Iterables.getOnlyElement(com.google.common.collect.Iterables.getOnlyElement) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) PrestoNode(com.facebook.presto.metadata.PrestoNode) StandardCharsets(java.nio.charset.StandardCharsets) PRESTO_MAX_WAIT(com.facebook.presto.client.PrestoHeaders.PRESTO_MAX_WAIT) String.format(java.lang.String.format) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) RemoteTask(com.facebook.presto.execution.RemoteTask) TaskTestUtils(com.facebook.presto.execution.TaskTestUtils) HttpStatus(io.airlift.http.client.HttpStatus) TaskId(com.facebook.presto.execution.TaskId) HttpRemoteTaskFactory(com.facebook.presto.server.HttpRemoteTaskFactory) ImmutableListMultimap(com.google.common.collect.ImmutableListMultimap) Response(io.airlift.http.client.Response) OutputBuffers.createInitialEmptyOutputBuffers(com.facebook.presto.OutputBuffers.createInitialEmptyOutputBuffers) Assert.assertTrue(org.testng.Assert.assertTrue) TaskUpdateRequest(com.facebook.presto.server.TaskUpdateRequest) TaskInfo(com.facebook.presto.execution.TaskInfo) SECONDS(java.util.concurrent.TimeUnit.SECONDS) JsonCodec(io.airlift.json.JsonCodec) TaskId(com.facebook.presto.execution.TaskId) TaskManagerConfig(com.facebook.presto.execution.TaskManagerConfig) TestSqlTaskManager(com.facebook.presto.execution.TestSqlTaskManager) PrestoNode(com.facebook.presto.metadata.PrestoNode) TaskInfo(com.facebook.presto.execution.TaskInfo) NodeVersion(com.facebook.presto.client.NodeVersion) CompletableFuture(java.util.concurrent.CompletableFuture) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) HttpRemoteTaskFactory(com.facebook.presto.server.HttpRemoteTaskFactory) NodeTaskMap(com.facebook.presto.execution.NodeTaskMap) TaskUpdateRequest(com.facebook.presto.server.TaskUpdateRequest) RemoteTask(com.facebook.presto.execution.RemoteTask) Duration(io.airlift.units.Duration) TaskStatus(com.facebook.presto.execution.TaskStatus) AtomicLong(java.util.concurrent.atomic.AtomicLong) QueryManagerConfig(com.facebook.presto.execution.QueryManagerConfig) Test(org.testng.annotations.Test)

Example 2 with TEST_SESSION

use of com.facebook.presto.SessionTestUtils.TEST_SESSION in project presto by prestodb.

the class TestHashPartitionMaskOperator method testHashPartitionMaskWithMask.

@Test(dataProvider = "hashEnabledValues")
public void testHashPartitionMaskWithMask(boolean hashEnabled) throws Exception {
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT, BOOLEAN, BOOLEAN);
    List<Page> input = rowPagesBuilder.addSequencePage(ROW_COUNT, 0, 0, 1).build();
    OperatorFactory operatorFactory = new HashPartitionMaskOperatorFactory(0, new PlanNodeId("test"), PARTITION_COUNT, rowPagesBuilder.getTypes(), ImmutableList.of(1, 2), ImmutableList.of(0), rowPagesBuilder.getHashChannel());
    int[] rowPartition = new int[ROW_COUNT];
    Arrays.fill(rowPartition, -1);
    for (int partition = 0; partition < PARTITION_COUNT; partition++) {
        MaterializedResult.Builder expected = resultBuilder(TEST_SESSION, BIGINT, BOOLEAN, BOOLEAN, BOOLEAN);
        for (int i = 0; i < ROW_COUNT; i++) {
            long rawHash = BigintOperators.hashCode(i);
            // mix the bits so we don't use the same hash used to distribute between stages
            rawHash = XxHash64.hash(Long.reverse(rawHash));
            rawHash &= Long.MAX_VALUE;
            boolean active = (rawHash % PARTITION_COUNT == partition);
            boolean maskValue = i % 2 == 0;
            expected.row((long) i, active && maskValue, active && !maskValue, active);
            if (active) {
                assertEquals(rowPartition[i], -1);
                rowPartition[i] = partition;
            }
        }
        OperatorAssertion.assertOperatorEqualsIgnoreOrder(operatorFactory, createDriverContext(), input, expected.build(), hashEnabled, Optional.of(3));
    }
    assertTrue(IntStream.of(rowPartition).noneMatch(partition -> partition == -1));
}
Also used : Page(com.facebook.presto.spi.Page) IntStream(java.util.stream.IntStream) Arrays(java.util.Arrays) BigintOperators(com.facebook.presto.type.BigintOperators) DataProvider(org.testng.annotations.DataProvider) MaterializedResult.resultBuilder(com.facebook.presto.testing.MaterializedResult.resultBuilder) Assert.assertEquals(org.testng.Assert.assertEquals) Test(org.testng.annotations.Test) BIGINT(com.facebook.presto.spi.type.BigintType.BIGINT) TEST_SESSION(com.facebook.presto.SessionTestUtils.TEST_SESSION) RowPagesBuilder(com.facebook.presto.RowPagesBuilder) ImmutableList(com.google.common.collect.ImmutableList) BOOLEAN(com.facebook.presto.spi.type.BooleanType.BOOLEAN) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) HashPartitionMaskOperatorFactory(com.facebook.presto.operator.HashPartitionMaskOperator.HashPartitionMaskOperatorFactory) PlanNodeId(com.facebook.presto.sql.planner.plan.PlanNodeId) ExecutorService(java.util.concurrent.ExecutorService) AfterClass(org.testng.annotations.AfterClass) TestingTaskContext.createTaskContext(com.facebook.presto.testing.TestingTaskContext.createTaskContext) BeforeClass(org.testng.annotations.BeforeClass) XxHash64(io.airlift.slice.XxHash64) Ints(com.google.common.primitives.Ints) MaterializedResult(com.facebook.presto.testing.MaterializedResult) List(java.util.List) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) Optional(java.util.Optional) Assert.assertTrue(org.testng.Assert.assertTrue) RowPagesBuilder.rowPagesBuilder(com.facebook.presto.RowPagesBuilder.rowPagesBuilder) RowPagesBuilder(com.facebook.presto.RowPagesBuilder) Page(com.facebook.presto.spi.Page) PlanNodeId(com.facebook.presto.sql.planner.plan.PlanNodeId) HashPartitionMaskOperatorFactory(com.facebook.presto.operator.HashPartitionMaskOperator.HashPartitionMaskOperatorFactory) HashPartitionMaskOperatorFactory(com.facebook.presto.operator.HashPartitionMaskOperator.HashPartitionMaskOperatorFactory) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Test(org.testng.annotations.Test)

Example 3 with TEST_SESSION

use of com.facebook.presto.SessionTestUtils.TEST_SESSION in project presto by prestodb.

the class TestHashPartitionMaskOperator method testHashPartitionMask.

@Test(dataProvider = "hashEnabledValues")
public void testHashPartitionMask(boolean hashEnabled) throws Exception {
    RowPagesBuilder rowPagesBuilder = rowPagesBuilder(hashEnabled, Ints.asList(0), BIGINT);
    List<Page> input = rowPagesBuilder.addSequencePage(ROW_COUNT, 0).build();
    OperatorFactory operatorFactory = new HashPartitionMaskOperatorFactory(0, new PlanNodeId("test"), PARTITION_COUNT, rowPagesBuilder.getTypes(), ImmutableList.of(), ImmutableList.of(0), rowPagesBuilder.getHashChannel());
    int[] rowPartition = new int[ROW_COUNT];
    Arrays.fill(rowPartition, -1);
    for (int partition = 0; partition < PARTITION_COUNT; partition++) {
        MaterializedResult.Builder expected = resultBuilder(TEST_SESSION, BIGINT, BOOLEAN);
        for (int i = 0; i < ROW_COUNT; i++) {
            long rawHash = BigintOperators.hashCode(i);
            // mix the bits so we don't use the same hash used to distribute between stages
            rawHash = XxHash64.hash(Long.reverse(rawHash));
            rawHash &= Long.MAX_VALUE;
            boolean active = (rawHash % PARTITION_COUNT == partition);
            expected.row((long) i, active);
            if (active) {
                assertEquals(rowPartition[i], -1);
                rowPartition[i] = partition;
            }
        }
        OperatorAssertion.assertOperatorEqualsIgnoreOrder(operatorFactory, createDriverContext(), input, expected.build(), hashEnabled, Optional.of(1));
    }
    assertTrue(IntStream.of(rowPartition).noneMatch(partition -> partition == -1));
}
Also used : Page(com.facebook.presto.spi.Page) IntStream(java.util.stream.IntStream) Arrays(java.util.Arrays) BigintOperators(com.facebook.presto.type.BigintOperators) DataProvider(org.testng.annotations.DataProvider) MaterializedResult.resultBuilder(com.facebook.presto.testing.MaterializedResult.resultBuilder) Assert.assertEquals(org.testng.Assert.assertEquals) Test(org.testng.annotations.Test) BIGINT(com.facebook.presto.spi.type.BigintType.BIGINT) TEST_SESSION(com.facebook.presto.SessionTestUtils.TEST_SESSION) RowPagesBuilder(com.facebook.presto.RowPagesBuilder) ImmutableList(com.google.common.collect.ImmutableList) BOOLEAN(com.facebook.presto.spi.type.BooleanType.BOOLEAN) Threads.daemonThreadsNamed(io.airlift.concurrent.Threads.daemonThreadsNamed) HashPartitionMaskOperatorFactory(com.facebook.presto.operator.HashPartitionMaskOperator.HashPartitionMaskOperatorFactory) PlanNodeId(com.facebook.presto.sql.planner.plan.PlanNodeId) ExecutorService(java.util.concurrent.ExecutorService) AfterClass(org.testng.annotations.AfterClass) TestingTaskContext.createTaskContext(com.facebook.presto.testing.TestingTaskContext.createTaskContext) BeforeClass(org.testng.annotations.BeforeClass) XxHash64(io.airlift.slice.XxHash64) Ints(com.google.common.primitives.Ints) MaterializedResult(com.facebook.presto.testing.MaterializedResult) List(java.util.List) Executors.newCachedThreadPool(java.util.concurrent.Executors.newCachedThreadPool) Optional(java.util.Optional) Assert.assertTrue(org.testng.Assert.assertTrue) RowPagesBuilder.rowPagesBuilder(com.facebook.presto.RowPagesBuilder.rowPagesBuilder) RowPagesBuilder(com.facebook.presto.RowPagesBuilder) Page(com.facebook.presto.spi.Page) PlanNodeId(com.facebook.presto.sql.planner.plan.PlanNodeId) HashPartitionMaskOperatorFactory(com.facebook.presto.operator.HashPartitionMaskOperator.HashPartitionMaskOperatorFactory) HashPartitionMaskOperatorFactory(com.facebook.presto.operator.HashPartitionMaskOperator.HashPartitionMaskOperatorFactory) MaterializedResult(com.facebook.presto.testing.MaterializedResult) Test(org.testng.annotations.Test)

Aggregations

TEST_SESSION (com.facebook.presto.SessionTestUtils.TEST_SESSION)3 Assert.assertEquals (org.testng.Assert.assertEquals)3 Assert.assertTrue (org.testng.Assert.assertTrue)3 Test (org.testng.annotations.Test)3 RowPagesBuilder (com.facebook.presto.RowPagesBuilder)2 RowPagesBuilder.rowPagesBuilder (com.facebook.presto.RowPagesBuilder.rowPagesBuilder)2 HashPartitionMaskOperatorFactory (com.facebook.presto.operator.HashPartitionMaskOperator.HashPartitionMaskOperatorFactory)2 Page (com.facebook.presto.spi.Page)2 BIGINT (com.facebook.presto.spi.type.BigintType.BIGINT)2 BOOLEAN (com.facebook.presto.spi.type.BooleanType.BOOLEAN)2 PlanNodeId (com.facebook.presto.sql.planner.plan.PlanNodeId)2 MaterializedResult (com.facebook.presto.testing.MaterializedResult)2 MaterializedResult.resultBuilder (com.facebook.presto.testing.MaterializedResult.resultBuilder)2 TestingTaskContext.createTaskContext (com.facebook.presto.testing.TestingTaskContext.createTaskContext)2 BigintOperators (com.facebook.presto.type.BigintOperators)2 ImmutableList (com.google.common.collect.ImmutableList)2 Ints (com.google.common.primitives.Ints)2 Threads.daemonThreadsNamed (io.airlift.concurrent.Threads.daemonThreadsNamed)2 XxHash64 (io.airlift.slice.XxHash64)2 Arrays (java.util.Arrays)2