Search in sources :

Example 6 with ExchangeManagerRegistry

use of io.trino.exchange.ExchangeManagerRegistry in project trino by trinodb.

the class TestDirectExchangeClient method testDeduplication.

@Test
public void testDeduplication() throws Exception {
    DataSize maxResponseSize = DataSize.of(10, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
    TaskId taskP0A0 = new TaskId(new StageId("query", 1), 0, 0);
    TaskId taskP1A0 = new TaskId(new StageId("query", 1), 1, 0);
    TaskId taskP0A1 = new TaskId(new StageId("query", 1), 0, 1);
    URI locationP0A0 = URI.create("http://localhost:8080/1");
    URI locationP1A0 = URI.create("http://localhost:8080/2");
    URI locationP0A1 = URI.create("http://localhost:8080/3");
    processor.addPage(locationP1A0, createSerializedPage(1));
    processor.addPage(locationP0A1, createSerializedPage(2));
    processor.addPage(locationP0A1, createSerializedPage(3));
    @SuppressWarnings("resource") DirectExchangeClient exchangeClient = new DirectExchangeClient("localhost", DataIntegrityVerification.ABORT, new DeduplicatingDirectExchangeBuffer(scheduler, DataSize.of(1, Unit.KILOBYTE), RetryPolicy.QUERY, new ExchangeManagerRegistry(new ExchangeHandleResolver()), new QueryId("query"), createRandomExchangeId()), maxResponseSize, 1, new Duration(1, SECONDS), true, new TestingHttpClient(processor, scheduler), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), pageBufferClientCallbackExecutor, (taskId, failure) -> {
    });
    exchangeClient.addLocation(taskP0A0, locationP0A0);
    exchangeClient.addLocation(taskP1A0, locationP1A0);
    exchangeClient.addLocation(taskP0A1, locationP0A1);
    processor.setComplete(locationP0A0);
    // Failing attempt 0. Results from all tasks for attempt 0 must be discarded.
    processor.setFailed(locationP1A0, new RuntimeException("failure"));
    processor.setComplete(locationP0A1);
    assertFalse(exchangeClient.isFinished());
    assertThatThrownBy(() -> exchangeClient.isBlocked().get(50, MILLISECONDS)).isInstanceOf(TimeoutException.class);
    exchangeClient.noMoreLocations();
    exchangeClient.isBlocked().get(10, SECONDS);
    List<Page> pages = new ArrayList<>();
    while (!exchangeClient.isFinished()) {
        Slice page = exchangeClient.pollPage();
        if (page == null) {
            break;
        }
        pages.add(PAGES_SERDE.deserialize(page));
    }
    assertThat(pages).hasSize(2);
    assertThat(pages.stream().map(Page::getPositionCount).collect(toImmutableSet())).containsAll(ImmutableList.of(2, 3));
    assertEventually(() -> assertTrue(exchangeClient.isFinished()));
    assertEventually(() -> {
        assertEquals(exchangeClient.getStatus().getPageBufferClientStatuses().get(0).getHttpRequestState(), "not scheduled", "httpRequestState");
        assertEquals(exchangeClient.getStatus().getPageBufferClientStatuses().get(1).getHttpRequestState(), "not scheduled", "httpRequestState");
        assertEquals(exchangeClient.getStatus().getPageBufferClientStatuses().get(2).getHttpRequestState(), "not scheduled", "httpRequestState");
    });
    exchangeClient.close();
}
Also used : TaskId(io.trino.execution.TaskId) SimpleLocalMemoryContext(io.trino.memory.context.SimpleLocalMemoryContext) StageId(io.trino.execution.StageId) QueryId(io.trino.spi.QueryId) ArrayList(java.util.ArrayList) Duration(io.airlift.units.Duration) Page(io.trino.spi.Page) URI(java.net.URI) ExchangeManagerRegistry(io.trino.exchange.ExchangeManagerRegistry) ExchangeHandleResolver(io.trino.metadata.ExchangeHandleResolver) Slice(io.airlift.slice.Slice) DataSize(io.airlift.units.DataSize) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) Test(org.testng.annotations.Test)

Example 7 with ExchangeManagerRegistry

use of io.trino.exchange.ExchangeManagerRegistry in project trino by trinodb.

the class TestMergeOperator method setUp.

@BeforeMethod
public void setUp() {
    executor = newSingleThreadScheduledExecutor(daemonThreadsNamed("test-merge-operator-%s"));
    serdeFactory = new TestingPagesSerdeFactory();
    taskBuffers = buildNonEvictableCache(CacheBuilder.newBuilder(), CacheLoader.from(TestingTaskBuffer::new));
    httpClient = new TestingHttpClient(new TestingExchangeHttpClientHandler(taskBuffers), executor);
    exchangeClientFactory = new DirectExchangeClientFactory(new NodeInfo("test"), new FeaturesConfig(), new DirectExchangeClientConfig(), httpClient, executor, new ExchangeManagerRegistry(new ExchangeHandleResolver()));
    orderingCompiler = new OrderingCompiler(new TypeOperators());
}
Also used : TestingPagesSerdeFactory(io.trino.execution.buffer.TestingPagesSerdeFactory) NodeInfo(io.airlift.node.NodeInfo) FeaturesConfig(io.trino.FeaturesConfig) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) OrderingCompiler(io.trino.sql.gen.OrderingCompiler) ExchangeManagerRegistry(io.trino.exchange.ExchangeManagerRegistry) ExchangeHandleResolver(io.trino.metadata.ExchangeHandleResolver) TypeOperators(io.trino.spi.type.TypeOperators) BeforeMethod(org.testng.annotations.BeforeMethod)

Example 8 with ExchangeManagerRegistry

use of io.trino.exchange.ExchangeManagerRegistry in project trino by trinodb.

the class TestExchangeOperator method createExchangeOperator.

private SourceOperator createExchangeOperator() {
    ExchangeOperatorFactory operatorFactory = new ExchangeOperatorFactory(0, new PlanNodeId("test"), directExchangeClientSupplier, SERDE_FACTORY, RetryPolicy.NONE, new ExchangeManagerRegistry(new ExchangeHandleResolver()));
    DriverContext driverContext = createTaskContext(scheduler, scheduledExecutor, TEST_SESSION).addPipelineContext(0, true, true, false).addDriverContext();
    SourceOperator operator = operatorFactory.createOperator(driverContext);
    assertEquals(getOnlyElement(operator.getOperatorContext().getNestedOperatorStats()).getUserMemoryReservation().toBytes(), 0);
    return operator;
}
Also used : PlanNodeId(io.trino.sql.planner.plan.PlanNodeId) ExchangeOperatorFactory(io.trino.operator.ExchangeOperator.ExchangeOperatorFactory) ExchangeManagerRegistry(io.trino.exchange.ExchangeManagerRegistry) ExchangeHandleResolver(io.trino.metadata.ExchangeHandleResolver)

Example 9 with ExchangeManagerRegistry

use of io.trino.exchange.ExchangeManagerRegistry in project trino by trinodb.

the class TestDirectExchangeClient method testDeduplicationTaskFailure.

@Test
public void testDeduplicationTaskFailure() throws Exception {
    DataSize maxResponseSize = DataSize.of(10, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
    TaskId attempt0Task1 = new TaskId(new StageId("query", 1), 0, 0);
    TaskId attempt1Task1 = new TaskId(new StageId("query", 1), 1, 0);
    TaskId attempt1Task2 = new TaskId(new StageId("query", 1), 2, 0);
    URI attempt0Task1Location = URI.create("http://localhost:8080/1/0");
    URI attempt1Task1Location = URI.create("http://localhost:8080/1/1");
    URI attempt1Task2Location = URI.create("http://localhost:8080/2/1");
    processor.setFailed(attempt0Task1Location, new RuntimeException("randomfailure"));
    processor.addPage(attempt1Task1Location, createPage(1));
    processor.setComplete(attempt1Task1Location);
    processor.setFailed(attempt1Task2Location, new RuntimeException("randomfailure"));
    DeduplicatingDirectExchangeBuffer buffer = new DeduplicatingDirectExchangeBuffer(scheduler, DataSize.of(1, Unit.MEGABYTE), RetryPolicy.QUERY, new ExchangeManagerRegistry(new ExchangeHandleResolver()), new QueryId("query"), createRandomExchangeId());
    DirectExchangeClient exchangeClient = new DirectExchangeClient("localhost", DataIntegrityVerification.ABORT, buffer, maxResponseSize, 1, new Duration(1, SECONDS), true, new TestingHttpClient(processor, scheduler), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), pageBufferClientCallbackExecutor, (taskId, failure) -> {
    });
    exchangeClient.addLocation(attempt0Task1, attempt0Task1Location);
    assertFalse(tryGetFutureValue(exchangeClient.isBlocked(), 10, MILLISECONDS).isPresent());
    exchangeClient.addLocation(attempt1Task1, attempt1Task1Location);
    exchangeClient.addLocation(attempt1Task2, attempt1Task2Location);
    assertFalse(tryGetFutureValue(exchangeClient.isBlocked(), 10, MILLISECONDS).isPresent());
    exchangeClient.noMoreLocations();
    exchangeClient.isBlocked().get(10, SECONDS);
    assertThatThrownBy(() -> getNextPage(exchangeClient)).hasMessageContaining("Encountered too many errors talking to a worker node");
    assertFalse(exchangeClient.isFinished());
}
Also used : TaskId(io.trino.execution.TaskId) SimpleLocalMemoryContext(io.trino.memory.context.SimpleLocalMemoryContext) StageId(io.trino.execution.StageId) QueryId(io.trino.spi.QueryId) Duration(io.airlift.units.Duration) URI(java.net.URI) ExchangeManagerRegistry(io.trino.exchange.ExchangeManagerRegistry) ExchangeHandleResolver(io.trino.metadata.ExchangeHandleResolver) DataSize(io.airlift.units.DataSize) TestingHttpClient(io.airlift.http.client.testing.TestingHttpClient) Test(org.testng.annotations.Test)

Aggregations

ExchangeManagerRegistry (io.trino.exchange.ExchangeManagerRegistry)9 ExchangeHandleResolver (io.trino.metadata.ExchangeHandleResolver)9 QueryId (io.trino.spi.QueryId)4 TestingHttpClient (io.airlift.http.client.testing.TestingHttpClient)3 TaskId (io.trino.execution.TaskId)3 URI (java.net.URI)3 Slice (io.airlift.slice.Slice)2 CounterStat (io.airlift.stats.CounterStat)2 DataSize (io.airlift.units.DataSize)2 Duration (io.airlift.units.Duration)2 StageId (io.trino.execution.StageId)2 QueryContext (io.trino.memory.QueryContext)2 SimpleLocalMemoryContext (io.trino.memory.context.SimpleLocalMemoryContext)2 Test (org.testng.annotations.Test)2 ImmutableList (com.google.common.collect.ImmutableList)1 ImmutableMap (com.google.common.collect.ImmutableMap)1 ObjectMapperProvider (io.airlift.json.ObjectMapperProvider)1 NodeInfo (io.airlift.node.NodeInfo)1 Slices.utf8Slice (io.airlift.slice.Slices.utf8Slice)1 TestingGcMonitor (io.airlift.stats.TestingGcMonitor)1