Examples with TaskId io.trino.execution.TaskId used on opensource projects

Example 56 with TaskId

use of io.trino.execution.TaskId in project trino by trinodb.

the class TestDirectExchangeClient method testStreamingHappyPath.

@Test
public void testStreamingHappyPath() {
    DataSize maxResponseSize = DataSize.of(10, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
    URI location = URI.create("http://localhost:8080");
    processor.addPage(location, createPage(1));
    processor.addPage(location, createPage(2));
    processor.addPage(location, createPage(3));
    processor.setComplete(location);
    @SuppressWarnings("resource") DirectExchangeClient exchangeClient = new DirectExchangeClient("localhost", DataIntegrityVerification.ABORT, new StreamingDirectExchangeBuffer(scheduler, DataSize.of(32, Unit.MEGABYTE)), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), true, new TestingHttpClient(processor, scheduler), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), pageBufferClientCallbackExecutor, (taskId, failure) -> {
    });
    exchangeClient.addLocation(new TaskId(new StageId("query", 1), 0, 0), location);
    exchangeClient.noMoreLocations();
    assertFalse(exchangeClient.isFinished());
    assertPageEquals(getNextPage(exchangeClient), createPage(1));
    assertFalse(exchangeClient.isFinished());
    assertPageEquals(getNextPage(exchangeClient), createPage(2));
    assertFalse(exchangeClient.isFinished());
    assertPageEquals(getNextPage(exchangeClient), createPage(3));
    assertNull(getNextPage(exchangeClient));
    assertTrue(exchangeClient.isFinished());
    DirectExchangeClientStatus status = exchangeClient.getStatus();
    assertEquals(status.getBufferedPages(), 0);
    // client should have sent only 3 requests: one to get all pages, one to acknowledge and one to get the done signal
    assertStatus(status.getPageBufferClientStatuses().get(0), location, "closed", 3, 3, 3, "not scheduled");
    exchangeClient.close();
}

Example 57 with TaskId

use of io.trino.execution.TaskId in project trino by trinodb.

the class TestDirectExchangeClient method testDeduplicationTaskFailure.

@Test
public void testDeduplicationTaskFailure() throws Exception {
    DataSize maxResponseSize = DataSize.of(10, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
    TaskId attempt0Task1 = new TaskId(new StageId("query", 1), 0, 0);
    TaskId attempt1Task1 = new TaskId(new StageId("query", 1), 1, 0);
    TaskId attempt1Task2 = new TaskId(new StageId("query", 1), 2, 0);
    URI attempt0Task1Location = URI.create("http://localhost:8080/1/0");
    URI attempt1Task1Location = URI.create("http://localhost:8080/1/1");
    URI attempt1Task2Location = URI.create("http://localhost:8080/2/1");
    processor.setFailed(attempt0Task1Location, new RuntimeException("randomfailure"));
    processor.addPage(attempt1Task1Location, createPage(1));
    processor.setComplete(attempt1Task1Location);
    processor.setFailed(attempt1Task2Location, new RuntimeException("randomfailure"));
    DeduplicatingDirectExchangeBuffer buffer = new DeduplicatingDirectExchangeBuffer(scheduler, DataSize.of(1, Unit.MEGABYTE), RetryPolicy.QUERY, new ExchangeManagerRegistry(new ExchangeHandleResolver()), new QueryId("query"), createRandomExchangeId());
    DirectExchangeClient exchangeClient = new DirectExchangeClient("localhost", DataIntegrityVerification.ABORT, buffer, maxResponseSize, 1, new Duration(1, SECONDS), true, new TestingHttpClient(processor, scheduler), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), pageBufferClientCallbackExecutor, (taskId, failure) -> {
    });
    exchangeClient.addLocation(attempt0Task1, attempt0Task1Location);
    assertFalse(tryGetFutureValue(exchangeClient.isBlocked(), 10, MILLISECONDS).isPresent());
    exchangeClient.addLocation(attempt1Task1, attempt1Task1Location);
    exchangeClient.addLocation(attempt1Task2, attempt1Task2Location);
    assertFalse(tryGetFutureValue(exchangeClient.isBlocked(), 10, MILLISECONDS).isPresent());
    exchangeClient.noMoreLocations();
    exchangeClient.isBlocked().get(10, SECONDS);
    assertThatThrownBy(() -> getNextPage(exchangeClient)).hasMessageContaining("Encountered too many errors talking to a worker node");
    assertFalse(exchangeClient.isFinished());
}

Example 58 with TaskId

use of io.trino.execution.TaskId in project trino by trinodb.

the class AbstractOperatorBenchmark method runOnce.

@Override
protected Map<String, Long> runOnce() {
    Session session = testSessionBuilder().setSystemProperty("optimizer.optimize-hash-generation", "true").setTransactionId(this.session.getRequiredTransactionId()).build();
    MemoryPool memoryPool = new MemoryPool(DataSize.of(1, GIGABYTE));
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(DataSize.of(1, GIGABYTE));
    TaskContext taskContext = new QueryContext(new QueryId("test"), DataSize.of(256, MEGABYTE), memoryPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), DataSize.of(256, MEGABYTE), spillSpaceTracker).addTaskContext(new TaskStateMachine(new TaskId(new StageId("query", 0), 0, 0), localQueryRunner.getExecutor()), session, () -> {
    }, false, false);
    CpuTimer cpuTimer = new CpuTimer();
    Map<String, Long> executionStats = execute(taskContext);
    CpuDuration executionTime = cpuTimer.elapsedTime();
    TaskStats taskStats = taskContext.getTaskStats();
    long inputRows = taskStats.getRawInputPositions();
    long inputBytes = taskStats.getRawInputDataSize().toBytes();
    long outputRows = taskStats.getOutputPositions();
    long outputBytes = taskStats.getOutputDataSize().toBytes();
    double inputMegaBytes = ((double) inputBytes) / MEGABYTE.inBytes();
    return ImmutableMap.<String, Long>builder().putAll(executionStats).put("elapsed_millis", executionTime.getWall().toMillis()).put("input_rows_per_second", (long) (inputRows / executionTime.getWall().getValue(SECONDS))).put("output_rows_per_second", (long) (outputRows / executionTime.getWall().getValue(SECONDS))).put("input_megabytes", (long) inputMegaBytes).put("input_megabytes_per_second", (long) (inputMegaBytes / executionTime.getWall().getValue(SECONDS))).put("wall_nanos", executionTime.getWall().roundTo(NANOSECONDS)).put("cpu_nanos", executionTime.getCpu().roundTo(NANOSECONDS)).put("user_nanos", executionTime.getUser().roundTo(NANOSECONDS)).put("input_rows", inputRows).put("input_bytes", inputBytes).put("output_rows", outputRows).put("output_bytes", outputBytes).buildOrThrow();
}

Example 59 with TaskId

use of io.trino.execution.TaskId in project trino by trinodb.

the class MemoryLocalQueryRunner method execute.

public List<Page> execute(@Language("SQL") String query) {
    MemoryPool memoryPool = new MemoryPool(DataSize.of(2, GIGABYTE));
    SpillSpaceTracker spillSpaceTracker = new SpillSpaceTracker(DataSize.of(1, GIGABYTE));
    QueryContext queryContext = new QueryContext(new QueryId("test"), DataSize.of(1, GIGABYTE), memoryPool, new TestingGcMonitor(), localQueryRunner.getExecutor(), localQueryRunner.getScheduler(), DataSize.of(4, GIGABYTE), spillSpaceTracker);
    TaskContext taskContext = queryContext.addTaskContext(new TaskStateMachine(new TaskId(new StageId("query", 0), 0, 0), localQueryRunner.getExecutor()), localQueryRunner.getDefaultSession(), () -> {
    }, false, false);
    // Use NullOutputFactory to avoid coping out results to avoid affecting benchmark results
    ImmutableList.Builder<Page> output = ImmutableList.builder();
    List<Driver> drivers = localQueryRunner.createDrivers(query, new PageConsumerOperator.PageConsumerOutputFactory(types -> output::add), taskContext);
    boolean done = false;
    while (!done) {
        boolean processed = false;
        for (Driver driver : drivers) {
            if (!driver.isFinished()) {
                driver.process();
                processed = true;
            }
        }
        done = !processed;
    }
    return output.build();
}