Examples with SimpleLocalMemoryContext io.prestosql.memory.context.SimpleLocalMemoryContext used on opensource projects

Example 1 with SimpleLocalMemoryContext

use of io.prestosql.memory.context.SimpleLocalMemoryContext in project hetu-core by openlookeng.

the class ExecutingStatementResource method getQuery.

protected Query getQuery(QueryId queryId, String slug) {
    Query query = queries.get(queryId);
    if (query != null) {
        if (!query.isSlugValid(slug)) {
            throw badRequest(NOT_FOUND, "Query not found");
        }
        return query;
    }
    // this is the first time the query has been accessed on this coordinator
    Session session;
    try {
        if (!queryManager.isQuerySlugValid(queryId, slug)) {
            throw badRequest(NOT_FOUND, "Query not found");
        }
        session = queryManager.getQuerySession(queryId);
    } catch (NoSuchElementException e) {
        throw badRequest(NOT_FOUND, "Query not found");
    }
    query = queries.computeIfAbsent(queryId, id -> {
        ExchangeClient exchangeClient = exchangeClientSupplier.get(new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), ExecutingStatementResource.class.getSimpleName()));
        if (SystemSessionProperties.isSnapshotEnabled(session)) {
            exchangeClient.setSnapshotEnabled(snapshotUtils.getOrCreateQuerySnapshotManager(queryId, session));
        }
        return Query.create(session, slug, queryManager, exchangeClient, responseExecutor, timeoutExecutor, blockEncodingSerde);
    });
    return query;
}

Example 2 with SimpleLocalMemoryContext

use of io.prestosql.memory.context.SimpleLocalMemoryContext in project hetu-core by openlookeng.

the class TestExchangeClient method testMarkers.

@Test
public void testMarkers() {
    DataSize maxResponseSize = new DataSize(10, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
    URI location = URI.create("http://localhost:8080");
    String instanceId = "testing instance id";
    processor.addPage(location, createPage(1));
    MarkerPage marker = MarkerPage.snapshotPage(1);
    processor.addPage(location, marker);
    processor.addPage(location, createPage(2));
    processor.setComplete(location);
    @SuppressWarnings("resource") ExchangeClient exchangeClient = new ExchangeClient(new DataSize(32, Unit.MEGABYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), true, new TestingHttpClient(processor, scheduler), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), pageBufferClientCallbackExecutor, new NoOpFailureDetector());
    exchangeClient.setSnapshotEnabled(NOOP_SNAPSHOT_UTILS.getQuerySnapshotManager(new QueryId("query")));
    final String target1 = "target1";
    final String target2 = "target2";
    exchangeClient.addLocation(new TaskLocation(location, instanceId));
    exchangeClient.noMoreLocations();
    exchangeClient.addTarget(target1);
    exchangeClient.addTarget(target2);
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient, target1), createPage(1));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient, target2), marker);
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient, target2), createPage(2));
    assertEquals(exchangeClient.isClosed(), false);
    // Target2 won't get any page, but because there are pending markers, it won't cause the client to close either
    assertTrue(exchangeClient.isBlocked().isDone());
    assertNull(getNextPage(exchangeClient, target2));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient, target1), marker);
    // New target receives pending marker
    assertEquals(exchangeClient.isClosed(), false);
    final String target3 = "target3";
    exchangeClient.addTarget(target3);
    assertPageEquals(getNextPage(exchangeClient, target3), marker);
    assertFalse(exchangeClient.isBlocked().isDone());
    assertEquals(exchangeClient.isClosed(), false);
    exchangeClient.noMoreTargets();
    assertNull(getNextPage(exchangeClient));
    assertEquals(exchangeClient.isClosed(), true);
    ExchangeClientStatus status = exchangeClient.getStatus();
    assertEquals(status.getBufferedPages(), 0);
    assertEquals(status.getBufferedBytes(), 0);
    // client should have sent only 2 requests: one to get all pages and once to get the done signal
    assertStatus(status.getPageBufferClientStatuses().get(0), location, "closed", 3, 3, 3, "not scheduled");
}

Example 3 with SimpleLocalMemoryContext

use of io.prestosql.memory.context.SimpleLocalMemoryContext in project hetu-core by openlookeng.

the class TestExchangeClient method testClose.

@Test
public void testClose() throws Exception {
    DataSize maxResponseSize = new DataSize(1, Unit.BYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
    URI location = URI.create("http://localhost:8080");
    String instanceId = "testing instance id";
    processor.addPage(location, createPage(1));
    processor.addPage(location, createPage(2));
    processor.addPage(location, createPage(3));
    @SuppressWarnings("resource") ExchangeClient exchangeClient = new ExchangeClient(new DataSize(1, Unit.BYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), true, new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), pageBufferClientCallbackExecutor, new NoOpFailureDetector());
    exchangeClient.addLocation(new TaskLocation(location, instanceId));
    exchangeClient.noMoreLocations();
    // fetch a page
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient), createPage(1));
    // close client while pages are still available
    exchangeClient.close();
    while (!exchangeClient.isFinished()) {
        MILLISECONDS.sleep(10);
    }
    assertEquals(exchangeClient.isClosed(), true);
    Pair<SerializedPage, String> pair = exchangeClient.pollPage(null);
    assertNotNull(pair);
    assertNull(pair.getLeft());
    assertEquals(exchangeClient.getStatus().getBufferedPages(), 0);
    assertEquals(exchangeClient.getStatus().getBufferedBytes(), 0);
    // client should have sent only 2 requests: one to get all pages and once to get the done signal
    PageBufferClientStatus clientStatus = exchangeClient.getStatus().getPageBufferClientStatuses().get(0);
    assertEquals(clientStatus.getUri(), location);
    assertEquals(clientStatus.getState(), "closed", "status");
    assertEquals(clientStatus.getHttpRequestState(), "not scheduled", "httpRequestState");
}

Example 4 with SimpleLocalMemoryContext

use of io.prestosql.memory.context.SimpleLocalMemoryContext in project hetu-core by openlookeng.

the class TestExchangeClient method testAddLocation.

@Test(timeOut = 10000)
public void testAddLocation() throws Exception {
    DataSize maxResponseSize = new DataSize(10, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
    @SuppressWarnings("resource") ExchangeClient exchangeClient = new ExchangeClient(new DataSize(32, Unit.MEGABYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), true, new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), pageBufferClientCallbackExecutor, new NoOpFailureDetector());
    URI location1 = URI.create("http://localhost:8081/foo");
    String instanceId1 = "testing instance id";
    processor.addPage(location1, createPage(1));
    processor.addPage(location1, createPage(2));
    processor.addPage(location1, createPage(3));
    processor.setComplete(location1);
    exchangeClient.addLocation(new TaskLocation(location1, instanceId1));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient), createPage(1));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient), createPage(2));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient), createPage(3));
    assertFalse(tryGetFutureValue(exchangeClient.isBlocked(), 10, MILLISECONDS).isPresent());
    assertEquals(exchangeClient.isClosed(), false);
    URI location2 = URI.create("http://localhost:8082/bar");
    String instanceId2 = "testing instance id2";
    processor.addPage(location2, createPage(4));
    processor.addPage(location2, createPage(5));
    processor.addPage(location2, createPage(6));
    processor.setComplete(location2);
    exchangeClient.addLocation(new TaskLocation(location2, instanceId2));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient), createPage(4));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient), createPage(5));
    assertEquals(exchangeClient.isClosed(), false);
    assertPageEquals(getNextPage(exchangeClient), createPage(6));
    assertFalse(tryGetFutureValue(exchangeClient.isBlocked(), 10, MILLISECONDS).isPresent());
    assertEquals(exchangeClient.isClosed(), false);
    exchangeClient.noMoreLocations();
    // receive a final GONE response, so just spin until it's closed or the test times out.
    while (!exchangeClient.isClosed()) {
        Thread.sleep(1);
    }
    ImmutableMap<URI, PageBufferClientStatus> statuses = uniqueIndex(exchangeClient.getStatus().getPageBufferClientStatuses(), PageBufferClientStatus::getUri);
    assertStatus(statuses.get(location1), location1, "closed", 3, 3, 3, "not scheduled");
    assertStatus(statuses.get(location2), location2, "closed", 3, 3, 3, "not scheduled");
}

Example 5 with SimpleLocalMemoryContext

use of io.prestosql.memory.context.SimpleLocalMemoryContext in project hetu-core by openlookeng.

the class TestExchangeClient method testAddTarget.

@Test
public void testAddTarget() {
    @SuppressWarnings("resource") ExchangeClient exchangeClient = new ExchangeClient(new DataSize(32, Unit.MEGABYTE), new DataSize(10, Unit.MEGABYTE), 1, new Duration(1, TimeUnit.MINUTES), true, mock(HttpClient.class), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), pageBufferClientCallbackExecutor, new NoOpFailureDetector());
    exchangeClient.setSnapshotEnabled(NOOP_SNAPSHOT_UTILS.getQuerySnapshotManager(new QueryId("query")));
    String origin1 = "location1";
    String origin2 = "location2";
    final String target1 = "target1";
    final String target2 = "target2";
    PagesSerde serde = testingPagesSerde();
    exchangeClient.addTarget(target1);
    exchangeClient.addPages(ImmutableList.of(serde.serialize(createPage(1))), origin1);
    MarkerPage marker = MarkerPage.snapshotPage(1);
    exchangeClient.addPages(ImmutableList.of(SerializedPage.forMarker(marker)), origin1);
    exchangeClient.addPages(ImmutableList.of(serde.serialize(createPage(2))), origin2);
    assertPageEquals(getNextPage(exchangeClient, target1), createPage(1));
    assertPageEquals(getNextPage(exchangeClient, target1), marker);
    exchangeClient.addTarget(target2);
    assertPageEquals(getNextPage(exchangeClient, target2, origin1), marker);
    assertPageEquals(getNextPage(exchangeClient, target1, origin2), createPage(2));
}