Examples with SimpleLocalMemoryContext io.prestosql.memory.context.SimpleLocalMemoryContext used on opensource projects

Example 6 with SimpleLocalMemoryContext

use of io.prestosql.memory.context.SimpleLocalMemoryContext in project hetu-core by openlookeng.

the class TestExchangeClient method testBufferLimit.

@Test
public void testBufferLimit() {
    DataSize maxResponseSize = new DataSize(1, Unit.BYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(maxResponseSize);
    URI location = URI.create("http://localhost:8080");
    String instanceId = "testing instance id";
    // add a pages
    processor.addPage(location, createPage(1));
    processor.addPage(location, createPage(2));
    processor.addPage(location, createPage(3));
    processor.setComplete(location);
    @SuppressWarnings("resource") ExchangeClient exchangeClient = new ExchangeClient(new DataSize(1, Unit.BYTE), maxResponseSize, 1, new Duration(1, TimeUnit.MINUTES), true, new TestingHttpClient(processor, newCachedThreadPool(daemonThreadsNamed("test-%s"))), scheduler, new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), pageBufferClientCallbackExecutor, new NoOpFailureDetector());
    exchangeClient.addLocation(new TaskLocation(location, instanceId));
    exchangeClient.noMoreLocations();
    assertEquals(exchangeClient.isClosed(), false);
    long start = System.nanoTime();
    // start fetching pages
    exchangeClient.scheduleRequestIfNecessary();
    // wait for a page to be fetched
    do {
        // there is no thread coordination here, so sleep is the best we can do
        assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
        sleepUninterruptibly(100, MILLISECONDS);
    } while (exchangeClient.getStatus().getBufferedPages() == 0);
    // client should have sent a single request for a single page
    assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
    assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 1, 1, 1, "not scheduled");
    // remove the page and wait for the client to fetch another page
    assertPageEquals(exchangeClient.pollPage(null).getLeft(), createPage(1));
    do {
        assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
        sleepUninterruptibly(100, MILLISECONDS);
    } while (exchangeClient.getStatus().getBufferedPages() == 0);
    // client should have sent a single request for a single page
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 2, 2, 2, "not scheduled");
    assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
    assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);
    // remove the page and wait for the client to fetch another page
    assertPageEquals(exchangeClient.pollPage(null).getLeft(), createPage(2));
    do {
        assertLessThan(Duration.nanosSince(start), new Duration(5, TimeUnit.SECONDS));
        sleepUninterruptibly(100, MILLISECONDS);
    } while (exchangeClient.getStatus().getBufferedPages() == 0);
    // client should have sent a single request for a single page
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "queued", 3, 3, 3, "not scheduled");
    assertEquals(exchangeClient.getStatus().getBufferedPages(), 1);
    assertTrue(exchangeClient.getStatus().getBufferedBytes() > 0);
    // remove last page
    assertPageEquals(getNextPage(exchangeClient), createPage(3));
    // wait for client to decide there are no more pages
    assertNull(getNextPage(exchangeClient));
    assertEquals(exchangeClient.getStatus().getBufferedPages(), 0);
    assertTrue(exchangeClient.getStatus().getBufferedBytes() == 0);
    assertEquals(exchangeClient.isClosed(), true);
    assertStatus(exchangeClient.getStatus().getPageBufferClientStatuses().get(0), location, "closed", 3, 5, 5, "not scheduled");
}

Example 7 with SimpleLocalMemoryContext

use of io.prestosql.memory.context.SimpleLocalMemoryContext in project hetu-core by openlookeng.

the class TestArbitraryOutputBuffer method createArbitraryBuffer.

private ArbitraryOutputBuffer createArbitraryBuffer(OutputBuffers buffers, DataSize dataSize) {
    ArbitraryOutputBuffer buffer = new ArbitraryOutputBuffer(new StateMachine<>("bufferState", stateNotificationExecutor, OPEN, TERMINAL_BUFFER_STATES), dataSize, () -> new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), stateNotificationExecutor);
    buffer.setOutputBuffers(buffers);
    return buffer;
}

Example 8 with SimpleLocalMemoryContext

use of io.prestosql.memory.context.SimpleLocalMemoryContext in project hetu-core by openlookeng.

the class TestBroadcastOutputBuffer method createBroadcastBuffer.

private BroadcastOutputBuffer createBroadcastBuffer(OutputBuffers outputBuffers, DataSize dataSize) {
    BroadcastOutputBuffer buffer = new BroadcastOutputBuffer(new StateMachine<>("bufferState", stateNotificationExecutor, OPEN, TERMINAL_BUFFER_STATES), dataSize, () -> new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), stateNotificationExecutor);
    buffer.setOutputBuffers(outputBuffers);
    return buffer;
}

Example 9 with SimpleLocalMemoryContext

use of io.prestosql.memory.context.SimpleLocalMemoryContext in project hetu-core by openlookeng.

the class TestArbitraryOutputBuffer method createOutputBuffer.

private OutputBuffer createOutputBuffer(OutputBuffers.BufferType bufferType) {
    OutputBuffers buffers = createInitialEmptyOutputBuffers(bufferType).withBuffer(FIRST, BROADCAST_PARTITION_ID).withNoMoreBufferIds();
    OutputBuffer buffer;
    switch(bufferType) {
        case ARBITRARY:
            buffer = new ArbitraryOutputBuffer(new StateMachine<>("bufferState", stateNotificationExecutor, OPEN, TERMINAL_BUFFER_STATES), sizeOfPages(5), () -> new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), stateNotificationExecutor);
            buffer.setOutputBuffers(buffers);
            break;
        case BROADCAST:
            buffer = new BroadcastOutputBuffer(new StateMachine<>("bufferState", stateNotificationExecutor, OPEN, TERMINAL_BUFFER_STATES), sizeOfPages(5), () -> new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), stateNotificationExecutor);
            buffer.setOutputBuffers(buffers);
            break;
        case PARTITIONED:
            buffer = new PartitionedOutputBuffer(new StateMachine<>("bufferState", stateNotificationExecutor, OPEN, TERMINAL_BUFFER_STATES), buffers, sizeOfPages(5), () -> new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), "test"), stateNotificationExecutor);
            break;
        default:
            throw new RuntimeException("Unexpected bufferType: " + bufferType);
    }
    return buffer;
}

Example 10 with SimpleLocalMemoryContext

use of io.prestosql.memory.context.SimpleLocalMemoryContext in project hetu-core by openlookeng.

the class PagePublisherQueryRunner method getQuery.

private Query getQuery(QueryId queryId, String slug) {
    // this is the first time the query has been accessed on this coordinator
    Session session;
    try {
        if (!queryManager.isQuerySlugValid(queryId, slug)) {
            throw badRequest(NOT_FOUND, "Query not found");
        }
        session = queryManager.getQuerySession(queryId);
    } catch (NoSuchElementException e) {
        throw badRequest(NOT_FOUND, "Query not found");
    }
    ExchangeClient exchangeClient = this.exchangeClientSupplier.get(new SimpleLocalMemoryContext(newSimpleAggregatedMemoryContext(), DataCenterStatementResource.class.getSimpleName()));
    return Query.create(session, slug, queryManager, exchangeClient, executor, timeoutExecutor, blockEncodingSerde);
}