Examples with Duration io.airlift.units.Duration used on opensource projects

Example 21 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class TestHttpPageBufferClient method testExceptionFromResponseHandler.

@Test
public void testExceptionFromResponseHandler() throws Exception {
    TestingTicker ticker = new TestingTicker();
    AtomicReference<Duration> tickerIncrement = new AtomicReference<>(new Duration(0, TimeUnit.SECONDS));
    TestingHttpClient.Processor processor = (input) -> {
        Duration delta = tickerIncrement.get();
        ticker.increment(delta.toMillis(), TimeUnit.MILLISECONDS);
        throw new RuntimeException("Foo");
    };
    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);
    URI location = URI.create("http://localhost:8080");
    HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, executor), new DataSize(10, Unit.MEGABYTE), new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), location, callback, executor, ticker);
    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");
    // request processor will throw exception, verify the request is marked a completed
    // this starts the error stopwatch
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 0);
    assertStatus(client, location, "queued", 0, 1, 1, 1, "not scheduled");
    // advance time forward, but not enough to fail the client
    tickerIncrement.set(new Duration(30, TimeUnit.SECONDS));
    // verify that the client has not failed
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 2);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 0);
    assertStatus(client, location, "queued", 0, 2, 2, 2, "not scheduled");
    // advance time forward beyond the minimum error duration
    tickerIncrement.set(new Duration(31, TimeUnit.SECONDS));
    // verify that the client has failed
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 3);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 1);
    assertInstanceOf(callback.getFailure(), PageTransportTimeoutException.class);
    assertContains(callback.getFailure().getMessage(), WORKER_NODE_ERROR + " (http://localhost:8080/0 - 3 failures, time since last success 61.00s)");
    assertStatus(client, location, "queued", 0, 3, 3, 3, "not scheduled");
}

Example 22 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class TestHttpPageBufferClient method testHappyPath.

@Test
public void testHappyPath() throws Exception {
    Page expectedPage = new Page(100);
    DataSize expectedMaxSize = new DataSize(11, Unit.MEGABYTE);
    MockExchangeRequestProcessor processor = new MockExchangeRequestProcessor(expectedMaxSize);
    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);
    URI location = URI.create("http://localhost:8080");
    HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, executor), expectedMaxSize, new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), location, callback, executor);
    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");
    // fetch a page and verify
    processor.addPage(location, expectedPage);
    callback.resetStats();
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 1);
    assertPageEquals(expectedPage, callback.getPages().get(0));
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertStatus(client, location, "queued", 1, 1, 1, 0, "not scheduled");
    // fetch no data and verify
    callback.resetStats();
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertStatus(client, location, "queued", 1, 2, 2, 0, "not scheduled");
    // fetch two more pages and verify
    processor.addPage(location, expectedPage);
    processor.addPage(location, expectedPage);
    callback.resetStats();
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getPages().size(), 2);
    assertPageEquals(expectedPage, callback.getPages().get(0));
    assertPageEquals(expectedPage, callback.getPages().get(1));
    assertEquals(callback.getCompletedRequests(), 1);
    assertEquals(callback.getFinishedBuffers(), 0);
    assertEquals(callback.getFailedBuffers(), 0);
    callback.resetStats();
    assertStatus(client, location, "queued", 3, 3, 3, 0, "not scheduled");
    // finish and verify
    callback.resetStats();
    processor.setComplete(location);
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    // get the buffer complete signal
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 1);
    // schedule the delete call to the buffer
    callback.resetStats();
    client.scheduleRequest();
    requestComplete.await(10, TimeUnit.SECONDS);
    assertEquals(callback.getFinishedBuffers(), 1);
    assertEquals(callback.getPages().size(), 0);
    assertEquals(callback.getCompletedRequests(), 0);
    assertEquals(callback.getFailedBuffers(), 0);
    assertStatus(client, location, "closed", 3, 5, 5, 0, "not scheduled");
}

Example 23 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class TestHttpPageBufferClient method testCloseDuringPendingRequest.

@Test
public void testCloseDuringPendingRequest() throws Exception {
    CyclicBarrier beforeRequest = new CyclicBarrier(2);
    CyclicBarrier afterRequest = new CyclicBarrier(2);
    StaticRequestProcessor processor = new StaticRequestProcessor(beforeRequest, afterRequest);
    processor.setResponse(new TestingResponse(HttpStatus.NO_CONTENT, ImmutableListMultimap.of(), new byte[0]));
    CyclicBarrier requestComplete = new CyclicBarrier(2);
    TestingClientCallback callback = new TestingClientCallback(requestComplete);
    URI location = URI.create("http://localhost:8080");
    HttpPageBufferClient client = new HttpPageBufferClient(new TestingHttpClient(processor, executor), new DataSize(10, Unit.MEGABYTE), new Duration(1, TimeUnit.MINUTES), new Duration(1, TimeUnit.MINUTES), location, callback, executor);
    assertStatus(client, location, "queued", 0, 0, 0, 0, "not scheduled");
    // send request
    client.scheduleRequest();
    beforeRequest.await(10, TimeUnit.SECONDS);
    assertStatus(client, location, "running", 0, 1, 0, 0, "PROCESSING_REQUEST");
    assertEquals(client.isRunning(), true);
    // request is pending, now close it
    client.close();
    try {
        requestComplete.await(10, TimeUnit.SECONDS);
    } catch (BrokenBarrierException ignored) {
    }
    try {
        afterRequest.await(10, TimeUnit.SECONDS);
    } catch (BrokenBarrierException ignored) {
        afterRequest.reset();
    }
    // client.close() triggers a DELETE request, so wait for it to finish
    beforeRequest.await(10, TimeUnit.SECONDS);
    afterRequest.await(10, TimeUnit.SECONDS);
    requestComplete.await(10, TimeUnit.SECONDS);
    assertStatus(client, location, "closed", 0, 1, 2, 1, "not scheduled");
}

Example 24 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class QueryStateMachine method transitionToFinishing.

public boolean transitionToFinishing() {
    Duration durationSinceCreation = nanosSince(createNanos).convertToMostSuccinctTimeUnit();
    queuedTime.compareAndSet(null, durationSinceCreation);
    totalPlanningStartNanos.compareAndSet(null, tickerNanos());
    totalPlanningTime.compareAndSet(null, nanosSince(totalPlanningStartNanos.get()));
    DateTime now = DateTime.now();
    executionStartTime.compareAndSet(null, now);
    finishingStartNanos.compareAndSet(null, tickerNanos());
    if (!queryState.setIf(FINISHING, currentState -> currentState != FINISHING && !currentState.isDone())) {
        return false;
    }
    if (autoCommit) {
        ListenableFuture<?> commitFuture = transactionManager.asyncCommit(session.getTransactionId().get());
        Futures.addCallback(commitFuture, new FutureCallback<Object>() {

            @Override
            public void onSuccess(@Nullable Object result) {
                transitionToFinished();
            }

            @Override
            public void onFailure(Throwable throwable) {
                transitionToFailed(throwable);
            }
        });
    } else {
        transitionToFinished();
    }
    return true;
}

Example 25 with Duration

use of io.airlift.units.Duration in project presto by prestodb.

the class ClusterMemoryManager method process.

public synchronized void process(Iterable<QueryExecution> queries) {
    if (!enabled) {
        return;
    }
    boolean outOfMemory = isClusterOutOfMemory();
    if (!outOfMemory) {
        lastTimeNotOutOfMemory = System.nanoTime();
    }
    boolean queryKilled = false;
    long totalBytes = 0;
    for (QueryExecution query : queries) {
        long bytes = query.getTotalMemoryReservation();
        DataSize sessionMaxQueryMemory = getQueryMaxMemory(query.getSession());
        long queryMemoryLimit = Math.min(maxQueryMemory.toBytes(), sessionMaxQueryMemory.toBytes());
        totalBytes += bytes;
        if (resourceOvercommit(query.getSession()) && outOfMemory) {
            // If a query has requested resource overcommit, only kill it if the cluster has run out of memory
            DataSize memory = succinctBytes(bytes);
            query.fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, format("The cluster is out of memory and %s=true, so this query was killed. It was using %s of memory", RESOURCE_OVERCOMMIT, memory)));
            queryKilled = true;
        }
        if (!resourceOvercommit(query.getSession()) && bytes > queryMemoryLimit) {
            DataSize maxMemory = succinctBytes(queryMemoryLimit);
            query.fail(exceededGlobalLimit(maxMemory));
            queryKilled = true;
        }
    }
    clusterMemoryUsageBytes.set(totalBytes);
    if (killOnOutOfMemory) {
        boolean shouldKillQuery = nanosSince(lastTimeNotOutOfMemory).compareTo(killOnOutOfMemoryDelay) > 0 && outOfMemory;
        boolean lastKilledQueryIsGone = (lastKilledQuery == null);
        if (!lastKilledQueryIsGone) {
            ClusterMemoryPool generalPool = pools.get(GENERAL_POOL);
            if (generalPool != null) {
                lastKilledQueryIsGone = generalPool.getQueryMemoryReservations().containsKey(lastKilledQuery);
            }
        }
        if (shouldKillQuery && lastKilledQueryIsGone && !queryKilled) {
            // Kill the biggest query in the general pool
            QueryExecution biggestQuery = null;
            long maxMemory = -1;
            for (QueryExecution query : queries) {
                long bytesUsed = query.getTotalMemoryReservation();
                if (bytesUsed > maxMemory && query.getMemoryPool().getId().equals(GENERAL_POOL)) {
                    biggestQuery = query;
                    maxMemory = bytesUsed;
                }
            }
            if (biggestQuery != null) {
                biggestQuery.fail(new PrestoException(CLUSTER_OUT_OF_MEMORY, "The cluster is out of memory, and your query was killed. Please try again in a few minutes."));
                queriesKilledDueToOutOfMemory.incrementAndGet();
                lastKilledQuery = biggestQuery.getQueryId();
            }
        }
    }
    Map<MemoryPoolId, Integer> countByPool = new HashMap<>();
    for (QueryExecution query : queries) {
        MemoryPoolId id = query.getMemoryPool().getId();
        countByPool.put(id, countByPool.getOrDefault(id, 0) + 1);
    }
    updatePools(countByPool);
    updateNodes(updateAssignments(queries));
    // check if CPU usage is over limit
    for (QueryExecution query : queries) {
        Duration cpuTime = query.getTotalCpuTime();
        Duration sessionLimit = getQueryMaxCpuTime(query.getSession());
        Duration limit = maxQueryCpuTime.compareTo(sessionLimit) < 0 ? maxQueryCpuTime : sessionLimit;
        if (cpuTime.compareTo(limit) > 0) {
            query.fail(new ExceededCpuLimitException(limit));
        }
    }
}