Examples with LocalMemoryContext com.facebook.presto.memory.context.LocalMemoryContext used on opensource projects

Example 21 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class TestMemoryTracking method testOperatorAllocations.

@Test
public void testOperatorAllocations() {
    MemoryTrackingContext operatorMemoryContext = operatorContext.getOperatorMemoryContext();
    LocalMemoryContext systemMemory = operatorContext.newLocalSystemMemoryContext("test");
    LocalMemoryContext userMemory = operatorContext.localUserMemoryContext();
    LocalMemoryContext revocableMemory = operatorContext.localRevocableMemoryContext();
    userMemory.setBytes(100);
    assertOperatorMemoryAllocations(operatorMemoryContext, 100, 0, 0);
    systemMemory.setBytes(1_000_000);
    assertOperatorMemoryAllocations(operatorMemoryContext, 100, 1_000_000, 0);
    systemMemory.setBytes(2_000_000);
    assertOperatorMemoryAllocations(operatorMemoryContext, 100, 2_000_000, 0);
    userMemory.setBytes(500);
    assertOperatorMemoryAllocations(operatorMemoryContext, 500, 2_000_000, 0);
    userMemory.setBytes(userMemory.getBytes() - 500);
    assertOperatorMemoryAllocations(operatorMemoryContext, 0, 2_000_000, 0);
    revocableMemory.setBytes(300);
    assertOperatorMemoryAllocations(operatorMemoryContext, 0, 2_000_000, 300);
    assertAllocationFails((ignored) -> userMemory.setBytes(userMemory.getBytes() - 500), "bytes cannot be negative");
    operatorContext.destroy();
    assertOperatorMemoryAllocations(operatorMemoryContext, 0, 0, 0);
}

Example 22 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class TestMemoryTracking method testStats.

@Test
public void testStats() {
    LocalMemoryContext systemMemory = operatorContext.newLocalSystemMemoryContext("test");
    LocalMemoryContext userMemory = operatorContext.localUserMemoryContext();
    userMemory.setBytes(100_000_000);
    systemMemory.setBytes(200_000_000);
    assertStats(operatorContext.getOperatorStats(), driverContext.getDriverStats(), pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 100_000_000, 0, 200_000_000);
    // allocate more and check peak memory reservation
    userMemory.setBytes(600_000_000);
    assertStats(operatorContext.getOperatorStats(), driverContext.getDriverStats(), pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 600_000_000, 0, 200_000_000);
    userMemory.setBytes(userMemory.getBytes() - 300_000_000);
    assertStats(operatorContext.getOperatorStats(), driverContext.getDriverStats(), pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 300_000_000, 0, 200_000_000);
    userMemory.setBytes(userMemory.getBytes() - 300_000_000);
    assertStats(operatorContext.getOperatorStats(), driverContext.getDriverStats(), pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 0, 0, 200_000_000);
    operatorContext.destroy();
    assertStats(operatorContext.getOperatorStats(), driverContext.getDriverStats(), pipelineContext.getPipelineStats(), taskContext.getTaskStats(), 0, 0, 0);
}

Example 23 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class TestMemoryTracking method testDestroy.

@Test
public void testDestroy() {
    LocalMemoryContext newLocalSystemMemoryContext = operatorContext.newLocalSystemMemoryContext("test");
    LocalMemoryContext newLocalUserMemoryContext = operatorContext.localUserMemoryContext();
    LocalMemoryContext newLocalRevocableMemoryContext = operatorContext.localRevocableMemoryContext();
    newLocalSystemMemoryContext.setBytes(100_000);
    newLocalRevocableMemoryContext.setBytes(200_000);
    newLocalUserMemoryContext.setBytes(400_000);
    assertEquals(operatorContext.getOperatorMemoryContext().getSystemMemory(), 100_000);
    assertEquals(operatorContext.getOperatorMemoryContext().getUserMemory(), 400_000);
    operatorContext.destroy();
    assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, 0, 0);
}

Example 24 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class TestMemoryTracking method testLocalTotalMemoryLimitExceeded.

@Test
public void testLocalTotalMemoryLimitExceeded() {
    LocalMemoryContext systemMemoryContext = operatorContext.newLocalSystemMemoryContext("test");
    systemMemoryContext.setBytes(100);
    assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, 100, 0);
    systemMemoryContext.setBytes(queryMaxTotalMemory.toBytes());
    assertOperatorMemoryAllocations(operatorContext.getOperatorMemoryContext(), 0, queryMaxTotalMemory.toBytes(), 0);
    try {
        systemMemoryContext.setBytes(queryMaxTotalMemory.toBytes() + 1);
        fail("allocation should hit the per-node total memory limit");
    } catch (ExceededMemoryLimitException e) {
        assertEquals(e.getMessage(), format("Query exceeded per-node total memory limit of %1$s [Allocated: %1$s, Delta: 1B, Top Consumers: {test=%1$s}]", queryMaxTotalMemory));
    }
}

Example 25 with LocalMemoryContext

use of com.facebook.presto.memory.context.LocalMemoryContext in project presto by prestodb.

the class TestMemoryTracking method testCumulativeTotalMemoryEstimation.

@Test
public void testCumulativeTotalMemoryEstimation() {
    LocalMemoryContext userMemory = operatorContext.localUserMemoryContext();
    LocalMemoryContext systemMemory = operatorContext.localSystemMemoryContext();
    long userMemoryBytes = 100_000_000;
    long systemMemoryBytes = 40_000_000;
    userMemory.setBytes(userMemoryBytes);
    systemMemory.setBytes(systemMemoryBytes);
    long startTime = System.nanoTime();
    double cumulativeTotalMemory = taskContext.getTaskStats().getCumulativeTotalMemory();
    long endTime = System.nanoTime();
    double elapsedTimeInMillis = (endTime - startTime) / 1_000_000.0;
    long averageMemoryForLastPeriod = (userMemoryBytes + systemMemoryBytes) / 2;
    assertTrue(cumulativeTotalMemory < elapsedTimeInMillis * averageMemoryForLastPeriod);
}