Examples with BufferAllocator org.apache.drill.exec.memory.BufferAllocator used on opensource projects

Example 6 with BufferAllocator

use of org.apache.drill.exec.memory.BufferAllocator in project drill by axbaretto.

the class TestMemoryRetention method main.

public static void main(String[] args) throws Exception {
    final DrillConfig config = DrillConfig.create();
    final BufferAllocator a = RootAllocatorFactory.newRoot(config);
    for (int i = 0; i < PARALLEL_THREADS; i++) {
        Alloc alloc = new Alloc(a);
        alloc.start();
    }
}

Example 7 with BufferAllocator

use of org.apache.drill.exec.memory.BufferAllocator in project drill by axbaretto.

the class TestLenientAllocation method testLenientLimit.

@Test
public void testLenientLimit() {
    LogFixtureBuilder logBuilder = LogFixture.builder().logger(Accountant.class, Level.WARN);
    try (LogFixture logFixture = logBuilder.build()) {
        // Test can't run without assertions
        assertTrue(AssertionUtil.isAssertionsEnabled());
        // Create a child allocator
        BufferAllocator allocator = fixture.allocator().newChildAllocator("test", 10 * ONE_MEG, 128 * ONE_MEG);
        ((Accountant) allocator).forceLenient();
        // Allocate most of the available memory
        DrillBuf buf1 = allocator.buffer(64 * ONE_MEG);
        // Oops, we did our math wrong; allocate too large a buffer.
        DrillBuf buf2 = allocator.buffer(128 * ONE_MEG);
        try {
            allocator.buffer(64 * ONE_MEG);
            fail();
        } catch (OutOfMemoryException e) {
        // Expected
        }
        // Clean up
        buf1.close();
        buf2.close();
        allocator.close();
    }
}

Example 8 with BufferAllocator

use of org.apache.drill.exec.memory.BufferAllocator in project drill by axbaretto.

the class TestLenientAllocation method testStrict.

/**
 * Test that the allocator is normally strict in debug mode.
 */
@Test
public void testStrict() {
    LogFixtureBuilder logBuilder = LogFixture.builder().logger(Accountant.class, Level.WARN);
    try (LogFixture logFixture = logBuilder.build()) {
        // Test can't run without assertions
        assertTrue(AssertionUtil.isAssertionsEnabled());
        // Create a child allocator
        BufferAllocator allocator = fixture.allocator().newChildAllocator("test", 10 * 1024, 128 * 1024);
        // Allocate most of the available memory
        DrillBuf buf1 = allocator.buffer(64 * 1024);
        try {
            allocator.buffer(128 * 1024);
            fail();
        } catch (OutOfMemoryException e) {
        // Expected
        }
        // Clean up
        buf1.close();
        allocator.close();
    }
}

Example 9 with BufferAllocator

use of org.apache.drill.exec.memory.BufferAllocator in project drill by axbaretto.

the class TestLenientAllocation method testLenient.

/**
 * Use a test-time hack to force the allocator to be lenient,
 * regardless of whether we are in debug mode or not.
 */
@Test
public void testLenient() {
    LogFixtureBuilder logBuilder = LogFixture.builder().logger(Accountant.class, Level.WARN);
    try (LogFixture logFixture = logBuilder.build()) {
        // Test can't run without assertions
        assertTrue(AssertionUtil.isAssertionsEnabled());
        // Create a child allocator
        BufferAllocator allocator = fixture.allocator().newChildAllocator("test", 10 * 1024, 128 * 1024);
        ((Accountant) allocator).forceLenient();
        // Allocate most of the available memory
        DrillBuf buf1 = allocator.buffer(64 * 1024);
        // Oops, we did our math wrong; allocate too large a buffer.
        DrillBuf buf2 = allocator.buffer(128 * 1024);
        assertEquals(192 * 1024, allocator.getAllocatedMemory());
        // We keep making mistakes.
        DrillBuf buf3 = allocator.buffer(32 * 1024);
        // Right up to the hard limit
        DrillBuf buf4 = allocator.buffer(32 * 1024);
        assertEquals(256 * 1024, allocator.getAllocatedMemory());
        try {
            allocator.buffer(8);
            fail();
        } catch (OutOfMemoryException e) {
        // Expected
        }
        // Recover from our excesses
        buf2.close();
        buf3.close();
        buf4.close();
        assertEquals(64 * 1024, allocator.getAllocatedMemory());
        // We're back in the good graces of the allocator,
        // can allocate more.
        DrillBuf buf5 = allocator.buffer(8);
        // Clean up
        buf1.close();
        buf5.close();
        allocator.close();
    }
}

Example 10 with BufferAllocator

use of org.apache.drill.exec.memory.BufferAllocator in project drill by axbaretto.

the class TestLoad method testSchemaChange.

@Test
public void testSchemaChange() throws SchemaChangeException {
    final BufferAllocator allocator = RootAllocatorFactory.newRoot(drillConfig);
    final RecordBatchLoader batchLoader = new RecordBatchLoader(allocator);
    // Initial schema: a: INT, b: VARCHAR
    // Schema change: N/A
    BatchSchema schema1 = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).build();
    {
        assertTrue(loadBatch(allocator, batchLoader, schema1));
        assertTrue(schema1.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    // Same schema
    // Schema change: No
    {
        assertFalse(loadBatch(allocator, batchLoader, schema1));
        assertTrue(schema1.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    // Reverse columns: b: VARCHAR, a: INT
    // Schema change: No
    {
        BatchSchema schema = new SchemaBuilder().add("b", MinorType.VARCHAR).add("a", MinorType.INT).build();
        assertFalse(loadBatch(allocator, batchLoader, schema));
        // Potential bug: see DRILL-5828
        assertTrue(schema.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    // Drop a column: a: INT
    // Schema change: Yes
    {
        BatchSchema schema = new SchemaBuilder().add("a", MinorType.INT).build();
        assertTrue(loadBatch(allocator, batchLoader, schema));
        assertTrue(schema.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    // Add a column: a: INT, b: VARCHAR, c: INT
    // Schema change: Yes
    {
        assertTrue(loadBatch(allocator, batchLoader, schema1));
        assertTrue(schema1.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
        BatchSchema schema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).add("c", MinorType.INT).build();
        assertTrue(loadBatch(allocator, batchLoader, schema));
        assertTrue(schema.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    // Change a column type: a: INT, b: VARCHAR, c: VARCHAR
    // Schema change: Yes
    {
        BatchSchema schema = new SchemaBuilder().add("a", MinorType.INT).add("b", MinorType.VARCHAR).add("c", MinorType.VARCHAR).build();
        assertTrue(loadBatch(allocator, batchLoader, schema));
        assertTrue(schema.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    // Empty schema
    // Schema change: Yes
    {
        BatchSchema schema = new SchemaBuilder().build();
        assertTrue(loadBatch(allocator, batchLoader, schema));
        assertTrue(schema.isEquivalent(batchLoader.getSchema()));
        batchLoader.getContainer().zeroVectors();
    }
    batchLoader.clear();
    allocator.close();
}