Examples with BufferManagerImpl org.teiid.common.buffer.impl.BufferManagerImpl used on opensource projects

Example 31 with BufferManagerImpl

use of org.teiid.common.buffer.impl.BufferManagerImpl in project teiid by teiid.

the class TestSortNode method helpTestSort.

private void helpTestSort(List elements, List[] data, List sortElements, List sortTypes, List[] expected, Mode mode) throws TeiidComponentException, TeiidProcessingException {
    BufferManagerImpl mgr = BufferManagerFactory.getTestBufferManager(10000, BATCH_SIZE);
    long reserve = mgr.getReserveBatchBytes();
    // $NON-NLS-1$ //$NON-NLS-2$
    CommandContext context = new CommandContext("pid", "test", null, null, 1);
    BlockingFakeRelationalNode dataNode = new BlockingFakeRelationalNode(2, data);
    dataNode.setReturnPeriod(3);
    dataNode.setElements(elements);
    dataNode.initialize(context, mgr, null);
    SortNode sortNode = new SortNode(1);
    sortNode.setSortElements(new OrderBy(sortElements, sortTypes).getOrderByItems());
    sortNode.setMode(mode);
    sortNode.setElements(elements);
    sortNode.addChild(dataNode);
    sortNode.initialize(context, mgr, null);
    sortNode.open();
    assertTrue(sortNode.hasBuffer());
    int currentRow = 1;
    while (true) {
        try {
            TupleBatch batch = sortNode.nextBatch();
            for (int row = currentRow; row <= batch.getEndRow(); row++) {
                // $NON-NLS-1$
                assertEquals("Rows don't match at " + row, expected[row - 1], batch.getTuple(row));
            }
            currentRow += batch.getRowCount();
            if (batch.getTerminationFlag()) {
                break;
            }
        } catch (BlockedException e) {
        }
    }
    assertEquals(expected.length, currentRow - 1);
    assertEquals(reserve, mgr.getReserveBatchBytes());
}

Example 32 with BufferManagerImpl

use of org.teiid.common.buffer.impl.BufferManagerImpl in project teiid by teiid.

the class TestLocalBufferService method testUseDiskFalse.

@Test
public void testUseDiskFalse() throws Exception {
    BufferServiceImpl svc = new BufferServiceImpl();
    svc.setDiskDirectory(UnitTestUtil.getTestScratchPath() + "/teiid/1");
    svc.setUseDisk(false);
    svc.start();
    // all the properties are set
    BufferManagerImpl mgr = svc.getBufferManager();
    FileStore f = mgr.getCache().createFileStore("x");
    f.write(new byte[1234], 0, 1234);
}

Example 33 with BufferManagerImpl

use of org.teiid.common.buffer.impl.BufferManagerImpl in project teiid by teiid.

the class TestLocalBufferService method testStateTransfer.

@Test
public void testStateTransfer() throws Exception {
    BufferServiceImpl svc = new BufferServiceImpl();
    svc.setDiskDirectory(UnitTestUtil.getTestScratchPath() + "/teiid/1");
    svc.setUseDisk(true);
    svc.start();
    BufferManager mgr = svc.getBufferManager();
    List<ElementSymbol> schema = new ArrayList<ElementSymbol>(2);
    // $NON-NLS-1$
    ElementSymbol es = new ElementSymbol("x");
    es.setType(DataTypeManager.getDataTypeClass(DefaultDataTypes.STRING));
    schema.add(es);
    // $NON-NLS-1$
    ElementSymbol es2 = new ElementSymbol("y");
    es2.setType(DataTypeManager.getDataTypeClass(DefaultDataTypes.INTEGER));
    schema.add(es2);
    // $NON-NLS-1$
    TupleBuffer buffer = mgr.createTupleBuffer(schema, "cached", TupleSourceType.FINAL);
    buffer.setBatchSize(50);
    buffer.setId("state_id");
    for (int batch = 0; batch < 3; batch++) {
        for (int row = 0; row < 50; row++) {
            int val = (batch * 50) + row;
            buffer.addTuple(Arrays.asList(new Object[] { "String" + val, new Integer(val) }));
        }
    }
    buffer.close();
    mgr.distributeTupleBuffer(buffer.getId(), buffer);
    FileOutputStream fo = new FileOutputStream(UnitTestUtil.getTestScratchPath() + "/teiid/statetest");
    ((BufferManagerImpl) mgr).getState(buffer.getId(), fo);
    fo.close();
    svc.stop();
    // now read back
    BufferServiceImpl svc2 = new BufferServiceImpl();
    svc2.setDiskDirectory(UnitTestUtil.getTestScratchPath() + "/teiid/2");
    svc2.setUseDisk(true);
    svc2.start();
    BufferManagerImpl mgr2 = svc2.getBufferManager();
    FileInputStream fis = new FileInputStream(UnitTestUtil.getTestScratchPath() + "/teiid/statetest");
    mgr2.setState(buffer.getId(), fis);
    fis.close();
    String id = "state_id";
    buffer = mgr2.getTupleBuffer(id);
    for (int batch = 0; batch < 3; batch++) {
        TupleBatch tb = buffer.getBatch((batch * 50) + 1);
        List[] rows = tb.getAllTuples();
        for (int row = 0; row < 50; row++) {
            int val = (batch * 50) + row;
            assertEquals("String" + val, rows[row].get(0));
            assertEquals(val, rows[row].get(1));
        }
    }
    svc2.stop();
}

Example 34 with BufferManagerImpl

use of org.teiid.common.buffer.impl.BufferManagerImpl in project teiid by teiid.

the class BufferServiceImpl method start.

public void start() {
    try {
        // Construct and initialize the buffer manager
        this.bufferMgr = new BufferManagerImpl(false);
        this.bufferMgr.setProcessorBatchSize(processorBatchSize);
        this.bufferMgr.setMaxReserveKB(this.maxReserveKb);
        this.bufferMgr.setMaxProcessingKB(this.maxProcessingKb);
        this.bufferMgr.setInlineLobs(inlineLobs);
        this.bufferMgr.initialize();
        // If necessary, add disk storage manager
        if (useDisk) {
            // $NON-NLS-1$
            LogManager.logDetail(LogConstants.CTX_DQP, "Starting BufferManager using", bufferDir);
            if (!bufferDir.exists()) {
                this.bufferDir.mkdirs();
            }
            // start the file storage manager in clean state
            // wise FileStorageManager is smart enough to clean up after itself
            cleanDirectory(bufferDir);
            // Get the properties for FileStorageManager and create.
            fsm = new FileStorageManager();
            fsm.setStorageDirectory(bufferDir.getCanonicalPath());
            fsm.setMaxOpenFiles(maxOpenFiles);
            fsm.setMaxBufferSpace(maxBufferSpace * MB);
            SplittableStorageManager ssm = new SplittableStorageManager(fsm);
            ssm.setMaxFileSize(maxFileSize);
            StorageManager sm = ssm;
            if (encryptFiles) {
                sm = new EncryptedStorageManager(ssm);
            }
            fsc = new BufferFrontedFileStoreCache();
            fsc.setBufferManager(this.bufferMgr);
            fsc.setMaxStorageObjectSize(maxStorageObjectSize);
            fsc.setDirect(memoryBufferOffHeap);
            // use approximately 40% of what's set aside for the reserved accounting for conversion from kb to bytes
            long autoMaxBufferSpace = 4 * (((long) this.bufferMgr.getMaxReserveKB()) << 10) / 10;
            // estimate inode/batch overhead
            if (memoryBufferSpace < 0) {
                fsc.setMemoryBufferSpace(autoMaxBufferSpace);
            } else {
                // scale from MB to bytes
                fsc.setMemoryBufferSpace(memoryBufferSpace << 20);
            }
            long batchAndInodeOverheadKB = fsc.getMemoryBufferSpace() >> (memoryBufferOffHeap ? 19 : 17);
            this.bufferMgr.setMaxReserveKB((int) Math.max(0, this.bufferMgr.getMaxReserveKB() - batchAndInodeOverheadKB));
            if (this.maxReserveKb < 0) {
                if (memoryBufferOffHeap) {
                    // the default is too large if off heap
                    this.bufferMgr.setMaxReserveKB(8 * this.bufferMgr.getMaxReserveKB() / 10);
                } else {
                    // adjust the value for the main memory buffer
                    this.bufferMgr.setMaxReserveKB((int) Math.max(0, this.bufferMgr.getMaxReserveKB() - (fsc.getMemoryBufferSpace() >> 10)));
                }
            }
            fsc.setStorageManager(sm);
            fsc.initialize();
            this.bufferMgr.setCache(fsc);
            this.workingMaxReserveKb = this.bufferMgr.getMaxReserveKB();
        } else {
            MemoryStorageManager msm = new MemoryStorageManager();
            SplittableStorageManager ssm = new SplittableStorageManager(msm);
            ssm.setMaxFileSizeDirect(MemoryStorageManager.MAX_FILE_SIZE);
            this.bufferMgr.setCache(msm);
            this.bufferMgr.setStorageManager(ssm);
        }
    } catch (TeiidComponentException e) {
        throw new TeiidRuntimeException(RuntimePlugin.Event.TEIID40039, e, RuntimePlugin.Util.gs(RuntimePlugin.Event.TEIID40039));
    } catch (IOException e) {
        throw new TeiidRuntimeException(RuntimePlugin.Event.TEIID40039, e, RuntimePlugin.Util.gs(RuntimePlugin.Event.TEIID40039));
    }
}