Examples with BufferFrontedFileStoreCache org.teiid.common.buffer.impl.BufferFrontedFileStoreCache used on opensource projects

Example 1 with BufferFrontedFileStoreCache

use of org.teiid.common.buffer.impl.BufferFrontedFileStoreCache in project teiid by teiid.

the class TestEnginePerformance method oneTimeSetup.

@BeforeClass
public static void oneTimeSetup() throws TeiidComponentException {
    bm = new BufferManagerImpl();
    bm.setMaxProcessingKB(1 << 12);
    bm.setMaxReserveKB((1 << 18) - (1 << 16));
    bm.setMaxActivePlans(20);
    cache = new BufferFrontedFileStoreCache();
    cache.setMemoryBufferSpace(1 << 26);
    FileStorageManager fsm = new FileStorageManager();
    fsm.setStorageDirectory(UnitTestUtil.getTestScratchPath() + "/data");
    cache.setStorageManager(fsm);
    cache.initialize();
    bm.setCache(cache);
    bm.initialize();
    es = Executors.newCachedThreadPool();
}

Example 2 with BufferFrontedFileStoreCache

use of org.teiid.common.buffer.impl.BufferFrontedFileStoreCache in project teiid by teiid.

the class TestSTree method testStorageWrites.

@Test
public void testStorageWrites() throws TeiidComponentException {
    BufferManagerImpl bm = BufferManagerFactory.createBufferManager();
    bm.setProcessorBatchSize(32);
    // force all to disk
    bm.setMaxReserveKB(0);
    BufferFrontedFileStoreCache fsc = (BufferFrontedFileStoreCache) bm.getCache();
    fsc.setMaxStorageObjectSize(1 << 19);
    fsc.setMemoryBufferSpace(1 << 19);
    fsc.initialize();
    bm.initialize();
    ElementSymbol e1 = new ElementSymbol("x");
    e1.setType(String.class);
    List<ElementSymbol> elements = Arrays.asList(e1);
    STree map = bm.createSTree(elements, "1", 1);
    int size = 1000;
    for (int i = 0; i < size; i++) {
        assertNull(map.insert(Arrays.asList(new String(new byte[1000])), InsertMode.ORDERED, size));
        assertEquals(i + 1, map.getRowCount());
    }
    for (int i = 0; i < size; i++) {
        assertNotNull(map.remove(Arrays.asList(new String(new byte[1000]))));
    }
    assertEquals(0, map.getRowCount());
    assertEquals(0, bm.getActiveBatchBytes());
    map.remove();
    assertEquals(0, bm.getActiveBatchBytes());
}

Example 3 with BufferFrontedFileStoreCache

use of org.teiid.common.buffer.impl.BufferFrontedFileStoreCache in project teiid by teiid.

the class BufferManagerFactory method initBufferManager.

public static BufferManagerImpl initBufferManager(BufferManagerImpl bufferManager) {
    try {
        bufferManager.initialize();
        bufferManager.setUseWeakReferences(false);
        MemoryStorageManager storageManager = new MemoryStorageManager();
        SplittableStorageManager ssm = new SplittableStorageManager(storageManager);
        ssm.setMaxFileSizeDirect(MemoryStorageManager.MAX_FILE_SIZE);
        BufferFrontedFileStoreCache fsc = new BufferFrontedFileStoreCache();
        fsc.setBufferManager(bufferManager);
        // use conservative allocations
        // allow the space to be GCed easily
        fsc.setDirect(false);
        fsc.setMaxStorageObjectSize(1 << 20);
        fsc.setMemoryBufferSpace(1 << 21);
        fsc.setStorageManager(ssm);
        fsc.initialize();
        bufferManager.setCache(fsc);
        return bufferManager;
    } catch (TeiidComponentException e) {
        throw new RuntimeException(e);
    }
}

Example 4 with BufferFrontedFileStoreCache

use of org.teiid.common.buffer.impl.BufferFrontedFileStoreCache in project teiid by teiid.

the class BufferServiceImpl method start.

public void start() {
    try {
        // Construct and initialize the buffer manager
        this.bufferMgr = new BufferManagerImpl(false);
        this.bufferMgr.setProcessorBatchSize(processorBatchSize);
        this.bufferMgr.setMaxReserveKB(this.maxReserveKb);
        this.bufferMgr.setMaxProcessingKB(this.maxProcessingKb);
        this.bufferMgr.setInlineLobs(inlineLobs);
        this.bufferMgr.initialize();
        // If necessary, add disk storage manager
        if (useDisk) {
            // $NON-NLS-1$
            LogManager.logDetail(LogConstants.CTX_DQP, "Starting BufferManager using", bufferDir);
            if (!bufferDir.exists()) {
                this.bufferDir.mkdirs();
            }
            // start the file storage manager in clean state
            // wise FileStorageManager is smart enough to clean up after itself
            cleanDirectory(bufferDir);
            // Get the properties for FileStorageManager and create.
            fsm = new FileStorageManager();
            fsm.setStorageDirectory(bufferDir.getCanonicalPath());
            fsm.setMaxOpenFiles(maxOpenFiles);
            fsm.setMaxBufferSpace(maxBufferSpace * MB);
            SplittableStorageManager ssm = new SplittableStorageManager(fsm);
            ssm.setMaxFileSize(maxFileSize);
            StorageManager sm = ssm;
            if (encryptFiles) {
                sm = new EncryptedStorageManager(ssm);
            }
            fsc = new BufferFrontedFileStoreCache();
            fsc.setBufferManager(this.bufferMgr);
            fsc.setMaxStorageObjectSize(maxStorageObjectSize);
            fsc.setDirect(memoryBufferOffHeap);
            // use approximately 40% of what's set aside for the reserved accounting for conversion from kb to bytes
            long autoMaxBufferSpace = 4 * (((long) this.bufferMgr.getMaxReserveKB()) << 10) / 10;
            // estimate inode/batch overhead
            if (memoryBufferSpace < 0) {
                fsc.setMemoryBufferSpace(autoMaxBufferSpace);
            } else {
                // scale from MB to bytes
                fsc.setMemoryBufferSpace(memoryBufferSpace << 20);
            }
            long batchAndInodeOverheadKB = fsc.getMemoryBufferSpace() >> (memoryBufferOffHeap ? 19 : 17);
            this.bufferMgr.setMaxReserveKB((int) Math.max(0, this.bufferMgr.getMaxReserveKB() - batchAndInodeOverheadKB));
            if (this.maxReserveKb < 0) {
                if (memoryBufferOffHeap) {
                    // the default is too large if off heap
                    this.bufferMgr.setMaxReserveKB(8 * this.bufferMgr.getMaxReserveKB() / 10);
                } else {
                    // adjust the value for the main memory buffer
                    this.bufferMgr.setMaxReserveKB((int) Math.max(0, this.bufferMgr.getMaxReserveKB() - (fsc.getMemoryBufferSpace() >> 10)));
                }
            }
            fsc.setStorageManager(sm);
            fsc.initialize();
            this.bufferMgr.setCache(fsc);
            this.workingMaxReserveKb = this.bufferMgr.getMaxReserveKB();
        } else {
            MemoryStorageManager msm = new MemoryStorageManager();
            SplittableStorageManager ssm = new SplittableStorageManager(msm);
            ssm.setMaxFileSizeDirect(MemoryStorageManager.MAX_FILE_SIZE);
            this.bufferMgr.setCache(msm);
            this.bufferMgr.setStorageManager(ssm);
        }
    } catch (TeiidComponentException e) {
        throw new TeiidRuntimeException(RuntimePlugin.Event.TEIID40039, e, RuntimePlugin.Util.gs(RuntimePlugin.Event.TEIID40039));
    } catch (IOException e) {
        throw new TeiidRuntimeException(RuntimePlugin.Event.TEIID40039, e, RuntimePlugin.Util.gs(RuntimePlugin.Event.TEIID40039));
    }
}