Examples with MemoryMR io.questdb.cairo.vm.api.MemoryMR used on opensource projects

Example 11 with MemoryMR

use of io.questdb.cairo.vm.api.MemoryMR in project questdb by bluestreak01.

the class SymbolCacheTest method testSimpleInteraction.

@Test
public void testSimpleInteraction() throws Exception {
    String tableName = "tb1";
    TestUtils.assertMemoryLeak(() -> {
        try (Path path = new Path();
            TableModel model = new TableModel(configuration, tableName, PartitionBy.DAY).col("symCol1", ColumnType.SYMBOL).col("symCol2", ColumnType.SYMBOL);
            SymbolCache cache = new SymbolCache(new DefaultLineTcpReceiverConfiguration() {

                @Override
                public long getSymbolCacheWaitUsBeforeReload() {
                    return 0;
                }
            })) {
            CairoTestUtils.create(model);
            try (TableWriter writer = new TableWriter(configuration, tableName);
                MemoryMR txMem = Vm.getMRInstance()) {
                int symColIndex1 = writer.getColumnIndex("symCol1");
                int symColIndex2 = writer.getColumnIndex("symCol2");
                long symCountOffset = TableUtils.getSymbolWriterIndexOffset(symColIndex2);
                long transientSymCountOffset = TableUtils.getSymbolWriterTransientIndexOffset(symColIndex2);
                path.of(configuration.getRoot()).concat(tableName);
                txMem.of(configuration.getFilesFacade(), path.concat(TableUtils.TXN_FILE_NAME).$(), transientSymCountOffset + Integer.BYTES, transientSymCountOffset + Integer.BYTES, MemoryTag.MMAP_DEFAULT);
                cache.of(configuration, path.of(configuration.getRoot()).concat(tableName), "symCol2", symColIndex2);
                TableWriter.Row r = writer.newRow();
                r.putSym(symColIndex1, "sym11");
                r.putSym(symColIndex2, "sym21");
                r.append();
                writer.commit();
                Assert.assertEquals(1, txMem.getInt(symCountOffset));
                Assert.assertEquals(1, txMem.getInt(transientSymCountOffset));
                int rc = cache.getSymbolKey("missing");
                Assert.assertEquals(SymbolTable.VALUE_NOT_FOUND, rc);
                Assert.assertEquals(0, cache.getCacheValueCount());
                rc = cache.getSymbolKey("sym21");
                Assert.assertEquals(0, rc);
                Assert.assertEquals(1, cache.getCacheValueCount());
                r = writer.newRow();
                r.putSym(symColIndex1, "sym12");
                r.putSym(symColIndex2, "sym21");
                r.append();
                writer.commit();
                Assert.assertEquals(1, txMem.getInt(symCountOffset));
                Assert.assertEquals(1, txMem.getInt(transientSymCountOffset));
                rc = cache.getSymbolKey("missing");
                Assert.assertEquals(SymbolTable.VALUE_NOT_FOUND, rc);
                Assert.assertEquals(1, cache.getCacheValueCount());
                rc = cache.getSymbolKey("sym21");
                Assert.assertEquals(0, rc);
                Assert.assertEquals(1, cache.getCacheValueCount());
                r = writer.newRow();
                r.putSym(symColIndex1, "sym12");
                r.putSym(symColIndex2, "sym22");
                r.append();
                Assert.assertEquals(1, txMem.getInt(symCountOffset));
                Assert.assertEquals(2, txMem.getInt(transientSymCountOffset));
                writer.commit();
                Assert.assertEquals(2, txMem.getInt(symCountOffset));
                Assert.assertEquals(2, txMem.getInt(transientSymCountOffset));
                rc = cache.getSymbolKey("sym21");
                Assert.assertEquals(0, rc);
                Assert.assertEquals(1, cache.getCacheValueCount());
                rc = cache.getSymbolKey("sym22");
                Assert.assertEquals(1, rc);
                Assert.assertEquals(2, cache.getCacheValueCount());
                // Test cached uncommitted symbols
                r = writer.newRow();
                r.putSym(symColIndex1, "sym12");
                r.putSym(symColIndex2, "sym23");
                r.append();
                r.putSym(symColIndex1, "sym12");
                r.putSym(symColIndex2, "sym24");
                r.append();
                r.putSym(symColIndex1, "sym12");
                r.putSym(symColIndex2, "sym25");
                r.append();
                Assert.assertEquals(2, txMem.getInt(symCountOffset));
                Assert.assertEquals(5, txMem.getInt(transientSymCountOffset));
                rc = cache.getSymbolKey("sym22");
                Assert.assertEquals(1, rc);
                Assert.assertEquals(2, cache.getCacheValueCount());
                rc = cache.getSymbolKey("sym24");
                Assert.assertEquals(3, rc);
                Assert.assertEquals(3, cache.getCacheValueCount());
                writer.commit();
                Assert.assertEquals(5, txMem.getInt(symCountOffset));
                Assert.assertEquals(5, txMem.getInt(transientSymCountOffset));
                // Test deleting a symbol column
                writer.removeColumn("symCol1");
                cache.close();
                txMem.close();
                symColIndex2 = writer.getColumnIndex("symCol2");
                symCountOffset = TableUtils.getSymbolWriterIndexOffset(symColIndex2);
                transientSymCountOffset = TableUtils.getSymbolWriterTransientIndexOffset(symColIndex2);
                path.of(configuration.getRoot()).concat(tableName);
                txMem.of(configuration.getFilesFacade(), path.concat(TableUtils.TXN_FILE_NAME).$(), transientSymCountOffset + Integer.BYTES, transientSymCountOffset + Integer.BYTES, MemoryTag.MMAP_DEFAULT);
                cache.of(configuration, path.of(configuration.getRoot()).concat(tableName), "symCol2", symColIndex2);
                Assert.assertEquals(5, txMem.getInt(symCountOffset));
                Assert.assertEquals(5, txMem.getInt(transientSymCountOffset));
                rc = cache.getSymbolKey("sym24");
                Assert.assertEquals(3, rc);
                Assert.assertEquals(1, cache.getCacheValueCount());
                r = writer.newRow();
                r.putSym(symColIndex2, "sym26");
                r.append();
                Assert.assertEquals(5, txMem.getInt(symCountOffset));
                Assert.assertEquals(6, txMem.getInt(transientSymCountOffset));
                rc = cache.getSymbolKey("sym26");
                Assert.assertEquals(5, rc);
                Assert.assertEquals(2, cache.getCacheValueCount());
                writer.commit();
                Assert.assertEquals(6, txMem.getInt(symCountOffset));
                Assert.assertEquals(6, txMem.getInt(transientSymCountOffset));
                rc = cache.getSymbolKey("sym26");
                Assert.assertEquals(5, rc);
                Assert.assertEquals(2, cache.getCacheValueCount());
            }
        }
    });
}

Example 12 with MemoryMR

use of io.questdb.cairo.vm.api.MemoryMR in project questdb by bluestreak01.

the class ExtendedOnePageMemoryTest method testFailOnGrow.

@Test
public void testFailOnGrow() throws IOException {
    createFile();
    try (MemoryMR mem = new MemoryCMRImpl()) {
        int sz = FILE_SIZE / 2;
        mem.of(ff, path, sz, sz, MemoryTag.MMAP_DEFAULT);
        FILE_MAP_FAIL.set(true);
        sz *= 2;
        try {
            mem.extend(sz);
            Assert.fail();
        } catch (CairoException ex) {
            TestUtils.assertContains(ex.getFlyweightMessage(), "could not remap");
        }
    }
}

Example 13 with MemoryMR

use of io.questdb.cairo.vm.api.MemoryMR in project questdb by bluestreak01.

the class ExtendedOnePageMemoryTest method testFailOnInitialMap.

@Test
public void testFailOnInitialMap() throws IOException {
    createFile();
    try (MemoryMR mem = new MemoryCMRImpl()) {
        FILE_MAP_FAIL.set(true);
        try {
            mem.smallFile(ff, path, MemoryTag.MMAP_DEFAULT);
            Assert.fail();
        } catch (CairoException ex) {
            TestUtils.assertContains(ex.getFlyweightMessage(), "could not mmap");
        }
    }
}

Example 14 with MemoryMR

use of io.questdb.cairo.vm.api.MemoryMR in project questdb by bluestreak01.

the class BinarySearchTest method testFindForwardBeforeRange.

@Test
public void testFindForwardBeforeRange() throws Exception {
    assertMemoryLeak(() -> {
        try (Path path = new Path()) {
            path.of(root).concat("binsearch.d").$();
            try (MemoryCMARW appendMem = Vm.getSmallCMARWInstance(FilesFacadeImpl.INSTANCE, path, MemoryTag.MMAP_DEFAULT)) {
                for (int i = 0; i < 100; i++) {
                    for (int j = 0; j < 3; j++) {
                        appendMem.putLong(i);
                    }
                }
                long max = 100 * 3 - 1;
                try (MemoryMR mem = Vm.getMRInstance(FilesFacadeImpl.INSTANCE, path, 400 * Long.BYTES, MemoryTag.MMAP_DEFAULT)) {
                    long index = BinarySearch.find(mem, -20, 0, max, BinarySearch.SCAN_DOWN);
                    Assert.assertEquals(-1, index);
                }
            }
        }
    });
}

Example 15 with MemoryMR

use of io.questdb.cairo.vm.api.MemoryMR in project questdb by bluestreak01.

the class TxSerializer method toJson.

public String toJson(String txPath) {
    TxFileStruct tx = new TxFileStruct();
    try (Path path = new Path()) {
        path.put(txPath).$();
        if (!ff.exists(path)) {
            System.out.println("error: " + txPath + " does not exist");
        }
        try (MemoryMR roTxMem = Vm.getMRInstance(ff, path, ff.length(path), MemoryTag.MMAP_DEFAULT)) {
            roTxMem.growToFileSize();
            tx.TX_OFFSET_TXN = roTxMem.getLong(TX_OFFSET_TXN);
            tx.TX_OFFSET_TRANSIENT_ROW_COUNT = roTxMem.getLong(TX_OFFSET_TRANSIENT_ROW_COUNT);
            tx.TX_OFFSET_FIXED_ROW_COUNT = roTxMem.getLong(TX_OFFSET_FIXED_ROW_COUNT);
            tx.TX_OFFSET_MIN_TIMESTAMP = roTxMem.getLong(TX_OFFSET_MIN_TIMESTAMP);
            tx.TX_OFFSET_MAX_TIMESTAMP = roTxMem.getLong(TX_OFFSET_MAX_TIMESTAMP);
            tx.TX_OFFSET_DATA_VERSION = roTxMem.getLong(TX_OFFSET_DATA_VERSION);
            tx.TX_OFFSET_STRUCT_VERSION = roTxMem.getLong(TX_OFFSET_STRUCT_VERSION);
            tx.TX_OFFSET_TXN_CHECK = roTxMem.getLong(TX_OFFSET_TXN_CHECK);
            tx.TX_OFFSET_MAP_WRITER_COUNT = roTxMem.getInt(TX_OFFSET_MAP_WRITER_COUNT);
            tx.TX_OFFSET_PARTITION_TABLE_VERSION = roTxMem.getLong(TX_OFFSET_PARTITION_TABLE_VERSION);
            int symbolsCount = tx.TX_OFFSET_MAP_WRITER_COUNT;
            tx.SYMBOLS = new ArrayList<>();
            long offset = getSymbolWriterIndexOffset(0);
            while (offset + 3 < Math.min(roTxMem.size(), getSymbolWriterIndexOffset(symbolsCount))) {
                TxFileStruct.SymbolInfo symbol = new TxFileStruct.SymbolInfo();
                tx.SYMBOLS.add(symbol);
                symbol.COUNT = roTxMem.getInt(offset);
                offset += 4;
                if (offset + 3 < roTxMem.size()) {
                    symbol.UNCOMMITTED_COUNT = roTxMem.getInt(offset);
                    offset += 4;
                }
            }
            int txAttachedPartitionsSize = roTxMem.getInt(getPartitionTableSizeOffset(symbolsCount)) / Long.BYTES / 4;
            tx.ATTACHED_PARTITION_SIZE = txAttachedPartitionsSize;
            tx.ATTACHED_PARTITIONS = new ArrayList<>();
            offset = getPartitionTableIndexOffset(symbolsCount, 0);
            while (offset + 7 < Math.min(roTxMem.size(), getPartitionTableIndexOffset(symbolsCount, txAttachedPartitionsSize * 4))) {
                TxFileStruct.AttachedPartition partition = new TxFileStruct.AttachedPartition();
                tx.ATTACHED_PARTITIONS.add(partition);
                partition.TS = roTxMem.getLong(offset);
                offset += 8;
                if (offset + 7 < roTxMem.size()) {
                    partition.SIZE = roTxMem.getLong(offset);
                    offset += 8;
                }
                if (offset + 7 < roTxMem.size()) {
                    partition.NAME_TX = roTxMem.getLong(offset);
                    offset += 8;
                }
                if (offset + 7 < roTxMem.size()) {
                    partition.DATA_TX = roTxMem.getLong(offset);
                    offset += 8;
                }
            }
        }
    }
    Gson gson = new GsonBuilder().setPrettyPrinting().create();
    return gson.toJson(tx);
}