Examples with MemoryCMARW io.questdb.cairo.vm.api.MemoryCMARW used on opensource projects

Example 6 with MemoryCMARW

use of io.questdb.cairo.vm.api.MemoryCMARW in project questdb by bluestreak01.

the class SymbolMapTest method create.

public static void create(Path path, CharSequence name, int symbolCapacity, boolean useCache) {
    int plen = path.length();
    try {
        try (MemoryCMARW mem = Vm.getSmallCMARWInstance(configuration.getFilesFacade(), path.concat(name).put(".o").$(), MemoryTag.MMAP_DEFAULT)) {
            mem.putInt(symbolCapacity);
            mem.putBool(useCache);
            mem.jumpTo(SymbolMapWriter.HEADER_SIZE);
        }
        configuration.getFilesFacade().touch(path.trimTo(plen).concat(name).put(".c").$());
        BitmapIndexTest.create(configuration, path.trimTo(plen), name, 4);
    } finally {
        path.trimTo(plen);
    }
}

Example 7 with MemoryCMARW

use of io.questdb.cairo.vm.api.MemoryCMARW in project questdb by bluestreak01.

the class TxSerializer method serializeJson.

public void serializeJson(String json, String target) {
    Gson des = new Gson();
    TxFileStruct tx = des.fromJson(json, TxFileStruct.class);
    if (tx.ATTACHED_PARTITION_SIZE != 0 && (tx.ATTACHED_PARTITIONS == null || tx.ATTACHED_PARTITION_SIZE != tx.ATTACHED_PARTITIONS.size())) {
        String arraySize = tx.ATTACHED_PARTITIONS == null ? "null" : Integer.toString(tx.ATTACHED_PARTITIONS.size());
        throw new IllegalArgumentException("ATTACHED_PARTITIONS array size of " + arraySize + " is different from ATTACHED_PARTITION_SIZE of " + tx.ATTACHED_PARTITION_SIZE);
    }
    if (tx.TX_OFFSET_MAP_WRITER_COUNT != 0 && (tx.SYMBOLS == null || tx.TX_OFFSET_MAP_WRITER_COUNT != tx.SYMBOLS.size())) {
        String arraySize = tx.SYMBOLS == null ? "null" : Integer.toString(tx.SYMBOLS.size());
        throw new IllegalArgumentException("SYMBOLS array size if " + arraySize + "is different from MAP_WRITER_COUNT of " + tx.TX_OFFSET_MAP_WRITER_COUNT);
    }
    long fileSize = tx.calculateFileSize();
    try (Path path = new Path()) {
        path.put(target).$();
        try (MemoryCMARW rwTxMem = Vm.getSmallCMARWInstance(ff, path, MemoryTag.MMAP_DEFAULT)) {
            Vect.memset(rwTxMem.addressOf(0), fileSize, 0);
            rwTxMem.setTruncateSize(fileSize);
            rwTxMem.putLong(TX_OFFSET_TXN, tx.TX_OFFSET_TXN);
            rwTxMem.putLong(TX_OFFSET_TRANSIENT_ROW_COUNT, tx.TX_OFFSET_TRANSIENT_ROW_COUNT);
            rwTxMem.putLong(TX_OFFSET_FIXED_ROW_COUNT, tx.TX_OFFSET_FIXED_ROW_COUNT);
            rwTxMem.putLong(TX_OFFSET_MIN_TIMESTAMP, tx.TX_OFFSET_MIN_TIMESTAMP);
            rwTxMem.putLong(TX_OFFSET_MAX_TIMESTAMP, tx.TX_OFFSET_MAX_TIMESTAMP);
            rwTxMem.putLong(TX_OFFSET_DATA_VERSION, tx.TX_OFFSET_DATA_VERSION);
            rwTxMem.putLong(TX_OFFSET_STRUCT_VERSION, tx.TX_OFFSET_STRUCT_VERSION);
            rwTxMem.putLong(TX_OFFSET_TXN_CHECK, tx.TX_OFFSET_TXN_CHECK);
            rwTxMem.putInt(TX_OFFSET_MAP_WRITER_COUNT, tx.TX_OFFSET_MAP_WRITER_COUNT);
            rwTxMem.putLong(TX_OFFSET_PARTITION_TABLE_VERSION, tx.TX_OFFSET_PARTITION_TABLE_VERSION);
            if (tx.TX_OFFSET_MAP_WRITER_COUNT != 0) {
                int isym = 0;
                for (TxFileStruct.SymbolInfo si : tx.SYMBOLS) {
                    long offset = getSymbolWriterIndexOffset(isym++);
                    rwTxMem.putInt(offset, si.COUNT);
                    offset += 4;
                    rwTxMem.putInt(offset, si.UNCOMMITTED_COUNT);
                }
            }
            rwTxMem.putInt(getPartitionTableSizeOffset(tx.TX_OFFSET_MAP_WRITER_COUNT), tx.ATTACHED_PARTITION_SIZE * 8 * 4);
            if (tx.ATTACHED_PARTITION_SIZE != 0) {
                int ipart = 0;
                for (TxFileStruct.AttachedPartition part : tx.ATTACHED_PARTITIONS) {
                    long offset = getPartitionTableIndexOffset(tx.TX_OFFSET_MAP_WRITER_COUNT, 4 * ipart++);
                    rwTxMem.putLong(offset, part.TS);
                    offset += 8;
                    rwTxMem.putLong(offset, part.SIZE);
                    offset += 8;
                    rwTxMem.putLong(offset, part.NAME_TX);
                    offset += 8;
                    rwTxMem.putLong(offset, part.DATA_TX);
                }
            }
        }
    }
}

Example 8 with MemoryCMARW

use of io.questdb.cairo.vm.api.MemoryCMARW in project questdb by bluestreak01.

the class TableWriterTest method testIncorrectTodoCode.

@Test
public void testIncorrectTodoCode() throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        CairoTestUtils.createAllTable(configuration, PartitionBy.NONE);
        try (MemoryCMARW mem = Vm.getCMARWInstance();
            Path path = new Path().of(root).concat("all").concat(TableUtils.TODO_FILE_NAME).$()) {
            mem.smallFile(FilesFacadeImpl.INSTANCE, path, MemoryTag.MMAP_DEFAULT);
            mem.putLong(32, 1);
            mem.putLong(40, 9990001L);
            mem.jumpTo(48);
        }
        try (TableWriter writer = new TableWriter(configuration, "all")) {
            Assert.assertNotNull(writer);
            Assert.assertTrue(writer.isOpen());
        }
        try (TableWriter writer = new TableWriter(configuration, "all")) {
            Assert.assertNotNull(writer);
            Assert.assertTrue(writer.isOpen());
        }
    });
}

Example 9 with MemoryCMARW

use of io.questdb.cairo.vm.api.MemoryCMARW in project questdb by bluestreak01.

the class CairoMemoryTest method testReadWriteCannotOpenFile.

@Test
public void testReadWriteCannotOpenFile() {
    long used = Unsafe.getMemUsed();
    class X extends FilesFacadeImpl {

        @Override
        public long openRW(LPSZ name) {
            int n = name.length();
            if (n > 5 && Chars.equals(".fail", name, n - 5, n)) {
                return -1;
            }
            return super.openRW(name);
        }
    }
    X ff = new X();
    long openFileCount = ff.getOpenFileCount();
    int successCount = 0;
    int failCount = 0;
    try (Path path = new Path()) {
        path.of(temp.getRoot().getAbsolutePath());
        int prefixLen = path.length();
        try (MemoryCMARW mem = Vm.getCMARWInstance()) {
            Rnd rnd = new Rnd();
            for (int k = 0; k < 10; k++) {
                path.trimTo(prefixLen).concat(rnd.nextString(10));
                boolean fail = rnd.nextBoolean();
                if (fail) {
                    path.put(".fail").$();
                    failCount++;
                } else {
                    path.put(".data").$();
                    successCount++;
                }
                if (fail) {
                    try {
                        mem.of(ff, path, 2 * ff.getPageSize(), -1, MemoryTag.MMAP_DEFAULT);
                        Assert.fail();
                    } catch (CairoException ignored) {
                    }
                } else {
                    mem.of(ff, path, 2 * ff.getPageSize(), -1, MemoryTag.MMAP_DEFAULT);
                    for (int i = 0; i < N; i++) {
                        mem.putLong(i);
                    }
                    Assert.assertEquals(N * 8, mem.getAppendOffset());
                }
            }
        }
    }
    Assert.assertEquals(used, Unsafe.getMemUsed());
    Assert.assertEquals(openFileCount, ff.getOpenFileCount());
    Assert.assertTrue(failCount > 0);
    Assert.assertTrue(successCount > 0);
}

Example 10 with MemoryCMARW

use of io.questdb.cairo.vm.api.MemoryCMARW in project questdb by bluestreak01.

the class TableReadFailTest method testReloadTimeout.

@Test
public void testReloadTimeout() throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        try (TableModel model = new TableModel(configuration, "x", PartitionBy.NONE).col("a", ColumnType.INT).col("b", ColumnType.LONG).timestamp()) {
            CairoTestUtils.create(model);
        }
        try (Path path = new Path();
            TableReader reader = new TableReader(configuration, "x");
            MemoryCMARW mem = Vm.getCMARWInstance()) {
            final Rnd rnd = new Rnd();
            final int N = 1000;
            // home path at txn file
            path.of(configuration.getRoot()).concat("x").concat(TableUtils.TXN_FILE_NAME).$();
            try (TableWriter w = new TableWriter(configuration, "x")) {
                for (int i = 0; i < N; i++) {
                    TableWriter.Row r = w.newRow();
                    r.putInt(0, rnd.nextInt());
                    r.putLong(1, rnd.nextLong());
                    r.append();
                }
                w.commit();
            }
            Assert.assertTrue(reader.reload());
            RecordCursor cursor = reader.getCursor();
            final Record record = cursor.getRecord();
            rnd.reset();
            int count = 0;
            while (cursor.hasNext()) {
                Assert.assertEquals(rnd.nextInt(), record.getInt(0));
                Assert.assertEquals(rnd.nextLong(), record.getLong(1));
                count++;
            }
            Assert.assertEquals(N, count);
            mem.smallFile(configuration.getFilesFacade(), path, MemoryTag.MMAP_DEFAULT);
            // keep txn file parameters
            long offset = configuration.getFilesFacade().length(mem.getFd());
            long txn = mem.getLong(TableUtils.TX_OFFSET_TXN);
            // corrupt the txn file
            mem.jumpTo(TableUtils.TX_OFFSET_TXN);
            mem.putLong(123);
            mem.jumpTo(offset);
            mem.close();
            // this should time out
            try {
                reader.reload();
                Assert.fail();
            } catch (CairoException e) {
                TestUtils.assertContains(e.getFlyweightMessage(), "timeout");
            }
            // restore txn file to its former glory
            mem.smallFile(configuration.getFilesFacade(), path, MemoryTag.MMAP_DEFAULT);
            mem.jumpTo(TableUtils.TX_OFFSET_TXN);
            mem.putLong(txn);
            mem.jumpTo(offset);
            mem.close();
            mem.close();
            // make sure reload functions correctly
            Assert.assertFalse(reader.reload());
            try (TableWriter w = new TableWriter(configuration, "x")) {
                // add more data
                for (int i = 0; i < N; i++) {
                    TableWriter.Row r = w.newRow();
                    r.putInt(0, rnd.nextInt());
                    r.putLong(1, rnd.nextLong());
                    r.append();
                }
                w.commit();
            }
            // does positive reload work?
            Assert.assertTrue(reader.reload());
            // can reader still see correct data?
            cursor = reader.getCursor();
            rnd.reset();
            count = 0;
            while (cursor.hasNext()) {
                Assert.assertEquals(rnd.nextInt(), record.getInt(0));
                Assert.assertEquals(rnd.nextLong(), record.getLong(1));
                count++;
            }
            Assert.assertEquals(2 * N, count);
        }
    });
}