Examples with MemoryARW io.questdb.cairo.vm.api.MemoryARW used on opensource projects

Example 1 with MemoryARW

use of io.questdb.cairo.vm.api.MemoryARW in project questdb by bluestreak01.

the class TableWriterTest method testCancelRowRecoveryFromAppendPosErrors.

@Test
public void testCancelRowRecoveryFromAppendPosErrors() throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        final Rnd rnd = new Rnd();
        class X extends FilesFacadeImpl {

            boolean fail = false;

            @Override
            public int rmdir(Path name) {
                if (fail) {
                    return -1;
                }
                return super.rmdir(name);
            }
        }
        X ff = new X();
        final int N = 10000;
        create(ff, PartitionBy.DAY, N);
        // supposed to be stored have matching partitions
        try (MemoryARW vmem = Vm.getARWInstance(ff.getPageSize(), Integer.MAX_VALUE, MemoryTag.NATIVE_DEFAULT)) {
            try (TableWriter writer = new TableWriter(new DefaultCairoConfiguration(root) {

                @Override
                public FilesFacade getFilesFacade() {
                    return ff;
                }
            }, PRODUCT)) {
                long ts = TimestampFormatUtils.parseTimestamp("2013-03-04T00:00:00.000Z");
                int i = 0;
                int cancelCount = 0;
                int failCount = 0;
                while (i < N) {
                    TableWriter.Row r = writer.newRow(ts += 60000 * 1000L);
                    r.putInt(0, rnd.nextPositiveInt());
                    r.putStr(1, rnd.nextString(7));
                    r.putSym(2, rnd.nextString(4));
                    r.putSym(3, rnd.nextString(11));
                    r.putDouble(4, rnd.nextDouble());
                    if (rnd.nextPositiveInt() % 50 == 0) {
                        ff.fail = true;
                        try {
                            r.cancel();
                        } catch (CairoException ignored) {
                            failCount++;
                            ff.fail = false;
                            r.cancel();
                        } finally {
                            ff.fail = false;
                        }
                        cancelCount++;
                    } else {
                        r.append();
                        // second append() is expected to be a NOOP
                        r.append();
                        vmem.putLong(ts);
                        i++;
                    }
                }
                writer.commit();
                Assert.assertEquals(N, writer.size());
                Assert.assertTrue(cancelCount > 0);
                Assert.assertTrue(failCount > 0);
                verifyTimestampPartitions(vmem);
            }
        }
    });
}

Example 2 with MemoryARW

use of io.questdb.cairo.vm.api.MemoryARW in project questdb by bluestreak01.

the class CairoMemoryTest method testVirtualMemoryJump.

private void testVirtualMemoryJump(VirtualMemoryFactory factory) throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        try (Path path = new Path().of(temp.newFile().getAbsolutePath()).$()) {
            try (MemoryARW mem = factory.newInstance(path)) {
                for (int i = 0; i < 100; i++) {
                    mem.putLong(i);
                }
                mem.jumpTo(0);
                for (int i = 0; i < 50; i++) {
                    mem.putLong(50 - i);
                }
                // keep previously written data
                mem.jumpTo(800);
            }
            try (MemoryMR roMem = new MemoryCMRImpl(FF, path, 800, MemoryTag.MMAP_DEFAULT)) {
                for (int i = 0; i < 50; i++) {
                    Assert.assertEquals(50 - i, roMem.getLong(i * 8));
                }
                for (int i = 50; i < 100; i++) {
                    Assert.assertEquals(i, roMem.getLong(i * 8));
                }
            }
        }
    });
}

Example 3 with MemoryARW

use of io.questdb.cairo.vm.api.MemoryARW in project questdb by bluestreak01.

the class O3PartitionJob method publishOpenColumnTasks.

private static void publishOpenColumnTasks(long txn, ObjList<MemoryMAR> columns, ObjList<MemoryCARW> oooColumns, CharSequence pathToTable, long srcOooLo, long srcOooHi, long srcOooMax, long oooTimestampMin, long oooTimestampMax, long oooTimestampLo, long partitionTimestamp, int prefixType, long prefixLo, long prefixHi, int mergeType, long mergeDataLo, long mergeDataHi, long mergeOOOLo, long mergeOOOHi, int suffixType, long suffixLo, long suffixHi, long srcDataMax, long srcDataTxn, int openColumnMode, long srcTimestampFd, long srcTimestampAddr, long srcTimestampSize, int timestampIndex, long sortedTimestampsAddr, TableWriter tableWriter, AtomicInteger columnCounter, O3Basket o3Basket, long tmpBuf) {
    LOG.debug().$("partition [ts=").$ts(oooTimestampLo).$(']').$();
    final long timestampMergeIndexAddr;
    if (mergeType == O3_BLOCK_MERGE) {
        timestampMergeIndexAddr = createMergeIndex(srcTimestampAddr, sortedTimestampsAddr, mergeDataLo, mergeDataHi, mergeOOOLo, mergeOOOHi);
    } else {
        timestampMergeIndexAddr = 0;
    }
    final RecordMetadata metadata = tableWriter.getMetadata();
    final int columnCount = metadata.getColumnCount();
    columnCounter.set(columnCount);
    int columnsInFlight = columnCount;
    try {
        for (int i = 0; i < columnCount; i++) {
            final int colOffset = TableWriter.getPrimaryColumnIndex(i);
            final boolean notTheTimestamp = i != timestampIndex;
            final int columnType = metadata.getColumnType(i);
            final MemoryARW oooMem1 = oooColumns.getQuick(colOffset);
            final MemoryARW oooMem2 = oooColumns.getQuick(colOffset + 1);
            final MemoryMA mem1 = columns.getQuick(colOffset);
            final MemoryMA mem2 = columns.getQuick(colOffset + 1);
            final long activeFixFd;
            final long activeVarFd;
            final long srcDataTop;
            final long srcOooFixAddr;
            final long srcOooVarAddr;
            if (!ColumnType.isVariableLength(columnType)) {
                activeFixFd = mem1.getFd();
                activeVarFd = 0;
                srcOooFixAddr = oooMem1.addressOf(0);
                srcOooVarAddr = 0;
            } else {
                activeFixFd = mem2.getFd();
                activeVarFd = mem1.getFd();
                srcOooFixAddr = oooMem2.addressOf(0);
                srcOooVarAddr = oooMem1.addressOf(0);
            }
            final CharSequence columnName = metadata.getColumnName(i);
            final boolean isIndexed = metadata.isColumnIndexed(i);
            if (openColumnMode == OPEN_LAST_PARTITION_FOR_APPEND || openColumnMode == OPEN_LAST_PARTITION_FOR_MERGE) {
                srcDataTop = tableWriter.getColumnTop(i);
            } else {
                // column open job will have to find out if top exists and its value
                srcDataTop = -1;
            }
            final BitmapIndexWriter indexWriter;
            if (isIndexed) {
                indexWriter = o3Basket.nextIndexer();
            } else {
                indexWriter = null;
            }
            try {
                final long cursor = tableWriter.getO3OpenColumnPubSeq().next();
                if (cursor > -1) {
                    publishOpenColumnTaskHarmonized(cursor, openColumnMode, pathToTable, columnName, columnCounter, o3Basket.nextPartCounter(), notTheTimestamp ? columnType : ColumnType.setDesignatedTimestampBit(columnType, true), timestampMergeIndexAddr, srcOooFixAddr, srcOooVarAddr, srcOooLo, srcOooHi, srcOooMax, oooTimestampMin, oooTimestampMax, oooTimestampLo, partitionTimestamp, srcDataTop, srcDataMax, srcDataTxn, txn, prefixType, prefixLo, prefixHi, mergeType, mergeDataLo, mergeDataHi, mergeOOOLo, mergeOOOHi, suffixType, suffixLo, suffixHi, isIndexed, srcTimestampFd, srcTimestampAddr, srcTimestampSize, activeFixFd, activeVarFd, tableWriter, indexWriter);
                } else {
                    publishOpenColumnTaskContended(tmpBuf, cursor, openColumnMode, pathToTable, columnName, columnCounter, o3Basket.nextPartCounter(), notTheTimestamp ? columnType : ColumnType.setDesignatedTimestampBit(columnType, true), timestampMergeIndexAddr, srcOooFixAddr, srcOooVarAddr, srcOooLo, srcOooHi, srcOooMax, oooTimestampMin, oooTimestampMax, oooTimestampLo, partitionTimestamp, srcDataTop, srcDataMax, srcDataTxn, txn, prefixType, prefixLo, prefixHi, mergeType, mergeDataLo, mergeDataHi, mergeOOOLo, mergeOOOHi, suffixType, suffixLo, suffixHi, srcTimestampFd, srcTimestampAddr, srcTimestampSize, isIndexed, activeFixFd, activeVarFd, tableWriter, indexWriter);
                }
            } catch (Throwable e) {
                tableWriter.o3BumpErrorCount();
                LOG.error().$("open column error [table=").$(tableWriter.getTableName()).$(", e=").$(e).I$();
                columnsInFlight = i + 1;
                throw e;
            }
        }
    } finally {
        final int delta = columnsInFlight - columnCount;
        LOG.debug().$("idle [delta=").$(delta).$(']').$();
        if (delta < 0 && columnCounter.addAndGet(delta) == 0) {
            O3CopyJob.closeColumnIdleQuick(timestampMergeIndexAddr, srcTimestampFd, srcTimestampAddr, srcTimestampSize, tableWriter);
        }
    }
}

Example 4 with MemoryARW

use of io.questdb.cairo.vm.api.MemoryARW in project questdb by bluestreak01.

the class MemoryCARWImplTest method testIntRndCompatibility.

@Test
public void testIntRndCompatibility() {
    long pageSize = 64;
    try (MemoryARW mem = new MemoryCARWImpl(pageSize, Integer.MAX_VALUE, MemoryTag.NATIVE_DEFAULT)) {
        // prime page
        mem.putByte(10, (byte) 22);
        mem.putInt(15, 1024);
        mem.putInt(55, 2048);
        assertEquals(1024, mem.getInt(15));
        assertEquals(2048, mem.getInt(55));
    }
}

Example 5 with MemoryARW

use of io.questdb.cairo.vm.api.MemoryARW in project questdb by bluestreak01.

the class MemoryCARWImplTest method testByteRnd.

@Test
public void testByteRnd() {
    try (MemoryARW mem = new MemoryCARWImpl(11, Integer.MAX_VALUE, MemoryTag.NATIVE_DEFAULT)) {
        int n = 120;
        long o = 0;
        for (int i = 0; i < n; i++, o++) {
            mem.putByte(o, (byte) i);
        }
        o = 0;
        for (int i = 0; i < n; i++) {
            assertEquals(i, mem.getByte(o++));
        }
    }
}