Examples with MemoryMA io.questdb.cairo.vm.api.MemoryMA used on opensource projects

Example 1 with MemoryMA

use of io.questdb.cairo.vm.api.MemoryMA in project questdb by bluestreak01.

the class TableReaderMetadataCorruptionTest method assertMetaConstructorFailure.

private void assertMetaConstructorFailure(String[] names, int[] types, int columnCount, int timestampIndex, String contains, long pageSize) throws Exception {
    TestUtils.assertMemoryLeak(() -> {
        try (Path path = new Path()) {
            path.of(root).concat("x");
            final int rootLen = path.length();
            if (FilesFacadeImpl.INSTANCE.mkdirs(path.slash$(), configuration.getMkDirMode()) == -1) {
                throw CairoException.instance(FilesFacadeImpl.INSTANCE.errno()).put("Cannot create dir: ").put(path);
            }
            try (MemoryMA mem = Vm.getMAInstance()) {
                mem.of(FilesFacadeImpl.INSTANCE, path.trimTo(rootLen).concat(TableUtils.META_FILE_NAME).$(), pageSize, MemoryTag.MMAP_DEFAULT);
                mem.putInt(columnCount);
                mem.putInt(PartitionBy.NONE);
                mem.putInt(timestampIndex);
                mem.putInt(ColumnType.VERSION);
                mem.jumpTo(TableUtils.META_OFFSET_COLUMN_TYPES);
                for (int i = 0; i < names.length; i++) {
                    mem.putInt(types[i]);
                    mem.putLong(0);
                    mem.putInt(0);
                    mem.skip(16);
                }
                for (int i = 0; i < names.length; i++) {
                    mem.putStr(names[i]);
                }
            }
            try {
                new TableReaderMetadata(FilesFacadeImpl.INSTANCE, path);
                Assert.fail();
            } catch (CairoException e) {
                TestUtils.assertContains(e.getFlyweightMessage(), contains);
            }
        }
    });
}

Example 2 with MemoryMA

use of io.questdb.cairo.vm.api.MemoryMA in project questdb by bluestreak01.

the class CairoMemoryTest method testAppendCannotOpenFile.

@Test
public void testAppendCannotOpenFile() {
    long used = Unsafe.getMemUsed();
    class X extends FilesFacadeImpl {

        @Override
        public long openRW(LPSZ name) {
            int n = name.length();
            if (n > 5 && Chars.equals(".fail", name, n - 5, n)) {
                return -1;
            }
            return super.openRW(name);
        }
    }
    X ff = new X();
    long openFileCount = ff.getOpenFileCount();
    int successCount = 0;
    int failCount = 0;
    try (Path path = new Path()) {
        path.of(temp.getRoot().getAbsolutePath());
        int prefixLen = path.length();
        try (MemoryMA mem = Vm.getMAInstance()) {
            Rnd rnd = new Rnd();
            for (int k = 0; k < 10; k++) {
                path.trimTo(prefixLen).concat(rnd.nextString(10));
                boolean fail = rnd.nextBoolean();
                if (fail) {
                    path.put(".fail").$();
                    failCount++;
                } else {
                    path.put(".data").$();
                    successCount++;
                }
                if (fail) {
                    try {
                        mem.of(ff, path, 2 * ff.getPageSize(), MemoryTag.MMAP_DEFAULT);
                        Assert.fail();
                    } catch (CairoException ignored) {
                    }
                } else {
                    mem.of(ff, path, 2 * ff.getPageSize(), MemoryTag.MMAP_DEFAULT);
                    for (int i = 0; i < N; i++) {
                        mem.putLong(i);
                    }
                    Assert.assertEquals(N * 8, mem.getAppendOffset());
                }
            }
        }
    }
    Assert.assertEquals(used, Unsafe.getMemUsed());
    Assert.assertEquals(openFileCount, ff.getOpenFileCount());
    Assert.assertTrue(failCount > 0);
    Assert.assertTrue(successCount > 0);
}

Example 3 with MemoryMA

use of io.questdb.cairo.vm.api.MemoryMA in project questdb by bluestreak01.

the class O3PartitionJob method publishOpenColumnTasks.

private static void publishOpenColumnTasks(long txn, ObjList<MemoryMAR> columns, ObjList<MemoryCARW> oooColumns, CharSequence pathToTable, long srcOooLo, long srcOooHi, long srcOooMax, long oooTimestampMin, long oooTimestampMax, long oooTimestampLo, long partitionTimestamp, int prefixType, long prefixLo, long prefixHi, int mergeType, long mergeDataLo, long mergeDataHi, long mergeOOOLo, long mergeOOOHi, int suffixType, long suffixLo, long suffixHi, long srcDataMax, long srcDataTxn, int openColumnMode, long srcTimestampFd, long srcTimestampAddr, long srcTimestampSize, int timestampIndex, long sortedTimestampsAddr, TableWriter tableWriter, AtomicInteger columnCounter, O3Basket o3Basket, long tmpBuf) {
    LOG.debug().$("partition [ts=").$ts(oooTimestampLo).$(']').$();
    final long timestampMergeIndexAddr;
    if (mergeType == O3_BLOCK_MERGE) {
        timestampMergeIndexAddr = createMergeIndex(srcTimestampAddr, sortedTimestampsAddr, mergeDataLo, mergeDataHi, mergeOOOLo, mergeOOOHi);
    } else {
        timestampMergeIndexAddr = 0;
    }
    final RecordMetadata metadata = tableWriter.getMetadata();
    final int columnCount = metadata.getColumnCount();
    columnCounter.set(columnCount);
    int columnsInFlight = columnCount;
    try {
        for (int i = 0; i < columnCount; i++) {
            final int colOffset = TableWriter.getPrimaryColumnIndex(i);
            final boolean notTheTimestamp = i != timestampIndex;
            final int columnType = metadata.getColumnType(i);
            final MemoryARW oooMem1 = oooColumns.getQuick(colOffset);
            final MemoryARW oooMem2 = oooColumns.getQuick(colOffset + 1);
            final MemoryMA mem1 = columns.getQuick(colOffset);
            final MemoryMA mem2 = columns.getQuick(colOffset + 1);
            final long activeFixFd;
            final long activeVarFd;
            final long srcDataTop;
            final long srcOooFixAddr;
            final long srcOooVarAddr;
            if (!ColumnType.isVariableLength(columnType)) {
                activeFixFd = mem1.getFd();
                activeVarFd = 0;
                srcOooFixAddr = oooMem1.addressOf(0);
                srcOooVarAddr = 0;
            } else {
                activeFixFd = mem2.getFd();
                activeVarFd = mem1.getFd();
                srcOooFixAddr = oooMem2.addressOf(0);
                srcOooVarAddr = oooMem1.addressOf(0);
            }
            final CharSequence columnName = metadata.getColumnName(i);
            final boolean isIndexed = metadata.isColumnIndexed(i);
            if (openColumnMode == OPEN_LAST_PARTITION_FOR_APPEND || openColumnMode == OPEN_LAST_PARTITION_FOR_MERGE) {
                srcDataTop = tableWriter.getColumnTop(i);
            } else {
                // column open job will have to find out if top exists and its value
                srcDataTop = -1;
            }
            final BitmapIndexWriter indexWriter;
            if (isIndexed) {
                indexWriter = o3Basket.nextIndexer();
            } else {
                indexWriter = null;
            }
            try {
                final long cursor = tableWriter.getO3OpenColumnPubSeq().next();
                if (cursor > -1) {
                    publishOpenColumnTaskHarmonized(cursor, openColumnMode, pathToTable, columnName, columnCounter, o3Basket.nextPartCounter(), notTheTimestamp ? columnType : ColumnType.setDesignatedTimestampBit(columnType, true), timestampMergeIndexAddr, srcOooFixAddr, srcOooVarAddr, srcOooLo, srcOooHi, srcOooMax, oooTimestampMin, oooTimestampMax, oooTimestampLo, partitionTimestamp, srcDataTop, srcDataMax, srcDataTxn, txn, prefixType, prefixLo, prefixHi, mergeType, mergeDataLo, mergeDataHi, mergeOOOLo, mergeOOOHi, suffixType, suffixLo, suffixHi, isIndexed, srcTimestampFd, srcTimestampAddr, srcTimestampSize, activeFixFd, activeVarFd, tableWriter, indexWriter);
                } else {
                    publishOpenColumnTaskContended(tmpBuf, cursor, openColumnMode, pathToTable, columnName, columnCounter, o3Basket.nextPartCounter(), notTheTimestamp ? columnType : ColumnType.setDesignatedTimestampBit(columnType, true), timestampMergeIndexAddr, srcOooFixAddr, srcOooVarAddr, srcOooLo, srcOooHi, srcOooMax, oooTimestampMin, oooTimestampMax, oooTimestampLo, partitionTimestamp, srcDataTop, srcDataMax, srcDataTxn, txn, prefixType, prefixLo, prefixHi, mergeType, mergeDataLo, mergeDataHi, mergeOOOLo, mergeOOOHi, suffixType, suffixLo, suffixHi, srcTimestampFd, srcTimestampAddr, srcTimestampSize, isIndexed, activeFixFd, activeVarFd, tableWriter, indexWriter);
                }
            } catch (Throwable e) {
                tableWriter.o3BumpErrorCount();
                LOG.error().$("open column error [table=").$(tableWriter.getTableName()).$(", e=").$(e).I$();
                columnsInFlight = i + 1;
                throw e;
            }
        }
    } finally {
        final int delta = columnsInFlight - columnCount;
        LOG.debug().$("idle [delta=").$(delta).$(']').$();
        if (delta < 0 && columnCounter.addAndGet(delta) == 0) {
            O3CopyJob.closeColumnIdleQuick(timestampMergeIndexAddr, srcTimestampFd, srcTimestampAddr, srcTimestampSize, tableWriter);
        }
    }
}

Example 4 with MemoryMA

use of io.questdb.cairo.vm.api.MemoryMA in project questdb by bluestreak01.

the class BinarySearchTest method testFindReverseTwoValues.

@Test
public void testFindReverseTwoValues() throws Exception {
    assertMemoryLeak(() -> {
        try (Path path = new Path()) {
            path.of(root).concat("binsearch.d").$();
            try (MemoryMA appendMem = Vm.getSmallCMARWInstance(FilesFacadeImpl.INSTANCE, path, MemoryTag.MMAP_DEFAULT)) {
                appendMem.putLong(1);
                appendMem.putLong(3);
                try (MemoryMR mem = Vm.getMRInstance(FilesFacadeImpl.INSTANCE, path, 400 * Long.BYTES, MemoryTag.MMAP_DEFAULT)) {
                    Assert.assertEquals(0, BinarySearch.find(mem, 2, 0, 1, BinarySearch.SCAN_UP));
                }
            }
        }
    });
}

Example 5 with MemoryMA

use of io.questdb.cairo.vm.api.MemoryMA in project questdb by bluestreak01.

the class BinarySearchTest method testFindForwardTwoValues.

@Test
public void testFindForwardTwoValues() {
    try (Path path = new Path()) {
        path.of(root).concat("binsearch.d").$();
        try (MemoryMA appendMem = Vm.getSmallCMARWInstance(FilesFacadeImpl.INSTANCE, path, MemoryTag.MMAP_DEFAULT)) {
            appendMem.putLong(1);
            appendMem.putLong(3);
            try (MemoryMR mem = Vm.getMRInstance(FilesFacadeImpl.INSTANCE, path, 400 * Long.BYTES, MemoryTag.MMAP_DEFAULT)) {
                Assert.assertEquals(0, BinarySearch.find(mem, 2, 0, 1, BinarySearch.SCAN_DOWN));
            }
        }
    }
}