Search in sources :

Example 1 with BooleanHolder

use of herddb.utils.BooleanHolder in project herddb by diennea.

the class TableManager method loadPageToMemory.

private DataPage loadPageToMemory(Long pageId, boolean recovery) throws DataStorageManagerException {
    DataPage result = pages.get(pageId);
    if (result != null) {
        pageReplacementPolicy.pageHit(result);
        return result;
    }
    long _start = System.currentTimeMillis();
    long _ioAndLock = 0;
    BooleanHolder computed = new BooleanHolder(false);
    try {
        result = pages.computeIfAbsent(pageId, (id) -> {
            try {
                computed.value = true;
                List<Record> page;
                maxCurrentPagesLoads.acquireUninterruptibly();
                try {
                    page = dataStorageManager.readPage(tableSpaceUUID, table.uuid, pageId);
                } finally {
                    maxCurrentPagesLoads.release();
                }
                loadedPagesCount.increment();
                return buildImmutableDataPage(pageId, page);
            } catch (DataStorageManagerException err) {
                throw new RuntimeException(err);
            }
        });
        if (computed.value) {
            _ioAndLock = System.currentTimeMillis();
            final Page.Metadata unload = pageReplacementPolicy.add(result);
            if (unload != null) {
                unload.owner.unload(unload.pageId);
            }
        }
    } catch (RuntimeException error) {
        if (error.getCause() != null) {
            Throwable cause = error.getCause();
            if (cause instanceof DataStorageManagerException) {
                if (cause instanceof DataPageDoesNotExistException) {
                    return null;
                }
                throw (DataStorageManagerException) cause;
            }
        }
        throw new DataStorageManagerException(error);
    }
    if (computed.value && LOGGER.isLoggable(Level.FINE)) {
        long _stop = System.currentTimeMillis();
        LOGGER.log(Level.FINE, "table {0}.{1}, loaded {2} records from page {3} in {4} ms, ({5} ms read + plock, {6} ms unlock)", new Object[] { table.tablespace, table.name, result.size(), pageId, (_stop - _start), (_ioAndLock - _start), (_stop - _ioAndLock) });
    }
    return result;
}
Also used : Arrays(java.util.Arrays) NullLockManager(herddb.utils.NullLockManager) Table(herddb.model.Table) TruncateTableStatement(herddb.model.commands.TruncateTableStatement) DuplicatePrimaryKeyException(herddb.model.DuplicatePrimaryKeyException) TableStatus(herddb.storage.TableStatus) Map(java.util.Map) DataAccessor(herddb.utils.DataAccessor) LogNotAvailableException(herddb.log.LogNotAvailableException) CommitLogResult(herddb.log.CommitLogResult) LogSequenceNumber(herddb.log.LogSequenceNumber) UniqueIndexContraintViolationException(herddb.model.UniqueIndexContraintViolationException) Set(java.util.Set) RecordSerializer(herddb.codec.RecordSerializer) JSQLParserPlanner.delimit(herddb.sql.JSQLParserPlanner.delimit) DataPageMetaData(herddb.core.PageSet.DataPageMetaData) ScanStatement(herddb.model.commands.ScanStatement) Stream(java.util.stream.Stream) StatsLogger(org.apache.bookkeeper.stats.StatsLogger) Bytes(herddb.utils.Bytes) Holder(herddb.utils.Holder) LockHandle(herddb.utils.LockHandle) ForeignKeyViolationException(herddb.model.ForeignKeyViolationException) LogEntry(herddb.log.LogEntry) ArrayList(java.util.ArrayList) TransactionContext(herddb.model.TransactionContext) Transaction(herddb.model.Transaction) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) Projection(herddb.model.Projection) ForeignKeyDef(herddb.model.ForeignKeyDef) EnsureLongIncrementAccumulator(herddb.utils.EnsureLongIncrementAccumulator) LogEntryType(herddb.log.LogEntryType) Record(herddb.model.Record) LogEntryFactory(herddb.log.LogEntryFactory) KeyToPageIndex(herddb.index.KeyToPageIndex) DataStorageManager(herddb.storage.DataStorageManager) ColumnTypes(herddb.model.ColumnTypes) ILocalLockManager(herddb.utils.ILocalLockManager) AtomicLong(java.util.concurrent.atomic.AtomicLong) Lock(java.util.concurrent.locks.Lock) IndexOperation(herddb.index.IndexOperation) Column(herddb.model.Column) StampedLock(java.util.concurrent.locks.StampedLock) UpdateStatement(herddb.model.commands.UpdateStatement) ScanLimitsImpl(herddb.model.ScanLimitsImpl) TupleComparator(herddb.model.TupleComparator) ServerConfiguration(herddb.server.ServerConfiguration) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RecordTooBigException(herddb.model.RecordTooBigException) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) LocalLockManager(herddb.utils.LocalLockManager) Futures(herddb.utils.Futures) DataStorageManagerException(herddb.storage.DataStorageManagerException) Index(herddb.model.Index) InsertStatement(herddb.model.commands.InsertStatement) DataScanner(herddb.model.DataScanner) DDLException(herddb.model.DDLException) RecordFunction(herddb.model.RecordFunction) StatementExecutionException(herddb.model.StatementExecutionException) TableContext(herddb.model.TableContext) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Logger(java.util.logging.Logger) Collectors(java.util.stream.Collectors) List(java.util.List) FullTableScanConsumer(herddb.storage.FullTableScanConsumer) GetStatement(herddb.model.commands.GetStatement) Entry(java.util.Map.Entry) Statement(herddb.model.Statement) LongAdder(java.util.concurrent.atomic.LongAdder) DataScannerException(herddb.model.DataScannerException) GetResult(herddb.model.GetResult) PrimaryIndexSeek(herddb.index.PrimaryIndexSeek) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Function(java.util.function.Function) BatchOrderedExecutor(herddb.utils.BatchOrderedExecutor) ConcurrentMap(java.util.concurrent.ConcurrentMap) Level(java.util.logging.Level) HashSet(java.util.HashSet) BooleanHolder(herddb.utils.BooleanHolder) ScanLimits(herddb.model.ScanLimits) DeleteStatement(herddb.model.commands.DeleteStatement) Iterator(java.util.Iterator) ReentrantLock(java.util.concurrent.locks.ReentrantLock) Semaphore(java.util.concurrent.Semaphore) DataPageDoesNotExistException(herddb.storage.DataPageDoesNotExistException) Counter(org.apache.bookkeeper.stats.Counter) StatementExecutionResult(herddb.model.StatementExecutionResult) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) CommitLog(herddb.log.CommitLog) AbstractMap(java.util.AbstractMap) TableConsistencyCheckStatement(herddb.model.commands.TableConsistencyCheckStatement) Predicate(herddb.model.Predicate) StatementEvaluationContext(herddb.model.StatementEvaluationContext) Comparator(java.util.Comparator) Collections(java.util.Collections) SECONDS(java.util.concurrent.TimeUnit.SECONDS) TableManagerStats(herddb.core.stats.TableManagerStats) SystemProperties(herddb.utils.SystemProperties) DataStorageManagerException(herddb.storage.DataStorageManagerException) BooleanHolder(herddb.utils.BooleanHolder) ArrayList(java.util.ArrayList) List(java.util.List) DataPageDoesNotExistException(herddb.storage.DataPageDoesNotExistException)

Example 2 with BooleanHolder

use of herddb.utils.BooleanHolder in project herddb by diennea.

the class ConcurrentMapKeyToPageIndex method put.

@Override
public boolean put(Bytes key, Long newPage, Long expectedPage) {
    if (expectedPage == null) {
        final Long opage = map.putIfAbsent(key, newPage);
        if (opage == null) {
            keyAdded(key);
            return true;
        }
        return false;
    } else {
        /*
             * We need to keep track if the update was really done. Reading computeIfPresent result won't
             * suffice, it can be equal to newPage even if no replacement was done (the map contained already
             * newPage mapping and expectedPage was different)
             */
        BooleanHolder holder = new BooleanHolder(false);
        map.computeIfPresent(key, (skey, spage) -> {
            if (spage.equals(expectedPage)) {
                holder.value = true;
                return newPage;
            }
            return spage;
        });
        if (holder.value) {
            keyAdded(key);
            return true;
        }
        return false;
    }
}
Also used : AtomicLong(java.util.concurrent.atomic.AtomicLong) BooleanHolder(herddb.utils.BooleanHolder)

Example 3 with BooleanHolder

use of herddb.utils.BooleanHolder in project herddb by diennea.

the class SimpleFollowerTest method testLeaderOnlineLogAvailableMultipleVersionsActivePages.

@Test
public void testLeaderOnlineLogAvailableMultipleVersionsActivePages() throws Exception {
    ServerConfiguration serverconfig_1 = newServerConfigurationWithAutoPort(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 1000);
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    // delete ledgers soon
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_LEDGERS_RETENTION_PERIOD, 1);
    serverconfig_1.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, 0);
    // disabled
    serverconfig_1.set(ServerConfiguration.PROPERTY_BOOKKEEPER_MAX_IDLE_TIME, 0);
    /*
         * Disable page compaction (avoid compaction of dirty page)
         */
    serverconfig_1.set(ServerConfiguration.PROPERTY_FILL_PAGE_THRESHOLD, 0.0D);
    ServerConfiguration serverconfig_2 = serverconfig_1.copy().set(ServerConfiguration.PROPERTY_NODEID, "server2").set(ServerConfiguration.PROPERTY_BOOT_FORCE_DOWNLOAD_SNAPSHOT, true).set(ServerConfiguration.PROPERTY_BASEDIR, folder.newFolder().toPath().toAbsolutePath());
    Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).column("s", ColumnTypes.STRING).primaryKey("c").build();
    try (Server server_1 = new Server(serverconfig_1)) {
        server_1.start();
        server_1.waitForStandaloneBoot();
        server_1.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        TableSpaceManager tableSpaceManager = server_1.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        AbstractTableManager tableManager = tableSpaceManager.getTableManager("t1");
        // fill table
        long tx = herddb.core.TestUtils.beginTransaction(server_1.getManager(), TableSpace.DEFAULT);
        for (int i = 0; i < 1000; i++) {
            server_1.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", i, "s", "1")), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(tx));
        }
        herddb.core.TestUtils.commitTransaction(server_1.getManager(), TableSpace.DEFAULT, tx);
        // when we send the dump to the follower we must send only the latest version of the record
        for (int i = 0; i < 10; i++) {
            tableManager.flush();
            server_1.getManager().executeUpdate(new UpdateStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1, "s", "2" + i), null), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        }
        BooleanHolder foundDuplicate = new BooleanHolder(false);
        server_1.getManager().getDataStorageManager().fullTableScan(tableSpaceManager.getTableSpaceUUID(), tableManager.getTable().uuid, new FullTableScanConsumer() {

            Map<Bytes, Long> recordPage = new HashMap<>();

            @Override
            public void acceptPage(long pageId, List<Record> records) {
                for (Record record : records) {
                    Long prev = recordPage.put(record.key, pageId);
                    if (prev != null) {
                        foundDuplicate.value = true;
                    }
                }
            }
        });
        assertTrue(foundDuplicate.value);
        // data will be downloaded from the server_1 (PROPERTY_BOOT_FORCE_DOWNLOAD_SNAPSHOT)
        try (Server server_2 = new Server(serverconfig_2)) {
            server_2.start();
            assertTrue(server_2.getManager().isTableSpaceLocallyRecoverable(server_1.getMetadataStorageManager().describeTableSpace(TableSpace.DEFAULT)));
            server_1.getManager().executeStatement(new AlterTableSpaceStatement(TableSpace.DEFAULT, new HashSet<>(Arrays.asList("server1", "server2")), "server1", 1, 0), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            assertTrue(server_2.getManager().waitForTablespace(TableSpace.DEFAULT, 60000, false));
        }
    }
}
Also used : UpdateStatement(herddb.model.commands.UpdateStatement) AlterTableSpaceStatement(herddb.model.commands.AlterTableSpaceStatement) Table(herddb.model.Table) Server(herddb.server.Server) HashMap(java.util.HashMap) ServerConfiguration(herddb.server.ServerConfiguration) CreateTableStatement(herddb.model.commands.CreateTableStatement) BooleanHolder(herddb.utils.BooleanHolder) InsertStatement(herddb.model.commands.InsertStatement) FullTableScanConsumer(herddb.storage.FullTableScanConsumer) Bytes(herddb.utils.Bytes) AbstractTableManager(herddb.core.AbstractTableManager) TransactionContext(herddb.model.TransactionContext) TableSpaceManager(herddb.core.TableSpaceManager) Record(herddb.model.Record) HashSet(java.util.HashSet) Test(org.junit.Test)

Aggregations

BooleanHolder (herddb.utils.BooleanHolder)3 Record (herddb.model.Record)2 Table (herddb.model.Table)2 TransactionContext (herddb.model.TransactionContext)2 InsertStatement (herddb.model.commands.InsertStatement)2 UpdateStatement (herddb.model.commands.UpdateStatement)2 ServerConfiguration (herddb.server.ServerConfiguration)2 FullTableScanConsumer (herddb.storage.FullTableScanConsumer)2 Bytes (herddb.utils.Bytes)2 HashMap (java.util.HashMap)2 HashSet (java.util.HashSet)2 AtomicLong (java.util.concurrent.atomic.AtomicLong)2 RecordSerializer (herddb.codec.RecordSerializer)1 AbstractTableManager (herddb.core.AbstractTableManager)1 DataPageMetaData (herddb.core.PageSet.DataPageMetaData)1 TableSpaceManager (herddb.core.TableSpaceManager)1 TableManagerStats (herddb.core.stats.TableManagerStats)1 IndexOperation (herddb.index.IndexOperation)1 KeyToPageIndex (herddb.index.KeyToPageIndex)1 PrimaryIndexSeek (herddb.index.PrimaryIndexSeek)1