Search in sources :

Example 66 with DataAccessor

use of herddb.utils.DataAccessor in project herddb by diennea.

the class DirectMultipleConcurrentUpdatesSuite method performTest.

protected void performTest(boolean useTransactions, long checkPointPeriod, boolean withIndexes, boolean uniqueIndexes) throws Exception {
    Path baseDir = folder.newFolder().toPath();
    ServerConfiguration serverConfiguration = newServerConfigurationWithAutoPort(baseDir);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024 / 4);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, checkPointPeriod);
    serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
    serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
    ConcurrentHashMap<String, Long> expectedValue = new ConcurrentHashMap<>();
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForStandaloneBoot();
        DBManager manager = server.getManager();
        execute(manager, "CREATE TABLE mytable (id string primary key, n1 long, n2 integer)", Collections.emptyList());
        if (withIndexes) {
            if (uniqueIndexes) {
                // use n1 + id in order to not have collisions and lock timeouts
                execute(manager, "CREATE UNIQUE INDEX theindex ON mytable (n1, id)", Collections.emptyList());
            } else {
                execute(manager, "CREATE INDEX theindex ON mytable (n1)", Collections.emptyList());
            }
        }
        long tx = TestUtils.beginTransaction(manager, TableSpace.DEFAULT);
        for (int i = 0; i < TABLESIZE; i++) {
            TestUtils.executeUpdate(manager, "INSERT INTO mytable (id,n1,n2) values(?,?,?)", Arrays.asList("test_" + i, 1, 2), new TransactionContext(tx));
            expectedValue.put("test_" + i, 1L);
        }
        TestUtils.commitTransaction(manager, TableSpace.DEFAULT, tx);
        ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
        try {
            List<Future> futures = new ArrayList<>();
            AtomicLong updates = new AtomicLong();
            AtomicLong skipped = new AtomicLong();
            AtomicLong gets = new AtomicLong();
            for (int i = 0; i < TABLESIZE * MULTIPLIER; i++) {
                futures.add(threadPool.submit(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            boolean update = ThreadLocalRandom.current().nextBoolean();
                            int k = ThreadLocalRandom.current().nextInt(TABLESIZE);
                            long value = ThreadLocalRandom.current().nextInt(TABLESIZE);
                            long transactionId;
                            String key = "test_" + k;
                            Long actual = expectedValue.remove(key);
                            if (actual == null) {
                                // another thread working on this entry, skip
                                skipped.incrementAndGet();
                                return;
                            }
                            if (update) {
                                updates.incrementAndGet();
                                DMLStatementExecutionResult updateResult = TestUtils.executeUpdate(manager, "UPDATE mytable set n1=? WHERE id=?", Arrays.asList(value, "test_" + k), new TransactionContext(useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID));
                                long count = updateResult.getUpdateCount();
                                transactionId = updateResult.transactionId;
                                if (count <= 0) {
                                    throw new RuntimeException("not updated ?");
                                }
                            } else {
                                gets.incrementAndGet();
                                DataScanner res = TestUtils.scanKeepReadLocks(manager, "SELECT * FROM mytable where id=?", Arrays.asList("test_" + k), new TransactionContext(useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID));
                                if (!res.hasNext()) {
                                    throw new RuntimeException("not found?");
                                }
                                res.close();
                                transactionId = res.getTransactionId();
                                // value did not change actually
                                value = actual;
                            }
                            if (useTransactions) {
                                if (transactionId <= 0) {
                                    throw new RuntimeException("no transaction ?");
                                }
                                commitTransaction(manager, TableSpace.DEFAULT, transactionId);
                            }
                            expectedValue.put(key, value);
                        } catch (Exception err) {
                            throw new RuntimeException(err);
                        }
                    }
                }));
            }
            for (Future f : futures) {
                f.get();
            }
            System.out.println("stats::updates:" + updates);
            System.out.println("stats::get:" + gets);
            assertTrue(updates.get() > 0);
            assertTrue(gets.get() > 0);
            List<String> erroredKeys = new ArrayList<>();
            for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
                List<DataAccessor> records;
                DataAccessor data;
                try (DataScanner res = scan(manager, "SELECT n1 FROM mytable where id=?", Arrays.asList(entry.getKey()))) {
                    records = res.consume();
                    data = records.get(0);
                }
                assertEquals(1, records.size());
                if (!entry.getValue().equals(data.get("n1"))) {
                    System.out.println("expected value " + data.get("n1") + ", but got " + Long.valueOf(entry.getValue()) + " for key " + entry.getKey());
                    erroredKeys.add(entry.getKey());
                }
            }
            assertTrue(erroredKeys.isEmpty());
            TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
            System.out.println("stats::tablesize:" + stats.getTablesize());
            System.out.println("stats::dirty records:" + stats.getDirtyrecords());
            System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
            System.out.println("stats::load count:" + stats.getLoadedPagesCount());
            System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
            // assertTrue(stats.getUnloadedPagesCount() > 0);
            assertEquals(TABLESIZE, stats.getTablesize());
        } finally {
            threadPool.shutdown();
            threadPool.awaitTermination(1, TimeUnit.MINUTES);
        }
    }
    // restart and recovery
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForTableSpaceBoot(TableSpace.DEFAULT, 300000, true);
        DBManager manager = server.getManager();
        List<String> erroredKeys = new ArrayList<>();
        for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
            List<DataAccessor> records;
            DataAccessor data;
            try (DataScanner res = scan(manager, "SELECT n1 FROM mytable where id=?", Arrays.asList(entry.getKey()))) {
                records = res.consume();
                data = records.get(0);
            }
            assertEquals(1, records.size());
            if (!entry.getValue().equals(data.get("n1"))) {
                System.out.println("expected value " + data.get("n1") + ", but got " + Long.valueOf(entry.getValue()) + " for key " + entry.getKey());
                erroredKeys.add(entry.getKey());
            }
        }
        assertTrue(erroredKeys.isEmpty());
    }
}
Also used : Server(herddb.server.Server) DataAccessor(herddb.utils.DataAccessor) ServerConfiguration(herddb.server.ServerConfiguration) ArrayList(java.util.ArrayList) DataScanner(herddb.model.DataScanner) TableManagerStats(herddb.core.stats.TableManagerStats) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Path(java.nio.file.Path) AtomicLong(java.util.concurrent.atomic.AtomicLong) DBManager(herddb.core.DBManager) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) TransactionContext(herddb.model.TransactionContext) AtomicLong(java.util.concurrent.atomic.AtomicLong) ExecutorService(java.util.concurrent.ExecutorService) Future(java.util.concurrent.Future) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap)

Example 67 with DataAccessor

use of herddb.utils.DataAccessor in project herddb by diennea.

the class TableManager method executeInsertAsync.

private CompletableFuture<StatementExecutionResult> executeInsertAsync(InsertStatement insert, Transaction transaction, StatementEvaluationContext context) {
    /*
         an insert can succeed only if the row is valid and the "keys" structure  does not contain the requested key
         the insert will add the row in the 'buffer' without assigning a page to it
         locks: the insert uses global 'insert' lock on the table
         the insert will update the 'maxKey' for auto_increment primary keys
         */
    Bytes key;
    byte[] value;
    try {
        key = Bytes.from_array(insert.getKeyFunction().computeNewValue(null, context, tableContext));
        value = insert.getValuesFunction().computeNewValue(new Record(key, null), context, tableContext);
    } catch (StatementExecutionException validationError) {
        return Futures.exception(validationError);
    } catch (Throwable validationError) {
        return Futures.exception(new StatementExecutionException(validationError));
    }
    List<UniqueIndexLockReference> uniqueIndexes = null;
    Map<String, AbstractIndexManager> indexes = tableSpaceManager.getIndexesOnTable(table.name);
    if (indexes != null || table.foreignKeys != null) {
        try {
            DataAccessor values = new Record(key, Bytes.from_array(value)).getDataAccessor(table);
            if (table.foreignKeys != null) {
                for (ForeignKeyDef fk : table.foreignKeys) {
                    checkForeignKeyConstraintsAsChildTable(fk, values, context, transaction);
                }
            }
            if (indexes != null) {
                for (AbstractIndexManager index : indexes.values()) {
                    if (index.isUnique()) {
                        Bytes indexKey = RecordSerializer.serializeIndexKey(values, index.getIndex(), index.getColumnNames());
                        if (uniqueIndexes == null) {
                            uniqueIndexes = new ArrayList<>(1);
                        }
                        uniqueIndexes.add(new UniqueIndexLockReference(index, indexKey));
                    } else {
                        RecordSerializer.validateIndexableValue(values, index.getIndex(), index.getColumnNames());
                    }
                }
            }
        } catch (IllegalArgumentException | herddb.utils.IllegalDataAccessException | StatementExecutionException err) {
            if (err instanceof StatementExecutionException) {
                return Futures.exception(err);
            } else {
                return Futures.exception(new StatementExecutionException(err.getMessage(), err));
            }
        }
    }
    final long size = DataPage.estimateEntrySize(key, value);
    if (size > maxLogicalPageSize) {
        return Futures.exception(new RecordTooBigException("New record " + key + " is to big to be inserted: size " + size + ", max size " + maxLogicalPageSize));
    }
    CompletableFuture<StatementExecutionResult> res = null;
    LockHandle lock = null;
    try {
        lock = lockForWrite(key, transaction);
        if (uniqueIndexes != null) {
            for (UniqueIndexLockReference uniqueIndexLock : uniqueIndexes) {
                AbstractIndexManager index = uniqueIndexLock.indexManager;
                LockHandle lockForIndex = lockForWrite(uniqueIndexLock.key, transaction, index.getIndexName(), index.getLockManager());
                if (transaction == null) {
                    uniqueIndexLock.lockHandle = lockForIndex;
                }
                if (index.valueAlreadyMapped(uniqueIndexLock.key, null)) {
                    res = Futures.exception(new UniqueIndexContraintViolationException(index.getIndexName(), key, "key " + key + ", already exists in table " + table.name + " on UNIQUE index " + index.getIndexName()));
                }
                if (res != null) {
                    break;
                }
            }
        }
    } catch (HerdDBInternalException err) {
        res = Futures.exception(err);
    }
    boolean fallbackToUpsert = false;
    if (res == null) {
        if (transaction != null) {
            if (transaction.recordDeleted(table.name, key)) {
            // OK, INSERT on a DELETED record inside this transaction
            } else if (transaction.recordInserted(table.name, key) != null) {
                // ERROR, INSERT on a INSERTED record inside this transaction
                res = Futures.exception(new DuplicatePrimaryKeyException(key, "key " + key + ", decoded as " + RecordSerializer.deserializePrimaryKey(key, table) + ", already exists in table " + table.name + " inside transaction " + transaction.transactionId));
            } else if (keyToPage.containsKey(key)) {
                if (insert.isUpsert()) {
                    fallbackToUpsert = true;
                } else {
                    res = Futures.exception(new DuplicatePrimaryKeyException(key, "key " + key + ", decoded as " + RecordSerializer.deserializePrimaryKey(key, table) + ", already exists in table " + table.name + " during transaction " + transaction.transactionId));
                }
            }
        } else if (keyToPage.containsKey(key)) {
            if (insert.isUpsert()) {
                fallbackToUpsert = true;
            } else {
                res = Futures.exception(new DuplicatePrimaryKeyException(key, "key " + key + ", decoded as " + RecordSerializer.deserializePrimaryKey(key, table) + ", already exists in table " + table.name));
            }
        }
    }
    if (res == null) {
        LogEntry entry;
        if (fallbackToUpsert) {
            entry = LogEntryFactory.update(table, key, Bytes.from_array(value), transaction);
        } else {
            entry = LogEntryFactory.insert(table, key, Bytes.from_array(value), transaction);
        }
        CommitLogResult pos = log.log(entry, entry.transactionId <= 0);
        res = pos.logSequenceNumber.thenApplyAsync((lsn) -> {
            apply(pos, entry, false);
            return new DMLStatementExecutionResult(entry.transactionId, 1, key, insert.isReturnValues() ? Bytes.from_array(value) : null);
        }, tableSpaceManager.getCallbacksExecutor());
    }
    if (uniqueIndexes != null) {
        // TODO: reverse order
        for (UniqueIndexLockReference uniqueIndexLock : uniqueIndexes) {
            res = releaseWriteLock(res, uniqueIndexLock.lockHandle, uniqueIndexLock.indexManager.getLockManager());
        }
    }
    if (transaction == null) {
        res = releaseWriteLock(res, lock);
    }
    return res;
}
Also used : Arrays(java.util.Arrays) NullLockManager(herddb.utils.NullLockManager) Table(herddb.model.Table) TruncateTableStatement(herddb.model.commands.TruncateTableStatement) DuplicatePrimaryKeyException(herddb.model.DuplicatePrimaryKeyException) TableStatus(herddb.storage.TableStatus) Map(java.util.Map) DataAccessor(herddb.utils.DataAccessor) LogNotAvailableException(herddb.log.LogNotAvailableException) CommitLogResult(herddb.log.CommitLogResult) LogSequenceNumber(herddb.log.LogSequenceNumber) UniqueIndexContraintViolationException(herddb.model.UniqueIndexContraintViolationException) Set(java.util.Set) RecordSerializer(herddb.codec.RecordSerializer) JSQLParserPlanner.delimit(herddb.sql.JSQLParserPlanner.delimit) DataPageMetaData(herddb.core.PageSet.DataPageMetaData) ScanStatement(herddb.model.commands.ScanStatement) Stream(java.util.stream.Stream) StatsLogger(org.apache.bookkeeper.stats.StatsLogger) Bytes(herddb.utils.Bytes) Holder(herddb.utils.Holder) LockHandle(herddb.utils.LockHandle) ForeignKeyViolationException(herddb.model.ForeignKeyViolationException) LogEntry(herddb.log.LogEntry) ArrayList(java.util.ArrayList) TransactionContext(herddb.model.TransactionContext) Transaction(herddb.model.Transaction) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) Projection(herddb.model.Projection) ForeignKeyDef(herddb.model.ForeignKeyDef) EnsureLongIncrementAccumulator(herddb.utils.EnsureLongIncrementAccumulator) LogEntryType(herddb.log.LogEntryType) Record(herddb.model.Record) LogEntryFactory(herddb.log.LogEntryFactory) KeyToPageIndex(herddb.index.KeyToPageIndex) DataStorageManager(herddb.storage.DataStorageManager) ColumnTypes(herddb.model.ColumnTypes) ILocalLockManager(herddb.utils.ILocalLockManager) AtomicLong(java.util.concurrent.atomic.AtomicLong) Lock(java.util.concurrent.locks.Lock) IndexOperation(herddb.index.IndexOperation) Column(herddb.model.Column) StampedLock(java.util.concurrent.locks.StampedLock) UpdateStatement(herddb.model.commands.UpdateStatement) ScanLimitsImpl(herddb.model.ScanLimitsImpl) TupleComparator(herddb.model.TupleComparator) ServerConfiguration(herddb.server.ServerConfiguration) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RecordTooBigException(herddb.model.RecordTooBigException) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) LocalLockManager(herddb.utils.LocalLockManager) Futures(herddb.utils.Futures) DataStorageManagerException(herddb.storage.DataStorageManagerException) Index(herddb.model.Index) InsertStatement(herddb.model.commands.InsertStatement) DataScanner(herddb.model.DataScanner) DDLException(herddb.model.DDLException) RecordFunction(herddb.model.RecordFunction) StatementExecutionException(herddb.model.StatementExecutionException) TableContext(herddb.model.TableContext) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Logger(java.util.logging.Logger) Collectors(java.util.stream.Collectors) List(java.util.List) FullTableScanConsumer(herddb.storage.FullTableScanConsumer) GetStatement(herddb.model.commands.GetStatement) Entry(java.util.Map.Entry) Statement(herddb.model.Statement) LongAdder(java.util.concurrent.atomic.LongAdder) DataScannerException(herddb.model.DataScannerException) GetResult(herddb.model.GetResult) PrimaryIndexSeek(herddb.index.PrimaryIndexSeek) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Function(java.util.function.Function) BatchOrderedExecutor(herddb.utils.BatchOrderedExecutor) ConcurrentMap(java.util.concurrent.ConcurrentMap) Level(java.util.logging.Level) HashSet(java.util.HashSet) BooleanHolder(herddb.utils.BooleanHolder) ScanLimits(herddb.model.ScanLimits) DeleteStatement(herddb.model.commands.DeleteStatement) Iterator(java.util.Iterator) ReentrantLock(java.util.concurrent.locks.ReentrantLock) Semaphore(java.util.concurrent.Semaphore) DataPageDoesNotExistException(herddb.storage.DataPageDoesNotExistException) Counter(org.apache.bookkeeper.stats.Counter) StatementExecutionResult(herddb.model.StatementExecutionResult) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) CommitLog(herddb.log.CommitLog) AbstractMap(java.util.AbstractMap) TableConsistencyCheckStatement(herddb.model.commands.TableConsistencyCheckStatement) Predicate(herddb.model.Predicate) StatementEvaluationContext(herddb.model.StatementEvaluationContext) Comparator(java.util.Comparator) Collections(java.util.Collections) SECONDS(java.util.concurrent.TimeUnit.SECONDS) TableManagerStats(herddb.core.stats.TableManagerStats) SystemProperties(herddb.utils.SystemProperties) DataAccessor(herddb.utils.DataAccessor) CommitLogResult(herddb.log.CommitLogResult) RecordTooBigException(herddb.model.RecordTooBigException) StatementExecutionException(herddb.model.StatementExecutionException) Bytes(herddb.utils.Bytes) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) StatementExecutionResult(herddb.model.StatementExecutionResult) DuplicatePrimaryKeyException(herddb.model.DuplicatePrimaryKeyException) Record(herddb.model.Record) LogEntry(herddb.log.LogEntry) LockHandle(herddb.utils.LockHandle) UniqueIndexContraintViolationException(herddb.model.UniqueIndexContraintViolationException) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) ForeignKeyDef(herddb.model.ForeignKeyDef)

Example 68 with DataAccessor

use of herddb.utils.DataAccessor in project herddb by diennea.

the class TableManager method checkForeignKeyConstraintsAsChildTable.

private void checkForeignKeyConstraintsAsChildTable(ForeignKeyDef fk, DataAccessor values, StatementEvaluationContext context, Transaction transaction) throws StatementExecutionException {
    // We are creating a SQL query and then using DBManager
    // using an SQL query will let us leverage the SQL Planner
    // and use the best index to perform the execution
    // the SQL Planner will cache the plan, and the plan will also be
    // invalidated consistently during DML operations.
    String query = childForeignKeyQueries.computeIfAbsent(fk.name, (l -> {
        Table parentTable = tableSpaceManager.getTableManagerByUUID(fk.parentTableId).getTable();
        // with '*' we are not going to perform projections or copies
        StringBuilder q = new StringBuilder("SELECT * FROM ");
        q.append(delimit(parentTable.tablespace));
        q.append(".");
        q.append(delimit(parentTable.name));
        q.append(" WHERE ");
        for (int i = 0; i < fk.parentTableColumns.length; i++) {
            if (i > 0) {
                q.append(" AND ");
            }
            q.append(delimit(fk.parentTableColumns[i]));
            q.append("=?");
        }
        return q.toString();
    }));
    final List<Object> valuesToMatch = new ArrayList<>(fk.columns.length);
    boolean allNulls = true;
    for (int i = 0; i < fk.columns.length; i++) {
        Object value = values.get(fk.columns[i]);
        allNulls = allNulls && value == null;
        valuesToMatch.add(value);
    }
    if (allNulls) {
        // all of the values are null, so no check on the parent table
        return;
    }
    TransactionContext tx = transaction != null ? new TransactionContext(transaction.transactionId) : TransactionContext.NO_TRANSACTION;
    boolean fkOk;
    try (DataScanner scan = tableSpaceManager.getDbmanager().executeSimpleQuery(tableSpaceManager.getTableSpaceName(), query, valuesToMatch, // only one record
    1, // keep read locks in TransactionContext
    true, tx, null)) {
        List<DataAccessor> resultSet = scan.consume();
        fkOk = !resultSet.isEmpty();
    } catch (DataScannerException err) {
        throw new StatementExecutionException(err);
    }
    if (!fkOk) {
        throw new ForeignKeyViolationException(fk.name, "foreignKey " + table.name + "." + fk.name + " violated");
    }
}
Also used : Arrays(java.util.Arrays) NullLockManager(herddb.utils.NullLockManager) Table(herddb.model.Table) TruncateTableStatement(herddb.model.commands.TruncateTableStatement) DuplicatePrimaryKeyException(herddb.model.DuplicatePrimaryKeyException) TableStatus(herddb.storage.TableStatus) Map(java.util.Map) DataAccessor(herddb.utils.DataAccessor) LogNotAvailableException(herddb.log.LogNotAvailableException) CommitLogResult(herddb.log.CommitLogResult) LogSequenceNumber(herddb.log.LogSequenceNumber) UniqueIndexContraintViolationException(herddb.model.UniqueIndexContraintViolationException) Set(java.util.Set) RecordSerializer(herddb.codec.RecordSerializer) JSQLParserPlanner.delimit(herddb.sql.JSQLParserPlanner.delimit) DataPageMetaData(herddb.core.PageSet.DataPageMetaData) ScanStatement(herddb.model.commands.ScanStatement) Stream(java.util.stream.Stream) StatsLogger(org.apache.bookkeeper.stats.StatsLogger) Bytes(herddb.utils.Bytes) Holder(herddb.utils.Holder) LockHandle(herddb.utils.LockHandle) ForeignKeyViolationException(herddb.model.ForeignKeyViolationException) LogEntry(herddb.log.LogEntry) ArrayList(java.util.ArrayList) TransactionContext(herddb.model.TransactionContext) Transaction(herddb.model.Transaction) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) Projection(herddb.model.Projection) ForeignKeyDef(herddb.model.ForeignKeyDef) EnsureLongIncrementAccumulator(herddb.utils.EnsureLongIncrementAccumulator) LogEntryType(herddb.log.LogEntryType) Record(herddb.model.Record) LogEntryFactory(herddb.log.LogEntryFactory) KeyToPageIndex(herddb.index.KeyToPageIndex) DataStorageManager(herddb.storage.DataStorageManager) ColumnTypes(herddb.model.ColumnTypes) ILocalLockManager(herddb.utils.ILocalLockManager) AtomicLong(java.util.concurrent.atomic.AtomicLong) Lock(java.util.concurrent.locks.Lock) IndexOperation(herddb.index.IndexOperation) Column(herddb.model.Column) StampedLock(java.util.concurrent.locks.StampedLock) UpdateStatement(herddb.model.commands.UpdateStatement) ScanLimitsImpl(herddb.model.ScanLimitsImpl) TupleComparator(herddb.model.TupleComparator) ServerConfiguration(herddb.server.ServerConfiguration) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RecordTooBigException(herddb.model.RecordTooBigException) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) LocalLockManager(herddb.utils.LocalLockManager) Futures(herddb.utils.Futures) DataStorageManagerException(herddb.storage.DataStorageManagerException) Index(herddb.model.Index) InsertStatement(herddb.model.commands.InsertStatement) DataScanner(herddb.model.DataScanner) DDLException(herddb.model.DDLException) RecordFunction(herddb.model.RecordFunction) StatementExecutionException(herddb.model.StatementExecutionException) TableContext(herddb.model.TableContext) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Logger(java.util.logging.Logger) Collectors(java.util.stream.Collectors) List(java.util.List) FullTableScanConsumer(herddb.storage.FullTableScanConsumer) GetStatement(herddb.model.commands.GetStatement) Entry(java.util.Map.Entry) Statement(herddb.model.Statement) LongAdder(java.util.concurrent.atomic.LongAdder) DataScannerException(herddb.model.DataScannerException) GetResult(herddb.model.GetResult) PrimaryIndexSeek(herddb.index.PrimaryIndexSeek) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Function(java.util.function.Function) BatchOrderedExecutor(herddb.utils.BatchOrderedExecutor) ConcurrentMap(java.util.concurrent.ConcurrentMap) Level(java.util.logging.Level) HashSet(java.util.HashSet) BooleanHolder(herddb.utils.BooleanHolder) ScanLimits(herddb.model.ScanLimits) DeleteStatement(herddb.model.commands.DeleteStatement) Iterator(java.util.Iterator) ReentrantLock(java.util.concurrent.locks.ReentrantLock) Semaphore(java.util.concurrent.Semaphore) DataPageDoesNotExistException(herddb.storage.DataPageDoesNotExistException) Counter(org.apache.bookkeeper.stats.Counter) StatementExecutionResult(herddb.model.StatementExecutionResult) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) CommitLog(herddb.log.CommitLog) AbstractMap(java.util.AbstractMap) TableConsistencyCheckStatement(herddb.model.commands.TableConsistencyCheckStatement) Predicate(herddb.model.Predicate) StatementEvaluationContext(herddb.model.StatementEvaluationContext) Comparator(java.util.Comparator) Collections(java.util.Collections) SECONDS(java.util.concurrent.TimeUnit.SECONDS) TableManagerStats(herddb.core.stats.TableManagerStats) SystemProperties(herddb.utils.SystemProperties) Table(herddb.model.Table) DataAccessor(herddb.utils.DataAccessor) ArrayList(java.util.ArrayList) StatementExecutionException(herddb.model.StatementExecutionException) DataScanner(herddb.model.DataScanner) ForeignKeyViolationException(herddb.model.ForeignKeyViolationException) TransactionContext(herddb.model.TransactionContext) DataScannerException(herddb.model.DataScannerException)

Example 69 with DataAccessor

use of herddb.utils.DataAccessor in project herddb by diennea.

the class TableManager method applyDelete.

private void applyDelete(Bytes key) throws DataStorageManagerException {
    /* This could be a normal or a temporary modifiable page */
    final Long pageId = keyToPage.remove(key);
    if (pageId == null) {
        throw new IllegalStateException("corrupted transaction log: key " + key + " is not present in table " + table.tablespace + "." + table.name);
    }
    if (LOGGER.isLoggable(Level.FINEST)) {
        LOGGER.log(Level.FINEST, "Deleted key " + key + " from page " + pageId + " from table " + table.tablespace + "." + table.name);
    }
    /*
         * We'll try to remove the record if in a writable page, otherwise we'll simply set the old page
         * as dirty.
         */
    final Map<String, AbstractIndexManager> indexes = tableSpaceManager.getIndexesOnTable(table.name);
    /*
         * When index is enabled we need the old value to update them, we'll force the page load only if that
         * record is really needed.
         */
    final DataPage page;
    final Record previous;
    if (indexes == null) {
        /* We don't need the page if isn't loaded or isn't a mutable new page */
        page = newPages.get(pageId);
        if (page != null) {
            pageReplacementPolicy.pageHit(page);
            previous = page.get(key);
            if (previous == null) {
                throw new IllegalStateException("corrupted PK: old page " + pageId + " for deleted record at " + key + " was not found in table " + table.tablespace + "." + table.name);
            }
        } else {
            previous = null;
        }
    } else {
        /* We really need the page for update index old values */
        page = loadPageToMemory(pageId, false);
        previous = page.get(key);
        if (previous == null) {
            throw new IllegalStateException("corrupted PK: old page " + pageId + " for deleted record at " + key + " was not found in table " + table.tablespace + "." + table.name);
        }
    }
    if (page == null || page.immutable) {
        /* Unloaded or immutable, set it as dirty */
        pageSet.setPageDirty(pageId, previous);
    } else {
        /* Mutable page, need to check if still modifiable or already unloaded */
        final Lock lock = page.pageLock.readLock();
        lock.lock();
        try {
            if (page.writable) {
                /* We can modify the page directly */
                page.remove(key);
            } else {
                /* Unfortunately is not writable (anymore), set it as dirty */
                pageSet.setPageDirty(pageId, previous);
            }
        } finally {
            lock.unlock();
        }
    }
    if (indexes != null) {
        /* If there are indexes e have already forced a page load and previous record has been loaded */
        DataAccessor values = previous.getDataAccessor(table);
        for (AbstractIndexManager index : indexes.values()) {
            Bytes indexKey = RecordSerializer.serializeIndexKey(values, index.getIndex(), index.getColumnNames());
            index.recordDeleted(key, indexKey);
        }
    }
}
Also used : Bytes(herddb.utils.Bytes) DataAccessor(herddb.utils.DataAccessor) AtomicLong(java.util.concurrent.atomic.AtomicLong) Record(herddb.model.Record) Lock(java.util.concurrent.locks.Lock) StampedLock(java.util.concurrent.locks.StampedLock) ReentrantLock(java.util.concurrent.locks.ReentrantLock)

Example 70 with DataAccessor

use of herddb.utils.DataAccessor in project herddb by diennea.

the class TableManager method executeUpdateAsync.

private CompletableFuture<StatementExecutionResult> executeUpdateAsync(UpdateStatement update, Transaction transaction, StatementEvaluationContext context) throws StatementExecutionException, DataStorageManagerException {
    // LOGGER.log(Level.SEVERE, "executeUpdateAsync, " + update + ", transaction " + transaction);
    AtomicInteger updateCount = new AtomicInteger();
    Holder<Bytes> lastKey = new Holder<>();
    Holder<byte[]> lastValue = new Holder<>();
    /*
         an update can succeed only if the row is valid, the key is contains in the "keys" structure
         the update will simply override the value of the row, assigning a null page to the row
         the update can have a 'where' predicate which is to be evaluated against the decoded row, the update will be executed only if the predicate returns boolean 'true' value  (CAS operation)
         locks: the update  uses a lock on the the key
         */
    RecordFunction function = update.getFunction();
    long transactionId = transaction != null ? transaction.transactionId : 0;
    Predicate predicate = update.getPredicate();
    Map<String, AbstractIndexManager> indexes = tableSpaceManager.getIndexesOnTable(table.name);
    ScanStatement scan = new ScanStatement(table.tablespace, table, predicate);
    List<CompletableFuture<PendingLogEntryWork>> writes = new ArrayList<>();
    try {
        accessTableData(scan, context, new ScanResultOperation() {

            @Override
            public void accept(Record current, LockHandle lockHandle) throws StatementExecutionException, LogNotAvailableException, DataStorageManagerException {
                List<UniqueIndexLockReference> uniqueIndexes = null;
                byte[] newValue;
                try {
                    if (childrenTables != null) {
                        DataAccessor currentValues = current.getDataAccessor(table);
                        for (Table childTable : childrenTables) {
                            executeForeignKeyConstraintsAsParentTable(childTable, currentValues, context, transaction, false);
                        }
                    }
                    newValue = function.computeNewValue(current, context, tableContext);
                    if (indexes != null || table.foreignKeys != null) {
                        DataAccessor values = new Record(current.key, Bytes.from_array(newValue)).getDataAccessor(table);
                        if (table.foreignKeys != null) {
                            for (ForeignKeyDef fk : table.foreignKeys) {
                                checkForeignKeyConstraintsAsChildTable(fk, values, context, transaction);
                            }
                        }
                        if (indexes != null) {
                            for (AbstractIndexManager index : indexes.values()) {
                                if (index.isUnique()) {
                                    Bytes indexKey = RecordSerializer.serializeIndexKey(values, index.getIndex(), index.getColumnNames());
                                    if (uniqueIndexes == null) {
                                        uniqueIndexes = new ArrayList<>(1);
                                    }
                                    UniqueIndexLockReference uniqueIndexLock = new UniqueIndexLockReference(index, indexKey);
                                    uniqueIndexes.add(uniqueIndexLock);
                                    LockHandle lockForIndex = lockForWrite(uniqueIndexLock.key, transaction, index.getIndexName(), index.getLockManager());
                                    if (transaction == null) {
                                        uniqueIndexLock.lockHandle = lockForIndex;
                                    }
                                    if (index.valueAlreadyMapped(indexKey, current.key)) {
                                        throw new UniqueIndexContraintViolationException(index.getIndexName(), indexKey, "Value " + indexKey + " already present in index " + index.getIndexName());
                                    }
                                } else {
                                    RecordSerializer.validateIndexableValue(values, index.getIndex(), index.getColumnNames());
                                }
                            }
                        }
                    }
                } catch (IllegalArgumentException | herddb.utils.IllegalDataAccessException | StatementExecutionException err) {
                    locksManager.releaseLock(lockHandle);
                    StatementExecutionException finalError;
                    if (!(err instanceof StatementExecutionException)) {
                        finalError = new StatementExecutionException(err.getMessage(), err);
                    } else {
                        finalError = (StatementExecutionException) err;
                    }
                    CompletableFuture<PendingLogEntryWork> res = Futures.exception(finalError);
                    if (uniqueIndexes != null) {
                        for (UniqueIndexLockReference lock : uniqueIndexes) {
                            res = releaseWriteLock(res, lock.lockHandle, lock.indexManager.getLockManager());
                        }
                    }
                    writes.add(res);
                    return;
                }
                final long size = DataPage.estimateEntrySize(current.key, newValue);
                if (size > maxLogicalPageSize) {
                    locksManager.releaseLock(lockHandle);
                    writes.add(Futures.exception(new RecordTooBigException("New version of record " + current.key + " is to big to be update: new size " + size + ", actual size " + DataPage.estimateEntrySize(current) + ", max size " + maxLogicalPageSize)));
                    return;
                }
                LogEntry entry = LogEntryFactory.update(table, current.key, Bytes.from_array(newValue), transaction);
                CommitLogResult pos = log.log(entry, entry.transactionId <= 0);
                final List<UniqueIndexLockReference> _uniqueIndexes = uniqueIndexes;
                writes.add(pos.logSequenceNumber.thenApply(lsn -> new PendingLogEntryWork(entry, pos, lockHandle, _uniqueIndexes)));
                lastKey.value = current.key;
                lastValue.value = newValue;
                updateCount.incrementAndGet();
            }
        }, transaction, true, true);
    } catch (HerdDBInternalException err) {
        LOGGER.log(Level.SEVERE, "bad error during an update", err);
        return Futures.exception(err);
    }
    if (writes.isEmpty()) {
        return CompletableFuture.completedFuture(new DMLStatementExecutionResult(transactionId, 0, null, null));
    }
    if (writes.size() == 1) {
        return writes.get(0).whenCompleteAsync((pending, error) -> {
            try {
                // apply any of the DML operations
                if (error == null) {
                    apply(pending.pos, pending.entry, false);
                }
            } finally {
                releaseMultiplePendingLogEntryWorks(writes);
            }
        }, tableSpaceManager.getCallbacksExecutor()).thenApply((pending) -> {
            return new DMLStatementExecutionResult(transactionId, updateCount.get(), lastKey.value, update.isReturnValues() ? (lastValue.value != null ? Bytes.from_array(lastValue.value) : null) : null);
        });
    } else {
        return Futures.collect(writes).whenCompleteAsync((pendings, error) -> {
            try {
                // apply any of the DML operations
                if (error == null) {
                    for (PendingLogEntryWork pending : pendings) {
                        apply(pending.pos, pending.entry, false);
                    }
                }
            } finally {
                releaseMultiplePendingLogEntryWorks(writes);
            }
        }, tableSpaceManager.getCallbacksExecutor()).thenApply((pendings) -> {
            return new DMLStatementExecutionResult(transactionId, updateCount.get(), lastKey.value, update.isReturnValues() ? (lastValue.value != null ? Bytes.from_array(lastValue.value) : null) : null);
        });
    }
}
Also used : Arrays(java.util.Arrays) NullLockManager(herddb.utils.NullLockManager) Table(herddb.model.Table) TruncateTableStatement(herddb.model.commands.TruncateTableStatement) DuplicatePrimaryKeyException(herddb.model.DuplicatePrimaryKeyException) TableStatus(herddb.storage.TableStatus) Map(java.util.Map) DataAccessor(herddb.utils.DataAccessor) LogNotAvailableException(herddb.log.LogNotAvailableException) CommitLogResult(herddb.log.CommitLogResult) LogSequenceNumber(herddb.log.LogSequenceNumber) UniqueIndexContraintViolationException(herddb.model.UniqueIndexContraintViolationException) Set(java.util.Set) RecordSerializer(herddb.codec.RecordSerializer) JSQLParserPlanner.delimit(herddb.sql.JSQLParserPlanner.delimit) DataPageMetaData(herddb.core.PageSet.DataPageMetaData) ScanStatement(herddb.model.commands.ScanStatement) Stream(java.util.stream.Stream) StatsLogger(org.apache.bookkeeper.stats.StatsLogger) Bytes(herddb.utils.Bytes) Holder(herddb.utils.Holder) LockHandle(herddb.utils.LockHandle) ForeignKeyViolationException(herddb.model.ForeignKeyViolationException) LogEntry(herddb.log.LogEntry) ArrayList(java.util.ArrayList) TransactionContext(herddb.model.TransactionContext) Transaction(herddb.model.Transaction) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) Projection(herddb.model.Projection) ForeignKeyDef(herddb.model.ForeignKeyDef) EnsureLongIncrementAccumulator(herddb.utils.EnsureLongIncrementAccumulator) LogEntryType(herddb.log.LogEntryType) Record(herddb.model.Record) LogEntryFactory(herddb.log.LogEntryFactory) KeyToPageIndex(herddb.index.KeyToPageIndex) DataStorageManager(herddb.storage.DataStorageManager) ColumnTypes(herddb.model.ColumnTypes) ILocalLockManager(herddb.utils.ILocalLockManager) AtomicLong(java.util.concurrent.atomic.AtomicLong) Lock(java.util.concurrent.locks.Lock) IndexOperation(herddb.index.IndexOperation) Column(herddb.model.Column) StampedLock(java.util.concurrent.locks.StampedLock) UpdateStatement(herddb.model.commands.UpdateStatement) ScanLimitsImpl(herddb.model.ScanLimitsImpl) TupleComparator(herddb.model.TupleComparator) ServerConfiguration(herddb.server.ServerConfiguration) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RecordTooBigException(herddb.model.RecordTooBigException) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) LocalLockManager(herddb.utils.LocalLockManager) Futures(herddb.utils.Futures) DataStorageManagerException(herddb.storage.DataStorageManagerException) Index(herddb.model.Index) InsertStatement(herddb.model.commands.InsertStatement) DataScanner(herddb.model.DataScanner) DDLException(herddb.model.DDLException) RecordFunction(herddb.model.RecordFunction) StatementExecutionException(herddb.model.StatementExecutionException) TableContext(herddb.model.TableContext) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Logger(java.util.logging.Logger) Collectors(java.util.stream.Collectors) List(java.util.List) FullTableScanConsumer(herddb.storage.FullTableScanConsumer) GetStatement(herddb.model.commands.GetStatement) Entry(java.util.Map.Entry) Statement(herddb.model.Statement) LongAdder(java.util.concurrent.atomic.LongAdder) DataScannerException(herddb.model.DataScannerException) GetResult(herddb.model.GetResult) PrimaryIndexSeek(herddb.index.PrimaryIndexSeek) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Function(java.util.function.Function) BatchOrderedExecutor(herddb.utils.BatchOrderedExecutor) ConcurrentMap(java.util.concurrent.ConcurrentMap) Level(java.util.logging.Level) HashSet(java.util.HashSet) BooleanHolder(herddb.utils.BooleanHolder) ScanLimits(herddb.model.ScanLimits) DeleteStatement(herddb.model.commands.DeleteStatement) Iterator(java.util.Iterator) ReentrantLock(java.util.concurrent.locks.ReentrantLock) Semaphore(java.util.concurrent.Semaphore) DataPageDoesNotExistException(herddb.storage.DataPageDoesNotExistException) Counter(org.apache.bookkeeper.stats.Counter) StatementExecutionResult(herddb.model.StatementExecutionResult) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) CommitLog(herddb.log.CommitLog) AbstractMap(java.util.AbstractMap) TableConsistencyCheckStatement(herddb.model.commands.TableConsistencyCheckStatement) Predicate(herddb.model.Predicate) StatementEvaluationContext(herddb.model.StatementEvaluationContext) Comparator(java.util.Comparator) Collections(java.util.Collections) SECONDS(java.util.concurrent.TimeUnit.SECONDS) TableManagerStats(herddb.core.stats.TableManagerStats) SystemProperties(herddb.utils.SystemProperties) DataStorageManagerException(herddb.storage.DataStorageManagerException) DataAccessor(herddb.utils.DataAccessor) ArrayList(java.util.ArrayList) CommitLogResult(herddb.log.CommitLogResult) RecordTooBigException(herddb.model.RecordTooBigException) StatementExecutionException(herddb.model.StatementExecutionException) Predicate(herddb.model.Predicate) Bytes(herddb.utils.Bytes) CompletableFuture(java.util.concurrent.CompletableFuture) Record(herddb.model.Record) ArrayList(java.util.ArrayList) List(java.util.List) RecordFunction(herddb.model.RecordFunction) LogEntry(herddb.log.LogEntry) ScanStatement(herddb.model.commands.ScanStatement) LockHandle(herddb.utils.LockHandle) Table(herddb.model.Table) Holder(herddb.utils.Holder) BooleanHolder(herddb.utils.BooleanHolder) UniqueIndexContraintViolationException(herddb.model.UniqueIndexContraintViolationException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) ForeignKeyDef(herddb.model.ForeignKeyDef) LogNotAvailableException(herddb.log.LogNotAvailableException)

Aggregations

DataAccessor (herddb.utils.DataAccessor)164 DataScanner (herddb.model.DataScanner)118 Test (org.junit.Test)110 MemoryCommitLogManager (herddb.mem.MemoryCommitLogManager)83 MemoryDataStorageManager (herddb.mem.MemoryDataStorageManager)82 MemoryMetadataStorageManager (herddb.mem.MemoryMetadataStorageManager)82 CreateTableSpaceStatement (herddb.model.commands.CreateTableSpaceStatement)76 RawString (herddb.utils.RawString)70 DBManager (herddb.core.DBManager)52 TransactionContext (herddb.model.TransactionContext)44 StatementExecutionException (herddb.model.StatementExecutionException)38 Table (herddb.model.Table)34 List (java.util.List)31 DataScannerException (herddb.model.DataScannerException)28 DMLStatementExecutionResult (herddb.model.DMLStatementExecutionResult)27 RuntimeProjectedDataAccessor (herddb.model.planner.ProjectOp.ZeroCopyProjection.RuntimeProjectedDataAccessor)27 Column (herddb.model.Column)22 ScanResult (herddb.model.ScanResult)22 ScanStatement (herddb.model.commands.ScanStatement)22 Bytes (herddb.utils.Bytes)22