Examples with TransactionContext herddb.model.TransactionContext used on opensource projects

Example 81 with TransactionContext

use of herddb.model.TransactionContext in project herddb by diennea.

the class DirectMultipleConcurrentUpdatesTest method performTest.

private void performTest(boolean useTransactions, long checkPointPeriod, boolean withIndexes) throws Exception {
    Path baseDir = folder.newFolder().toPath();
    ServerConfiguration serverConfiguration = new ServerConfiguration(baseDir);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_LOGICAL_PAGE_SIZE, 10 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_DATA_MEMORY, 1024 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_MAX_PK_MEMORY, 1024 * 1024);
    serverConfiguration.set(ServerConfiguration.PROPERTY_CHECKPOINT_PERIOD, checkPointPeriod);
    serverConfiguration.set(ServerConfiguration.PROPERTY_DATADIR, folder.newFolder().getAbsolutePath());
    serverConfiguration.set(ServerConfiguration.PROPERTY_LOGDIR, folder.newFolder().getAbsolutePath());
    ConcurrentHashMap<String, Long> expectedValue = new ConcurrentHashMap<>();
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForStandaloneBoot();
        DBManager manager = server.getManager();
        execute(manager, "CREATE TABLE mytable (id string primary key, n1 long, n2 integer)", Collections.emptyList());
        if (withIndexes) {
            execute(manager, "CREATE INDEX theindex ON mytable (n1 long)", Collections.emptyList());
        }
        long tx = TestUtils.beginTransaction(manager, TableSpace.DEFAULT);
        for (int i = 0; i < TABLESIZE; i++) {
            TestUtils.executeUpdate(manager, "INSERT INTO mytable (id,n1,n2) values(?,?,?)", Arrays.asList("test_" + i, 1, 2), new TransactionContext(tx));
            expectedValue.put("test_" + i, 1L);
        }
        TestUtils.commitTransaction(manager, TableSpace.DEFAULT, tx);
        ExecutorService threadPool = Executors.newFixedThreadPool(THREADPOLSIZE);
        try {
            List<Future> futures = new ArrayList<>();
            AtomicLong updates = new AtomicLong();
            AtomicLong skipped = new AtomicLong();
            AtomicLong gets = new AtomicLong();
            for (int i = 0; i < TABLESIZE * MULTIPLIER; i++) {
                futures.add(threadPool.submit(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            boolean update = ThreadLocalRandom.current().nextBoolean();
                            int k = ThreadLocalRandom.current().nextInt(TABLESIZE);
                            long value = ThreadLocalRandom.current().nextInt(TABLESIZE);
                            long transactionId;
                            String key = "test_" + k;
                            Long actual = expectedValue.remove(key);
                            if (actual == null) {
                                // another thread working on this entry, skip
                                skipped.incrementAndGet();
                                return;
                            }
                            if (update) {
                                updates.incrementAndGet();
                                DMLStatementExecutionResult updateResult = TestUtils.executeUpdate(manager, "UPDATE mytable set n1=? WHERE id=?", Arrays.asList(value, "test_" + k), new TransactionContext(useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID));
                                long count = updateResult.getUpdateCount();
                                transactionId = updateResult.transactionId;
                                if (count <= 0) {
                                    throw new RuntimeException("not updated ?");
                                }
                            } else {
                                gets.incrementAndGet();
                                DataScanner res = TestUtils.scan(manager, "SELECT * FROM mytable where id=?", Arrays.asList("test_" + k), new TransactionContext(useTransactions ? TransactionContext.AUTOTRANSACTION_ID : TransactionContext.NOTRANSACTION_ID));
                                if (!res.hasNext()) {
                                    throw new RuntimeException("not found?");
                                }
                                res.close();
                                transactionId = res.transactionId;
                                // value did not change actually
                                value = actual;
                            }
                            if (useTransactions) {
                                if (transactionId <= 0) {
                                    throw new RuntimeException("no transaction ?");
                                }
                                commitTransaction(manager, TableSpace.DEFAULT, transactionId);
                            }
                            expectedValue.put(key, value);
                        } catch (Exception err) {
                            throw new RuntimeException(err);
                        }
                    }
                }));
            }
            for (Future f : futures) {
                f.get();
            }
            System.out.println("stats::updates:" + updates);
            System.out.println("stats::get:" + gets);
            assertTrue(updates.get() > 0);
            assertTrue(gets.get() > 0);
            List<String> erroredKeys = new ArrayList<>();
            for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
                List<DataAccessor> records;
                DataAccessor data;
                try (DataScanner res = scan(manager, "SELECT n1 FROM mytable where id=?", Arrays.asList(entry.getKey()))) {
                    records = res.consume();
                    data = records.get(0);
                }
                assertEquals(1, records.size());
                if (!entry.getValue().equals(data.get("n1"))) {
                    System.out.println("expected value " + data.get("n1") + ", but got " + Long.valueOf(entry.getValue()) + " for key " + entry.getKey());
                    erroredKeys.add(entry.getKey());
                }
            }
            assertTrue(erroredKeys.isEmpty());
            TableManagerStats stats = server.getManager().getTableSpaceManager(TableSpace.DEFAULT).getTableManager("mytable").getStats();
            System.out.println("stats::tablesize:" + stats.getTablesize());
            System.out.println("stats::dirty records:" + stats.getDirtyrecords());
            System.out.println("stats::unload count:" + stats.getUnloadedPagesCount());
            System.out.println("stats::load count:" + stats.getLoadedPagesCount());
            System.out.println("stats::buffers used mem:" + stats.getBuffersUsedMemory());
            // assertTrue(stats.getUnloadedPagesCount() > 0);
            assertEquals(TABLESIZE, stats.getTablesize());
        } finally {
            threadPool.shutdown();
            threadPool.awaitTermination(1, TimeUnit.MINUTES);
        }
    }
    // restart and recovery
    try (Server server = new Server(serverConfiguration)) {
        server.start();
        server.waitForStandaloneBoot();
        DBManager manager = server.getManager();
        List<String> erroredKeys = new ArrayList<>();
        for (Map.Entry<String, Long> entry : expectedValue.entrySet()) {
            List<DataAccessor> records;
            DataAccessor data;
            try (DataScanner res = scan(manager, "SELECT n1 FROM mytable where id=?", Arrays.asList(entry.getKey()))) {
                records = res.consume();
                data = records.get(0);
            }
            assertEquals(1, records.size());
            if (!entry.getValue().equals(data.get("n1"))) {
                System.out.println("expected value " + data.get("n1") + ", but got " + Long.valueOf(entry.getValue()) + " for key " + entry.getKey());
                erroredKeys.add(entry.getKey());
            }
        }
        assertTrue(erroredKeys.isEmpty());
    }
}

Example 82 with TransactionContext

use of herddb.model.TransactionContext in project herddb by diennea.

the class TruncateTableSQLTest method truncateTableTransactionTest.

@Test
public void truncateTableTransactionTest() throws Exception {
    String nodeId = "localhost";
    try (DBManager manager = new DBManager("localhost", new MemoryMetadataStorageManager(), new MemoryDataStorageManager(), new MemoryCommitLogManager(), null, null)) {
        manager.start();
        CreateTableSpaceStatement st1 = new CreateTableSpaceStatement("tblspace1", Collections.singleton(nodeId), nodeId, 1, 0, 0);
        manager.executeStatement(st1, StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        manager.waitForTablespace("tblspace1", 10000);
        execute(manager, "CREATE TABLE tblspace1.tsql (k1 string primary key,n1 int,s1 string)", Collections.emptyList());
        execute(manager, "CREATE BRIN INDEX test1 ON tblspace1.tsql (k1)", Collections.emptyList());
        execute(manager, "CREATE HASH INDEX test2 ON tblspace1.tsql (k1)", Collections.emptyList());
        long tx1 = TestUtils.beginTransaction(manager, "tblspace1");
        try {
            // forbidden, transactions not allowed
            execute(manager, "truncate TABLE tblspace1.tsql", Collections.emptyList(), new TransactionContext(tx1));
            fail();
        } catch (StatementExecutionException ok) {
            assertEquals("TRUNCATE TABLE cannot be executed within the context of a Transaction", ok.getMessage());
        }
        long txId = execute(manager, "INSERT INTO tblspace1.tsql (k1) values('a')", Collections.emptyList(), TransactionContext.AUTOTRANSACTION_TRANSACTION).transactionId;
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.tsql ", Collections.emptyList(), new TransactionContext(txId))) {
            assertEquals(1, scan.consume().size());
        }
        try {
            // forbidden, a transaction is running on table
            execute(manager, "TRUNCATE TABLE tblspace1.tsql", Collections.emptyList(), TransactionContext.NO_TRANSACTION);
            fail();
        } catch (StatementExecutionException ok) {
            assertEquals("TRUNCATE TABLE cannot be executed table tsql: at least one transaction is pending on it", ok.getCause().getMessage());
        }
        TestUtils.commitTransaction(manager, "tblspace1", txId);
        execute(manager, "TRUNCATE TABLE tblspace1.tsql", Collections.emptyList(), TransactionContext.NO_TRANSACTION);
        try (DataScanner scan = scan(manager, "SELECT * FROM tblspace1.tsql ", Collections.emptyList())) {
            assertEquals(0, scan.consume().size());
        }
    }
}

Example 83 with TransactionContext

use of herddb.model.TransactionContext in project herddb by diennea.

the class TableSpaceManager method executeStatementAsyncInternal.

private CompletableFuture<StatementExecutionResult> executeStatementAsyncInternal(Statement statement, StatementEvaluationContext context, TransactionContext transactionContext, boolean rollbackOnError) throws StatementExecutionException {
    Transaction transaction = transactions.get(transactionContext.transactionId);
    if (transaction != null && !transaction.tableSpace.equals(tableSpaceName)) {
        return Futures.exception(new StatementExecutionException("transaction " + transaction.transactionId + " is for tablespace " + transaction.tableSpace + ", not for " + tableSpaceName));
    }
    if (transactionContext.transactionId > 0 && transaction == null) {
        return Futures.exception(new StatementExecutionException("transaction " + transactionContext.transactionId + " not found on tablespace " + tableSpaceName));
    }
    boolean isTransactionCommand = statement instanceof CommitTransactionStatement || statement instanceof RollbackTransactionStatement || // AlterTable implictly commits the transaction
    statement instanceof AlterTableStatement;
    if (transaction != null) {
        transaction.touch();
        if (!isTransactionCommand) {
            transaction.increaseRefcount();
        }
    }
    CompletableFuture<StatementExecutionResult> res;
    try {
        if (statement instanceof TableAwareStatement) {
            res = executeTableAwareStatement(statement, transaction, context);
        } else if (statement instanceof SQLPlannedOperationStatement) {
            res = executePlannedOperationStatement(statement, transactionContext, context);
        } else if (statement instanceof BeginTransactionStatement) {
            if (transaction != null) {
                res = Futures.exception(new StatementExecutionException("transaction already started"));
            } else {
                res = beginTransactionAsync(context, true);
            }
        } else if (statement instanceof CommitTransactionStatement) {
            res = commitTransaction((CommitTransactionStatement) statement, context);
        } else if (statement instanceof RollbackTransactionStatement) {
            res = rollbackTransaction((RollbackTransactionStatement) statement, context);
        } else if (statement instanceof CreateTableStatement) {
            res = CompletableFuture.completedFuture(createTable((CreateTableStatement) statement, transaction, context));
        } else if (statement instanceof CreateIndexStatement) {
            res = CompletableFuture.completedFuture(createIndex((CreateIndexStatement) statement, transaction, context));
        } else if (statement instanceof DropTableStatement) {
            res = CompletableFuture.completedFuture(dropTable((DropTableStatement) statement, transaction, context));
        } else if (statement instanceof DropIndexStatement) {
            res = CompletableFuture.completedFuture(dropIndex((DropIndexStatement) statement, transaction, context));
        } else if (statement instanceof AlterTableStatement) {
            res = CompletableFuture.completedFuture(alterTable((AlterTableStatement) statement, transactionContext, context));
        } else {
            res = Futures.exception(new StatementExecutionException("unsupported statement " + statement).fillInStackTrace());
        }
    } catch (StatementExecutionException error) {
        res = Futures.exception(error);
    }
    if (transaction != null && !isTransactionCommand) {
        res = res.whenComplete((a, b) -> {
            transaction.decreaseRefCount();
        });
    }
    if (rollbackOnError) {
        long txId = transactionContext.transactionId;
        if (txId > 0) {
            res = res.whenComplete((xx, error) -> {
                if (error != null) {
                    LOGGER.log(Level.FINE, tableSpaceName + " force rollback of implicit transaction " + txId, error);
                    try {
                        rollbackTransaction(new RollbackTransactionStatement(tableSpaceName, txId), context).get();
                    } catch (InterruptedException ex) {
                        Thread.currentThread().interrupt();
                        error.addSuppressed(ex);
                    } catch (ExecutionException ex) {
                        error.addSuppressed(ex.getCause());
                    }
                    throw new HerdDBInternalException(error);
                }
            });
        }
    }
    return res;
}

Example 84 with TransactionContext

use of herddb.model.TransactionContext in project herddb by diennea.

the class DeleteOp method executeAsync.

@Override
public CompletableFuture<StatementExecutionResult> executeAsync(TableSpaceManager tableSpaceManager, TransactionContext transactionContext, StatementEvaluationContext context, boolean lockRequired, boolean forWrite) {
    // not supported for deletes
    final boolean returnValues = false;
    StatementExecutionResult input = this.input.execute(tableSpaceManager, transactionContext, context, true, true);
    ScanResult downstreamScanResult = (ScanResult) input;
    final Table table = tableSpaceManager.getTableManager(tableName).getTable();
    long transactionId = transactionContext.transactionId;
    List<DMLStatement> statements = new ArrayList<>();
    try (DataScanner inputScanner = downstreamScanResult.dataScanner) {
        while (inputScanner.hasNext()) {
            DataAccessor row = inputScanner.next();
            long transactionIdFromScanner = inputScanner.getTransactionId();
            if (transactionIdFromScanner > 0 && transactionIdFromScanner != transactionId) {
                transactionId = transactionIdFromScanner;
                transactionContext = new TransactionContext(transactionId);
            }
            Bytes key = RecordSerializer.serializeIndexKey(row, table, table.getPrimaryKey());
            DMLStatement deleteStatement = new DeleteStatement(tableSpace, tableName, null, new RawKeyEquals(key));
            statements.add(deleteStatement);
        }
        if (statements.isEmpty()) {
            return CompletableFuture.completedFuture(new DMLStatementExecutionResult(transactionId, 0, null, null));
        }
        if (statements.size() == 1) {
            return tableSpaceManager.executeStatementAsync(statements.get(0), context, transactionContext);
        }
        CompletableFuture<StatementExecutionResult> finalResult = new CompletableFuture<>();
        AtomicInteger updateCounts = new AtomicInteger();
        AtomicReference<Bytes> lastKey = new AtomicReference<>();
        AtomicReference<Bytes> lastNewValue = new AtomicReference<>();
        class ComputeNext implements BiConsumer<StatementExecutionResult, Throwable> {

            int current;

            public ComputeNext(int current) {
                this.current = current;
            }

            @Override
            public void accept(StatementExecutionResult res, Throwable error) {
                if (error != null) {
                    finalResult.completeExceptionally(error);
                    return;
                }
                DMLStatementExecutionResult dml = (DMLStatementExecutionResult) res;
                updateCounts.addAndGet(dml.getUpdateCount());
                if (returnValues) {
                    lastKey.set(dml.getKey());
                    lastNewValue.set(dml.getNewvalue());
                }
                long newTransactionId = res.transactionId;
                if (current == statements.size()) {
                    DMLStatementExecutionResult finalDMLResult = new DMLStatementExecutionResult(newTransactionId, updateCounts.get(), lastKey.get(), lastNewValue.get());
                    finalResult.complete(finalDMLResult);
                    return;
                }
                DMLStatement nextStatement = statements.get(current);
                TransactionContext transactionContext = new TransactionContext(newTransactionId);
                CompletableFuture<StatementExecutionResult> nextPromise = tableSpaceManager.executeStatementAsync(nextStatement, context, transactionContext);
                nextPromise.whenComplete(new ComputeNext(current + 1));
            }
        }
        DMLStatement firstStatement = statements.get(0);
        tableSpaceManager.executeStatementAsync(firstStatement, context, transactionContext).whenComplete(new ComputeNext(1));
        return finalResult;
    } catch (DataScannerException err) {
        throw new StatementExecutionException(err);
    }
}

Example 85 with TransactionContext

use of herddb.model.TransactionContext in project herddb by diennea.

the class InsertOp method executeAsync.

@Override
public CompletableFuture<StatementExecutionResult> executeAsync(TableSpaceManager tableSpaceManager, TransactionContext transactionContext, StatementEvaluationContext context, boolean lockRequired, boolean forWrite) {
    StatementExecutionResult input = this.input.execute(tableSpaceManager, transactionContext, context, true, true);
    ScanResult downstreamScanResult = (ScanResult) input;
    final Table table = tableSpaceManager.getTableManager(tableName).getTable();
    long transactionId = transactionContext.transactionId;
    List<DMLStatement> statements = new ArrayList<>();
    try (DataScanner inputScanner = downstreamScanResult.dataScanner) {
        while (inputScanner.hasNext()) {
            DataAccessor row = inputScanner.next();
            long transactionIdFromScanner = inputScanner.getTransactionId();
            if (transactionIdFromScanner > 0 && transactionIdFromScanner != transactionId) {
                transactionId = transactionIdFromScanner;
                transactionContext = new TransactionContext(transactionId);
            }
            int index = 0;
            List<CompiledSQLExpression> keyValueExpression = new ArrayList<>();
            List<String> keyExpressionToColumn = new ArrayList<>();
            List<CompiledSQLExpression> valuesExpressions = new ArrayList<>();
            List<String> valuesColumns = new ArrayList<>();
            for (Column column : table.getColumns()) {
                Object value = row.get(index++);
                if (value != null) {
                    ConstantExpression exp = new ConstantExpression(value, column.type);
                    if (table.isPrimaryKeyColumn(column.name)) {
                        keyExpressionToColumn.add(column.name);
                        keyValueExpression.add(exp);
                    }
                    valuesColumns.add(column.name);
                    valuesExpressions.add(exp);
                }
            }
            RecordFunction keyfunction;
            if (keyValueExpression.isEmpty() && table.auto_increment) {
                keyfunction = new AutoIncrementPrimaryKeyRecordFunction();
            } else {
                if (keyValueExpression.size() != table.primaryKey.length) {
                    throw new StatementExecutionException("you must set a value for the primary key (expressions=" + keyValueExpression.size() + ")");
                }
                keyfunction = new SQLRecordKeyFunction(keyExpressionToColumn, keyValueExpression, table);
            }
            RecordFunction valuesfunction = new SQLRecordFunction(valuesColumns, table, valuesExpressions);
            DMLStatement insertStatement = new InsertStatement(tableSpace, tableName, keyfunction, valuesfunction, upsert).setReturnValues(returnValues);
            statements.add(insertStatement);
        }
        if (statements.isEmpty()) {
            return CompletableFuture.completedFuture(new DMLStatementExecutionResult(transactionId, 0, null, null));
        }
        if (statements.size() == 1) {
            return tableSpaceManager.executeStatementAsync(statements.get(0), context, transactionContext);
        }
        if (returnValues) {
            return Futures.exception(new StatementExecutionException("cannot 'return values' on multi-values insert"));
        }
        CompletableFuture<StatementExecutionResult> finalResult = new CompletableFuture<>();
        AtomicInteger updateCounts = new AtomicInteger();
        AtomicReference<Bytes> lastKey = new AtomicReference<>();
        AtomicReference<Bytes> lastNewValue = new AtomicReference<>();
        class ComputeNext implements BiConsumer<StatementExecutionResult, Throwable> {

            int current;

            public ComputeNext(int current) {
                this.current = current;
            }

            @Override
            public void accept(StatementExecutionResult res, Throwable error) {
                if (error != null) {
                    finalResult.completeExceptionally(error);
                    return;
                }
                DMLStatementExecutionResult dml = (DMLStatementExecutionResult) res;
                updateCounts.addAndGet(dml.getUpdateCount());
                if (returnValues) {
                    lastKey.set(dml.getKey());
                    lastNewValue.set(dml.getNewvalue());
                }
                long newTransactionId = res.transactionId;
                if (current == statements.size()) {
                    DMLStatementExecutionResult finalDMLResult = new DMLStatementExecutionResult(newTransactionId, updateCounts.get(), lastKey.get(), lastNewValue.get());
                    finalResult.complete(finalDMLResult);
                    return;
                }
                DMLStatement nextStatement = statements.get(current);
                TransactionContext transactionContext = new TransactionContext(newTransactionId);
                CompletableFuture<StatementExecutionResult> nextPromise = tableSpaceManager.executeStatementAsync(nextStatement, context, transactionContext);
                nextPromise.whenComplete(new ComputeNext(current + 1));
            }
        }
        DMLStatement firstStatement = statements.get(0);
        tableSpaceManager.executeStatementAsync(firstStatement, context, transactionContext).whenComplete(new ComputeNext(1));
        return finalResult;
    } catch (DataScannerException err) {
        throw new StatementExecutionException(err);
    }
}