Search in sources :

Example 1 with Transaction

use of herddb.model.Transaction in project herddb by diennea.

the class BookkeeperFailuresTest method testBookieNotAvailableDuringTransaction.

@Test
public void testBookieNotAvailableDuringTransaction() throws Exception {
    ServerConfiguration serverconfig_1 = new ServerConfiguration(folder.newFolder().toPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_NODEID, "server1");
    serverconfig_1.set(ServerConfiguration.PROPERTY_PORT, 7867);
    serverconfig_1.set(ServerConfiguration.PROPERTY_MODE, ServerConfiguration.PROPERTY_MODE_CLUSTER);
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_ADDRESS, testEnv.getAddress());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_PATH, testEnv.getPath());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ZOOKEEPER_SESSIONTIMEOUT, testEnv.getTimeout());
    serverconfig_1.set(ServerConfiguration.PROPERTY_ENFORCE_LEADERSHIP, false);
    try (Server server = new Server(serverconfig_1)) {
        server.start();
        server.waitForStandaloneBoot();
        Table table = Table.builder().name("t1").column("c", ColumnTypes.INTEGER).primaryKey("c").build();
        // create table is done out of the transaction (this is very like autocommit=true)
        server.getManager().executeStatement(new CreateTableStatement(table), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        StatementExecutionResult executeStatement = server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 1)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.AUTOTRANSACTION_TRANSACTION);
        long transactionId = executeStatement.transactionId;
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 2)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 3)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
        TableSpaceManager tableSpaceManager = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        BookkeeperCommitLog log = (BookkeeperCommitLog) tableSpaceManager.getLog();
        long ledgerId = log.getLastSequenceNumber().ledgerId;
        assertTrue(ledgerId >= 1);
        Transaction transaction = tableSpaceManager.getTransactions().stream().filter(t -> t.transactionId == transactionId).findFirst().get();
        // Transaction will synch, so every addEntry will be acked, but will not be "confirmed" yet
        transaction.synch();
        try (DataScanner scan = scan(server.getManager(), "select * from t1", Collections.emptyList(), new TransactionContext(transactionId))) {
            assertEquals(3, scan.consume().size());
        }
        try (DataScanner scan = scan(server.getManager(), "select * from t1", Collections.emptyList(), TransactionContext.NO_TRANSACTION)) {
            // no record, but the table exists!
            assertEquals(0, scan.consume().size());
        }
        // we do not want auto-recovery
        server.getManager().setActivatorPauseStatus(true);
        testEnv.stopBookie();
        // transaction will continue and see the failure only the time of the commit
        try {
            server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
            // this will piggyback the LAC for the transaction
            System.out.println("Insert of c,4 OK");
        } catch (StatementExecutionException expected) {
            System.out.println("Insert of c,4 failed " + expected);
            // in can happen that the log gets closed
            assertEquals(herddb.log.LogNotAvailableException.class, expected.getCause().getClass());
        }
        try {
            server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 5)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
            // this will piggyback the LAC for the transaction
            System.out.println("Insert of c,5 OK");
        } catch (StatementExecutionException expected) {
            System.out.println("Insert of c,5 failed " + expected);
            // in can happen that the log gets closed
            assertEquals(herddb.log.LogNotAvailableException.class, expected.getCause().getClass());
        }
        try {
            server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 6)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), new TransactionContext(transactionId));
            // this will piggyback the LAC for the transaction
            System.out.println("Insert of c,6 OK");
        } catch (StatementExecutionException expected) {
            System.out.println("Insert of c,6 failed " + expected);
            // in can happen that the log gets closed
            assertEquals(herddb.log.LogNotAvailableException.class, expected.getCause().getClass());
        }
        try {
            server.getManager().executeStatement(new CommitTransactionStatement(TableSpace.DEFAULT, transactionId), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
            // this will fail alweays
            fail();
        } catch (StatementExecutionException expected) {
            System.out.println("Commit failed as expected:" + expected);
        }
        testEnv.startBookie(false);
        while (true) {
            System.out.println("status leader:" + tableSpaceManager.isLeader() + " failed:" + tableSpaceManager.isFailed());
            if (tableSpaceManager.isFailed()) {
                break;
            }
            Thread.sleep(100);
        }
        try (BookKeeper bk = createBookKeeper();
            LedgerHandle handle = bk.openLedgerNoRecovery(ledgerId, BookKeeper.DigestType.CRC32, "herddb".getBytes(StandardCharsets.UTF_8))) {
            BookKeeperAdmin admin = new BookKeeperAdmin(bk);
            try {
                LedgerMetadata ledgerMetadata = admin.getLedgerMetadata(handle);
                System.out.println("current ledger metadata before recovery: " + ledgerMetadata);
            } finally {
                admin.close();
            }
        }
        server.getManager().setActivatorPauseStatus(false);
        server.getManager().triggerActivator(ActivatorRunRequest.TABLESPACEMANAGEMENT);
        while (true) {
            TableSpaceManager tableSpaceManager_after_failure = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
            System.out.println("tableSpaceManager_after_failure:" + tableSpaceManager_after_failure);
            System.out.println("tableSpaceManager:" + tableSpaceManager);
            if (tableSpaceManager_after_failure != null && tableSpaceManager_after_failure != tableSpaceManager) {
                break;
            }
            Thread.sleep(1000);
            server.getManager().triggerActivator(ActivatorRunRequest.TABLESPACEMANAGEMENT);
        }
        TableSpaceManager tableSpaceManager_after_failure = server.getManager().getTableSpaceManager(TableSpace.DEFAULT);
        Assert.assertNotNull(tableSpaceManager_after_failure);
        assertNotSame(tableSpaceManager_after_failure, tableSpaceManager);
        assertTrue(!tableSpaceManager_after_failure.isFailed());
        // the insert should succeed because the trasaction has been rolledback automatically
        server.getManager().executeUpdate(new InsertStatement(TableSpace.DEFAULT, "t1", RecordSerializer.makeRecord(table, "c", 4)), StatementEvaluationContext.DEFAULT_EVALUATION_CONTEXT(), TransactionContext.NO_TRANSACTION);
        try (DataScanner scan = scan(server.getManager(), "select * from t1", Collections.emptyList())) {
            assertEquals(1, scan.consume().size());
        }
    }
}
Also used : Table(herddb.model.Table) CommitTransactionStatement(herddb.model.commands.CommitTransactionStatement) LedgerHandle(org.apache.bookkeeper.client.LedgerHandle) CreateTableStatement(herddb.model.commands.CreateTableStatement) BookKeeper(org.apache.bookkeeper.client.BookKeeper) InsertStatement(herddb.model.commands.InsertStatement) StatementExecutionException(herddb.model.StatementExecutionException) DataScanner(herddb.model.DataScanner) Transaction(herddb.model.Transaction) LedgerMetadata(org.apache.bookkeeper.client.LedgerMetadata) TransactionContext(herddb.model.TransactionContext) StatementExecutionResult(herddb.model.StatementExecutionResult) TableSpaceManager(herddb.core.TableSpaceManager) BookkeeperCommitLog(herddb.cluster.BookkeeperCommitLog) BookKeeperAdmin(org.apache.bookkeeper.client.BookKeeperAdmin) Test(org.junit.Test)

Example 2 with Transaction

use of herddb.model.Transaction in project herddb by diennea.

the class TableSpaceManager method dumpTableSpace.

void dumpTableSpace(String dumpId, Channel _channel, int fetchSize, boolean includeLog) throws DataStorageManagerException, LogNotAvailableException {
    LOGGER.log(Level.SEVERE, "dumpTableSpace dumpId:" + dumpId + " channel " + _channel + " fetchSize:" + fetchSize + ", includeLog:" + includeLog);
    TableSpaceCheckpoint checkpoint;
    List<DumpedLogEntry> txlogentries = new CopyOnWriteArrayList<>();
    CommitLogListener logDumpReceiver = new CommitLogListener() {

        @Override
        public void logEntry(LogSequenceNumber logPos, LogEntry data) {
            // we are going to capture all the changes to the tablespace during the dump, in order to replay
            // eventually 'missed' changes during the dump
            txlogentries.add(new DumpedLogEntry(logPos, data.serialize()));
            LOGGER.log(Level.SEVERE, "dumping entry " + logPos + ", " + data + " nentries: " + txlogentries.size());
        }
    };
    generalLock.writeLock().lock();
    try {
        if (includeLog) {
            log.attachCommitLogListener(logDumpReceiver);
        }
        checkpoint = checkpoint(true, true);
        /* Downgrade lock */
        generalLock.readLock().lock();
    } finally {
        generalLock.writeLock().unlock();
    }
    try {
        final int timeout = 60000;
        Map<String, Object> startData = new HashMap<>();
        startData.put("command", "start");
        LogSequenceNumber logSequenceNumber = log.getLastSequenceNumber();
        startData.put("ledgerid", logSequenceNumber.ledgerId);
        startData.put("offset", logSequenceNumber.offset);
        Message response_to_start = _channel.sendMessageWithReply(Message.TABLESPACE_DUMP_DATA(null, tableSpaceName, dumpId, startData), timeout);
        if (response_to_start.type != Message.TYPE_ACK) {
            LOGGER.log(Level.SEVERE, "error response at start command: " + response_to_start.parameters);
            return;
        }
        if (includeLog) {
            List<Transaction> transactionsSnapshot = new ArrayList<>();
            dataStorageManager.loadTransactions(logSequenceNumber, tableSpaceUUID, transactionsSnapshot::add);
            List<Transaction> batch = new ArrayList<>();
            for (Transaction t : transactionsSnapshot) {
                batch.add(t);
                if (batch.size() == 10) {
                    sendTransactionsDump(batch, _channel, dumpId, timeout, response_to_start);
                }
            }
            sendTransactionsDump(batch, _channel, dumpId, timeout, response_to_start);
        }
        for (Entry<String, LogSequenceNumber> entry : checkpoint.tablesCheckpoints.entrySet()) {
            final AbstractTableManager tableManager = tables.get(entry.getKey());
            final LogSequenceNumber sequenceNumber = entry.getValue();
            if (tableManager.isSystemTable()) {
                continue;
            }
            try {
                FullTableScanConsumer sink = new SingleTableDumper(tableSpaceName, tableManager, _channel, dumpId, timeout, fetchSize);
                tableManager.dump(sequenceNumber, sink);
            } catch (DataStorageManagerException err) {
                Map<String, Object> errorOnData = new HashMap<>();
                errorOnData.put("command", "error");
                _channel.sendMessageWithReply(Message.TABLESPACE_DUMP_DATA(null, tableSpaceName, dumpId, errorOnData), timeout);
                LOGGER.log(Level.SEVERE, "error sending dump id " + dumpId, err);
                return;
            }
        }
        if (!txlogentries.isEmpty()) {
            txlogentries.sort(Comparator.naturalOrder());
            sendDumpedCommitLog(txlogentries, _channel, dumpId, timeout);
        }
        Map<String, Object> finishData = new HashMap<>();
        LogSequenceNumber finishLogSequenceNumber = log.getLastSequenceNumber();
        finishData.put("ledgerid", finishLogSequenceNumber.ledgerId);
        finishData.put("offset", finishLogSequenceNumber.offset);
        finishData.put("command", "finish");
        _channel.sendOneWayMessage(Message.TABLESPACE_DUMP_DATA(null, tableSpaceName, dumpId, finishData), new SendResultCallback() {

            @Override
            public void messageSent(Message originalMessage, Throwable error) {
            }
        });
    } catch (InterruptedException | TimeoutException error) {
        LOGGER.log(Level.SEVERE, "error sending dump id " + dumpId);
    } finally {
        generalLock.readLock().unlock();
        if (includeLog) {
            log.removeCommitLogListener(logDumpReceiver);
        }
        for (Entry<String, LogSequenceNumber> entry : checkpoint.tablesCheckpoints.entrySet()) {
            dataStorageManager.unPinTableCheckpoint(tableSpaceUUID, entry.getKey(), entry.getValue());
        }
    }
}
Also used : DataStorageManagerException(herddb.storage.DataStorageManagerException) DumpedLogEntry(herddb.backup.DumpedLogEntry) Message(herddb.network.Message) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) ArrayList(java.util.ArrayList) FullTableScanConsumer(herddb.storage.FullTableScanConsumer) SendResultCallback(herddb.network.SendResultCallback) LogEntry(herddb.log.LogEntry) DumpedLogEntry(herddb.backup.DumpedLogEntry) TimeoutException(java.util.concurrent.TimeoutException) CommitLogListener(herddb.log.CommitLogListener) LogSequenceNumber(herddb.log.LogSequenceNumber) TableCheckpoint(herddb.core.AbstractTableManager.TableCheckpoint) Transaction(herddb.model.Transaction) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList)

Example 3 with Transaction

use of herddb.model.Transaction in project herddb by diennea.

the class TableSpaceManager method commitTransaction.

private StatementExecutionResult commitTransaction(CommitTransactionStatement commitTransactionStatement) throws StatementExecutionException {
    long txId = commitTransactionStatement.getTransactionId();
    LogEntry entry = LogEntryFactory.commitTransaction(txId);
    generalLock.readLock().lock();
    try {
        Transaction tx = transactions.get(txId);
        if (tx == null) {
            throw new StatementExecutionException("no such transaction " + commitTransactionStatement.getTransactionId());
        }
        CommitLogResult pos = log.log(entry, true);
        apply(pos, entry, false);
    } catch (Exception err) {
        throw new StatementExecutionException(err);
    } finally {
        generalLock.readLock().unlock();
    }
    return new TransactionResult(txId, TransactionResult.OutcomeType.COMMIT);
}
Also used : TransactionResult(herddb.model.TransactionResult) Transaction(herddb.model.Transaction) CommitLogResult(herddb.log.CommitLogResult) LogEntry(herddb.log.LogEntry) DumpedLogEntry(herddb.backup.DumpedLogEntry) StatementExecutionException(herddb.model.StatementExecutionException) HDBException(herddb.client.HDBException) IndexAlreadyExistsException(herddb.model.IndexAlreadyExistsException) LogNotAvailableException(herddb.log.LogNotAvailableException) ClientSideMetadataProviderException(herddb.client.ClientSideMetadataProviderException) IndexDoesNotExistException(herddb.model.IndexDoesNotExistException) FullRecoveryNeededException(herddb.log.FullRecoveryNeededException) TableAlreadyExistsException(herddb.model.TableAlreadyExistsException) TimeoutException(java.util.concurrent.TimeoutException) DataStorageManagerException(herddb.storage.DataStorageManagerException) TableDoesNotExistException(herddb.model.TableDoesNotExistException) DDLException(herddb.model.DDLException) StatementExecutionException(herddb.model.StatementExecutionException) EOFException(java.io.EOFException) MetadataStorageManagerException(herddb.metadata.MetadataStorageManagerException)

Example 4 with Transaction

use of herddb.model.Transaction in project herddb by diennea.

the class TableSpaceManager method scan.

public DataScanner scan(ScanStatement statement, StatementEvaluationContext context, TransactionContext transactionContext, boolean lockRequired, boolean forWrite) throws StatementExecutionException {
    boolean rollbackOnError = false;
    if (transactionContext.transactionId == TransactionContext.AUTOTRANSACTION_ID) {
        StatementExecutionResult newTransaction = beginTransaction();
        transactionContext = new TransactionContext(newTransaction.transactionId);
        rollbackOnError = true;
    }
    Transaction transaction = transactions.get(transactionContext.transactionId);
    if (transaction != null && !transaction.tableSpace.equals(tableSpaceName)) {
        throw new StatementExecutionException("transaction " + transaction.transactionId + " is for tablespace " + transaction.tableSpace + ", not for " + tableSpaceName);
    }
    try {
        String table = statement.getTable();
        AbstractTableManager tableManager = tables.get(table);
        if (tableManager == null) {
            throw new TableDoesNotExistException("no table " + table + " in tablespace " + tableSpaceName);
        }
        if (tableManager.getCreatedInTransaction() > 0) {
            if (transaction == null || transaction.transactionId != tableManager.getCreatedInTransaction()) {
                throw new TableDoesNotExistException("no table " + table + " in tablespace " + tableSpaceName + ". created temporary in transaction " + tableManager.getCreatedInTransaction());
            }
        }
        return tableManager.scan(statement, context, transaction, lockRequired, forWrite);
    } catch (StatementExecutionException error) {
        if (rollbackOnError) {
            rollbackTransaction(new RollbackTransactionStatement(tableSpaceName, transactionContext.transactionId));
        }
        throw error;
    }
}
Also used : TableDoesNotExistException(herddb.model.TableDoesNotExistException) Transaction(herddb.model.Transaction) TransactionContext(herddb.model.TransactionContext) DDLStatementExecutionResult(herddb.model.DDLStatementExecutionResult) StatementExecutionResult(herddb.model.StatementExecutionResult) RollbackTransactionStatement(herddb.model.commands.RollbackTransactionStatement) StatementExecutionException(herddb.model.StatementExecutionException)

Example 5 with Transaction

use of herddb.model.Transaction in project herddb by diennea.

the class TableSpaceManager method rollbackTransaction.

private StatementExecutionResult rollbackTransaction(RollbackTransactionStatement rollbackTransactionStatement) throws StatementExecutionException {
    long txId = rollbackTransactionStatement.getTransactionId();
    LogEntry entry = LogEntryFactory.rollbackTransaction(txId);
    generalLock.readLock().lock();
    try {
        Transaction tx = transactions.get(txId);
        if (tx == null) {
            throw new StatementExecutionException("no such transaction " + rollbackTransactionStatement.getTransactionId());
        }
        CommitLogResult pos = log.log(entry, true);
        apply(pos, entry, false);
    } catch (Exception err) {
        throw new StatementExecutionException(err);
    } finally {
        generalLock.readLock().unlock();
    }
    return new TransactionResult(txId, TransactionResult.OutcomeType.ROLLBACK);
}
Also used : TransactionResult(herddb.model.TransactionResult) Transaction(herddb.model.Transaction) CommitLogResult(herddb.log.CommitLogResult) LogEntry(herddb.log.LogEntry) DumpedLogEntry(herddb.backup.DumpedLogEntry) StatementExecutionException(herddb.model.StatementExecutionException) HDBException(herddb.client.HDBException) IndexAlreadyExistsException(herddb.model.IndexAlreadyExistsException) LogNotAvailableException(herddb.log.LogNotAvailableException) ClientSideMetadataProviderException(herddb.client.ClientSideMetadataProviderException) IndexDoesNotExistException(herddb.model.IndexDoesNotExistException) FullRecoveryNeededException(herddb.log.FullRecoveryNeededException) TableAlreadyExistsException(herddb.model.TableAlreadyExistsException) TimeoutException(java.util.concurrent.TimeoutException) DataStorageManagerException(herddb.storage.DataStorageManagerException) TableDoesNotExistException(herddb.model.TableDoesNotExistException) DDLException(herddb.model.DDLException) StatementExecutionException(herddb.model.StatementExecutionException) EOFException(java.io.EOFException) MetadataStorageManagerException(herddb.metadata.MetadataStorageManagerException)

Aggregations

Transaction (herddb.model.Transaction)16 DataStorageManagerException (herddb.storage.DataStorageManagerException)11 StatementExecutionException (herddb.model.StatementExecutionException)8 ArrayList (java.util.ArrayList)8 LogSequenceNumber (herddb.log.LogSequenceNumber)6 StatementExecutionResult (herddb.model.StatementExecutionResult)6 Table (herddb.model.Table)6 HashMap (java.util.HashMap)6 Map (java.util.Map)6 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)6 DumpedLogEntry (herddb.backup.DumpedLogEntry)5 LogEntry (herddb.log.LogEntry)5 Index (herddb.model.Index)5 TimeoutException (java.util.concurrent.TimeoutException)5 CommitLogResult (herddb.log.CommitLogResult)4 LogNotAvailableException (herddb.log.LogNotAvailableException)4 DDLException (herddb.model.DDLException)4 DataScanner (herddb.model.DataScanner)3 Record (herddb.model.Record)3 TableDoesNotExistException (herddb.model.TableDoesNotExistException)3