Examples with DataStorageManagerException herddb.storage.DataStorageManagerException used on opensource projects

Example 46 with DataStorageManagerException

use of herddb.storage.DataStorageManagerException in project herddb by diennea.

the class TableSpaceManager method checkpoint.

TableSpaceCheckpoint checkpoint(boolean full, boolean pin) throws DataStorageManagerException, LogNotAvailableException {
    if (virtual) {
        return null;
    }
    long _start = System.currentTimeMillis();
    LogSequenceNumber logSequenceNumber;
    LogSequenceNumber _logSequenceNumber;
    List<PostCheckpointAction> actions = new ArrayList<>();
    Map<String, LogSequenceNumber> checkpoints = new HashMap<>();
    generalLock.writeLock().lock();
    try {
        logSequenceNumber = log.getLastSequenceNumber();
        if (logSequenceNumber.isStartOfTime()) {
            LOGGER.log(Level.INFO, nodeId + " checkpoint " + tableSpaceName + " at " + logSequenceNumber + ". skipped (no write ever issued to log)");
            return new TableSpaceCheckpoint(logSequenceNumber, checkpoints);
        }
        LOGGER.log(Level.INFO, nodeId + " checkpoint start " + tableSpaceName + " at " + logSequenceNumber);
        if (actualLogSequenceNumber == null) {
            throw new DataStorageManagerException("actualLogSequenceNumber cannot be null");
        }
        // TODO: transactions checkpoint is not atomic
        actions.addAll(dataStorageManager.writeTransactionsAtCheckpoint(tableSpaceUUID, logSequenceNumber, new ArrayList<>(transactions.values())));
        actions.addAll(writeTablesOnDataStorageManager(new CommitLogResult(logSequenceNumber, false)));
        // we checkpoint all data to disk and save the actual log sequence number
        for (AbstractTableManager tableManager : tables.values()) {
            if (!tableManager.isSystemTable()) {
                TableCheckpoint checkpoint = full ? tableManager.fullCheckpoint(pin) : tableManager.checkpoint(pin);
                if (checkpoint != null) {
                    actions.addAll(checkpoint.actions);
                    checkpoints.put(checkpoint.tableName, checkpoint.sequenceNumber);
                    if (afterTableCheckPointAction != null) {
                        afterTableCheckPointAction.run();
                    }
                }
            }
        }
        // we are sure that all data as been flushed. upon recovery we will replay the log starting from this position
        actions.addAll(dataStorageManager.writeCheckpointSequenceNumber(tableSpaceUUID, logSequenceNumber));
        /* Indexes checkpoint is handled by TableManagers */
        log.dropOldLedgers(logSequenceNumber);
        _logSequenceNumber = log.getLastSequenceNumber();
    } finally {
        generalLock.writeLock().unlock();
    }
    for (PostCheckpointAction action : actions) {
        try {
            action.run();
        } catch (Exception error) {
            LOGGER.log(Level.SEVERE, "postcheckpoint error:" + error, error);
        }
    }
    long _stop = System.currentTimeMillis();
    LOGGER.log(Level.INFO, "{0} checkpoint finish {1} started ad {2}, finished at {3}, total time {4} ms", new Object[] { nodeId, tableSpaceName, logSequenceNumber, _logSequenceNumber, Long.toString(_stop - _start) });
    return new TableSpaceCheckpoint(logSequenceNumber, checkpoints);
}

Example 47 with DataStorageManagerException

use of herddb.storage.DataStorageManagerException in project herddb by diennea.

the class TableSpaceManager method dropTable.

private StatementExecutionResult dropTable(DropTableStatement statement, Transaction transaction) throws StatementExecutionException {
    generalLock.writeLock().lock();
    try {
        if (!tables.containsKey(statement.getTable())) {
            if (statement.isIfExists()) {
                return new DDLStatementExecutionResult(transaction != null ? transaction.transactionId : 0);
            }
            throw new TableDoesNotExistException("table does not exist " + statement.getTable() + " on tableSpace " + statement.getTableSpace());
        }
        if (transaction != null && transaction.isTableDropped(statement.getTable())) {
            if (statement.isIfExists()) {
                return new DDLStatementExecutionResult(transaction.transactionId);
            }
            throw new TableDoesNotExistException("table does not exist " + statement.getTable() + " on tableSpace " + statement.getTableSpace());
        }
        Map<String, AbstractIndexManager> indexesOnTable = indexesByTable.get(statement.getTable());
        if (indexesOnTable != null) {
            for (String index : indexesOnTable.keySet()) {
                LogEntry entry = LogEntryFactory.dropIndex(index, transaction);
                CommitLogResult pos = log.log(entry, entry.transactionId <= 0);
                apply(pos, entry, false);
            }
        }
        LogEntry entry = LogEntryFactory.dropTable(statement.getTable(), transaction);
        CommitLogResult pos = log.log(entry, entry.transactionId <= 0);
        apply(pos, entry, false);
        return new DDLStatementExecutionResult(entry.transactionId);
    } catch (DataStorageManagerException | LogNotAvailableException err) {
        throw new StatementExecutionException(err);
    } finally {
        generalLock.writeLock().unlock();
    }
}

Example 48 with DataStorageManagerException

use of herddb.storage.DataStorageManagerException in project herddb by diennea.

the class TableSpaceManager method createTable.

private StatementExecutionResult createTable(CreateTableStatement statement, Transaction transaction) throws StatementExecutionException {
    generalLock.writeLock().lock();
    try {
        if (tables.containsKey(statement.getTableDefinition().name)) {
            throw new TableAlreadyExistsException(statement.getTableDefinition().name);
        }
        for (Index additionalIndex : statement.getAdditionalIndexes()) {
            if (indexes.containsKey(additionalIndex.name)) {
                throw new IndexAlreadyExistsException(additionalIndex.name);
            }
        }
        LogEntry entry = LogEntryFactory.createTable(statement.getTableDefinition(), transaction);
        CommitLogResult pos = log.log(entry, entry.transactionId <= 0);
        apply(pos, entry, false);
        for (Index additionalIndex : statement.getAdditionalIndexes()) {
            LogEntry index_entry = LogEntryFactory.createIndex(additionalIndex, transaction);
            CommitLogResult index_pos = log.log(index_entry, index_entry.transactionId <= 0);
            apply(index_pos, index_entry, false);
        }
        return new DDLStatementExecutionResult(entry.transactionId);
    } catch (DataStorageManagerException | LogNotAvailableException err) {
        throw new StatementExecutionException(err);
    } finally {
        generalLock.writeLock().unlock();
    }
}

Example 49 with DataStorageManagerException

use of herddb.storage.DataStorageManagerException in project herddb by diennea.

the class TableSpaceManager method apply.

void apply(CommitLogResult position, LogEntry entry, boolean recovery) throws DataStorageManagerException, DDLException {
    if (!position.deferred) {
        this.actualLogSequenceNumber = position.getLogSequenceNumber();
    }
    if (LOGGER.isLoggable(Level.FINEST)) {
        LOGGER.log(Level.FINEST, "apply entry {0} {1}", new Object[] { position, entry });
    }
    switch(entry.type) {
        case LogEntryType.BEGINTRANSACTION:
            {
                long id = entry.transactionId;
                Transaction transaction = new Transaction(id, tableSpaceName, position);
                transactions.put(id, transaction);
            }
            break;
        case LogEntryType.ROLLBACKTRANSACTION:
            {
                long id = entry.transactionId;
                Transaction transaction = transactions.get(id);
                if (transaction == null) {
                    throw new DataStorageManagerException("invalid transaction id " + id + ", only " + transactions.keySet());
                }
                List<AbstractTableManager> managers = new ArrayList<>(tables.values());
                for (AbstractTableManager manager : managers) {
                    if (transaction.isNewTable(manager.getTable().name)) {
                        LOGGER.log(Level.SEVERE, "rollback CREATE TABLE " + manager.getTable().name);
                        manager.dropTableData();
                        manager.close();
                        tables.remove(manager.getTable().name);
                    } else {
                        manager.onTransactionRollback(transaction);
                    }
                }
                transactions.remove(transaction.transactionId);
            }
            break;
        case LogEntryType.COMMITTRANSACTION:
            {
                long id = entry.transactionId;
                Transaction transaction = transactions.get(id);
                if (transaction == null) {
                    throw new DataStorageManagerException("invalid transaction id " + id);
                }
                transaction.synch();
                List<AbstractTableManager> managers = new ArrayList<>(tables.values());
                for (AbstractTableManager manager : managers) {
                    manager.onTransactionCommit(transaction, recovery);
                }
                List<AbstractIndexManager> indexManagers = new ArrayList<>(indexes.values());
                for (AbstractIndexManager indexManager : indexManagers) {
                    indexManager.onTransactionCommit(transaction, recovery);
                }
                if ((transaction.droppedTables != null && !transaction.droppedTables.isEmpty()) || (transaction.droppedIndexes != null && !transaction.droppedIndexes.isEmpty())) {
                    if (transaction.droppedTables != null) {
                        for (String dropped : transaction.droppedTables) {
                            for (AbstractTableManager manager : managers) {
                                if (manager.getTable().name.equals(dropped)) {
                                    manager.dropTableData();
                                    manager.close();
                                    tables.remove(manager.getTable().name);
                                }
                            }
                        }
                    }
                    if (transaction.droppedIndexes != null) {
                        for (String dropped : transaction.droppedIndexes) {
                            for (AbstractIndexManager manager : indexManagers) {
                                if (manager.getIndex().name.equals(dropped)) {
                                    manager.dropIndexData();
                                    manager.close();
                                    indexes.remove(manager.getIndex().name);
                                    Map<String, AbstractIndexManager> indexesForTable = indexesByTable.get(manager.getIndex().table);
                                    if (indexesForTable != null) {
                                        indexesForTable.remove(manager.getIndex().name);
                                    }
                                }
                            }
                        }
                    }
                }
                if ((transaction.newTables != null && !transaction.newTables.isEmpty()) || (transaction.droppedTables != null && !transaction.droppedTables.isEmpty()) || (transaction.newIndexes != null && !transaction.newIndexes.isEmpty()) || (transaction.droppedIndexes != null && !transaction.droppedIndexes.isEmpty())) {
                    writeTablesOnDataStorageManager(position);
                    dbmanager.getPlanner().clearCache();
                }
                transactions.remove(transaction.transactionId);
            }
            break;
        case LogEntryType.CREATE_TABLE:
            {
                Table table = Table.deserialize(entry.value);
                if (entry.transactionId > 0) {
                    long id = entry.transactionId;
                    Transaction transaction = transactions.get(id);
                    transaction.registerNewTable(table, position);
                }
                bootTable(table, entry.transactionId, null);
                if (entry.transactionId <= 0) {
                    writeTablesOnDataStorageManager(position);
                }
            }
            break;
        case LogEntryType.CREATE_INDEX:
            {
                Index index = Index.deserialize(entry.value);
                if (entry.transactionId > 0) {
                    long id = entry.transactionId;
                    Transaction transaction = transactions.get(id);
                    transaction.registerNewIndex(index, position);
                }
                AbstractTableManager tableManager = tables.get(index.table);
                if (tableManager == null) {
                    throw new RuntimeException("table " + index.table + " does not exists");
                }
                bootIndex(index, tableManager, entry.transactionId, true);
                if (entry.transactionId <= 0) {
                    writeTablesOnDataStorageManager(position);
                }
            }
            break;
        case LogEntryType.DROP_TABLE:
            {
                String tableName = entry.tableName;
                if (entry.transactionId > 0) {
                    long id = entry.transactionId;
                    Transaction transaction = transactions.get(id);
                    transaction.registerDropTable(tableName, position);
                } else {
                    AbstractTableManager manager = tables.get(tableName);
                    if (manager != null) {
                        manager.dropTableData();
                        manager.close();
                        tables.remove(manager.getTable().name);
                    }
                }
                if (entry.transactionId <= 0) {
                    writeTablesOnDataStorageManager(position);
                }
            }
            break;
        case LogEntryType.DROP_INDEX:
            {
                String indexName = Bytes.from_array(entry.value).to_string();
                if (entry.transactionId > 0) {
                    long id = entry.transactionId;
                    Transaction transaction = transactions.get(id);
                    transaction.registerDropIndex(indexName, position);
                } else {
                    AbstractIndexManager manager = indexes.get(indexName);
                    if (manager != null) {
                        manager.dropIndexData();
                        manager.close();
                        indexes.remove(manager.getIndexName());
                        Map<String, AbstractIndexManager> indexesForTable = indexesByTable.get(manager.getIndex().table);
                        if (indexesForTable != null) {
                            indexesForTable.remove(manager.getIndex().name);
                        }
                    }
                }
                if (entry.transactionId <= 0) {
                    writeTablesOnDataStorageManager(position);
                    dbmanager.getPlanner().clearCache();
                }
            }
            break;
        case LogEntryType.ALTER_TABLE:
            {
                Table table = Table.deserialize(entry.value);
                alterTable(table, null);
                writeTablesOnDataStorageManager(position);
            }
            break;
        default:
            // other entry types are not important for the tablespacemanager
            break;
    }
    if (entry.tableName != null && entry.type != LogEntryType.CREATE_TABLE && entry.type != LogEntryType.CREATE_INDEX && entry.type != LogEntryType.ALTER_TABLE && entry.type != LogEntryType.DROP_TABLE) {
        AbstractTableManager tableManager = tables.get(entry.tableName);
        tableManager.apply(position, entry, recovery);
    }
}

Example 50 with DataStorageManagerException

use of herddb.storage.DataStorageManagerException in project herddb by diennea.

the class TableSpaceManager method recover.

void recover(TableSpace tableSpaceInfo) throws DataStorageManagerException, LogNotAvailableException, MetadataStorageManagerException {
    LogSequenceNumber logSequenceNumber = dataStorageManager.getLastcheckpointSequenceNumber(tableSpaceUUID);
    actualLogSequenceNumber = logSequenceNumber;
    LOGGER.log(Level.INFO, "{0} recover {1}, logSequenceNumber from DataStorage: {2}", new Object[] { nodeId, tableSpaceName, logSequenceNumber });
    List<Table> tablesAtBoot = dataStorageManager.loadTables(logSequenceNumber, tableSpaceUUID);
    List<Index> indexesAtBoot = dataStorageManager.loadIndexes(logSequenceNumber, tableSpaceUUID);
    String tableNames = tablesAtBoot.stream().map(t -> {
        return t.name;
    }).collect(Collectors.joining(","));
    String indexNames = indexesAtBoot.stream().map(t -> {
        return t.name + " on table " + t.table;
    }).collect(Collectors.joining(","));
    LOGGER.log(Level.INFO, "{0} {1} tablesAtBoot {2}, indexesAtBoot {3}", new Object[] { nodeId, tableSpaceName, tableNames, indexNames });
    for (Table table : tablesAtBoot) {
        TableManager tableManager = bootTable(table, 0, null);
        for (Index index : indexesAtBoot) {
            if (index.table.equals(table.name)) {
                bootIndex(index, tableManager, 0, false);
            }
        }
    }
    dataStorageManager.loadTransactions(logSequenceNumber, tableSpaceUUID, t -> {
        transactions.put(t.transactionId, t);
        LOGGER.log(Level.FINER, "{0} {1} tx {2} at boot", new Object[] { nodeId, tableSpaceName, t.transactionId });
        try {
            if (t.newTables != null) {
                for (Table table : t.newTables.values()) {
                    if (!tables.containsKey(table.name)) {
                        bootTable(table, t.transactionId, null);
                    }
                }
            }
            if (t.newIndexes != null) {
                for (Index index : t.newIndexes.values()) {
                    if (!indexes.containsKey(index.name)) {
                        AbstractTableManager tableManager = tables.get(index.table);
                        bootIndex(index, tableManager, t.transactionId, false);
                    }
                }
            }
        } catch (Exception err) {
            LOGGER.log(Level.SEVERE, "error while booting tmp tables " + err, err);
            throw new RuntimeException(err);
        }
    });
    try {
        log.recovery(logSequenceNumber, new ApplyEntryOnRecovery(), false);
    } catch (FullRecoveryNeededException fullRecoveryNeeded) {
        LOGGER.log(Level.SEVERE, nodeId + " full recovery of data is needed for tableSpace " + tableSpaceName, fullRecoveryNeeded);
        downloadTableSpaceData();
        log.recovery(actualLogSequenceNumber, new ApplyEntryOnRecovery(), false);
    }
    checkpoint(false, false);
}