Search in sources :

Example 1 with DDLException

use of herddb.model.DDLException in project herddb by diennea.

the class DBManager method manageTableSpaces.

private boolean manageTableSpaces() {
    Collection<String> actualTablesSpaces;
    try {
        actualTablesSpaces = metadataStorageManager.listTableSpaces();
    } catch (MetadataStorageManagerException error) {
        LOGGER.log(Level.SEVERE, "cannot access tablespaces metadata", error);
        return true;
    }
    Map<String, TableSpace> actualTableSpaceMetadata = new HashMap<>();
    generalLock.writeLock().lock();
    try {
        for (String tableSpace : actualTablesSpaces) {
            TableSpace tableSpaceMetadata = metadataStorageManager.describeTableSpace(tableSpace);
            actualTableSpaceMetadata.put(tableSpaceMetadata.uuid, tableSpaceMetadata);
            try {
                handleTableSpace(tableSpaceMetadata);
            } catch (Exception err) {
                LOGGER.log(Level.SEVERE, "cannot handle tablespace " + tableSpace, err);
                if (haltOnTableSpaceBootError && haltProcedure != null) {
                    err.printStackTrace();
                    haltProcedure.run();
                }
            }
        }
    } catch (MetadataStorageManagerException error) {
        LOGGER.log(Level.SEVERE, "cannot access tablespaces metadata", error);
        return true;
    } finally {
        generalLock.writeLock().unlock();
    }
    List<TableSpaceManager> followingActiveTableSpaces = new ArrayList<>();
    Set<String> failedTableSpaces = new HashSet<>();
    for (Map.Entry<String, TableSpaceManager> entry : tablesSpaces.entrySet()) {
        try {
            String tableSpaceUuid = entry.getValue().getTableSpaceUUID();
            if (entry.getValue().isFailed()) {
                failedTableSpaces.add(entry.getKey());
            } else if (!entry.getKey().equals(virtualTableSpaceId) && !actualTablesSpaces.contains(entry.getKey())) {
                failedTableSpaces.add(entry.getKey());
            } else if (entry.getValue().isLeader()) {
                metadataStorageManager.updateTableSpaceReplicaState(TableSpaceReplicaState.builder().mode(TableSpaceReplicaState.MODE_LEADER).nodeId(nodeId).uuid(tableSpaceUuid).timestamp(System.currentTimeMillis()).build());
            } else {
                metadataStorageManager.updateTableSpaceReplicaState(TableSpaceReplicaState.builder().mode(TableSpaceReplicaState.MODE_FOLLOWER).nodeId(nodeId).uuid(tableSpaceUuid).timestamp(System.currentTimeMillis()).build());
                followingActiveTableSpaces.add(entry.getValue());
            }
        } catch (MetadataStorageManagerException error) {
            LOGGER.log(Level.SEVERE, "cannot access tablespace " + entry.getKey() + " metadata", error);
            return true;
        }
    }
    if (!failedTableSpaces.isEmpty()) {
        generalLock.writeLock().lock();
        try {
            for (String tableSpace : failedTableSpaces) {
                stopTableSpace(tableSpace, null);
            }
        } catch (MetadataStorageManagerException error) {
            LOGGER.log(Level.SEVERE, "cannot access tablespace metadata", error);
            return true;
        } finally {
            generalLock.writeLock().unlock();
        }
    }
    if (!followingActiveTableSpaces.isEmpty()) {
        long now = System.currentTimeMillis();
        try {
            for (TableSpaceManager tableSpaceManager : followingActiveTableSpaces) {
                String tableSpaceUuid = tableSpaceManager.getTableSpaceUUID();
                TableSpace tableSpaceInfo = actualTableSpaceMetadata.get(tableSpaceUuid);
                if (tableSpaceInfo != null && tableSpaceInfo.maxLeaderInactivityTime > 0 && !tableSpaceManager.isFailed()) {
                    List<TableSpaceReplicaState> allReplicas = metadataStorageManager.getTableSpaceReplicaState(tableSpaceUuid);
                    TableSpaceReplicaState leaderState = allReplicas.stream().filter(t -> t.mode == TableSpaceReplicaState.MODE_LEADER && !t.nodeId.equals(nodeId)).findAny().orElse(null);
                    if (leaderState == null) {
                        LOGGER.log(Level.SEVERE, "Leader for " + tableSpaceUuid + " should be " + tableSpaceInfo.leaderId + ", but it never sent pings or it disappeared");
                        tryBecomeLeaderFor(tableSpaceInfo);
                    } else {
                        long delta = now - leaderState.timestamp;
                        if (tableSpaceInfo.maxLeaderInactivityTime > delta) {
                            LOGGER.log(Level.FINER, "Leader for " + tableSpaceUuid + " is " + leaderState.nodeId + ", last ping " + new java.sql.Timestamp(leaderState.timestamp) + ". leader is healty");
                        } else {
                            LOGGER.log(Level.SEVERE, "Leader for " + tableSpaceUuid + " is " + leaderState.nodeId + ", last ping " + new java.sql.Timestamp(leaderState.timestamp) + ". leader is failed. trying to take leadership");
                            tryBecomeLeaderFor(tableSpaceInfo);
                            // only one change at a time
                            break;
                        }
                    }
                }
            }
        } catch (MetadataStorageManagerException | DDLException error) {
            LOGGER.log(Level.SEVERE, "cannot access tablespace metadata", error);
            return true;
        }
    }
    return false;
}
Also used : TableSpace(herddb.model.TableSpace) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) DataStorageManagerException(herddb.storage.DataStorageManagerException) LogNotAvailableException(herddb.log.LogNotAvailableException) DDLException(herddb.model.DDLException) StatementExecutionException(herddb.model.StatementExecutionException) NotLeaderException(herddb.model.NotLeaderException) MetadataStorageManagerException(herddb.metadata.MetadataStorageManagerException) DataScannerException(herddb.model.DataScannerException) RejectedExecutionException(java.util.concurrent.RejectedExecutionException) TableSpaceDoesNotExistException(herddb.model.TableSpaceDoesNotExistException) MetadataStorageManagerException(herddb.metadata.MetadataStorageManagerException) TableSpaceReplicaState(herddb.model.TableSpaceReplicaState) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) DDLException(herddb.model.DDLException) HashSet(java.util.HashSet)

Example 2 with DDLException

use of herddb.model.DDLException in project herddb by diennea.

the class DBManager method start.

/**
 * Initial boot of the system
 *
 * @throws herddb.storage.DataStorageManagerException
 * @throws herddb.log.LogNotAvailableException
 * @throws herddb.metadata.MetadataStorageManagerException
 */
public void start() throws DataStorageManagerException, LogNotAvailableException, MetadataStorageManagerException {
    if (serverConfiguration.getBoolean(ServerConfiguration.PROPERTY_JMX_ENABLE, ServerConfiguration.PROPERTY_JMX_ENABLE_DEFAULT)) {
        JMXUtils.registerDBManagerStatsMXBean(stats);
    }
    final long maxHeap = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
    /* If max memory isn't configured or is too high default it to maximum heap */
    if (maxMemoryReference == 0 || maxMemoryReference > maxHeap) {
        maxMemoryReference = maxHeap;
    }
    LOGGER.log(Level.INFO, ServerConfiguration.PROPERTY_MEMORY_LIMIT_REFERENCE + "= {0} bytes", Long.toString(maxMemoryReference));
    /* If max data memory for pages isn't configured or is too high default it to 0.3 maxMemoryReference */
    if (maxDataUsedMemory == 0 || maxDataUsedMemory > maxMemoryReference) {
        maxDataUsedMemory = (long) (0.3F * maxMemoryReference);
    }
    /* If max index memory for pages isn't configured or is too high default it to 0.2 maxMemoryReference */
    if (maxPKUsedMemory == 0 || maxPKUsedMemory > maxMemoryReference) {
        maxPKUsedMemory = (long) (0.2F * maxMemoryReference);
    }
    /* If max used memory is too high lower index and data accordingly */
    if (maxDataUsedMemory + maxPKUsedMemory > maxMemoryReference) {
        long data = (int) ((double) maxDataUsedMemory / ((double) (maxDataUsedMemory + maxPKUsedMemory)) * maxMemoryReference);
        long pk = (int) ((double) maxPKUsedMemory / ((double) (maxDataUsedMemory + maxPKUsedMemory)) * maxMemoryReference);
        maxDataUsedMemory = data;
        maxPKUsedMemory = pk;
    }
    memoryManager = new MemoryManager(maxDataUsedMemory, maxPKUsedMemory, maxLogicalPageSize);
    metadataStorageManager.start();
    if (clearAtBoot) {
        metadataStorageManager.clear();
    }
    metadataStorageManager.setMetadataChangeListener(this);
    NodeMetadata nodeMetadata = NodeMetadata.builder().host(hostData.getHost()).port(hostData.getPort()).ssl(hostData.isSsl()).nodeId(nodeId).build();
    LOGGER.log(Level.SEVERE, "Registering on metadata storage manager my data: {0}", nodeMetadata);
    metadataStorageManager.registerNode(nodeMetadata);
    try {
        TableSpaceManager local_node_virtual_tables_manager = new TableSpaceManager(nodeId, virtualTableSpaceId, virtualTableSpaceId, metadataStorageManager, dataStorageManager, null, this, true);
        tablesSpaces.put(virtualTableSpaceId, local_node_virtual_tables_manager);
        local_node_virtual_tables_manager.start();
    } catch (DDLException | DataStorageManagerException | LogNotAvailableException | MetadataStorageManagerException error) {
        throw new IllegalStateException("cannot boot local virtual tablespace manager");
    }
    metadataStorageManager.ensureDefaultTableSpace(nodeId);
    commitLogManager.start();
    generalLock.writeLock().lock();
    try {
        dataStorageManager.start();
    } finally {
        generalLock.writeLock().unlock();
    }
    activator.start();
    triggerActivator(ActivatorRunRequest.FULL);
}
Also used : NodeMetadata(herddb.model.NodeMetadata) MetadataStorageManagerException(herddb.metadata.MetadataStorageManagerException) DataStorageManagerException(herddb.storage.DataStorageManagerException) DDLException(herddb.model.DDLException) LogNotAvailableException(herddb.log.LogNotAvailableException)

Example 3 with DDLException

use of herddb.model.DDLException in project herddb by diennea.

the class DBManager method handleTableSpace.

private void handleTableSpace(TableSpace tableSpace) throws DataStorageManagerException, LogNotAvailableException, MetadataStorageManagerException, DDLException {
    String tableSpaceName = tableSpace.name;
    TableSpaceManager actual_manager = tablesSpaces.get(tableSpaceName);
    if (actual_manager != null && actual_manager.isFailed()) {
        LOGGER.log(Level.INFO, "Tablespace {0} is in 'Failed' status", new Object[] { tableSpaceName, nodeId });
        return;
    }
    if (actual_manager != null && actual_manager.isLeader() && !tableSpace.leaderId.equals(nodeId)) {
        LOGGER.log(Level.SEVERE, "Tablespace {0} leader is no more {1}, it changed to {2}", new Object[] { tableSpaceName, nodeId, tableSpace.leaderId });
        stopTableSpace(tableSpaceName, tableSpace.uuid);
        return;
    }
    if (actual_manager != null && !actual_manager.isLeader() && tableSpace.leaderId.equals(nodeId)) {
        LOGGER.log(Level.SEVERE, "Tablespace {0} need to switch to leadership on node {1}", new Object[] { tableSpaceName, nodeId });
        stopTableSpace(tableSpaceName, tableSpace.uuid);
        return;
    }
    if (tableSpace.replicas.contains(nodeId) && !tablesSpaces.containsKey(tableSpaceName)) {
        LOGGER.log(Level.SEVERE, "Booting tablespace {0} on {1}, uuid {2}", new Object[] { tableSpaceName, nodeId, tableSpace.uuid });
        long _start = System.currentTimeMillis();
        CommitLog commitLog = commitLogManager.createCommitLog(tableSpace.uuid);
        TableSpaceManager manager = new TableSpaceManager(nodeId, tableSpaceName, tableSpace.uuid, metadataStorageManager, dataStorageManager, commitLog, this, false);
        try {
            manager.start();
            LOGGER.log(Level.SEVERE, "Boot success tablespace {0} on {1}, uuid {2}, time {3} ms", new Object[] { tableSpaceName, nodeId, tableSpace.uuid, (System.currentTimeMillis() - _start) + "" });
            tablesSpaces.put(tableSpaceName, manager);
            if (serverConfiguration.getBoolean(ServerConfiguration.PROPERTY_JMX_ENABLE, ServerConfiguration.PROPERTY_JMX_ENABLE_DEFAULT)) {
                JMXUtils.registerTableSpaceManagerStatsMXBean(tableSpaceName, manager.getStats());
            }
        } catch (DataStorageManagerException | LogNotAvailableException | MetadataStorageManagerException | DDLException t) {
            LOGGER.log(Level.SEVERE, "Error Booting tablespace {0} on {1}", new Object[] { tableSpaceName, nodeId });
            LOGGER.log(Level.SEVERE, "Error", t);
            try {
                manager.close();
            } catch (Throwable t2) {
                LOGGER.log(Level.SEVERE, "Other Error", t2);
            }
            throw t;
        }
        return;
    }
    if (tablesSpaces.containsKey(tableSpaceName) && !tableSpace.replicas.contains(nodeId)) {
        LOGGER.log(Level.SEVERE, "Tablespace {0} on {1} is not more in replica list {3}, uuid {2}", new Object[] { tableSpaceName, nodeId, tableSpace.uuid, tableSpace.replicas + "" });
        stopTableSpace(tableSpaceName, tableSpace.uuid);
        return;
    }
    if (tableSpace.replicas.size() < tableSpace.expectedReplicaCount) {
        List<NodeMetadata> nodes = metadataStorageManager.listNodes();
        LOGGER.log(Level.SEVERE, "Tablespace {0} is underreplicated expectedReplicaCount={1}, replicas={2}, nodes={3}", new Object[] { tableSpaceName, tableSpace.expectedReplicaCount, tableSpace.replicas, nodes });
        List<String> availableOtherNodes = nodes.stream().map(n -> {
            return n.nodeId;
        }).filter(n -> {
            return !tableSpace.replicas.contains(n);
        }).collect(Collectors.toList());
        Collections.shuffle(availableOtherNodes);
        LOGGER.log(Level.SEVERE, "Tablespace {0} is underreplicated expectedReplicaCount={1}, replicas={2}, availableOtherNodes={3}", new Object[] { tableSpaceName, tableSpace.expectedReplicaCount, tableSpace.replicas, availableOtherNodes });
        if (!availableOtherNodes.isEmpty()) {
            int countMissing = tableSpace.expectedReplicaCount - tableSpace.replicas.size();
            TableSpace.Builder newTableSpaceBuilder = TableSpace.builder().cloning(tableSpace);
            while (!availableOtherNodes.isEmpty() && countMissing > 0) {
                String node = availableOtherNodes.remove(0);
                newTableSpaceBuilder.replica(node);
            }
            TableSpace newTableSpace = newTableSpaceBuilder.build();
            boolean ok = metadataStorageManager.updateTableSpace(newTableSpace, tableSpace);
            if (!ok) {
                LOGGER.log(Level.SEVERE, "updating tableSpace " + tableSpaceName + " metadata failed");
            }
        }
    }
}
Also used : ExecutionPlan(herddb.model.ExecutionPlan) SQLStatementEvaluationContext(herddb.sql.SQLStatementEvaluationContext) Arrays(java.util.Arrays) FileMetadataStorageManager(herddb.file.FileMetadataStorageManager) JMXUtils(herddb.jmx.JMXUtils) ClientConfiguration(herddb.client.ClientConfiguration) MetadataStorageManager(herddb.metadata.MetadataStorageManager) Channel(herddb.network.Channel) ServerConfiguration(herddb.server.ServerConfiguration) Map(java.util.Map) DDLStatementExecutionResult(herddb.model.DDLStatementExecutionResult) ConnectionsInfoProvider(herddb.core.stats.ConnectionsInfoProvider) DMLStatementExecutionResult(herddb.model.DMLStatementExecutionResult) ThreadFactory(java.util.concurrent.ThreadFactory) Path(java.nio.file.Path) DataStorageManagerException(herddb.storage.DataStorageManagerException) DropTableSpaceStatement(herddb.model.commands.DropTableSpaceStatement) ScanResult(herddb.model.ScanResult) DataAccessor(herddb.utils.DataAccessor) LogNotAvailableException(herddb.log.LogNotAvailableException) DataScannerJoinExecutor(herddb.core.join.DataScannerJoinExecutor) InsertStatement(herddb.model.commands.InsertStatement) DataScanner(herddb.model.DataScanner) DDLException(herddb.model.DDLException) StatementExecutionException(herddb.model.StatementExecutionException) Collection(java.util.Collection) AbstractSQLPlanner(herddb.sql.AbstractSQLPlanner) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) BlockingQueue(java.util.concurrent.BlockingQueue) Logger(java.util.logging.Logger) Collectors(java.util.stream.Collectors) Executors(java.util.concurrent.Executors) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) ScanStatement(herddb.model.commands.ScanStatement) List(java.util.List) Message(herddb.network.Message) NotLeaderException(herddb.model.NotLeaderException) GetStatement(herddb.model.commands.GetStatement) NodeMetadata(herddb.model.NodeMetadata) DefaultJVMHalt(herddb.utils.DefaultJVMHalt) DDLStatement(herddb.model.DDLStatement) TableSpace(herddb.model.TableSpace) SuppressFBWarnings(edu.umd.cs.findbugs.annotations.SuppressFBWarnings) Statement(herddb.model.Statement) AlterTableSpaceStatement(herddb.model.commands.AlterTableSpaceStatement) CalcitePlanner(herddb.sql.CalcitePlanner) CreateTableSpaceStatement(herddb.model.commands.CreateTableSpaceStatement) MetadataStorageManagerException(herddb.metadata.MetadataStorageManagerException) DataScannerException(herddb.model.DataScannerException) MetadataChangeListener(herddb.metadata.MetadataChangeListener) GetResult(herddb.model.GetResult) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) TableSpaceReplicaState(herddb.model.TableSpaceReplicaState) ArrayList(java.util.ArrayList) Level(java.util.logging.Level) HashSet(java.util.HashSet) RejectedExecutionException(java.util.concurrent.RejectedExecutionException) TableSpaceDoesNotExistException(herddb.model.TableSpaceDoesNotExistException) SQLPlanner(herddb.sql.SQLPlanner) TransactionContext(herddb.model.TransactionContext) CommitLogManager(herddb.log.CommitLogManager) ManagementFactory(java.lang.management.ManagementFactory) ExecutorService(java.util.concurrent.ExecutorService) ReentrantLock(java.util.concurrent.locks.ReentrantLock) StatementExecutionResult(herddb.model.StatementExecutionResult) DMLStatement(herddb.model.DMLStatement) LimitedDataScanner(herddb.model.LimitedDataScanner) DataStorageManager(herddb.storage.DataStorageManager) TimeUnit(java.util.concurrent.TimeUnit) AtomicLong(java.util.concurrent.atomic.AtomicLong) DBManagerStatsMXBean(herddb.jmx.DBManagerStatsMXBean) Condition(java.util.concurrent.locks.Condition) Lock(java.util.concurrent.locks.Lock) CommitLog(herddb.log.CommitLog) TuplePredicate(herddb.model.TuplePredicate) Column(herddb.model.Column) StatementEvaluationContext(herddb.model.StatementEvaluationContext) Collections(java.util.Collections) MemoryMetadataStorageManager(herddb.mem.MemoryMetadataStorageManager) ServerHostData(herddb.network.ServerHostData) DataStorageManagerException(herddb.storage.DataStorageManagerException) TableSpace(herddb.model.TableSpace) MetadataStorageManagerException(herddb.metadata.MetadataStorageManagerException) NodeMetadata(herddb.model.NodeMetadata) CommitLog(herddb.log.CommitLog) DDLException(herddb.model.DDLException) LogNotAvailableException(herddb.log.LogNotAvailableException)

Example 4 with DDLException

use of herddb.model.DDLException in project herddb by diennea.

the class FileMetadataStorageManager method ensureDefaultTableSpace.

@Override
public void ensureDefaultTableSpace(String localNodeId) throws MetadataStorageManagerException {
    lock.writeLock().lock();
    try {
        TableSpace exists = tableSpaces.get(TableSpace.DEFAULT);
        if (exists == null) {
            TableSpace defaultTableSpace = TableSpace.builder().leader(localNodeId).replica(localNodeId).name(TableSpace.DEFAULT).build();
            registerTableSpace(defaultTableSpace);
        }
    } catch (DDLException err) {
        throw new MetadataStorageManagerException(err);
    } finally {
        lock.writeLock().unlock();
    }
}
Also used : MetadataStorageManagerException(herddb.metadata.MetadataStorageManagerException) TableSpace(herddb.model.TableSpace) DDLException(herddb.model.DDLException)

Example 5 with DDLException

use of herddb.model.DDLException in project herddb by diennea.

the class MemoryMetadataStorageManager method ensureDefaultTableSpace.

@Override
public void ensureDefaultTableSpace(String localNodeId) throws MetadataStorageManagerException {
    lock.writeLock().lock();
    try {
        TableSpace exists = tableSpaces.get(TableSpace.DEFAULT);
        if (exists == null) {
            TableSpace defaultTableSpace = TableSpace.builder().leader(localNodeId).replica(localNodeId).name(TableSpace.DEFAULT).build();
            registerTableSpace(defaultTableSpace);
        }
    } catch (DDLException err) {
        throw new MetadataStorageManagerException(err);
    } finally {
        lock.writeLock().unlock();
    }
}
Also used : MetadataStorageManagerException(herddb.metadata.MetadataStorageManagerException) TableSpace(herddb.model.TableSpace) DDLException(herddb.model.DDLException)

Aggregations

MetadataStorageManagerException (herddb.metadata.MetadataStorageManagerException)5 DDLException (herddb.model.DDLException)5 TableSpace (herddb.model.TableSpace)4 LogNotAvailableException (herddb.log.LogNotAvailableException)3 DataStorageManagerException (herddb.storage.DataStorageManagerException)3 DataScannerException (herddb.model.DataScannerException)2 NodeMetadata (herddb.model.NodeMetadata)2 NotLeaderException (herddb.model.NotLeaderException)2 StatementExecutionException (herddb.model.StatementExecutionException)2 TableSpaceDoesNotExistException (herddb.model.TableSpaceDoesNotExistException)2 TableSpaceReplicaState (herddb.model.TableSpaceReplicaState)2 ArrayList (java.util.ArrayList)2 HashMap (java.util.HashMap)2 HashSet (java.util.HashSet)2 Map (java.util.Map)2 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)2 RejectedExecutionException (java.util.concurrent.RejectedExecutionException)2 SuppressFBWarnings (edu.umd.cs.findbugs.annotations.SuppressFBWarnings)1 ClientConfiguration (herddb.client.ClientConfiguration)1 DataScannerJoinExecutor (herddb.core.join.DataScannerJoinExecutor)1