Examples with MetadataStorageManagerException herddb.metadata.MetadataStorageManagerException used on opensource projects

Example 1 with MetadataStorageManagerException

use of herddb.metadata.MetadataStorageManagerException in project herddb by diennea.

the class DBManager method manageTableSpaces.

private boolean manageTableSpaces() {
    Collection<String> actualTablesSpaces;
    try {
        actualTablesSpaces = metadataStorageManager.listTableSpaces();
    } catch (MetadataStorageManagerException error) {
        LOGGER.log(Level.SEVERE, "cannot access tablespaces metadata", error);
        return true;
    }
    Map<String, TableSpace> actualTableSpaceMetadata = new HashMap<>();
    generalLock.writeLock().lock();
    try {
        for (String tableSpace : actualTablesSpaces) {
            TableSpace tableSpaceMetadata = metadataStorageManager.describeTableSpace(tableSpace);
            actualTableSpaceMetadata.put(tableSpaceMetadata.uuid, tableSpaceMetadata);
            try {
                handleTableSpace(tableSpaceMetadata);
            } catch (Exception err) {
                LOGGER.log(Level.SEVERE, "cannot handle tablespace " + tableSpace, err);
                if (haltOnTableSpaceBootError && haltProcedure != null) {
                    err.printStackTrace();
                    haltProcedure.run();
                }
            }
        }
    } catch (MetadataStorageManagerException error) {
        LOGGER.log(Level.SEVERE, "cannot access tablespaces metadata", error);
        return true;
    } finally {
        generalLock.writeLock().unlock();
    }
    List<TableSpaceManager> followingActiveTableSpaces = new ArrayList<>();
    Set<String> failedTableSpaces = new HashSet<>();
    for (Map.Entry<String, TableSpaceManager> entry : tablesSpaces.entrySet()) {
        try {
            String tableSpaceUuid = entry.getValue().getTableSpaceUUID();
            if (entry.getValue().isFailed()) {
                failedTableSpaces.add(entry.getKey());
            } else if (!entry.getKey().equals(virtualTableSpaceId) && !actualTablesSpaces.contains(entry.getKey())) {
                failedTableSpaces.add(entry.getKey());
            } else if (entry.getValue().isLeader()) {
                metadataStorageManager.updateTableSpaceReplicaState(TableSpaceReplicaState.builder().mode(TableSpaceReplicaState.MODE_LEADER).nodeId(nodeId).uuid(tableSpaceUuid).timestamp(System.currentTimeMillis()).build());
            } else {
                metadataStorageManager.updateTableSpaceReplicaState(TableSpaceReplicaState.builder().mode(TableSpaceReplicaState.MODE_FOLLOWER).nodeId(nodeId).uuid(tableSpaceUuid).timestamp(System.currentTimeMillis()).build());
                followingActiveTableSpaces.add(entry.getValue());
            }
        } catch (MetadataStorageManagerException error) {
            LOGGER.log(Level.SEVERE, "cannot access tablespace " + entry.getKey() + " metadata", error);
            return true;
        }
    }
    if (!failedTableSpaces.isEmpty()) {
        generalLock.writeLock().lock();
        try {
            for (String tableSpace : failedTableSpaces) {
                stopTableSpace(tableSpace, null);
            }
        } catch (MetadataStorageManagerException error) {
            LOGGER.log(Level.SEVERE, "cannot access tablespace metadata", error);
            return true;
        } finally {
            generalLock.writeLock().unlock();
        }
    }
    if (!followingActiveTableSpaces.isEmpty()) {
        long now = System.currentTimeMillis();
        try {
            for (TableSpaceManager tableSpaceManager : followingActiveTableSpaces) {
                String tableSpaceUuid = tableSpaceManager.getTableSpaceUUID();
                TableSpace tableSpaceInfo = actualTableSpaceMetadata.get(tableSpaceUuid);
                if (tableSpaceInfo != null && tableSpaceInfo.maxLeaderInactivityTime > 0 && !tableSpaceManager.isFailed()) {
                    List<TableSpaceReplicaState> allReplicas = metadataStorageManager.getTableSpaceReplicaState(tableSpaceUuid);
                    TableSpaceReplicaState leaderState = allReplicas.stream().filter(t -> t.mode == TableSpaceReplicaState.MODE_LEADER && !t.nodeId.equals(nodeId)).findAny().orElse(null);
                    if (leaderState == null) {
                        LOGGER.log(Level.SEVERE, "Leader for " + tableSpaceUuid + " should be " + tableSpaceInfo.leaderId + ", but it never sent pings or it disappeared");
                        tryBecomeLeaderFor(tableSpaceInfo);
                    } else {
                        long delta = now - leaderState.timestamp;
                        if (tableSpaceInfo.maxLeaderInactivityTime > delta) {
                            LOGGER.log(Level.FINER, "Leader for " + tableSpaceUuid + " is " + leaderState.nodeId + ", last ping " + new java.sql.Timestamp(leaderState.timestamp) + ". leader is healty");
                        } else {
                            LOGGER.log(Level.SEVERE, "Leader for " + tableSpaceUuid + " is " + leaderState.nodeId + ", last ping " + new java.sql.Timestamp(leaderState.timestamp) + ". leader is failed. trying to take leadership");
                            tryBecomeLeaderFor(tableSpaceInfo);
                            // only one change at a time
                            break;
                        }
                    }
                }
            }
        } catch (MetadataStorageManagerException | DDLException error) {
            LOGGER.log(Level.SEVERE, "cannot access tablespace metadata", error);
            return true;
        }
    }
    return false;
}

Example 2 with MetadataStorageManagerException

use of herddb.metadata.MetadataStorageManagerException in project herddb by diennea.

the class DBManager method start.

/**
 * Initial boot of the system
 *
 * @throws herddb.storage.DataStorageManagerException
 * @throws herddb.log.LogNotAvailableException
 * @throws herddb.metadata.MetadataStorageManagerException
 */
public void start() throws DataStorageManagerException, LogNotAvailableException, MetadataStorageManagerException {
    if (serverConfiguration.getBoolean(ServerConfiguration.PROPERTY_JMX_ENABLE, ServerConfiguration.PROPERTY_JMX_ENABLE_DEFAULT)) {
        JMXUtils.registerDBManagerStatsMXBean(stats);
    }
    final long maxHeap = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
    /* If max memory isn't configured or is too high default it to maximum heap */
    if (maxMemoryReference == 0 || maxMemoryReference > maxHeap) {
        maxMemoryReference = maxHeap;
    }
    LOGGER.log(Level.INFO, ServerConfiguration.PROPERTY_MEMORY_LIMIT_REFERENCE + "= {0} bytes", Long.toString(maxMemoryReference));
    /* If max data memory for pages isn't configured or is too high default it to 0.3 maxMemoryReference */
    if (maxDataUsedMemory == 0 || maxDataUsedMemory > maxMemoryReference) {
        maxDataUsedMemory = (long) (0.3F * maxMemoryReference);
    }
    /* If max index memory for pages isn't configured or is too high default it to 0.2 maxMemoryReference */
    if (maxPKUsedMemory == 0 || maxPKUsedMemory > maxMemoryReference) {
        maxPKUsedMemory = (long) (0.2F * maxMemoryReference);
    }
    /* If max used memory is too high lower index and data accordingly */
    if (maxDataUsedMemory + maxPKUsedMemory > maxMemoryReference) {
        long data = (int) ((double) maxDataUsedMemory / ((double) (maxDataUsedMemory + maxPKUsedMemory)) * maxMemoryReference);
        long pk = (int) ((double) maxPKUsedMemory / ((double) (maxDataUsedMemory + maxPKUsedMemory)) * maxMemoryReference);
        maxDataUsedMemory = data;
        maxPKUsedMemory = pk;
    }
    memoryManager = new MemoryManager(maxDataUsedMemory, maxPKUsedMemory, maxLogicalPageSize);
    metadataStorageManager.start();
    if (clearAtBoot) {
        metadataStorageManager.clear();
    }
    metadataStorageManager.setMetadataChangeListener(this);
    NodeMetadata nodeMetadata = NodeMetadata.builder().host(hostData.getHost()).port(hostData.getPort()).ssl(hostData.isSsl()).nodeId(nodeId).build();
    LOGGER.log(Level.SEVERE, "Registering on metadata storage manager my data: {0}", nodeMetadata);
    metadataStorageManager.registerNode(nodeMetadata);
    try {
        TableSpaceManager local_node_virtual_tables_manager = new TableSpaceManager(nodeId, virtualTableSpaceId, virtualTableSpaceId, metadataStorageManager, dataStorageManager, null, this, true);
        tablesSpaces.put(virtualTableSpaceId, local_node_virtual_tables_manager);
        local_node_virtual_tables_manager.start();
    } catch (DDLException | DataStorageManagerException | LogNotAvailableException | MetadataStorageManagerException error) {
        throw new IllegalStateException("cannot boot local virtual tablespace manager");
    }
    metadataStorageManager.ensureDefaultTableSpace(nodeId);
    commitLogManager.start();
    generalLock.writeLock().lock();
    try {
        dataStorageManager.start();
    } finally {
        generalLock.writeLock().unlock();
    }
    activator.start();
    triggerActivator(ActivatorRunRequest.FULL);
}

Example 3 with MetadataStorageManagerException

use of herddb.metadata.MetadataStorageManagerException in project herddb by diennea.

the class DBManager method handleTableSpace.

private void handleTableSpace(TableSpace tableSpace) throws DataStorageManagerException, LogNotAvailableException, MetadataStorageManagerException, DDLException {
    String tableSpaceName = tableSpace.name;
    TableSpaceManager actual_manager = tablesSpaces.get(tableSpaceName);
    if (actual_manager != null && actual_manager.isFailed()) {
        LOGGER.log(Level.INFO, "Tablespace {0} is in 'Failed' status", new Object[] { tableSpaceName, nodeId });
        return;
    }
    if (actual_manager != null && actual_manager.isLeader() && !tableSpace.leaderId.equals(nodeId)) {
        LOGGER.log(Level.SEVERE, "Tablespace {0} leader is no more {1}, it changed to {2}", new Object[] { tableSpaceName, nodeId, tableSpace.leaderId });
        stopTableSpace(tableSpaceName, tableSpace.uuid);
        return;
    }
    if (actual_manager != null && !actual_manager.isLeader() && tableSpace.leaderId.equals(nodeId)) {
        LOGGER.log(Level.SEVERE, "Tablespace {0} need to switch to leadership on node {1}", new Object[] { tableSpaceName, nodeId });
        stopTableSpace(tableSpaceName, tableSpace.uuid);
        return;
    }
    if (tableSpace.replicas.contains(nodeId) && !tablesSpaces.containsKey(tableSpaceName)) {
        LOGGER.log(Level.SEVERE, "Booting tablespace {0} on {1}, uuid {2}", new Object[] { tableSpaceName, nodeId, tableSpace.uuid });
        long _start = System.currentTimeMillis();
        CommitLog commitLog = commitLogManager.createCommitLog(tableSpace.uuid);
        TableSpaceManager manager = new TableSpaceManager(nodeId, tableSpaceName, tableSpace.uuid, metadataStorageManager, dataStorageManager, commitLog, this, false);
        try {
            manager.start();
            LOGGER.log(Level.SEVERE, "Boot success tablespace {0} on {1}, uuid {2}, time {3} ms", new Object[] { tableSpaceName, nodeId, tableSpace.uuid, (System.currentTimeMillis() - _start) + "" });
            tablesSpaces.put(tableSpaceName, manager);
            if (serverConfiguration.getBoolean(ServerConfiguration.PROPERTY_JMX_ENABLE, ServerConfiguration.PROPERTY_JMX_ENABLE_DEFAULT)) {
                JMXUtils.registerTableSpaceManagerStatsMXBean(tableSpaceName, manager.getStats());
            }
        } catch (DataStorageManagerException | LogNotAvailableException | MetadataStorageManagerException | DDLException t) {
            LOGGER.log(Level.SEVERE, "Error Booting tablespace {0} on {1}", new Object[] { tableSpaceName, nodeId });
            LOGGER.log(Level.SEVERE, "Error", t);
            try {
                manager.close();
            } catch (Throwable t2) {
                LOGGER.log(Level.SEVERE, "Other Error", t2);
            }
            throw t;
        }
        return;
    }
    if (tablesSpaces.containsKey(tableSpaceName) && !tableSpace.replicas.contains(nodeId)) {
        LOGGER.log(Level.SEVERE, "Tablespace {0} on {1} is not more in replica list {3}, uuid {2}", new Object[] { tableSpaceName, nodeId, tableSpace.uuid, tableSpace.replicas + "" });
        stopTableSpace(tableSpaceName, tableSpace.uuid);
        return;
    }
    if (tableSpace.replicas.size() < tableSpace.expectedReplicaCount) {
        List<NodeMetadata> nodes = metadataStorageManager.listNodes();
        LOGGER.log(Level.SEVERE, "Tablespace {0} is underreplicated expectedReplicaCount={1}, replicas={2}, nodes={3}", new Object[] { tableSpaceName, tableSpace.expectedReplicaCount, tableSpace.replicas, nodes });
        List<String> availableOtherNodes = nodes.stream().map(n -> {
            return n.nodeId;
        }).filter(n -> {
            return !tableSpace.replicas.contains(n);
        }).collect(Collectors.toList());
        Collections.shuffle(availableOtherNodes);
        LOGGER.log(Level.SEVERE, "Tablespace {0} is underreplicated expectedReplicaCount={1}, replicas={2}, availableOtherNodes={3}", new Object[] { tableSpaceName, tableSpace.expectedReplicaCount, tableSpace.replicas, availableOtherNodes });
        if (!availableOtherNodes.isEmpty()) {
            int countMissing = tableSpace.expectedReplicaCount - tableSpace.replicas.size();
            TableSpace.Builder newTableSpaceBuilder = TableSpace.builder().cloning(tableSpace);
            while (!availableOtherNodes.isEmpty() && countMissing > 0) {
                String node = availableOtherNodes.remove(0);
                newTableSpaceBuilder.replica(node);
            }
            TableSpace newTableSpace = newTableSpaceBuilder.build();
            boolean ok = metadataStorageManager.updateTableSpace(newTableSpace, tableSpace);
            if (!ok) {
                LOGGER.log(Level.SEVERE, "updating tableSpace " + tableSpaceName + " metadata failed");
            }
        }
    }
}

Example 4 with MetadataStorageManagerException

use of herddb.metadata.MetadataStorageManagerException in project herddb by diennea.

the class FileMetadataStorageManager method persistTableSpaceOnDisk.

private void persistTableSpaceOnDisk(TableSpace tableSpace) throws MetadataStorageManagerException {
    Path tablespaceMetaTmp = baseDirectory.resolve(tableSpace.name + "." + System.nanoTime() + ".tmpmetadata");
    Path tablespaceMeta = baseDirectory.resolve(tableSpace.name + ".metadata");
    try (ManagedFile file = ManagedFile.open(tablespaceMetaTmp);
        SimpleBufferedOutputStream buffer = new SimpleBufferedOutputStream(file.getOutputStream(), FileDataStorageManager.COPY_BUFFERS_SIZE);
        XXHash64Utils.HashingOutputStream oo = new XXHash64Utils.HashingOutputStream(buffer);
        ExtendedDataOutputStream dout = new ExtendedDataOutputStream(oo)) {
        // version
        dout.writeVLong(1);
        // flags for future implementations
        dout.writeVLong(0);
        tableSpace.serialize(dout);
        // footer
        dout.writeLong(oo.hash());
        dout.flush();
        file.sync();
    } catch (IOException err) {
        throw new MetadataStorageManagerException(err);
    }
    try {
        Files.move(tablespaceMetaTmp, tablespaceMeta, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
    } catch (IOException err) {
        throw new MetadataStorageManagerException(err);
    }
}

Example 5 with MetadataStorageManagerException

use of herddb.metadata.MetadataStorageManagerException in project herddb by diennea.

the class FileMetadataStorageManager method clear.

@Override
public void clear() throws MetadataStorageManagerException {
    lock.writeLock().lock();
    try {
        FileUtils.cleanDirectory(baseDirectory);
        Files.createDirectories(baseDirectory);
        nodes.clear();
        tableSpaces.clear();
    } catch (IOException err) {
        LOGGER.log(Level.SEVERE, "cannot clear local data", err);
        throw new MetadataStorageManagerException(err);
    } finally {
        lock.writeLock().unlock();
    }
}