Examples with NodeMetadata herddb.model.NodeMetadata used on opensource projects

Example 1 with NodeMetadata

use of herddb.model.NodeMetadata in project herddb by diennea.

the class DBManager method start.

/**
 * Initial boot of the system
 *
 * @throws herddb.storage.DataStorageManagerException
 * @throws herddb.log.LogNotAvailableException
 * @throws herddb.metadata.MetadataStorageManagerException
 */
public void start() throws DataStorageManagerException, LogNotAvailableException, MetadataStorageManagerException {
    if (serverConfiguration.getBoolean(ServerConfiguration.PROPERTY_JMX_ENABLE, ServerConfiguration.PROPERTY_JMX_ENABLE_DEFAULT)) {
        JMXUtils.registerDBManagerStatsMXBean(stats);
    }
    final long maxHeap = ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax();
    /* If max memory isn't configured or is too high default it to maximum heap */
    if (maxMemoryReference == 0 || maxMemoryReference > maxHeap) {
        maxMemoryReference = maxHeap;
    }
    LOGGER.log(Level.INFO, ServerConfiguration.PROPERTY_MEMORY_LIMIT_REFERENCE + "= {0} bytes", Long.toString(maxMemoryReference));
    /* If max data memory for pages isn't configured or is too high default it to 0.3 maxMemoryReference */
    if (maxDataUsedMemory == 0 || maxDataUsedMemory > maxMemoryReference) {
        maxDataUsedMemory = (long) (0.3F * maxMemoryReference);
    }
    /* If max index memory for pages isn't configured or is too high default it to 0.2 maxMemoryReference */
    if (maxPKUsedMemory == 0 || maxPKUsedMemory > maxMemoryReference) {
        maxPKUsedMemory = (long) (0.2F * maxMemoryReference);
    }
    /* If max used memory is too high lower index and data accordingly */
    if (maxDataUsedMemory + maxPKUsedMemory > maxMemoryReference) {
        long data = (int) ((double) maxDataUsedMemory / ((double) (maxDataUsedMemory + maxPKUsedMemory)) * maxMemoryReference);
        long pk = (int) ((double) maxPKUsedMemory / ((double) (maxDataUsedMemory + maxPKUsedMemory)) * maxMemoryReference);
        maxDataUsedMemory = data;
        maxPKUsedMemory = pk;
    }
    memoryManager = new MemoryManager(maxDataUsedMemory, maxPKUsedMemory, maxLogicalPageSize);
    metadataStorageManager.start();
    if (clearAtBoot) {
        metadataStorageManager.clear();
    }
    metadataStorageManager.setMetadataChangeListener(this);
    NodeMetadata nodeMetadata = NodeMetadata.builder().host(hostData.getHost()).port(hostData.getPort()).ssl(hostData.isSsl()).nodeId(nodeId).build();
    LOGGER.log(Level.SEVERE, "Registering on metadata storage manager my data: {0}", nodeMetadata);
    metadataStorageManager.registerNode(nodeMetadata);
    try {
        TableSpaceManager local_node_virtual_tables_manager = new TableSpaceManager(nodeId, virtualTableSpaceId, virtualTableSpaceId, metadataStorageManager, dataStorageManager, null, this, true);
        tablesSpaces.put(virtualTableSpaceId, local_node_virtual_tables_manager);
        local_node_virtual_tables_manager.start();
    } catch (DDLException | DataStorageManagerException | LogNotAvailableException | MetadataStorageManagerException error) {
        throw new IllegalStateException("cannot boot local virtual tablespace manager");
    }
    metadataStorageManager.ensureDefaultTableSpace(nodeId);
    commitLogManager.start();
    generalLock.writeLock().lock();
    try {
        dataStorageManager.start();
    } finally {
        generalLock.writeLock().unlock();
    }
    activator.start();
    triggerActivator(ActivatorRunRequest.FULL);
}

Example 2 with NodeMetadata

use of herddb.model.NodeMetadata in project herddb by diennea.

the class DBManager method handleTableSpace.

private void handleTableSpace(TableSpace tableSpace) throws DataStorageManagerException, LogNotAvailableException, MetadataStorageManagerException, DDLException {
    String tableSpaceName = tableSpace.name;
    TableSpaceManager actual_manager = tablesSpaces.get(tableSpaceName);
    if (actual_manager != null && actual_manager.isFailed()) {
        LOGGER.log(Level.INFO, "Tablespace {0} is in 'Failed' status", new Object[] { tableSpaceName, nodeId });
        return;
    }
    if (actual_manager != null && actual_manager.isLeader() && !tableSpace.leaderId.equals(nodeId)) {
        LOGGER.log(Level.SEVERE, "Tablespace {0} leader is no more {1}, it changed to {2}", new Object[] { tableSpaceName, nodeId, tableSpace.leaderId });
        stopTableSpace(tableSpaceName, tableSpace.uuid);
        return;
    }
    if (actual_manager != null && !actual_manager.isLeader() && tableSpace.leaderId.equals(nodeId)) {
        LOGGER.log(Level.SEVERE, "Tablespace {0} need to switch to leadership on node {1}", new Object[] { tableSpaceName, nodeId });
        stopTableSpace(tableSpaceName, tableSpace.uuid);
        return;
    }
    if (tableSpace.replicas.contains(nodeId) && !tablesSpaces.containsKey(tableSpaceName)) {
        LOGGER.log(Level.SEVERE, "Booting tablespace {0} on {1}, uuid {2}", new Object[] { tableSpaceName, nodeId, tableSpace.uuid });
        long _start = System.currentTimeMillis();
        CommitLog commitLog = commitLogManager.createCommitLog(tableSpace.uuid);
        TableSpaceManager manager = new TableSpaceManager(nodeId, tableSpaceName, tableSpace.uuid, metadataStorageManager, dataStorageManager, commitLog, this, false);
        try {
            manager.start();
            LOGGER.log(Level.SEVERE, "Boot success tablespace {0} on {1}, uuid {2}, time {3} ms", new Object[] { tableSpaceName, nodeId, tableSpace.uuid, (System.currentTimeMillis() - _start) + "" });
            tablesSpaces.put(tableSpaceName, manager);
            if (serverConfiguration.getBoolean(ServerConfiguration.PROPERTY_JMX_ENABLE, ServerConfiguration.PROPERTY_JMX_ENABLE_DEFAULT)) {
                JMXUtils.registerTableSpaceManagerStatsMXBean(tableSpaceName, manager.getStats());
            }
        } catch (DataStorageManagerException | LogNotAvailableException | MetadataStorageManagerException | DDLException t) {
            LOGGER.log(Level.SEVERE, "Error Booting tablespace {0} on {1}", new Object[] { tableSpaceName, nodeId });
            LOGGER.log(Level.SEVERE, "Error", t);
            try {
                manager.close();
            } catch (Throwable t2) {
                LOGGER.log(Level.SEVERE, "Other Error", t2);
            }
            throw t;
        }
        return;
    }
    if (tablesSpaces.containsKey(tableSpaceName) && !tableSpace.replicas.contains(nodeId)) {
        LOGGER.log(Level.SEVERE, "Tablespace {0} on {1} is not more in replica list {3}, uuid {2}", new Object[] { tableSpaceName, nodeId, tableSpace.uuid, tableSpace.replicas + "" });
        stopTableSpace(tableSpaceName, tableSpace.uuid);
        return;
    }
    if (tableSpace.replicas.size() < tableSpace.expectedReplicaCount) {
        List<NodeMetadata> nodes = metadataStorageManager.listNodes();
        LOGGER.log(Level.SEVERE, "Tablespace {0} is underreplicated expectedReplicaCount={1}, replicas={2}, nodes={3}", new Object[] { tableSpaceName, tableSpace.expectedReplicaCount, tableSpace.replicas, nodes });
        List<String> availableOtherNodes = nodes.stream().map(n -> {
            return n.nodeId;
        }).filter(n -> {
            return !tableSpace.replicas.contains(n);
        }).collect(Collectors.toList());
        Collections.shuffle(availableOtherNodes);
        LOGGER.log(Level.SEVERE, "Tablespace {0} is underreplicated expectedReplicaCount={1}, replicas={2}, availableOtherNodes={3}", new Object[] { tableSpaceName, tableSpace.expectedReplicaCount, tableSpace.replicas, availableOtherNodes });
        if (!availableOtherNodes.isEmpty()) {
            int countMissing = tableSpace.expectedReplicaCount - tableSpace.replicas.size();
            TableSpace.Builder newTableSpaceBuilder = TableSpace.builder().cloning(tableSpace);
            while (!availableOtherNodes.isEmpty() && countMissing > 0) {
                String node = availableOtherNodes.remove(0);
                newTableSpaceBuilder.replica(node);
            }
            TableSpace newTableSpace = newTableSpaceBuilder.build();
            boolean ok = metadataStorageManager.updateTableSpace(newTableSpace, tableSpace);
            if (!ok) {
                LOGGER.log(Level.SEVERE, "updating tableSpace " + tableSpaceName + " metadata failed");
            }
        }
    }
}

Example 3 with NodeMetadata

use of herddb.model.NodeMetadata in project herddb by diennea.

the class ZookeeperClientSideMetadataProvider method readAsNode.

private String readAsNode(ZooKeeper zooKeeper, String tableSpace) throws IOException, InterruptedException, KeeperException {
    Stat stat = new Stat();
    byte[] result = zooKeeper.getData(basePath + "/nodes/" + tableSpace, false, stat);
    NodeMetadata md = NodeMetadata.deserialize(result, stat.getVersion());
    LOG.severe("node metadata:" + md);
    String leader = md.nodeId;
    tableSpaceLeaders.put(tableSpace, leader);
    return leader;
}

Example 4 with NodeMetadata

use of herddb.model.NodeMetadata in project herddb by diennea.

the class ZookeeperClientSideMetadataProvider method getServerHostData.

@Override
public ServerHostData getServerHostData(String nodeId) throws ClientSideMetadataProviderException {
    ServerHostData cached = servers.get(nodeId);
    if (cached != null) {
        return cached;
    }
    ZooKeeper zooKeeper = getZooKeeper();
    try {
        for (int i = 0; i < MAX_TRIALS; i++) {
            try {
                Stat stat = new Stat();
                byte[] node = zooKeeper.getData(basePath + "/nodes/" + nodeId, null, stat);
                NodeMetadata nodeMetadata = NodeMetadata.deserialize(node, stat.getVersion());
                ServerHostData result = new ServerHostData(nodeMetadata.host, nodeMetadata.port, "?", nodeMetadata.ssl, new HashMap<>());
                servers.put(nodeId, result);
                return result;
            } catch (KeeperException.NoNodeException ex) {
                return null;
            } catch (KeeperException.ConnectionLossException ex) {
                LOG.log(Level.SEVERE, "tmp error getServerHostData for " + nodeId + ": " + ex);
                try {
                    Thread.sleep(i * 500 + 1000);
                } catch (InterruptedException exit) {
                    throw new ClientSideMetadataProviderException(exit);
                }
            } catch (KeeperException | InterruptedException | IOException ex) {
                throw new ClientSideMetadataProviderException(ex);
            } finally {
                if (ownZooKeeper) {
                    try {
                        zooKeeper.close();
                    } catch (InterruptedException ex) {
                        throw new ClientSideMetadataProviderException(ex);
                    }
                }
            }
        }
    } finally {
        if (ownZooKeeper) {
            try {
                zooKeeper.close();
            } catch (InterruptedException ex) {
                throw new ClientSideMetadataProviderException(ex);
            }
        }
    }
    throw new ClientSideMetadataProviderException("Could not find a server info for node " + nodeId + " in time");
}

Example 5 with NodeMetadata

use of herddb.model.NodeMetadata in project herddb by diennea.

the class ZookeeperMetadataStorageManager method listNodes.

@Override
public List<NodeMetadata> listNodes() throws MetadataStorageManagerException {
    try {
        List<String> children = ensureZooKeeper().getChildren(nodesPath, mainWatcher, null);
        LOGGER.severe("listNodes: for " + nodesPath + ": " + children);
        List<NodeMetadata> result = new ArrayList<>();
        for (String child : children) {
            NodeMetadata nodeMetadata = getNode(child);
            result.add(nodeMetadata);
        }
        return result;
    } catch (IOException | InterruptedException | KeeperException err) {
        handleSessionExpiredError(err);
        throw new MetadataStorageManagerException(err);
    }
}