Examples with HostGroupTable com.ms.silverking.cloud.config.HostGroupTable used on opensource projects

Example 6 with HostGroupTable

use of com.ms.silverking.cloud.config.HostGroupTable in project SilverKing by Morgan-Stanley.

the class SKAdmin method execClusterCommandGroup.

private boolean execClusterCommandGroup(SKAdminCommand[] commands) throws IOException, KeeperException {
    /*
		 * Each DHT consists of active + passive nodes
		 * Filter active&passive by particular host groups
		 * 	any servers that aren't in the included host groups won't be used
		 * Fetch all host group tables
		 * Fetch all class variables for the host groups
		 * For ChecSKFS, fetch the skfs environment
		 * Wait for all fetches to complete
		 * Create map of servers->commands to run
		 * Pass the command map to TwoLevelParallelSSH and run
		 * Run/wait until complete
		 */
    // Map<String,HostGroupTable>	hostGroupTables;
    // hostGroupTables = getHostGroupTables(hostGroups, dhtMC.getZooKeeper().getZKConfig());
    Set<String> activeHostGroupNames;
    Map<String, ClassVars> hostGroupToClassVars;
    HostGroupTable hostGroupTable;
    Set<String> validActiveServers;
    Set<String> validPassiveServers;
    String hostGroupTableName;
    Map<String, String[]> serverCommands;
    Set<String> passiveNodeHostGroupNames;
    boolean result;
    Set<String> targetServers;
    Set<String> passiveTargetServers;
    targetServers = CollectionUtil.parseSet(options.targets, ",");
    activeHostGroupNames = dhtConfig.getHostGroups();
    Log.warning("hostGroupNames: ", CollectionUtil.toString(activeHostGroupNames));
    hostGroupToClassVars = getHostGroupToClassVarsMap(dhtConfig);
    Log.warning("hostGroupToClassVars: ", CollectionUtil.mapToString(hostGroupToClassVars));
    Log.warning("ringConfig: ", ringConfig);
    hostGroupTableName = ringConfig.getCloudConfiguration().getHostGroupTableName();
    Log.warning("hostGroupTableName: ", hostGroupTableName);
    hostGroupTable = getHostGroupTable(hostGroupTableName, dhtMC.getZooKeeper().getZKConfig());
    // FUTURE - Do more validation of configuration. E.g. prevent a server from being both
    // active and passive, the ring from containing servers without class vars, etc.
    validActiveServers = findValidActiveServers(activeHostGroupNames, hostGroupTable, ringTree);
    validActiveServers = retainOnlySpecifiedAndNonExcludedServers(validActiveServers, targetServers);
    verifyServerEligibility(validActiveServers, commands);
    Log.warning("validActiveServers: ", CollectionUtil.toString(validActiveServers));
    // Allow StopNodes with empty validActiveServers if the target is activeDaemons
    if (options.targetsEqualsActiveDaemonsTarget() && validActiveServers.isEmpty()) {
        boolean exitOK;
        exitOK = true;
        for (SKAdminCommand command : commands) {
            if (command != SKAdminCommand.StopNodes) {
                exitOK = false;
            }
        }
        if (exitOK) {
            return true;
        }
    }
    passiveTargetServers = new HashSet<>();
    passiveTargetServers.addAll(targetServers);
    passiveTargetServers.removeAll(validActiveServers);
    passiveNodeHostGroupNames = dhtConfig.getPassiveNodeHostGroupsAsSet();
    Log.warning("passiveNodeHostGroupNames: ", CollectionUtil.toString(passiveNodeHostGroupNames));
    if (passiveTargetServers.size() > 0) {
        validPassiveServers = ImmutableSet.copyOf(passiveTargetServers);
    } else {
        validPassiveServers = findValidPassiveServers(passiveNodeHostGroupNames, hostGroupTable);
    }
    validPassiveServers = retainOnlySpecifiedAndNonExcludedServers(validPassiveServers, passiveTargetServers);
    Log.warning("validPassiveServers: ", CollectionUtil.toString(validPassiveServers));
    if (Arrays.contains(commands, SKAdminCommand.ClearData) && !options.targetsEqualsExclusionsTarget()) {
        Log.countdownWarning("*** Clearing ALL data ***", unsafeWarningCountdown);
    }
    result = true;
    for (SKAdminCommand command : commands) {
        boolean _result;
        Log.warning("Executing cluster command: ", command);
        serverCommands = createServerCommands(command, validActiveServers, validPassiveServers, hostGroupTable, hostGroupToClassVars, activeHostGroupNames, passiveNodeHostGroupNames);
        displayCommandMap(serverCommands);
        if (!options.displayOnly) {
            _result = execCommandMap(serverCommands, validActiveServers.size() > 0 ? validActiveServers : validPassiveServers, hostGroupTable);
            result = result && _result;
            if (!result) {
                break;
            }
        }
        if (command.equals(SKAdminCommand.StartNodes)) {
            int[] timeouts;
            boolean running;
            int attemptIndex;
            Log.warning("Waiting for nodes to enter running state...");
            timeouts = NumUtil.parseIntArray(options.timeoutSeconds, ",");
            running = false;
            attemptIndex = 0;
            do {
                Pair<Set<IPAndPort>, Boolean> waitResult;
                Set<IPAndPort> failedServers;
                Log.warningf("attemptIndex: %d\ttimeout: %d", attemptIndex, timeouts[attemptIndex]);
                if (replicaSetExcludedByExclusions(exclusionSet)) {
                    return false;
                }
                waitResult = waitUntilRunning(IPAndPort.set(validActiveServers, dhtConfig.getPort()), timeouts[attemptIndex]);
                failedServers = waitResult.getV1();
                if (waitResult.getV2()) {
                    running = true;
                } else {
                    ++attemptIndex;
                    if (attemptIndex < timeouts.length) {
                        Log.warningf("Adding to instance exclusion set: %s", failedServers);
                        if (options.excludeInstanceExclusions) {
                            exclusionSet = exclusionSet.addByIPAndPort(failedServers);
                        }
                        addToInstanceExclusions(failedServers);
                        validActiveServers = removeServers(validActiveServers, failedServers);
                    }
                }
            } while (!running && attemptIndex < timeouts.length);
            if (!running) {
                return false;
            }
        }
    }
    return result;
}

Example 7 with HostGroupTable

use of com.ms.silverking.cloud.config.HostGroupTable in project SilverKing by Morgan-Stanley.

the class CentralConvergenceController method getPassiveNodes.

private Set<IPAndPort> getPassiveNodes() throws ConvergenceException {
    try {
        HostGroupTable hostGroupTable;
        Set<IPAndPort> validPassiveServers;
        String hostGroupTableName;
        Set<String> passiveNodeHostGroupNames;
        Set<IPAndPort> currentReplicas;
        Set<IPAndPort> targetReplicas;
        Set<IPAndPort> currentButNonTargetReplicas;
        hostGroupTableName = currentRingConfig.getCloudConfiguration().getHostGroupTableName();
        Log.warningAsync("hostGroupTableName: ", hostGroupTableName);
        hostGroupTable = getHostGroupTable(hostGroupTableName, ringMC.getZooKeeper().getZKConfig());
        // FUTURE - Do more validation of configuration. E.g. prevent a server from being both
        // active and passive, the ring from containing servers without class vars, etc.
        passiveNodeHostGroupNames = dhtConfig.getPassiveNodeHostGroupsAsSet();
        Log.warningAsync("passiveNodeHostGroupNames: ", CollectionUtil.toString(passiveNodeHostGroupNames));
        currentReplicas = getCurrentReplicasWithPorts();
        targetReplicas = getTargetReplicasWithPorts();
        currentButNonTargetReplicas = new HashSet<>(currentReplicas);
        currentButNonTargetReplicas.removeAll(targetReplicas);
        validPassiveServers = new HashSet<>(findValidPassiveServers(passiveNodeHostGroupNames, hostGroupTable));
        validPassiveServers.addAll(currentButNonTargetReplicas);
        validPassiveServers.removeAll(IPAndPort.set(exclusionSet.getServers(), dhtConfig.getPort()));
        validPassiveServers.removeAll(targetReplicas);
        Log.warningAsync("validPassiveServers: ", CollectionUtil.toString(validPassiveServers));
        return validPassiveServers;
    } catch (IOException | KeeperException e) {
        throw new ConvergenceException("Exception reading passive nodes", e);
    }
}

Example 8 with HostGroupTable

use of com.ms.silverking.cloud.config.HostGroupTable in project SilverKing by Morgan-Stanley.

the class SKAdmin method execCreateSKFSns.

private boolean execCreateSKFSns() throws IOException, ClientException, KeeperException {
    SKFSNamespaceCreator nsCreator;
    String preferredServer;
    Pair<RingConfiguration, InstantiatedRingTree> ringConfigAndTree;
    HostGroupTable hostGroupTable;
    String hostGroupTableName;
    ringConfigAndTree = getRing(dhtConfig, dhtMC);
    Log.warning("ringConfig: ", ringConfigAndTree.getV1());
    hostGroupTableName = ringConfigAndTree.getV1().getCloudConfiguration().getHostGroupTableName();
    Log.warning("hostGroupTableName: ", hostGroupTableName);
    hostGroupTable = getHostGroupTable(hostGroupTableName, dhtMC.getZooKeeper().getZKConfig());
    if (options.preferredServer == null) {
        preferredServer = findArbitraryActiveServer(dhtConfig.getHostGroups(), hostGroupTable);
    } else {
        preferredServer = options.preferredServer;
    }
    Log.warning("Using preferredServer ", preferredServer);
    nsCreator = new SKFSNamespaceCreator(gc.getClientDHTConfiguration(), preferredServer);
    nsCreator.createNamespaces(skfsNamespaces, skfsNSOptions);
    nsCreator.createNamespaces(skfsMutableNamespaces, skfsMutableNSOptions);
    nsCreator.createNamespaces(skfsFileBlockNamespaces, skfsFileBlockNSOptions);
    nsCreator.createNamespaces(skfsDirNamespaces, skfsDirNSOptions);
    return true;
}