Examples with DaemonState com.ms.silverking.cloud.dht.daemon.DaemonState used on opensource projects

Example 1 with DaemonState

use of com.ms.silverking.cloud.dht.daemon.DaemonState in project SilverKing by Morgan-Stanley.

the class DaemonStateZK method waitForQuorumState.

public Map<IPAndPort, DaemonState> waitForQuorumState(Set<IPAndPort> members, DaemonState targetState, int inactiveNodeTimeoutSeconds, boolean exitOnTimeout) {
    boolean targetReached;
    int lastNotReadyCount;
    int notReadyCount;
    int inactiveCount;
    Stopwatch sw;
    Set<IPAndPort> incompleteMembers;
    incompleteMembers = new HashSet<>(members);
    if (verbose) {
        System.out.printf("Waiting for quorum state: %s\n", targetState);
    }
    sw = new SimpleStopwatch();
    lastNotReadyCount = Integer.MAX_VALUE;
    targetReached = false;
    randomAwait(0, minQuorumStatePollIntervalMillis);
    while (!targetReached) {
        StringBuilder sb;
        sb = new StringBuilder();
        try {
            if (mc.getZooKeeper().getState() == States.CONNECTED) {
                Map<IPAndPort, DaemonState> quorumState;
                Stopwatch qsSW;
                Set<IPAndPort> newlyCompleteMembers;
                Pair<Boolean, Map<IPAndPort, DaemonState>> rVal;
                boolean allRead;
                qsSW = new SimpleStopwatch();
                rVal = getQuorumState(incompleteMembers, targetState);
                qsSW.stop();
                Log.info("getQuorumState elapsed ", qsSW.getElapsedSeconds());
                allRead = rVal.getV1();
                quorumState = rVal.getV2();
                targetReached = true;
                notReadyCount = 0;
                inactiveCount = 0;
                newlyCompleteMembers = new HashSet<>();
                for (IPAndPort incompleteMember : incompleteMembers) {
                    DaemonState memberState;
                    memberState = quorumState.get(incompleteMember);
                    if (verbose && allRead) {
                        sb.append(String.format("%s\t%s\n", incompleteMember, memberState));
                    }
                    if (memberState == null) {
                        if (allRead) {
                            ++inactiveCount;
                            if ((int) sw.getSplitSeconds() < inactiveNodeTimeoutSeconds) {
                                ++notReadyCount;
                                targetReached = false;
                            } else {
                                sb.append(String.format("%s\t%s\tinactive node timed out\n", incompleteMember, memberState));
                                if (exitOnTimeout) {
                                    Map<IPAndPort, DaemonState> stateMap;
                                    System.out.print(sb);
                                    Log.warningf("Timeout: %s", incompleteMember);
                                    stateMap = fetchAndDisplayIncompleteState(incompleteMembers, targetState);
                                    return stateMap;
                                }
                            }
                        } else {
                            targetReached = false;
                        }
                    } else if (memberState.ordinal() < targetState.ordinal()) {
                        targetReached = false;
                        ++notReadyCount;
                    } else {
                        newlyCompleteMembers.add(incompleteMember);
                    }
                }
                incompleteMembers.removeAll(newlyCompleteMembers);
                if (verbose) {
                    if (allRead) {
                        if (notReadyCount != lastNotReadyCount) {
                            System.out.print(sb);
                        } else {
                            System.out.printf("Waiting for %d nodes\n", notReadyCount);
                        }
                    } else {
                        System.out.printf("waitForQuorumState > %d incomplete\n", maxIncompletePerFetch);
                    }
                }
                lastNotReadyCount = notReadyCount;
            }
        } catch (KeeperException ke) {
            Log.logErrorWarning(ke);
        }
        if (!targetReached) {
            int maxSleepMillis;
            if ((int) sw.getSplitSeconds() > inactiveNodeTimeoutSeconds) {
                if (exitOnTimeout) {
                    Map<IPAndPort, DaemonState> stateMap;
                    Log.warningf("Timeout in targetState: %s", targetState);
                    try {
                        stateMap = fetchAndDisplayIncompleteState(incompleteMembers, targetState);
                    } catch (KeeperException e) {
                        e.printStackTrace();
                        stateMap = ImmutableMap.of();
                    }
                    return stateMap;
                }
            }
            maxSleepMillis = (int) Math.max((double) maxQuorumStatePollIntervalMillis * ((double) incompleteMembers.size() / (double) maxQuorumStatePollThreshold), minQuorumStatePollIntervalMillis);
            maxSleepMillis = Math.min(maxSleepMillis, maxQuorumStatePollIntervalMillis);
            Log.info("waitForQuorumState maxSleepMillis: ", maxSleepMillis);
            expectedSignals = incompleteMembers.size();
            randomAwait(minQuorumStatePollIntervalMillis, maxSleepMillis);
            Log.info("waitForQuorumState awake");
        }
    }
    if (verbose) {
        System.out.printf("Quorum state reached: %s\n", targetState);
    }
    return ImmutableMap.of();
}

Example 2 with DaemonState

use of com.ms.silverking.cloud.dht.daemon.DaemonState in project SilverKing by Morgan-Stanley.

the class DaemonStateZK method fetchAndDisplayIncompleteState.

private Map<IPAndPort, DaemonState> fetchAndDisplayIncompleteState(Set<IPAndPort> members, DaemonState targetState) throws KeeperException {
    Pair<Boolean, Map<IPAndPort, DaemonState>> sp;
    Map<IPAndPort, DaemonState> qs;
    sp = getAllQuorumState(members, targetState);
    qs = sp.getV2();
    for (IPAndPort im : members) {
        DaemonState state;
        state = qs.get(im);
        if (state == null || state != targetState) {
            System.out.printf("Incomplete:\t%s\t%s\n", im, state);
        }
    }
    return qs;
}

Example 3 with DaemonState

use of com.ms.silverking.cloud.dht.daemon.DaemonState in project SilverKing by Morgan-Stanley.

the class DaemonStateZK method _getQuorumState.

private Map<IPAndPort, DaemonState> _getQuorumState(Set<IPAndPort> members) throws KeeperException {
    ZooKeeperExtended zk;
    Set<String> daemonStatePaths;
    Map<String, Integer> daemonStates;
    Map<IPAndPort, DaemonState> quorumState;
    daemonStatePaths = new HashSet<>(members.size());
    for (IPAndPort member : members) {
        daemonStatePaths.add(getMemberDaemonStatePath(member));
    }
    zk = mc.getZooKeeper();
    daemonStates = zk.getInts(daemonStatePaths, this);
    quorumState = new HashMap<>();
    for (IPAndPort member : members) {
        Integer rawState;
        rawState = daemonStates.get(getMemberDaemonStatePath(member));
        if (rawState != null) {
            DaemonState ds;
            ds = DaemonState.values()[rawState];
            quorumState.put(member, ds);
        }
    }
    return quorumState;
}

Example 4 with DaemonState

use of com.ms.silverking.cloud.dht.daemon.DaemonState in project SilverKing by Morgan-Stanley.

the class DaemonStateZK method getQuorumState.

private Pair<Boolean, Map<IPAndPort, DaemonState>> getQuorumState(Set<IPAndPort> members, DaemonState targetState, int _maxIncompletePerFetch) throws KeeperException {
    Map<IPAndPort, DaemonState> quorumState;
    boolean allRead;
    int incomplete;
    quorumState = new HashMap<>();
    allRead = true;
    incomplete = 0;
    for (IPAndPort member : members) {
        DaemonState state;
        state = getMemberState(member);
        Log.warningf("member %s state %s", member, state);
        quorumState.put(member, state);
        if (state == null || state.ordinal() < targetState.ordinal()) {
            ++incomplete;
            if (incomplete >= _maxIncompletePerFetch && incomplete < members.size()) {
                allRead = false;
                break;
            }
        }
    }
    return new Pair<>(allRead, quorumState);
}

Example 5 with DaemonState

use of com.ms.silverking.cloud.dht.daemon.DaemonState in project SilverKing by Morgan-Stanley.

the class SKAdmin method waitUntilRunning.

// /////////////////////////////////////////
public Pair<Set<IPAndPort>, Boolean> waitUntilRunning(Set<IPAndPort> activeNodes, int timeoutSeconds) {
    DaemonStateZK daemonStateZK;
    Map<IPAndPort, DaemonState> daemonState;
    daemonStateZK = new DaemonStateZK(dhtMC);
    daemonState = daemonStateZK.waitForQuorumState(activeNodes, DaemonState.RUNNING, timeoutSeconds, true);
    Log.warningf("daemonState: %s", daemonState);
    if (daemonState.isEmpty()) {
        return new Pair<>(ImmutableSet.of(), true);
    } else {
        HashSet<IPAndPort> failedDaemons;
        boolean running;
        running = true;
        failedDaemons = new HashSet<>();
        for (IPAndPort activeNode : activeNodes) {
            DaemonState ds;
            ds = daemonState.get(activeNode);
            System.out.printf("Node: %s\tstate: %s\n", activeNode, ds);
            if (ds == null) {
                failedDaemons.add(activeNode);
                running = false;
            } else {
                if (ds != DaemonState.RUNNING) {
                    running = false;
                }
            }
        }
        return new Pair<>(failedDaemons, running);
    }
}