Examples with StateWaiter com.yahoo.vespa.clustercontroller.core.testutils.StateWaiter used on opensource projects

Example 1 with StateWaiter

use of com.yahoo.vespa.clustercontroller.core.testutils.StateWaiter in project vespa by vespa-engine.

the class StateChangeTest method testSystemStateSentWhenNodesReplied.

@Test
public void testSystemStateSentWhenNodesReplied() throws Exception {
    startingTest("StateChangeTest::testSystemStateSentWhenNodesReplied()");
    final FleetControllerOptions options = new FleetControllerOptions("mycluster", createNodes(10));
    options.minTimeBeforeFirstSystemStateBroadcast = 300 * 60 * 1000;
    setUpSystem(true, options);
    setUpVdsNodes(true, new DummyVdsNodeOptions(), true);
    for (int i = 0; i < nodes.size(); ++i) {
        nodes.get(i).connect();
    }
    // Marking one node as 'initializing' improves testing of state later on.
    nodes.get(3).setNodeState(State.INITIALIZING);
    setUpFleetController(true, options);
    final StateWaiter waiter = new StateWaiter(timer);
    fleetController.addSystemStateListener(waiter);
    waiter.waitForState("version:\\d+ distributor:10 storage:10 .1.s:i .1.i:1.0", timeoutMS);
    waitForCompleteCycle();
    final int version = waiter.getCurrentSystemState().getVersion();
    fleetController.waitForNodesHavingSystemStateVersionEqualToOrAbove(version, 20, timeoutMS);
    // The last two versions of the cluster state should be seen (all nodes up,
    // zero out timestate)
    new StateMessageChecker(nodes) {

        @Override
        int expectedMessageCount(final DummyVdsNode node) {
            return 2;
        }
    };
}

Example 2 with StateWaiter

use of com.yahoo.vespa.clustercontroller.core.testutils.StateWaiter in project vespa by vespa-engine.

the class StateChangeTest method testNoSystemStateBeforeInitialTimePeriod.

@Test
public void testNoSystemStateBeforeInitialTimePeriod() throws Exception {
    startingTest("StateChangeTest::testNoSystemStateBeforeInitialTimePeriod()");
    FleetControllerOptions options = new FleetControllerOptions("mycluster", createNodes(10));
    options.minTimeBeforeFirstSystemStateBroadcast = 3 * 60 * 1000;
    setUpSystem(true, options);
    setUpVdsNodes(true, new DummyVdsNodeOptions(), true);
    // Leave one node down to avoid sending cluster state due to having seen all node states.
    for (int i = 0; i < nodes.size(); ++i) {
        if (i != 3) {
            nodes.get(i).connect();
        }
    }
    setUpFleetController(true, options);
    StateWaiter waiter = new StateWaiter(timer);
    fleetController.addSystemStateListener(waiter);
    // Ensure all nodes have been seen by fleetcontroller and that it has had enough time to possibly have sent a cluster state
    waiter.waitForState("version:\\d+ distributor:10 (\\.\\d+\\.t:\\d+ )*storage:10 (\\.\\d+\\.t:\\d+ )*.1.s:d( \\.\\d+\\.t:\\d+)*", timeoutMS);
    waitForCompleteCycle();
    new StateMessageChecker(nodes) {

        @Override
        int expectedMessageCount(final DummyVdsNode node) {
            return 0;
        }
    };
    // Pass time and see that the nodes get state
    timer.advanceTime(3 * 60 * 1000);
    waiter.waitForState("version:\\d+ distributor:10 storage:10 .1.s:d", timeoutMS);
    int version = waiter.getCurrentSystemState().getVersion();
    fleetController.waitForNodesHavingSystemStateVersionEqualToOrAbove(version, 19, timeoutMS);
    new StateMessageChecker(nodes) {

        @Override
        int expectedMessageCount(final DummyVdsNode node) {
            return node.getNode().equals(new Node(NodeType.STORAGE, 1)) ? 0 : 2;
        }
    };
    assertEquals(version, waiter.getCurrentSystemState().getVersion());
}

Example 3 with StateWaiter

use of com.yahoo.vespa.clustercontroller.core.testutils.StateWaiter in project vespa by vespa-engine.

the class StateChangeTest method testSetAllTimestampsAfterDowntime.

@Test
public void testSetAllTimestampsAfterDowntime() throws Exception {
    startingTest("StateChangeTest::testSetAllTimestampsAfterDowntime");
    FleetControllerOptions options = new FleetControllerOptions("mycluster", createNodes(10));
    setUpFleetController(true, options);
    setUpVdsNodes(true, new DummyVdsNodeOptions());
    waitForStableSystem();
    StateWaiter waiter = new StateWaiter(timer);
    fleetController.addSystemStateListener(waiter);
    // Simulate netsplit. Take node down without node booting
    assertEquals(true, nodes.get(0).isDistributor());
    nodes.get(0).disconnectImmediately();
    waiter.waitForState("version:\\d+ distributor:10 .0.s:d storage:10", timeoutMS);
    // Add node back.
    nodes.get(0).connect();
    waitForStableSystem();
    // At this time, node taken down should have cluster states with all starting timestamps set. Others node should not.
    for (DummyVdsNode node : nodes) {
        node.waitForSystemStateVersion(waiter.getCurrentSystemState().getVersion(), timeoutMS);
        List<ClusterState> states = node.getSystemStatesReceived();
        ClusterState lastState = states.get(0);
        StringBuilder stateHistory = new StringBuilder();
        for (ClusterState state : states) {
            stateHistory.append(state.toString()).append("\n");
        }
        if (node.getNode().equals(new Node(NodeType.DISTRIBUTOR, 0))) {
            for (ConfiguredNode i : options.nodes) {
                Node nodeId = new Node(NodeType.STORAGE, i.index());
                long ts = lastState.getNodeState(nodeId).getStartTimestamp();
                assertTrue(nodeId + "\n" + stateHistory + "\nWas " + ts + " should be " + fleetController.getCluster().getNodeInfo(nodeId).getStartTimestamp(), ts > 0);
            }
        } else {
            for (ConfiguredNode i : options.nodes) {
                Node nodeId = new Node(NodeType.STORAGE, i.index());
                assertTrue(nodeId.toString(), lastState.getNodeState(nodeId).getStartTimestamp() == 0);
            }
        }
        for (ConfiguredNode i : options.nodes) {
            Node nodeId = new Node(NodeType.DISTRIBUTOR, i.index());
            assertTrue(nodeId.toString(), lastState.getNodeState(nodeId).getStartTimestamp() == 0);
        }
    }
}

Example 4 with StateWaiter

use of com.yahoo.vespa.clustercontroller.core.testutils.StateWaiter in project vespa by vespa-engine.

the class StateChangeTest method testDontTagFailingSetSystemStateOk.

@Test
public void testDontTagFailingSetSystemStateOk() throws Exception {
    startingTest("StateChangeTest::testDontTagFailingSetSystemStateOk()");
    FleetControllerOptions options = new FleetControllerOptions("mycluster", createNodes(10));
    setUpFleetController(true, options);
    setUpVdsNodes(true, new DummyVdsNodeOptions());
    waitForStableSystem();
    StateWaiter waiter = new StateWaiter(timer);
    fleetController.addSystemStateListener(waiter);
    nodes.get(1).failSetSystemState(true);
    int versionBeforeChange = nodes.get(1).getSystemStatesReceived().get(0).getVersion();
    // cause a new state
    nodes.get(2).disconnect();
    waiter.waitForState("version:\\d+ distributor:10 .1.s:d storage:10", timeoutMS);
    int versionAfterChange = waiter.getCurrentSystemState().getVersion();
    assertTrue(versionAfterChange > versionBeforeChange);
    fleetController.waitForNodesHavingSystemStateVersionEqualToOrAbove(versionAfterChange, 18, timeoutMS);
    // Assert that the failed node has not acknowledged the latest version.
    // (The version may still be larger than versionBeforeChange if the fleet controller sends a
    // "stable system" update without timestamps in the meantime
    assertTrue(fleetController.getCluster().getNodeInfo(nodes.get(1).getNode()).getSystemStateVersionAcknowledged() < versionAfterChange);
    // Ensure non-concurrent access to getNewestSystemStateVersionSent
    synchronized (timer) {
        int sentVersion = fleetController.getCluster().getNodeInfo(nodes.get(1).getNode()).getNewestSystemStateVersionSent();
        assertTrue(sentVersion == -1 || sentVersion == versionAfterChange);
    }
}