Examples with FaultMessage org.voltcore.messaging.FaultMessage used on opensource projects

Example 1 with FaultMessage

use of org.voltcore.messaging.FaultMessage in project voltdb by VoltDB.

the class MeshArbiter method notifyOnKill.

/**
     * Notify all survivors when you are closing links to nodes
     * @param decision map where the keys contain the kill sites
     *   and its values are their last known safe transaction ids
     * @return true if successfully confirmed that all survivors
     * agree on the decision, false otherwise.
     */
protected boolean notifyOnKill(Set<Long> hsIds, Map<Long, Long> decision) {
    SiteFailureMessage.Builder sfmb = SiteFailureMessage.builder().decisions(decision.keySet()).failures(decision.keySet());
    Set<Long> dests = Sets.filter(m_seeker.getSurvivors(), not(equalTo(m_hsId)));
    if (dests.isEmpty())
        return true;
    sfmb.survivors(Sets.difference(m_seeker.getSurvivors(), decision.keySet()));
    sfmb.safeTxnIds(getSafeTxnIdsForSites(hsIds));
    SiteFailureMessage sfm = sfmb.build();
    m_mailbox.send(Longs.toArray(dests), sfm);
    m_recoveryLog.info("Agreement, Sending [" + CoreUtils.hsIdCollectionToString(dests) + "]  " + sfm);
    // that we've entered a loop, exit here.
    if (m_localHistoricDecisions.size() >= 100) {
        // Too many decisions have been made without converging
        RateLimitedLogger.tryLogForMessage(System.currentTimeMillis(), 10, TimeUnit.SECONDS, m_recoveryLog, Level.WARN, "Agreement, %d local decisions have been made without converging", m_localHistoricDecisions.size());
    }
    for (SiteFailureMessage lhd : m_localHistoricDecisions) {
        if (lhd.m_survivors.equals(sfm.m_survivors)) {
            m_recoveryLog.info("Agreement, detected decision loop. Exiting");
            return true;
        }
    }
    m_localHistoricDecisions.add(sfm);
    // Wait for all survivors in the local decision to send their decisions over.
    // If one of the host's decision conflicts with ours, remove that host's link
    // and repeat the decision process.
    final Set<Long> expectedSurvivors = Sets.filter(sfm.m_survivors, not(equalTo(m_hsId)));
    m_recoveryLog.info("Agreement, Waiting for agreement on decision from survivors " + CoreUtils.hsIdCollectionToString(expectedSurvivors));
    final Iterator<SiteFailureMessage> iter = m_decidedSurvivors.values().iterator();
    while (iter.hasNext()) {
        final SiteFailureMessage remoteDecision = iter.next();
        if (expectedSurvivors.contains(remoteDecision.m_sourceHSId)) {
            if (remoteDecision.m_decision.contains(m_hsId)) {
                iter.remove();
                m_recoveryLog.info("Agreement, Received inconsistent decision from " + CoreUtils.hsIdToString(remoteDecision.m_sourceHSId) + ", " + remoteDecision);
                final FaultMessage localFault = new FaultMessage(m_hsId, remoteDecision.m_sourceHSId);
                localFault.m_sourceHSId = m_hsId;
                m_mailbox.deliverFront(localFault);
                return false;
            }
        }
    }
    long start = System.currentTimeMillis();
    boolean allDecisionsMatch = true;
    do {
        final VoltMessage msg = m_mailbox.recvBlocking(receiveSubjects, 5);
        if (msg == null) {
            // Send a heartbeat to keep the dead host timeout active.
            m_meshAide.sendHeartbeats(m_seeker.getSurvivors());
            final long duration = System.currentTimeMillis() - start;
            if (duration > 20000) {
                m_recoveryLog.error("Agreement, Still waiting for decisions from " + CoreUtils.hsIdCollectionToString(Sets.difference(expectedSurvivors, m_decidedSurvivors.keySet())) + " after " + TimeUnit.MILLISECONDS.toSeconds(duration) + " seconds");
                start = System.currentTimeMillis();
            }
            continue;
        }
        if (m_hsId != msg.m_sourceHSId && !expectedSurvivors.contains(msg.m_sourceHSId)) {
            // Ignore messages from failed sites
            continue;
        }
        if (msg.getSubject() == Subject.SITE_FAILURE_UPDATE.getId()) {
            final SiteFailureMessage fm = (SiteFailureMessage) msg;
            if (!fm.m_decision.isEmpty()) {
                if (expectedSurvivors.contains(fm.m_sourceHSId)) {
                    if (fm.m_decision.contains(m_hsId)) {
                        m_decidedSurvivors.remove(fm.m_sourceHSId);
                        // The remote host has decided that we are gone, remove the remote host
                        final FaultMessage localFault = new FaultMessage(m_hsId, fm.m_sourceHSId);
                        localFault.m_sourceHSId = m_hsId;
                        m_mailbox.deliverFront(localFault);
                        return false;
                    } else {
                        m_decidedSurvivors.put(fm.m_sourceHSId, fm);
                    }
                }
            } else {
                m_mailbox.deliverFront(fm);
                return false;
            }
        } else if (msg.getSubject() == Subject.FAILURE.getId()) {
            final FaultMessage fm = (FaultMessage) msg;
            if (!fm.decided) {
                // In case of concurrent fault, handle it
                m_mailbox.deliverFront(msg);
                return false;
            } else if (mayIgnore(hsIds, fm) == Discard.DoNot) {
                m_mailbox.deliverFront(msg);
                return false;
            }
        }
        for (SiteFailureMessage remoteDecision : m_decidedSurvivors.values()) {
            if (!sfm.m_survivors.equals(remoteDecision.m_survivors)) {
                allDecisionsMatch = false;
            }
        }
    } while (!m_decidedSurvivors.keySet().containsAll(expectedSurvivors) && allDecisionsMatch);
    return true;
}

Example 2 with FaultMessage

use of org.voltcore.messaging.FaultMessage in project voltdb by VoltDB.

the class MeshArbiter method discoverGlobalFaultData_rcv.

/**
     * Collect the failure site update messages from all sites This site sent
     * its own mailbox the above broadcast the maximum is local to this site.
     * This also ensures at least one response.
     *
     * Concurrent failures can be detected by additional reports from the FaultDistributor
     * or a mismatch in the set of failed hosts reported in a message from another site
     */
private boolean discoverGlobalFaultData_rcv(Set<Long> hsIds) {
    long blockedOnReceiveStart = System.currentTimeMillis();
    long lastReportTime = 0;
    boolean haveEnough = false;
    int[] forwardStallCount = new int[] { FORWARD_STALL_COUNT };
    do {
        VoltMessage m = m_mailbox.recvBlocking(receiveSubjects, 5);
        /*
             * If fault resolution takes longer then 10 seconds start logging
             */
        final long now = System.currentTimeMillis();
        if (now - blockedOnReceiveStart > 10000) {
            if (now - lastReportTime > 60000) {
                lastReportTime = System.currentTimeMillis();
                haveNecessaryFaultInfo(m_seeker.getSurvivors(), true);
            }
        }
        if (m == null) {
            // Send a heartbeat to keep the dead host timeout active.  Needed because IV2 doesn't
            // generate its own heartbeats to keep this running.
            m_meshAide.sendHeartbeats(m_seeker.getSurvivors());
        } else if (m.getSubject() == Subject.SITE_FAILURE_UPDATE.getId()) {
            SiteFailureMessage sfm = (SiteFailureMessage) m;
            if (!m_seeker.getSurvivors().contains(m.m_sourceHSId) || m_failedSites.contains(m.m_sourceHSId) || m_failedSites.containsAll(sfm.getFailedSites()))
                continue;
            if (!sfm.m_decision.isEmpty()) {
                m_decidedSurvivors.put(sfm.m_sourceHSId, sfm);
            }
            updateFailedSitesLedger(hsIds, sfm);
            m_seeker.add(sfm);
            addForwardCandidate(new SiteFailureForwardMessage(sfm));
            m_recoveryLog.info("Agreement, Received " + sfm);
        } else if (m.getSubject() == Subject.SITE_FAILURE_FORWARD.getId()) {
            SiteFailureForwardMessage fsfm = (SiteFailureForwardMessage) m;
            addForwardCandidate(fsfm);
            if (!hsIds.contains(fsfm.m_sourceHSId) || m_seeker.getSurvivors().contains(fsfm.m_reportingHSId) || m_failedSites.contains(fsfm.m_reportingHSId) || m_failedSites.containsAll(fsfm.getFailedSites()))
                continue;
            m_seeker.add(fsfm);
            m_recoveryLog.info("Agreement, Received forward " + fsfm);
            forwardStallCount[0] = FORWARD_STALL_COUNT;
        } else if (m.getSubject() == Subject.FAILURE.getId()) {
            /*
                 * If the fault distributor reports a new fault, ignore it if it is known , otherwise
                 * re-deliver the message to ourself and then abort so that the process can restart.
                 */
            FaultMessage fm = (FaultMessage) m;
            Discard ignoreIt = mayIgnore(hsIds, fm);
            if (Discard.DoNot == ignoreIt) {
                m_mailbox.deliverFront(m);
                m_recoveryLog.info("Agreement, Detected a concurrent failure from FaultDistributor, new failed site " + CoreUtils.hsIdToString(fm.failedSite));
                return false;
            } else {
                if (m_recoveryLog.isDebugEnabled()) {
                    ignoreIt.log(fm);
                }
            }
        }
        haveEnough = haveEnough || haveNecessaryFaultInfo(m_seeker.getSurvivors(), false);
        if (haveEnough) {
            Iterator<Map.Entry<Long, SiteFailureForwardMessage>> itr = m_forwardCandidates.entrySet().iterator();
            while (itr.hasNext()) {
                Map.Entry<Long, SiteFailureForwardMessage> e = itr.next();
                Set<Long> unseenBy = m_seeker.forWhomSiteIsDead(e.getKey());
                if (unseenBy.size() > 0) {
                    m_mailbox.send(Longs.toArray(unseenBy), e.getValue());
                    m_recoveryLog.info("Agreement, fowarding to " + CoreUtils.hsIdCollectionToString(unseenBy) + " " + e.getValue());
                }
                itr.remove();
            }
        }
    } while (!haveEnough || m_seeker.needForward(forwardStallCount));
    return true;
}

Example 3 with FaultMessage

use of org.voltcore.messaging.FaultMessage in project voltdb by VoltDB.

the class AgreementSite method reportFault.

public void reportFault(long faultingSite) {
    FaultMessage fm = new FaultMessage(m_hsId, faultingSite);
    fm.m_sourceHSId = m_hsId;
    m_mailbox.deliver(fm);
}

Example 4 with FaultMessage

use of org.voltcore.messaging.FaultMessage in project voltdb by VoltDB.

the class MiniSite method processMessage.

private void processMessage(VoltMessage msg) {
    if (!m_currentHSIds.contains(msg.m_sourceHSId)) {
        if (m_siteLog.isDebugEnabled()) {
            m_siteLog.debug("Dropping message " + msg + " because it is not from a known up site");
        }
        return;
    }
    // need heartbeat something in here?
    if (msg instanceof FaultMessage) {
        FaultMessage fm = (FaultMessage) msg;
        discoverGlobalFaultData(fm);
    }
}

Example 5 with FaultMessage

use of org.voltcore.messaging.FaultMessage in project voltdb by VoltDB.

the class TestMeshArbiter method testMixOfWitnessedAndNon.

@Test
public void testMixOfWitnessedAndNon() throws Exception {
    Maker<SiteFailureMessage> um = a(SiteFailureMessage, with(sfmSurvivors, Longs.asList(0, 2, 3)), with(sfmFailures, sfmFailed(1)), with(sfmSafeTxns, sfmSafe(0, 10, 1, 11, 2, 22, 3, 33)));
    when(aide.getNewestSafeTransactionForInitiator(1L)).thenReturn(11L);
    when(mbox.recvBlocking(any(Subject[].class), eq(5L))).thenReturn(make(um.but(with(sfmSource, 2L)))).thenReturn(new FaultMessage(0, 1));
    Map<Long, Long> decision = arbiter.reconfigureOnFault(hsids, new FaultMessage(2, 1, ImmutableSet.of(0L, 2L, 3L)));
    verify(mbox, times(1)).deliverFront(any(VoltMessage.class));
    verify(mbox, times(1)).send(any(long[].class), any(VoltMessage.class));
    verify(mbox).send(any(long[].class), argThat(siteFailureIs(sfmFailed(1), sfmSurvived(0, 1, 2, 3))));
    assertEquals(decision, ImmutableMap.<Long, Long>of());
    reset(mbox);
    when(mbox.recvBlocking(any(Subject[].class), eq(5L))).thenReturn(make(um.but(with(sfmSource, 0L)))).thenReturn(make(um.but(with(sfmSource, 3L))));
    decision = arbiter.reconfigureOnFault(hsids, new FaultMessage(0, 1));
    verify(mbox, never()).deliverFront(any(VoltMessage.class));
    verify(mbox, times(2)).send(any(long[].class), argThat(siteFailureIs(sfmFailed(1), sfmSurvived(0, 2, 3))));
    assertEquals(decision, ImmutableMap.<Long, Long>of(1L, 11L));
}