Examples with FragmentTaskMessage org.voltdb.messaging.FragmentTaskMessage used on opensource projects

Example 16 with FragmentTaskMessage

use of org.voltdb.messaging.FragmentTaskMessage in project voltdb by VoltDB.

the class SpScheduler method handleFragmentTaskMessageRepair.

private void handleFragmentTaskMessageRepair(List<Long> needsRepair, FragmentTaskMessage message) {
    // set up duplicate counter. expect exactly the responses corresponding
    // to needsRepair. These may, or may not, include the local site.
    List<Long> expectedHSIds = new ArrayList<Long>(needsRepair);
    DuplicateCounter counter = new DuplicateCounter(// Assume that the MPI's HSID hasn't changed
    message.getCoordinatorHSId(), message.getTxnId(), expectedHSIds, message);
    safeAddToDuplicateCounterMap(new DuplicateCounterKey(message.getTxnId(), message.getSpHandle()), counter);
    // is local repair necessary?
    if (needsRepair.contains(m_mailbox.getHSId())) {
        // Sanity check that we really need repair.
        if (m_outstandingTxns.get(message.getTxnId()) != null) {
            hostLog.warn("SPI repair attempted to repair a fragment which it has already seen. " + "This shouldn't be possible.");
            // Not sure what to do in this event.  Crash for now
            throw new RuntimeException("Attempted to repair with a fragment we've already seen.");
        }
        needsRepair.remove(m_mailbox.getHSId());
        // make a copy because handleIv2 non-repair case does?
        FragmentTaskMessage localWork = new FragmentTaskMessage(message.getInitiatorHSId(), message.getCoordinatorHSId(), message);
        doLocalFragmentOffer(localWork);
    }
    // is remote repair necessary?
    if (!needsRepair.isEmpty()) {
        FragmentTaskMessage replmsg = new FragmentTaskMessage(m_mailbox.getHSId(), m_mailbox.getHSId(), message);
        m_mailbox.send(com.google_voltpatches.common.primitives.Longs.toArray(needsRepair), replmsg);
    }
}

Example 17 with FragmentTaskMessage

use of org.voltdb.messaging.FragmentTaskMessage in project voltdb by VoltDB.

the class SpScheduler method handleFragmentTaskMessage.

// SpSchedulers will see FragmentTaskMessage for:
// - The scatter fragment(s) of a multi-part transaction (normal or sysproc)
// - Borrow tasks to do the local fragment work if this partition is the
//   buddy of the MPI.  Borrow tasks may include input dependency tables for
//   aggregation fragments, or not, if it's a replicated table read.
// For multi-batch MP transactions, we'll need to look up the transaction state
// that gets created when the first batch arrives.
// During command log replay a new SP handle is going to be generated, but it really
// doesn't matter, it isn't going to be used for anything.
void handleFragmentTaskMessage(FragmentTaskMessage message) {
    FragmentTaskMessage msg = message;
    long newSpHandle;
    if (m_isLeader) {
        // Quick hack to make progress...we need to copy the FragmentTaskMessage
        // before we start mucking with its state (SPHANDLE).  We need to revisit
        // all the messaging mess at some point.
        msg = new FragmentTaskMessage(message.getInitiatorHSId(), message.getCoordinatorHSId(), message);
        if (!message.isReadOnly()) {
            TxnEgo ego = advanceTxnEgo();
            newSpHandle = ego.getTxnId();
            if (m_outstandingTxns.get(msg.getTxnId()) == null) {
                updateMaxScheduledTransactionSpHandle(newSpHandle);
            }
        } else {
            newSpHandle = getMaxScheduledTxnSpHandle();
        }
        msg.setSpHandle(newSpHandle);
        logRepair(msg);
        if (msg.getInitiateTask() != null) {
            //set the handle
            msg.getInitiateTask().setSpHandle(newSpHandle);
            //Trigger reserialization so the new handle is used
            msg.setStateForDurability(msg.getInitiateTask(), msg.getInvolvedPartitions());
        }
        /*
             * If there a replicas to send it to, forward it!
             * Unless... it's read only AND not a sysproc. Read only sysprocs may expect to be sent
             * everywhere.
             * In that case don't propagate it to avoid a determinism check and extra messaging overhead
             */
        if (m_sendToHSIds.length > 0 && (!message.isReadOnly() || msg.isSysProcTask())) {
            for (long hsId : m_sendToHSIds) {
                FragmentTaskMessage finalMsg = msg;
                final VoltTrace.TraceEventBatch traceLog = VoltTrace.log(VoltTrace.Category.SPI);
                if (traceLog != null) {
                    traceLog.add(() -> VoltTrace.beginAsync("replicatefragment", MiscUtils.hsIdPairTxnIdToString(m_mailbox.getHSId(), hsId, finalMsg.getSpHandle(), finalMsg.getTxnId()), "txnId", TxnEgo.txnIdToString(finalMsg.getTxnId()), "dest", CoreUtils.hsIdToString(hsId)));
                }
            }
            FragmentTaskMessage replmsg = new FragmentTaskMessage(m_mailbox.getHSId(), m_mailbox.getHSId(), msg);
            m_mailbox.send(m_sendToHSIds, replmsg);
            DuplicateCounter counter;
            /*
                 * Non-determinism should be impossible to happen with MP fragments.
                 * if you see "MP_DETERMINISM_ERROR" as procedure name in the crash logs
                 * something has horribly gone wrong.
                 */
            if (message.getFragmentTaskType() != FragmentTaskMessage.SYS_PROC_PER_SITE) {
                counter = new DuplicateCounter(msg.getCoordinatorHSId(), msg.getTxnId(), m_replicaHSIds, message);
            } else {
                counter = new SysProcDuplicateCounter(msg.getCoordinatorHSId(), msg.getTxnId(), m_replicaHSIds, message);
            }
            safeAddToDuplicateCounterMap(new DuplicateCounterKey(message.getTxnId(), newSpHandle), counter);
        }
    } else {
        newSpHandle = msg.getSpHandle();
        logRepair(msg);
        setMaxSeenTxnId(newSpHandle);
    }
    Iv2Trace.logFragmentTaskMessage(message, m_mailbox.getHSId(), newSpHandle, false);
    doLocalFragmentOffer(msg);
}

Example 18 with FragmentTaskMessage

use of org.voltdb.messaging.FragmentTaskMessage in project voltdb by VoltDB.

the class MpTransactionState method recursableRun.

@Override
public Map<Integer, List<VoltTable>> recursableRun(SiteProcedureConnection siteConnection) {
    final VoltTrace.TraceEventBatch traceLog = VoltTrace.log(VoltTrace.Category.MPSITE);
    // if we're restarting this transaction, and we only have local work, add some dummy
    // remote work so that we can avoid injecting a borrow task into the local buddy site
    // before the CompleteTransactionMessage with the restart flag reaches it.
    // Right now, any read on a replicated table which has no distributed work will
    // generate these null fragments in the restarted transaction.
    boolean usedNullFragment = false;
    if (m_isRestart && m_remoteWork == null) {
        usedNullFragment = true;
        m_remoteWork = new FragmentTaskMessage(m_localWork.getInitiatorHSId(), m_localWork.getCoordinatorHSId(), m_localWork.getTxnId(), m_localWork.getUniqueId(), m_localWork.isReadOnly(), false, false);
        m_remoteWork.setEmptyForRestart(getNextDependencyId());
        if (!m_haveDistributedInitTask && !isForReplay() && !isReadOnly()) {
            m_haveDistributedInitTask = true;
            m_remoteWork.setStateForDurability((Iv2InitiateTaskMessage) getNotice(), m_masterHSIds.keySet());
        }
        // Distribute fragments to remote destinations.
        long[] non_local_hsids = new long[m_useHSIds.size()];
        for (int i = 0; i < m_useHSIds.size(); i++) {
            non_local_hsids[i] = m_useHSIds.get(i);
        }
        // send to all non-local sites
        if (non_local_hsids.length > 0) {
            m_mbox.send(non_local_hsids, m_remoteWork);
        }
    }
    // Do distributed fragments, if any
    if (m_remoteWork != null) {
        // Create some record of expected dependencies for tracking
        m_remoteDeps = createTrackedDependenciesFromTask(m_remoteWork, m_useHSIds);
        // cause ProcedureRunner to do the right thing and cause rollback.
        while (!checkDoneReceivingFragResponses()) {
            FragmentResponseMessage msg = pollForResponses();
            if (traceLog != null) {
                final int batchIdx = m_remoteWork.getCurrentBatchIndex();
                traceLog.add(() -> VoltTrace.endAsync("sendfragment", MiscUtils.hsIdPairTxnIdToString(m_mbox.getHSId(), msg.m_sourceHSId, txnId, batchIdx), "status", Byte.toString(msg.getStatusCode())));
            }
            boolean expectedMsg = handleReceivedFragResponse(msg);
            if (expectedMsg) {
                // Will roll-back and throw if this message has an exception
                checkForException(msg);
            }
        }
    }
    // satisified. Clear this defensively. Procedure runner is sloppy with
    // cleaning up if it decides new work is necessary that is local-only.
    m_remoteWork = null;
    BorrowTaskMessage borrowmsg = new BorrowTaskMessage(m_localWork);
    m_localWork.setCoordinatorTask(true);
    m_localWork.m_sourceHSId = m_mbox.getHSId();
    // don't include the empty dependencies we got back in the borrow fragment.
    if (!usedNullFragment) {
        borrowmsg.addInputDepMap(m_remoteDepTables);
    }
    if (traceLog != null) {
        final int batchIdx = m_localWork.getCurrentBatchIndex();
        traceLog.add(() -> VoltTrace.beginAsync("sendborrow", MiscUtils.hsIdPairTxnIdToString(m_mbox.getHSId(), m_buddyHSId, txnId, batchIdx), "txnId", TxnEgo.txnIdToString(txnId), "dest", CoreUtils.hsIdToString(m_buddyHSId)));
    }
    m_mbox.send(m_buddyHSId, borrowmsg);
    FragmentResponseMessage msg;
    while (true) {
        msg = pollForResponses();
        final FragmentResponseMessage finalMsg = msg;
        if (traceLog != null) {
            final int batchIdx = m_localWork.getCurrentBatchIndex();
            traceLog.add(() -> VoltTrace.endAsync("sendborrow", MiscUtils.hsIdPairTxnIdToString(m_mbox.getHSId(), m_buddyHSId, txnId, batchIdx), "status", Byte.toString(finalMsg.getStatusCode())));
        }
        assert (msg.getTableCount() > 0);
        // If this is a restarted TXN, verify that this is not a stale message from a different Dependency
        if (!m_isRestart || (msg.m_sourceHSId == m_buddyHSId && msg.getTableDependencyIdAtIndex(0) == m_localWork.getOutputDepId(0))) {
            // Will roll-back and throw if this message has an exception
            checkForException(msg);
            break;
        } else {
            // ignore those
            assert m_isRestart;
        }
    }
    m_localWork = null;
    // Build results from the FragmentResponseMessage
    // This is similar to dependency tracking...maybe some
    // sane way to merge it
    Map<Integer, List<VoltTable>> results = new HashMap<Integer, List<VoltTable>>();
    for (int i = 0; i < msg.getTableCount(); i++) {
        int this_depId = msg.getTableDependencyIdAtIndex(i);
        VoltTable this_dep = msg.getTableAtIndex(i);
        List<VoltTable> tables = results.get(this_depId);
        if (tables == null) {
            tables = new ArrayList<VoltTable>();
            results.put(this_depId, tables);
        }
        tables.add(this_dep);
    }
    // Need some sanity check that we got all of the expected output dependencies?
    return results;
}

Example 19 with FragmentTaskMessage

use of org.voltdb.messaging.FragmentTaskMessage in project voltdb by VoltDB.

the class TestTransactionTaskQueue method createFrag.

// Create the first fragment of a MP txn
private FragmentTask createFrag(long localTxnId, long mpTxnId, TransactionTaskQueue queue, boolean forReplay) {
    FragmentTaskMessage msg = mock(FragmentTaskMessage.class);
    when(msg.getTxnId()).thenReturn(mpTxnId);
    when(msg.isForReplay()).thenReturn(forReplay);
    InitiatorMailbox mbox = mock(InitiatorMailbox.class);
    when(mbox.getHSId()).thenReturn(1337l);
    ParticipantTransactionState pft = new ParticipantTransactionState(localTxnId, msg);
    FragmentTask task = new FragmentTask(mbox, pft, queue, msg, null);
    return task;
}

Example 20 with FragmentTaskMessage

use of org.voltdb.messaging.FragmentTaskMessage in project voltdb by VoltDB.

the class TestMpTransactionState method testTruncationHandleForwarding.

@Test
public void testTruncationHandleForwarding() throws IOException {
    long truncPt = 100L;
    Iv2InitiateTaskMessage taskmsg = new Iv2InitiateTaskMessage(0, 0, truncPt, 101L, System.currentTimeMillis(), true, false, new StoredProcedureInvocation(), 0, 0, false);
    assertEquals(truncPt, taskmsg.getTruncationHandle());
    FragmentTaskMessage localFrag = mock(FragmentTaskMessage.class);
    FragmentTaskMessage remoteFrag = mock(FragmentTaskMessage.class);
    when(remoteFrag.getFragmentCount()).thenReturn(1);
    buddyHSId = 0;
    Mailbox mailbox = mock(Mailbox.class);
    MpTransactionState dut = new MpTransactionState(mailbox, taskmsg, allHsids, partMasters, buddyHSId, false);
    // create local work and verify the created localwork has the
    // expected truncation point.
    dut.createLocalFragmentWork(localFrag, false);
    verify(dut.m_localWork).setTruncationHandle(truncPt);
    // same with partcipating work.
    dut.createAllParticipatingFragmentWork(remoteFrag);
    verify(dut.m_remoteWork).setTruncationHandle(truncPt);
}