Examples with VoltMessage org.voltcore.messaging.VoltMessage used on opensource projects

Example 26 with VoltMessage

use of org.voltcore.messaging.VoltMessage in project voltdb by VoltDB.

the class TestMeshArbiter method testPingsOnLongReceives.

@Test
public void testPingsOnLongReceives() throws Exception {
    Maker<SiteFailureMessage> siteOneSfm = a(SiteFailureMessage, with(sfmSurvivors, Longs.asList(0, 2, 3)), with(sfmFailures, sfmFailed(1)), with(sfmSafeTxns, sfmSafe(0, 10, 1, 11, 2, 22, 3, 33)));
    when(aide.getNewestSafeTransactionForInitiator(1L)).thenReturn(11L);
    when(mbox.recvBlocking(any(Subject[].class), eq(5L))).thenReturn((VoltMessage) null).thenReturn((VoltMessage) null).thenReturn((VoltMessage) null).thenReturn(make(siteOneSfm.but(with(sfmSource, 0L)))).thenReturn(make(siteOneSfm.but(with(sfmSource, 2L)))).thenReturn(make(siteOneSfm.but(with(sfmSource, 3L))));
    Map<Long, Long> decision = arbiter.reconfigureOnFault(hsids, new FaultMessage(0, 1));
    verify(mbox, times(2)).send(any(long[].class), argThat(siteFailureIs(sfmFailed(1), sfmSurvived(0, 2, 3))));
    verify(aide, atLeast(2)).sendHeartbeats(destinationCaptor.capture());
    assertEquals(destinationCaptor.getValue(), sfmSurvived(0, 2, 3));
    assertEquals(decision, ImmutableMap.<Long, Long>of(1L, 11L));
}

Example 27 with VoltMessage

use of org.voltcore.messaging.VoltMessage in project voltdb by VoltDB.

the class TestMpPromoteAlgo method testFuzz.

@Test
public void testFuzz() throws Exception {
    System.out.println("Running testFuzz");
    InitiatorMailbox mbox = mock(InitiatorMailbox.class);
    Random rand = new Random(System.currentTimeMillis());
    // Generate a random message stream to several "replicas", interrupted
    // at random points to all but one.  Validate that promotion repair
    // results in identical, correct, repair streams to all replicas.
    TxnEgo sphandle = TxnEgo.makeZero(0);
    UniqueIdGenerator uig = new UniqueIdGenerator(0, 0);
    sphandle = sphandle.makeNext();
    RandomMsgGenerator msgGen = new RandomMsgGenerator();
    boolean[] stops = new boolean[3];
    RepairLog[] logs = new RepairLog[3];
    for (int i = 0; i < 3; i++) {
        logs[i] = new RepairLog();
        stops[i] = false;
    }
    for (int i = 0; i < 4000; i++) {
        // get next message, update the sphandle according to SpScheduler rules,
        // but only submit messages that would have been forwarded by the master
        // to the repair log.
        TransactionInfoBaseMessage msg = msgGen.generateRandomMessageInStream();
        msg.setSpHandle(sphandle.getTxnId());
        sphandle = sphandle.makeNext();
        if (!msg.isReadOnly() || msg instanceof CompleteTransactionMessage) {
            if (!stops[0]) {
                logs[0].deliver(msg);
            }
            if (!stops[1]) {
                logs[1].deliver(msg);
            }
            logs[2].deliver(msg);
            // be fed any transactions
            for (int j = 0; j < 2; j++) {
                // Hacky way to get spaced failures
                if (rand.nextDouble() < (.01 / ((j + 1) * 5))) {
                    stops[j] = true;
                }
            }
        }
    }
    List<Long> survivors = new ArrayList<Long>();
    survivors.add(0l);
    survivors.add(1l);
    survivors.add(2l);
    MpPromoteAlgo dut = new MpPromoteAlgo(survivors, mbox, "bleh ");
    Future<RepairResult> result = dut.start();
    for (int i = 0; i < 3; i++) {
        List<Iv2RepairLogResponseMessage> stuff = logs[i].contents(dut.getRequestId(), true);
        System.out.println("Repair log size from: " + i + ": " + stuff.size());
        for (Iv2RepairLogResponseMessage msg : stuff) {
            msg.m_sourceHSId = (long) i;
            dut.deliver(msg);
        }
    }
    result.get();
    assertFalse(result.isCancelled());
    assertTrue(result.isDone());
    // Unfortunately, it's painful to try to stub things to make repairSurvivors() work, so we'll
    // go and inspect the guts of SpPromoteAlgo instead.  This iteration is largely a copy of the inner loop
    // of repairSurvivors()
    List<TransactionInfoBaseMessage> finalStream = new ArrayList<TransactionInfoBaseMessage>();
    for (Iv2RepairLogResponseMessage li : dut.m_repairLogUnion) {
        VoltMessage msg = dut.createRepairMessage(li);
        finalStream.add((TransactionInfoBaseMessage) msg);
    }
    // Check the sanity of the repair stream generated by the MPI.
    long lastTxnId = Long.MIN_VALUE;
    boolean seenFrag = false;
    for (TransactionInfoBaseMessage msg : finalStream) {
        if (lastTxnId == Long.MIN_VALUE) {
            lastTxnId = msg.getTxnId();
        } else {
            assertTrue(msg.getTxnId() > lastTxnId);
            lastTxnId = msg.getTxnId();
        }
        if (msg instanceof FragmentTaskMessage) {
            assertFalse(seenFrag);
            seenFrag = true;
        }
    }
}

Example 28 with VoltMessage

use of org.voltcore.messaging.VoltMessage in project voltdb by VoltDB.

the class ExportGeneration method createAndRegisterAckMailboxes.

private void createAndRegisterAckMailboxes(final Set<Integer> localPartitions, HostMessenger messenger) {
    m_mailboxesZKPath = VoltZK.exportGenerations + "/" + m_timestamp + "/" + "mailboxes";
    m_mbox = new LocalMailbox(messenger) {

        @Override
        public void deliver(VoltMessage message) {
            if (message instanceof BinaryPayloadMessage) {
                BinaryPayloadMessage bpm = (BinaryPayloadMessage) message;
                ByteBuffer buf = ByteBuffer.wrap(bpm.m_payload);
                final int partition = buf.getInt();
                final int length = buf.getInt();
                byte[] stringBytes = new byte[length];
                buf.get(stringBytes);
                String signature = new String(stringBytes, Constants.UTF8ENCODING);
                final long ackUSO = buf.getLong();
                final boolean runEveryWhere = (buf.getShort() == (short) 1);
                final Map<String, ExportDataSource> partitionSources = m_dataSourcesByPartition.get(partition);
                if (partitionSources == null) {
                    exportLog.error("Received an export ack for partition " + partition + " which does not exist on this node, partitions = " + m_dataSourcesByPartition);
                    return;
                }
                final ExportDataSource eds = partitionSources.get(signature);
                if (eds == null) {
                    exportLog.warn("Received an export ack for partition " + partition + " source signature " + signature + " which does not exist on this node, sources = " + partitionSources);
                    return;
                }
                try {
                    eds.ack(ackUSO, runEveryWhere);
                } catch (RejectedExecutionException ignoreIt) {
                // ignore it: as it is already shutdown
                }
            } else {
                exportLog.error("Receive unexpected message " + message + " in export subsystem");
            }
        }
    };
    messenger.createMailbox(null, m_mbox);
    for (Integer partition : localPartitions) {
        final String partitionDN = m_mailboxesZKPath + "/" + partition;
        ZKUtil.asyncMkdirs(messenger.getZK(), partitionDN);
        ZKUtil.StringCallback cb = new ZKUtil.StringCallback();
        messenger.getZK().create(partitionDN + "/" + m_mbox.getHSId(), null, Ids.OPEN_ACL_UNSAFE, CreateMode.EPHEMERAL, cb, null);
    }
    ListenableFuture<?> fut = m_childUpdatingThread.submit(new Runnable() {

        @Override
        public void run() {
            List<Pair<Integer, ZKUtil.ChildrenCallback>> callbacks = new ArrayList<Pair<Integer, ZKUtil.ChildrenCallback>>();
            for (Integer partition : localPartitions) {
                ZKUtil.ChildrenCallback callback = new ZKUtil.ChildrenCallback();
                messenger.getZK().getChildren(m_mailboxesZKPath + "/" + partition, constructMailboxChildWatcher(messenger), callback, null);
                callbacks.add(Pair.of(partition, callback));
            }
            for (Pair<Integer, ZKUtil.ChildrenCallback> p : callbacks) {
                final Integer partition = p.getFirst();
                List<String> children = null;
                try {
                    children = p.getSecond().getChildren();
                } catch (InterruptedException e) {
                    Throwables.propagate(e);
                } catch (KeeperException e) {
                    Throwables.propagate(e);
                }
                ImmutableList.Builder<Long> mailboxes = ImmutableList.builder();
                for (String child : children) {
                    if (child.equals(Long.toString(m_mbox.getHSId())))
                        continue;
                    mailboxes.add(Long.valueOf(child));
                }
                ImmutableList<Long> mailboxHsids = mailboxes.build();
                for (ExportDataSource eds : m_dataSourcesByPartition.get(partition).values()) {
                    eds.updateAckMailboxes(Pair.of(m_mbox, mailboxHsids));
                }
            }
        }
    });
    try {
        fut.get();
    } catch (Throwable t) {
        Throwables.propagate(t);
    }
}

Example 29 with VoltMessage

use of org.voltcore.messaging.VoltMessage in project voltdb by VoltDB.

the class AgreementSite method recoveryRunLoop.

public void recoveryRunLoop() throws Exception {
    long lastHeartbeatTime = System.currentTimeMillis();
    while (m_recovering && m_shouldContinue) {
        if (m_recoveryStage == RecoveryStage.WAITING_FOR_SAFETY) {
            Long safeTxnId = m_txnQueue.safeToRecover();
            if (safeTxnId != null) {
                m_recoveryStage = RecoveryStage.SENT_PROPOSAL;
                m_recoverBeforeTxn = safeTxnId;
                long sourceHSId = 0;
                for (Long hsId : m_hsIds) {
                    if (hsId != m_hsId) {
                        sourceHSId = hsId;
                        break;
                    }
                }
                RecoveryMessage recoveryMessage = new RecoveryMessage(m_hsId, safeTxnId, -1);
                m_mailbox.send(sourceHSId, recoveryMessage);
            }
        }
        VoltMessage message = m_mailbox.recvBlocking(5);
        if (message != null) {
            processMessage(message);
        }
        final long now = System.currentTimeMillis();
        if (now - lastHeartbeatTime > 5) {
            lastHeartbeatTime = now;
            sendHeartbeats();
        }
        if (m_recoverBeforeTxn == null) {
            continue;
        }
        if (m_txnQueue.peek() != null && m_txnQueue.peek().txnId < m_recoverBeforeTxn.longValue()) {
            m_transactionsById.remove(m_txnQueue.poll().txnId);
        } else if (m_recoveryStage == RecoveryStage.RECEIVED_SNAPSHOT) {
            processZKSnapshot();
            return;
        }
    }
}

Example 30 with VoltMessage

use of org.voltcore.messaging.VoltMessage in project voltdb by VoltDB.

the class AgreementSite method run.

@Override
public void run() {
    try {
        if (m_recovering) {
            recoveryRunLoop();
        }
        long lastHeartbeatTime = System.currentTimeMillis();
        while (m_shouldContinue) {
            VoltMessage message = m_mailbox.recvBlocking(5);
            if (message != null) {
                processMessage(message);
            }
            final long now = System.currentTimeMillis();
            if (now - lastHeartbeatTime > 5) {
                lastHeartbeatTime = now;
                sendHeartbeats();
            }
            if (m_recovering) {
                continue;
            }
            OrderableTransaction ot = m_txnQueue.poll();
            if (ot != null) {
                if (m_recoverBeforeTxn != null) {
                    assert (m_recoveryStage == RecoveryStage.RECOVERED);
                    assert (m_recovering == false);
                    assert (m_siteRequestingRecovery != null);
                    if (ot.txnId >= m_recoverBeforeTxn) {
                        shipZKDatabaseSnapshot(m_siteRequestingRecovery, ot.txnId);
                    }
                }
                if (ot.txnId < m_minTxnIdAfterRecovery) {
                    String errMsg = "Transaction queue released a transaction from before this " + " node was recovered was complete";
                    org.voltdb.VoltDB.crashLocalVoltDB(errMsg, false, null);
                }
                m_transactionsById.remove(ot.txnId);
                if (ot instanceof AgreementRejoinTransactionState) {
                    AgreementRejoinTransactionState txnState = (AgreementRejoinTransactionState) ot;
                    try {
                        processJoin(txnState.m_rejoiningSite);
                    } finally {
                        if (txnState.m_onCompletion != null) {
                            txnState.m_onCompletion.countDown();
                        }
                    }
                } else if (ot instanceof AgreementTransactionState) {
                    AgreementTransactionState txnState = (AgreementTransactionState) ot;
                    //Owner is what associates the session with a specific initiator
                    //only used for createSession
                    txnState.m_request.setOwner(txnState.initiatorHSId);
                    /*
                         * We may pull reads out of the priority queue outside the global
                         * order. This means the txnid might be wrong so just sub
                         * the last used txnid from a write that is guaranteed to have been globally
                         * ordered properly
                         *
                         * It doesn't matter for the most part, but the ZK code we give the ID to expects to
                         * it to always increase and if we pull reads in early that will not always be true.
                         */
                    long txnIdToUse = txnState.txnId;
                    switch(txnState.m_request.type) {
                        case OpCode.exists:
                        case OpCode.getChildren:
                        case OpCode.getChildren2:
                        case OpCode.getData:
                            //Don't use the txnid generated for the read since
                            //it may not be globally ordered with writes
                            txnIdToUse = m_lastUsedTxnId;
                            break;
                        default:
                            //This is a write, stash away the txnid for use
                            //for future reads
                            m_lastUsedTxnId = txnState.txnId;
                            break;
                    }
                    m_server.prepRequest(txnState.m_request, txnIdToUse);
                }
            } else if (m_recoverBeforeTxn != null) {
                assert (m_recoveryStage == RecoveryStage.RECOVERED);
                assert (m_recovering == false);
                assert (m_siteRequestingRecovery != null);
                Long foo = m_txnQueue.safeToRecover();
                if (foo != null && foo.longValue() >= m_recoverBeforeTxn.longValue()) {
                    shipZKDatabaseSnapshot(m_siteRequestingRecovery, foo);
                }
            }
        }
    } catch (Throwable e) {
        org.voltdb.VoltDB.crashLocalVoltDB("Error in agreement site", true, e);
    } finally {
        try {
            shutdownInternal();
        } catch (Exception e) {
            m_agreementLog.warn("Exception during agreement internal shutdown.", e);
        } finally {
            m_shutdownComplete.countDown();
        }
    }
}