Examples with JSONObject org.json_voltpatches.JSONObject used on opensource projects

Example 91 with JSONObject

use of org.json_voltpatches.JSONObject in project voltdb by VoltDB.

the class SnapshotDaemon method initiateNextSnapshot.

private void initiateNextSnapshot(long now) {
    setState(State.SNAPSHOTTING);
    m_lastSysprocInvocation = now;
    final Date nowDate = new Date(now);
    final String dateString = m_dateFormat.format(nowDate);
    final String nonce = m_prefix + dateString;
    JSONObject jsObj = new JSONObject();
    try {
        jsObj.put(SnapshotUtil.JSON_PATH, m_path);
        jsObj.put(SnapshotUtil.JSON_PATH_TYPE, SnapshotPathType.SNAP_AUTO.toString());
        jsObj.put(SnapshotUtil.JSON_NONCE, nonce);
        jsObj.put("perPartitionTxnIds", retrievePerPartitionTransactionIds());
        m_snapshots.offer(new Snapshot(m_path, SnapshotPathType.SNAP_AUTO, nonce, now));
        long handle = m_nextCallbackHandle++;
        m_procedureCallbacks.put(handle, new ProcedureCallback() {

            @Override
            public void clientCallback(final ClientResponse clientResponse) throws Exception {
                m_lastInitiationTs = null;
                processClientResponsePrivate(clientResponse);
            }
        });
        SNAP_LOG.info("Requesting auto snapshot to path " + m_path + " nonce " + nonce);
        initiateSnapshotSave(handle, new Object[] { jsObj.toString(4) }, false);
    } catch (JSONException e) {
        /*
             * Should never happen, so fail fast
             */
        VoltDB.crashLocalVoltDB("", false, e);
    }
}

Example 92 with JSONObject

use of org.json_voltpatches.JSONObject in project voltdb by VoltDB.

the class SnapshotDaemon method processSnapshotTruncationRequestCreated.

/*
     * A ZK event occured requestion a truncation snapshot be taken
     */
private void processSnapshotTruncationRequestCreated(final WatchedEvent event) {
    loggingLog.info("Snapshot truncation leader received snapshot truncation request");
    // Get the truncation request ID which is the truncation request node path.
    final String truncReqId;
    try {
        List<String> children = m_zk.getChildren(event.getPath(), false);
        if (children.isEmpty()) {
            loggingLog.error("Unable to retrieve truncation snapshot request id from ZK, log can't be truncated");
            return;
        }
        truncReqId = ZKUtil.joinZKPath(event.getPath(), Collections.max(children));
    } catch (Exception e) {
        loggingLog.error("Unable to retrieve truncation snapshot request ID from ZK, log can't be truncated");
        return;
    }
    final long now = System.currentTimeMillis();
    final String nonce = Long.toString(now);
    // TRAIL [TruncSnap:7] write current ts to request zk node data
    try {
        ByteBuffer payload = ByteBuffer.allocate(8);
        payload.putLong(0, now);
        m_zk.setData(VoltZK.request_truncation_snapshot, payload.array(), -1);
    } catch (Exception e) {
        //Cause a cascading failure?
        VoltDB.crashLocalVoltDB("Setting data on the truncation snapshot request in ZK should never fail", true, e);
    }
    // for the snapshot save invocations
    JSONObject jsObj = new JSONObject();
    try {
        assert truncReqId != null;
        String sData = "";
        JSONObject jsData = new JSONObject();
        jsData.put("truncReqId", truncReqId);
        sData = jsData.toString();
        jsObj.put(SnapshotUtil.JSON_PATH, VoltDB.instance().getCommandLogSnapshotPath());
        jsObj.put(SnapshotUtil.JSON_NONCE, nonce);
        jsObj.put(SnapshotUtil.JSON_PATH_TYPE, SnapshotPathType.SNAP_CL);
        jsObj.put("perPartitionTxnIds", retrievePerPartitionTransactionIds());
        jsObj.put("data", sData);
    } catch (JSONException e) {
        /*
             * Should never happen, so fail fast
             */
        VoltDB.crashLocalVoltDB("", true, e);
    }
    // for the snapshot save invocations
    long handle = m_nextCallbackHandle++;
    // for the snapshot save invocation
    m_procedureCallbacks.put(handle, new ProcedureCallback() {

        @Override
        public void clientCallback(ClientResponse clientResponse) throws Exception {
            m_lastInitiationTs = null;
            if (clientResponse.getStatus() != ClientResponse.SUCCESS) {
                loggingLog.warn("Attempt to initiate a truncation snapshot was not successful: " + clientResponse.getStatusString());
                loggingLog.warn("Retrying log truncation snapshot in 5 minutes");
                /*
                     * TRAIL [TruncSnap:8] (callback) on failed response try again in a few minute
                     */
                m_es.schedule(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            processTruncationRequestEvent(event);
                        } catch (Exception e) {
                            VoltDB.crashLocalVoltDB("Error processing snapshot truncation request event", true, e);
                        }
                    }
                }, 5, TimeUnit.MINUTES);
                return;
            }
            final VoltTable[] results = clientResponse.getResults();
            final VoltTable result = results[0];
            boolean success = true;
            final String err = SnapshotUtil.didSnapshotRequestFailWithErr(results);
            if (err != null) {
                if (err.trim().equalsIgnoreCase("SNAPSHOT IN PROGRESS")) {
                    loggingLog.info("Snapshot is in progress");
                } else {
                    loggingLog.warn("Snapshot failed with failure response: " + err);
                }
                success = false;
            }
            //assert(result.getColumnName(1).equals("TABLE"));
            if (success) {
                while (result.advanceRow()) {
                    if (!result.getString("RESULT").equals("SUCCESS")) {
                        success = false;
                        loggingLog.warn("Snapshot save feasibility test failed for host " + result.getLong("HOST_ID") + " table " + result.getString("TABLE") + " with error message " + result.getString("ERR_MSG"));
                    }
                }
            }
            if (success) {
                loggingLog.info("Snapshot initiation for log truncation was successful");
                JSONObject obj = new JSONObject(clientResponse.getAppStatusString());
                final long snapshotTxnId = Long.valueOf(obj.getLong("txnId"));
                try {
                    boolean found = false;
                    ZKUtil.VoidCallback lastCallback = null;
                    for (String child : m_zk.getChildren(event.getPath(), false)) {
                        String requestId = ZKUtil.joinZKPath(event.getPath(), child);
                        found = found || requestId.equals(truncReqId);
                        lastCallback = new ZKUtil.VoidCallback();
                        m_zk.delete(requestId, -1, lastCallback, null);
                    }
                    if (lastCallback != null) {
                        try {
                            lastCallback.get();
                        } catch (KeeperException.NoNodeException ignoreIt) {
                        }
                    }
                    if (!found) {
                        VoltDB.crashLocalVoltDB("Could not match truncations snapshot request id while atepting its removal", true, null);
                    }
                } catch (Exception e) {
                    VoltDB.crashLocalVoltDB("Unexpected error deleting truncation snapshot request", true, e);
                }
                try {
                    TruncationSnapshotAttempt snapshotAttempt = m_truncationSnapshotAttempts.get(snapshotTxnId);
                    if (snapshotAttempt == null) {
                        snapshotAttempt = new TruncationSnapshotAttempt();
                        m_truncationSnapshotAttempts.put(snapshotTxnId, snapshotAttempt);
                        snapshotAttempt.pathType = SnapshotPathType.SNAP_CL.toString();
                    }
                    snapshotAttempt.nonce = nonce;
                    snapshotAttempt.path = VoltDB.instance().getCommandLogSnapshotPath();
                } finally {
                    // TRAIL [TruncSnap:9] (callback) restart the whole request check cycle
                    try {
                        truncationRequestExistenceCheck();
                    } catch (Exception e) {
                        VoltDB.crashLocalVoltDB("Unexpected error checking for existence of truncation snapshot request", true, e);
                    }
                }
            } else {
                loggingLog.info("Retrying log truncation snapshot in 60 seconds");
                /*
                     * TRAIL [TruncSnap:10] (callback) on table reported failure try again in a few minutes
                     */
                m_es.schedule(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            processTruncationRequestEvent(event);
                        } catch (Exception e) {
                            VoltDB.crashLocalVoltDB("Exception processing truncation request event", true, e);
                        }
                    }
                }, 1, TimeUnit.MINUTES);
            }
        }
    });
    try {
        loggingLog.info("Initiating @SnapshotSave for log truncation");
        initiateSnapshotSave(handle, new Object[] { jsObj.toString(4) }, false);
    } catch (JSONException e) {
        /*
             * Should never happen, so fail fast
             */
        VoltDB.crashLocalVoltDB("", true, e);
    }
    return;
}

Example 93 with JSONObject

use of org.json_voltpatches.JSONObject in project voltdb by VoltDB.

the class AgreementSite method processMessage.

private void processMessage(VoltMessage message) throws Exception {
    if (!m_hsIds.contains(message.m_sourceHSId)) {
        m_recoveryLog.info("Dropping message " + message + " because it is not from a known up site");
        return;
    }
    if (message instanceof TransactionInfoBaseMessage) {
        TransactionInfoBaseMessage info = (TransactionInfoBaseMessage) message;
        // Special case heartbeats which only update RPQ
        if (info instanceof HeartbeatMessage) {
            // use the heartbeat to unclog the priority queue if clogged
            long lastSeenTxnFromInitiator = m_txnQueue.noteTransactionRecievedAndReturnLastSeen(info.getInitiatorHSId(), info.getTxnId(), ((HeartbeatMessage) info).getLastSafeTxnId());
            // respond to the initiator with the last seen transaction
            HeartbeatResponseMessage response = new HeartbeatResponseMessage(m_hsId, lastSeenTxnFromInitiator, m_txnQueue.getQueueState() == RestrictedPriorityQueue.QueueState.BLOCKED_SAFETY);
            m_mailbox.send(info.getInitiatorHSId(), response);
            // we're done here (in the case of heartbeats)
            return;
        }
        assert (false);
    } else if (message instanceof HeartbeatResponseMessage) {
        HeartbeatResponseMessage hrm = (HeartbeatResponseMessage) message;
        m_safetyState.updateLastSeenTxnIdFromExecutorBySiteId(hrm.getExecHSId(), hrm.getLastReceivedTxnId());
    } else if (message instanceof LocalObjectMessage) {
        LocalObjectMessage lom = (LocalObjectMessage) message;
        if (lom.payload instanceof Runnable) {
            ((Runnable) lom.payload).run();
        } else if (lom.payload instanceof Request) {
            Request r = (Request) lom.payload;
            long txnId = 0;
            boolean isRead = false;
            switch(r.type) {
                case OpCode.createSession:
                    txnId = r.sessionId;
                    break;
                //For reads see if we can skip global agreement and just do the read
                case OpCode.exists:
                case OpCode.getChildren:
                case OpCode.getChildren2:
                case OpCode.getData:
                    //in this case because ordering of reads and writes matters
                    if (m_txnQueue.isEmpty()) {
                        r.setOwner(m_hsId);
                        m_server.prepRequest(new Request(r), m_lastUsedTxnId);
                        return;
                    }
                    isRead = true;
                //it in the global order
                default:
                    txnId = m_idManager.getNextUniqueTransactionId();
                    break;
            }
            /*
                 * Don't send the whole request if this is a read blocked on a write
                 * We may send a heartbeat instead of propagating a useless read transaction
                 * at the end of this block
                 */
            if (!isRead) {
                for (long initiatorHSId : m_hsIds) {
                    if (initiatorHSId == m_hsId)
                        continue;
                    AgreementTaskMessage atm = new AgreementTaskMessage(r, txnId, m_hsId, m_safetyState.getNewestGloballySafeTxnId());
                    m_mailbox.send(initiatorHSId, atm);
                }
            }
            //Process the ATM eagerly locally to aid
            //in having a complete set of stuff to ship
            //to a recovering agreement site
            AgreementTaskMessage atm = new AgreementTaskMessage(new Request(r), txnId, m_hsId, m_safetyState.getNewestGloballySafeTxnId());
            atm.m_sourceHSId = m_hsId;
            processMessage(atm);
            /*
                 * Don't send a heartbeat out for ever single blocked read that occurs
                 * Try and limit to 2000 a second which is a lot and should be pretty
                 * close to the previous behavior of propagating all reads. My measurements
                 * don't show the old behavior is better than none at all, but I fear
                 * change.
                 */
            if (isRead) {
                final long now = System.nanoTime();
                if (TimeUnit.NANOSECONDS.toMicros(now - m_lastHeartbeatTime) > 500) {
                    m_lastHeartbeatTime = now;
                    sendHeartbeats();
                }
            }
        }
    } else if (message instanceof AgreementTaskMessage) {
        AgreementTaskMessage atm = (AgreementTaskMessage) message;
        if (!m_transactionsById.containsKey(atm.m_txnId) && atm.m_txnId >= m_minTxnIdAfterRecovery) {
            m_txnQueue.noteTransactionRecievedAndReturnLastSeen(atm.m_initiatorHSId, atm.m_txnId, atm.m_lastSafeTxnId);
            AgreementTransactionState transactionState = new AgreementTransactionState(atm.m_txnId, atm.m_initiatorHSId, atm.m_request);
            if (m_txnQueue.add(transactionState)) {
                m_transactionsById.put(transactionState.txnId, transactionState);
            } else {
                m_agreementLog.info("Dropping txn " + transactionState.txnId + " data from failed initiatorSiteId: " + transactionState.initiatorHSId);
            }
        } else {
            m_recoveryLog.info("Agreement, discarding duplicate txn during recovery, txnid is " + atm.m_txnId + " this should only occur during recovery. minTxnIdAfterRecovery " + m_minTxnIdAfterRecovery + " and  dup is " + m_transactionsById.containsKey(atm.m_txnId));
        }
    } else if (message instanceof BinaryPayloadMessage) {
        BinaryPayloadMessage bpm = (BinaryPayloadMessage) message;
        ByteBuffer metadata = ByteBuffer.wrap(bpm.m_metadata);
        final byte type = metadata.get();
        if (type == BINARY_PAYLOAD_SNAPSHOT) {
            assert (m_recovering);
            assert (m_recoveryStage == RecoveryStage.SENT_PROPOSAL);
            if (m_recoveryStage != RecoveryStage.SENT_PROPOSAL) {
                org.voltdb.VoltDB.crashLocalVoltDB("Received a recovery snapshot in stage " + m_recoveryStage.toString(), true, null);
            }
            long selectedRecoverBeforeTxn = metadata.getLong();
            if (selectedRecoverBeforeTxn < m_recoverBeforeTxn) {
                org.voltdb.VoltDB.crashLocalVoltDB("Selected recover before txn was earlier than the  proposed recover before txn", true, null);
            }
            m_recoverBeforeTxn = selectedRecoverBeforeTxn;
            //anything before this precedes the snapshot
            m_minTxnIdAfterRecovery = m_recoverBeforeTxn;
            try {
                m_recoverySnapshot = org.xerial.snappy.Snappy.uncompress(bpm.m_payload);
            } catch (IOException e) {
                org.voltdb.VoltDB.crashLocalVoltDB("Unable to decompress ZK snapshot", true, e);
            }
            m_recoveryStage = RecoveryStage.RECEIVED_SNAPSHOT;
            /*
                 * Clean out all txns from before the snapshot
                 */
            Iterator<Map.Entry<Long, OrderableTransaction>> iter = m_transactionsById.entrySet().iterator();
            while (iter.hasNext()) {
                final Map.Entry<Long, OrderableTransaction> entry = iter.next();
                if (entry.getKey() < m_minTxnIdAfterRecovery) {
                    m_txnQueue.faultTransaction(entry.getValue());
                    iter.remove();
                }
            }
        } else if (type == BINARY_PAYLOAD_JOIN_REQUEST) {
            JSONObject jsObj = new JSONObject(new String(bpm.m_payload, "UTF-8"));
            final long initiatorHSId = jsObj.getLong("initiatorHSId");
            final long txnId = jsObj.getLong("txnId");
            final long lastSafeTxnId = jsObj.getLong("lastSafeTxnId");
            final long joiningHSId = jsObj.getLong("joiningHSId");
            if (m_recovering) {
                org.voltdb.VoltDB.crashLocalVoltDB("Received a join request during recovery for " + CoreUtils.hsIdToString(joiningHSId) + " from " + CoreUtils.hsIdToString(initiatorHSId), true, null);
            }
            m_txnQueue.noteTransactionRecievedAndReturnLastSeen(initiatorHSId, txnId, lastSafeTxnId);
            AgreementRejoinTransactionState transactionState = new AgreementRejoinTransactionState(txnId, initiatorHSId, joiningHSId, null);
            if (m_txnQueue.add(transactionState)) {
                m_transactionsById.put(transactionState.txnId, transactionState);
            } else {
                m_agreementLog.info("Dropping txn " + transactionState.txnId + " data from failed initiatorSiteId: " + transactionState.initiatorHSId);
            }
        }
    } else if (message instanceof FaultMessage) {
        FaultMessage fm = (FaultMessage) message;
        discoverGlobalFaultData(fm);
    } else if (message instanceof RecoveryMessage) {
        RecoveryMessage rm = (RecoveryMessage) message;
        assert (m_recoverBeforeTxn == null);
        assert (m_siteRequestingRecovery == null);
        assert (m_recovering == false);
        assert (m_recoveryStage == RecoveryStage.RECOVERED);
        m_recoverBeforeTxn = rm.txnId();
        m_siteRequestingRecovery = rm.sourceSite();
    }
}

Example 94 with JSONObject

use of org.json_voltpatches.JSONObject in project voltdb by VoltDB.

the class AgreementSite method requestJoin.

/*
     * Construct a ZK transaction that will add the initiator to the cluster
     */
public CountDownLatch requestJoin(final long joiningSite) throws Exception {
    final CountDownLatch cdl = new CountDownLatch(1);
    final Runnable r = new Runnable() {

        @Override
        public void run() {
            try {
                final long txnId = m_idManager.getNextUniqueTransactionId();
                for (long initiatorHSId : m_hsIds) {
                    if (initiatorHSId == m_hsId)
                        continue;
                    JSONObject jsObj = new JSONObject();
                    jsObj.put("txnId", txnId);
                    jsObj.put("initiatorHSId", m_hsId);
                    jsObj.put("joiningHSId", joiningSite);
                    jsObj.put("lastSafeTxnId", m_safetyState.getNewestSafeTxnIdForExecutorBySiteId(initiatorHSId));
                    byte[] payload = jsObj.toString(4).getBytes("UTF-8");
                    ByteBuffer metadata = ByteBuffer.allocate(1);
                    metadata.put(BINARY_PAYLOAD_JOIN_REQUEST);
                    BinaryPayloadMessage bpm = new BinaryPayloadMessage(metadata.array(), payload);
                    m_mailbox.send(initiatorHSId, bpm);
                }
                m_txnQueue.noteTransactionRecievedAndReturnLastSeen(m_hsId, txnId, m_safetyState.getNewestGloballySafeTxnId());
                AgreementRejoinTransactionState arts = new AgreementRejoinTransactionState(txnId, m_hsId, joiningSite, cdl);
                if (!m_txnQueue.add(arts)) {
                    org.voltdb.VoltDB.crashLocalVoltDB("Shouldn't have failed to add txn", true, null);
                }
                m_transactionsById.put(arts.txnId, arts);
            } catch (Throwable e) {
                org.voltdb.VoltDB.crashLocalVoltDB("Error constructing JSON", false, e);
            }
        }
    };
    LocalObjectMessage lom = new LocalObjectMessage(r);
    lom.m_sourceHSId = m_hsId;
    m_mailbox.deliver(lom);
    return cdl;
}

Example 95 with JSONObject

use of org.json_voltpatches.JSONObject in project voltdb by VoltDB.

the class AdHocPlannedStmtBatch method explainStatement.

/**
     * Return the "EXPLAIN" string of the batched statement at the index
     * @param i the index
     * @param db the database context (for adding catalog details).
     */
public String explainStatement(int i, Database db) {
    AdHocPlannedStatement plannedStatement = plannedStatements.get(i);
    String aggplan = new String(plannedStatement.core.aggregatorFragment, Constants.UTF8ENCODING);
    PlanNodeTree pnt = new PlanNodeTree();
    try {
        JSONObject jobj = new JSONObject(aggplan);
        pnt.loadFromJSONPlan(jobj, db);
        if (plannedStatement.core.collectorFragment != null) {
            // multi-partition query plan
            String collplan = new String(plannedStatement.core.collectorFragment, Constants.UTF8ENCODING);
            PlanNodeTree collpnt = new PlanNodeTree();
            // reattach plan fragments
            JSONObject jobMP = new JSONObject(collplan);
            collpnt.loadFromJSONPlan(jobMP, db);
            assert (collpnt.getRootPlanNode() instanceof SendPlanNode);
            pnt.getRootPlanNode().reattachFragment(collpnt.getRootPlanNode());
        }
        String result = pnt.getRootPlanNode().toExplainPlanString();
        return result;
    } catch (JSONException e) {
        System.out.println(e);
        return "Internal Error (JSONException): " + e.getMessage();
    }
}