Examples with SnapshotPathType org.voltdb.sysprocs.saverestore.SnapshotPathType used on opensource projects

Example 1 with SnapshotPathType

use of org.voltdb.sysprocs.saverestore.SnapshotPathType in project voltdb by VoltDB.

the class InvocationDispatcher method getSnapshotCatalogFile.

private final File getSnapshotCatalogFile(JSONObject snapJo) throws JSONException {
    NodeSettings paths = m_catalogContext.get().getNodeSettings();
    String catFN = snapJo.getString(SnapshotUtil.JSON_NONCE) + ".jar";
    SnapshotPathType pathType = SnapshotPathType.valueOf(snapJo.optString(SnapshotUtil.JSON_PATH_TYPE, SnapshotPathType.SNAP_PATH.name()));
    switch(pathType) {
        case SNAP_AUTO:
            return new File(paths.resolve(paths.getSnapshoth()), catFN);
        case SNAP_CL:
            return new File(paths.resolve(paths.getCommandLogSnapshot()), catFN);
        default:
            File snapDH = new VoltFile(snapJo.getString(SnapshotUtil.JSON_PATH));
            return new File(snapDH, catFN);
    }
}

Example 2 with SnapshotPathType

use of org.voltdb.sysprocs.saverestore.SnapshotPathType in project voltdb by VoltDB.

the class SnapshotDaemon method initiateSnapshotSave.

private void initiateSnapshotSave(final long handle, final Object[] params, boolean blocking) {
    boolean success = true;
    VoltTable checkResult = SnapshotUtil.constructNodeResultsTable();
    final String jsString = String.class.cast(params[0]);
    if (m_lastInitiationTs != null) {
        final long elapsedMs = System.currentTimeMillis() - m_lastInitiationTs.getFirst();
        // Blocking snapshot may take a long time to finish, don't time it out if it's blocking
        if (!m_lastInitiationTs.getSecond() && elapsedMs > INITIATION_RESPONSE_TIMEOUT_MS) {
            SNAP_LOG.warn(String.format("A snapshot was initiated %d minutes ago and hasn't received a response yet.", TimeUnit.MILLISECONDS.toMinutes(elapsedMs)));
            m_lastInitiationTs = null;
        } else {
            checkResult.addRow(CoreUtils.getHostIdFromHSId(m_mb.getHSId()), CoreUtils.getHostnameOrAddress(), null, "FAILURE", "SNAPSHOT IN PROGRESS");
            success = false;
        }
    }
    if (success) {
        try {
            final JSONObject jsObj = new JSONObject(jsString);
            boolean initiateSnapshot;
            // Do scan work on all known live hosts
            VoltMessage msg = new SnapshotCheckRequestMessage(jsString);
            SnapshotPathType pathType = SnapshotPathType.valueOf(jsObj.getString(SnapshotUtil.JSON_PATH_TYPE));
            Set<Integer> liveHosts = VoltDB.instance().getHostMessenger().getLiveHostIds();
            for (int hostId : liveHosts) {
                m_mb.send(CoreUtils.getHSIdFromHostAndSite(hostId, HostMessenger.SNAPSHOT_IO_AGENT_ID), msg);
            }
            // Wait for responses from all hosts for a certain amount of time
            Map<Integer, VoltTable> responses = Maps.newHashMap();
            // 10s timeout
            final long timeoutMs = 10 * 1000;
            final long endTime = System.currentTimeMillis() + timeoutMs;
            SnapshotCheckResponseMessage response;
            while ((response = (SnapshotCheckResponseMessage) m_mb.recvBlocking(timeoutMs)) != null) {
                final String nonce = jsObj.getString(SnapshotUtil.JSON_NONCE);
                boolean nonceFound = false;
                if (pathType == SnapshotPathType.SNAP_PATH) {
                    // If request was explicitely PATH check path too.
                    if (nonce.equals(response.getNonce()) && response.getPath().equals(jsObj.getString(SnapshotUtil.JSON_PATH))) {
                        nonceFound = true;
                    }
                } else {
                    // If request is with type other than path just check type.
                    if (nonce.equals(response.getNonce()) && response.getSnapshotPathType() == pathType) {
                        nonceFound = true;
                    }
                }
                if (nonceFound) {
                    responses.put(CoreUtils.getHostIdFromHSId(response.m_sourceHSId), response.getResponse());
                }
                if (responses.size() == liveHosts.size() || System.currentTimeMillis() > endTime) {
                    break;
                }
            }
            if (responses.size() != liveHosts.size()) {
                checkResult.addRow(CoreUtils.getHostIdFromHSId(m_mb.getHSId()), CoreUtils.getHostnameOrAddress(), null, "FAILURE", "TIMED OUT CHECKING SNAPSHOT FEASIBILITY");
                success = false;
            }
            if (success) {
                // TRAIL [TruncSnap:12] all participating nodes have initiated successfully
                // Call @SnapshotSave if check passed, return the failure otherwise
                checkResult = VoltTableUtil.unionTables(responses.values());
                initiateSnapshot = SnapshotUtil.didSnapshotRequestSucceed(new VoltTable[] { checkResult });
                if (initiateSnapshot) {
                    m_lastInitiationTs = Pair.of(System.currentTimeMillis(), blocking);
                    m_initiator.initiateSnapshotDaemonWork("@SnapshotSave", handle, params);
                } else {
                    success = false;
                }
            }
        } catch (JSONException e) {
            success = false;
            checkResult.addRow(CoreUtils.getHostIdFromHSId(m_mb.getHSId()), CoreUtils.getHostnameOrAddress(), null, "FAILURE", "ERROR PARSING JSON");
            SNAP_LOG.warn("Error parsing JSON string: " + jsString, e);
        }
    }
    if (!success) {
        final ClientResponseImpl failureResponse = new ClientResponseImpl(ClientResponseImpl.SUCCESS, new VoltTable[] { checkResult }, null);
        failureResponse.setClientHandle(handle);
        processClientResponse(Callables.returning(failureResponse));
    }
}

Example 3 with SnapshotPathType

use of org.voltdb.sysprocs.saverestore.SnapshotPathType in project voltdb by VoltDB.

the class SnapshotCompletionMonitor method processSnapshotData.

private void processSnapshotData(byte[] data) throws Exception {
    if (data == null) {
        return;
    }
    JSONObject jsonObj = new JSONObject(new String(data, "UTF-8"));
    long txnId = jsonObj.getLong("txnId");
    int hostCount = jsonObj.getInt("hostCount");
    String path = jsonObj.getString(SnapshotUtil.JSON_PATH);
    SnapshotPathType stype = SnapshotPathType.valueOf(jsonObj.getString(SnapshotUtil.JSON_PATH_TYPE));
    String nonce = jsonObj.getString(SnapshotUtil.JSON_NONCE);
    boolean truncation = jsonObj.getBoolean("isTruncation");
    boolean didSucceed = jsonObj.getBoolean("didSucceed");
    // A truncation request ID is not always provided. It's used for
    // snapshots triggered indirectly via ZooKeeper so that the
    // triggerer can recognize the snapshot when it finishes.
    String truncReqId = jsonObj.optString("truncReqId");
    if (hostCount == 0) {
        /*
             * Convert the JSON object containing the export sequence numbers for each
             * table and partition to a regular map
             */
        Map<String, Map<Integer, Pair<Long, Long>>> exportSequenceNumbers = null;
        final JSONObject exportSequenceJSON = jsonObj.getJSONObject("exportSequenceNumbers");
        final ImmutableMap.Builder<String, Map<Integer, Pair<Long, Long>>> builder = ImmutableMap.builder();
        @SuppressWarnings("unchecked") final Iterator<String> tableKeys = exportSequenceJSON.keys();
        while (tableKeys.hasNext()) {
            final String tableName = tableKeys.next();
            final JSONObject tableSequenceNumbers = exportSequenceJSON.getJSONObject(tableName);
            ImmutableMap.Builder<Integer, Pair<Long, Long>> tableBuilder = ImmutableMap.builder();
            @SuppressWarnings("unchecked") final Iterator<String> partitionKeys = tableSequenceNumbers.keys();
            while (partitionKeys.hasNext()) {
                final String partitionString = partitionKeys.next();
                final Integer partitionId = Integer.valueOf(partitionString);
                JSONObject sequenceNumbers = tableSequenceNumbers.getJSONObject(partitionString);
                final Long ackOffset = sequenceNumbers.getLong("ackOffset");
                final Long sequenceNumber = sequenceNumbers.getLong("sequenceNumber");
                tableBuilder.put(partitionId, Pair.of(ackOffset, sequenceNumber));
            }
            builder.put(tableName, tableBuilder.build());
        }
        exportSequenceNumbers = builder.build();
        long clusterCreateTime = jsonObj.optLong("clusterCreateTime", -1);
        Map<Integer, Long> drSequenceNumbers = new HashMap<>();
        JSONObject drTupleStreamJSON = jsonObj.getJSONObject("drTupleStreamStateInfo");
        Iterator<String> partitionKeys = drTupleStreamJSON.keys();
        int drVersion = 0;
        while (partitionKeys.hasNext()) {
            String partitionIdString = partitionKeys.next();
            JSONObject stateInfo = drTupleStreamJSON.getJSONObject(partitionIdString);
            drVersion = (int) stateInfo.getLong("drVersion");
            drSequenceNumbers.put(Integer.valueOf(partitionIdString), stateInfo.getLong("sequenceNumber"));
        }
        Map<Integer, Long> partitionTxnIdsMap = ImmutableMap.of();
        synchronized (m_snapshotTxnIdsToPartitionTxnIds) {
            Map<Integer, Long> partitionTxnIdsList = m_snapshotTxnIdsToPartitionTxnIds.get(txnId);
            if (partitionTxnIdsList != null) {
                partitionTxnIdsMap = ImmutableMap.copyOf(partitionTxnIdsList);
            }
        }
        /*
             * Collect all the last seen ids from the remote data centers so they can
             * be used by live rejoin to initialize a starting state for applying DR
             * data
             */
        Map<Integer, Map<Integer, Map<Integer, DRConsumerDrIdTracker>>> drMixedClusterSizeConsumerState = new HashMap<>();
        JSONObject consumerPartitions = jsonObj.getJSONObject("drMixedClusterSizeConsumerState");
        Iterator<String> cpKeys = consumerPartitions.keys();
        while (cpKeys.hasNext()) {
            final String consumerPartitionIdStr = cpKeys.next();
            final Integer consumerPartitionId = Integer.valueOf(consumerPartitionIdStr);
            JSONObject siteInfo = consumerPartitions.getJSONObject(consumerPartitionIdStr);
            drMixedClusterSizeConsumerState.put(consumerPartitionId, ExtensibleSnapshotDigestData.buildConsumerSiteDrIdTrackersFromJSON(siteInfo));
        }
        Iterator<SnapshotCompletionInterest> iter = m_interests.iterator();
        while (iter.hasNext()) {
            SnapshotCompletionInterest interest = iter.next();
            try {
                interest.snapshotCompleted(new SnapshotCompletionEvent(path, stype, nonce, txnId, partitionTxnIdsMap, truncation, didSucceed, truncReqId, exportSequenceNumbers, Collections.unmodifiableMap(drSequenceNumbers), Collections.unmodifiableMap(drMixedClusterSizeConsumerState), drVersion, clusterCreateTime));
            } catch (Exception e) {
                SNAP_LOG.warn("Exception while executing snapshot completion interest", e);
            }
        }
    }
}

Example 4 with SnapshotPathType

use of org.voltdb.sysprocs.saverestore.SnapshotPathType in project voltdb by VoltDB.

the class RestoreAgent method getSnapshots.

/**
     * Finds all the snapshots in all the places we know of which could possibly
     * store snapshots, like command log snapshots, auto snapshots, etc.
     *
     * @return All snapshots
     */
private Map<String, Snapshot> getSnapshots() {
    /*
         * Use the individual snapshot directories instead of voltroot, because
         * they can be set individually
         */
    Map<String, SnapshotPathType> paths = new HashMap<String, SnapshotPathType>();
    if (VoltDB.instance().getConfig().m_isEnterprise) {
        if (m_clSnapshotPath != null) {
            paths.put(m_clSnapshotPath, SnapshotPathType.SNAP_CL);
        }
    }
    if (m_snapshotPath != null) {
        paths.put(m_snapshotPath, SnapshotPathType.SNAP_AUTO);
    }
    HashMap<String, Snapshot> snapshots = new HashMap<String, Snapshot>();
    FileFilter filter = new SnapshotUtil.SnapshotFilter();
    for (String path : paths.keySet()) {
        SnapshotUtil.retrieveSnapshotFiles(new File(path), snapshots, filter, false, paths.get(path), LOG);
    }
    return snapshots;
}

Example 5 with SnapshotPathType

use of org.voltdb.sysprocs.saverestore.SnapshotPathType in project voltdb by VoltDB.

the class SnapshotDeleteAgent method dispatchSnapshotDelete.

private VoltTable[] dispatchSnapshotDelete(JSONObject obj) throws JSONException {
    VoltTable result = constructFragmentResultsTable();
    int length = obj.getJSONArray("paths").length();
    final String[] paths = new String[length];
    for (int i = 0; i < length; i++) {
        paths[i] = obj.getJSONArray("paths").getString(i);
    }
    length = obj.getJSONArray("nonces").length();
    final String[] nonces = new String[length];
    for (int i = 0; i < length; i++) {
        nonces[i] = obj.getJSONArray("nonces").getString(i);
    }
    final SnapshotPathType stype = SnapshotPathType.valueOf(obj.getString(SnapshotUtil.JSON_PATH_TYPE));
    new Thread("Async snapshot deletion thread") {

        @Override
        public void run() {
            StringBuilder sb = new StringBuilder();
            sb.append("Deleting files: ");
            for (int ii = 0; ii < paths.length; ii++) {
                //When user calls @SnapshotDelete this will be set to SNAP_PATH so we will delete what user requested.
                //If its SNAP_AUTO then its coming from periodic delete task.
                String path = SnapshotUtil.getRealPath(stype, paths[ii]);
                List<File> relevantFiles = retrieveRelevantFiles(path, nonces[ii]);
                if (relevantFiles != null) {
                    for (final File f : relevantFiles) {
                        sb.append(f.getPath());
                        sb.append(',');
                        //long size = f.length();
                        f.delete();
                    }
                }
            }
            SNAP_LOG.info(sb.toString());
        }
    }.start();
    return new VoltTable[] { result };
}