Examples with Snapshot org.voltdb.sysprocs.saverestore.SnapshotUtil.Snapshot used on opensource projects

Example 1 with Snapshot

use of org.voltdb.sysprocs.saverestore.SnapshotUtil.Snapshot in project voltdb by VoltDB.

the class RestoreAgent method generatePlans.

/**
     * Generate restore and replay plans.
     *
     * @return The snapshot to restore, or null if there is none.
     * @throws Exception
     *             If any exception is thrown, it means that the plan generation
     *             has failed. Should crash the cluster.
     */
SnapshotInfo generatePlans() throws Exception {
    Map<String, Snapshot> snapshots = new HashMap<String, SnapshotUtil.Snapshot>();
    // Only scan if startup might require a snapshot restore.
    if (m_action.doesRecover()) {
        snapshots = getSnapshots();
    }
    final Long maxLastSeenTxn = m_replayAgent.getMaxLastSeenTxn();
    Set<SnapshotInfo> snapshotInfos = new HashSet<SnapshotInfo>();
    for (Snapshot e : snapshots.values()) {
        SnapshotInfo info = checkSnapshotIsComplete(e.getTxnId(), e);
        // if the cluster instance IDs in the snapshot and command log don't match, just move along
        if (m_replayAgent.getInstanceId() != null && info != null && !m_replayAgent.getInstanceId().equals(info.instanceId)) {
            // Exceptions are not well tolerated here, so don't throw over something
            // as trivial as error message formatting.
            String agentIdString;
            String infoIdString;
            try {
                agentIdString = m_replayAgent.getInstanceId().serializeToJSONObject().toString();
            } catch (JSONException e1) {
                agentIdString = "<failed to serialize id>";
            }
            try {
                infoIdString = info.instanceId.serializeToJSONObject().toString();
            } catch (JSONException e1) {
                infoIdString = "<failed to serialize id>";
            }
            m_snapshotErrLogStr.append("\nRejected snapshot ").append(info.nonce).append(" due to mismatching instance IDs.").append(" Command log ID: ").append(agentIdString).append(" Snapshot ID: ").append(infoIdString);
            continue;
        }
        if (info != null) {
            final Map<Integer, Long> cmdlogmap = m_replayAgent.getMaxLastSeenTxnByPartition();
            final Map<Integer, Long> snapmap = info.partitionToTxnId;
            // don't do any TXN ID consistency checking between command log and snapshot
            if (cmdlogmap != null) {
                for (Integer cmdpart : cmdlogmap.keySet()) {
                    Long snaptxnId = snapmap.get(cmdpart);
                    if (snaptxnId == null) {
                        m_snapshotErrLogStr.append("\nRejected snapshot ").append(info.nonce).append(" due to missing partition: ").append(cmdpart);
                        info = null;
                        break;
                    } else if (snaptxnId < cmdlogmap.get(cmdpart)) {
                        m_snapshotErrLogStr.append("\nRejected snapshot ").append(info.nonce).append(" because it does not overlap the command log").append(" for partition: ").append(cmdpart).append(" command log txn ID: ").append(cmdlogmap.get(cmdpart)).append(" snapshot txn ID: ").append(snaptxnId);
                        info = null;
                        break;
                    }
                }
            }
        }
        if (info != null) {
            snapshotInfos.add(info);
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Gathered " + snapshotInfos.size() + " snapshot information");
    }
    sendLocalRestoreInformation(maxLastSeenTxn, snapshotInfos);
    // Negotiate with other hosts about which snapshot to restore
    SnapshotInfo infoWithMinHostId = getRestorePlan();
    if (infoWithMinHostId != null && infoWithMinHostId.nonce.equals(m_terminusNonce)) {
        m_replayAgent.returnAllSegments();
        initialize(StartAction.CREATE);
        m_planned = true;
        return infoWithMinHostId;
    }
    /*
         * Generate the replay plan here so that we don't have to wait until the
         * snapshot restore finishes.
         */
    if (m_action.doesRecover()) {
        if (infoWithMinHostId != null) {
            // The expected partition count could be determined by the new partition count recorded
            // in the truncation snapshot. Truncation snapshot taken at the end of the join process
            // actually records the new partition count in the digest.
            m_replayAgent.generateReplayPlan(infoWithMinHostId.instanceId.getTimestamp(), infoWithMinHostId.txnId, infoWithMinHostId.newPartitionCount, m_isLeader);
        }
    }
    m_planned = true;
    return infoWithMinHostId;
}

Example 2 with Snapshot

use of org.voltdb.sysprocs.saverestore.SnapshotUtil.Snapshot in project voltdb by VoltDB.

the class RestoreAgent method getSnapshots.

/**
     * Finds all the snapshots in all the places we know of which could possibly
     * store snapshots, like command log snapshots, auto snapshots, etc.
     *
     * @return All snapshots
     */
private Map<String, Snapshot> getSnapshots() {
    /*
         * Use the individual snapshot directories instead of voltroot, because
         * they can be set individually
         */
    Map<String, SnapshotPathType> paths = new HashMap<String, SnapshotPathType>();
    if (VoltDB.instance().getConfig().m_isEnterprise) {
        if (m_clSnapshotPath != null) {
            paths.put(m_clSnapshotPath, SnapshotPathType.SNAP_CL);
        }
    }
    if (m_snapshotPath != null) {
        paths.put(m_snapshotPath, SnapshotPathType.SNAP_AUTO);
    }
    HashMap<String, Snapshot> snapshots = new HashMap<String, Snapshot>();
    FileFilter filter = new SnapshotUtil.SnapshotFilter();
    for (String path : paths.keySet()) {
        SnapshotUtil.retrieveSnapshotFiles(new File(path), snapshots, filter, false, paths.get(path), LOG);
    }
    return snapshots;
}

Example 3 with Snapshot

use of org.voltdb.sysprocs.saverestore.SnapshotUtil.Snapshot in project voltdb by VoltDB.

the class SnapshotConverter method main.

/**
     * @param args
     */
public static void main(String[] args) {
    String snapshotName = null;
    ArrayList<File> directories = new ArrayList<File>();
    ArrayList<String> tables = new ArrayList<String>();
    File outdir = null;
    String type = null;
    char delimiter = '\0';
    for (int ii = 0; ii < args.length; ii++) {
        String arg = args[ii];
        if (arg.equals("--help")) {
            printHelpAndQuit(0);
        } else if (arg.equals("--dir")) {
            if (args.length < ii + 1) {
                System.err.println("Error: Not enough args following --dirs");
                printHelpAndQuit(-1);
            }
            boolean invalidDir = false;
            String dir = args[ii + 1];
            ii++;
            File f = new File(dir);
            if (!f.exists()) {
                System.err.println("Error: " + dir + " does not exist");
                invalidDir = true;
            }
            if (!f.canRead()) {
                System.err.println("Error: " + dir + " does not have read permission set");
                invalidDir = true;
            }
            if (!f.canExecute()) {
                System.err.println("Error: " + dir + " does not have execute permission set");
                invalidDir = true;
            }
            directories.add(f);
            if (invalidDir) {
                System.exit(-1);
            }
        } else if (arg.equals("--timezone")) {
            if (args.length < ii + 1) {
                System.err.println("Error: Not enough args following --timezone");
                printHelpAndQuit(-1);
            }
            String tzId = args[ii + 1];
            ii++;
            VoltTableUtil.tz = TimeZone.getTimeZone(tzId);
        } else if (arg.equals("--table")) {
            if (args.length < ii + 1) {
                System.err.println("Error: Not enough args following --tables");
                printHelpAndQuit(-1);
            }
            tables.add(args[ii + 1].toUpperCase());
            ii++;
        } else if (arg.equals("--outdir")) {
            if (args.length < ii + 1) {
                System.err.println("Error: Not enough args following --outdir");
                printHelpAndQuit(-1);
            }
            boolean invalidDir = false;
            outdir = new File(args[ii + 1]);
            if (!outdir.exists()) {
                System.err.println("Error: " + outdir.getPath() + " does not exist");
                invalidDir = true;
            }
            if (!outdir.canRead()) {
                System.err.println("Error: " + outdir.getPath() + " does not have read permission set");
                invalidDir = true;
            }
            if (!outdir.canExecute()) {
                System.err.println("Error: " + outdir.getPath() + " does not have execute permission set");
                invalidDir = true;
            }
            if (!outdir.canWrite()) {
                System.err.println("Error: " + outdir.getPath() + " does not have write permission set");
                invalidDir = true;
            }
            if (invalidDir) {
                System.exit(-1);
            }
            ii++;
        } else if (arg.equals("--type")) {
            if (args.length < ii + 1) {
                System.err.println("Error: Not enough args following --type");
                printHelpAndQuit(-1);
            }
            type = args[ii + 1];
            if (type.equalsIgnoreCase("csv")) {
                delimiter = ',';
            } else if (type.equalsIgnoreCase("tsv")) {
                delimiter = '\t';
            } else {
                System.err.println("Error: --type must be one of CSV or TSV");
                printHelpAndQuit(-1);
            }
            ii++;
        } else {
            if (snapshotName != null) {
                System.err.println("Error: Multiple snapshots specified for conversion. First - " + snapshotName + " second " + args[ii]);
                printHelpAndQuit(-1);
            }
            snapshotName = args[ii];
        }
    }
    boolean fail = false;
    if (snapshotName == null) {
        System.err.println("Error: No --name specified");
        fail = true;
    }
    if (directories.isEmpty()) {
        directories.add(new File("."));
    }
    if (tables.isEmpty()) {
        System.err.println("Error: No --tables specified");
        fail = true;
    }
    if (outdir == null) {
        outdir = new File(".");
    }
    if (type == null) {
        System.err.println("Error: No --type specified");
        fail = true;
    }
    if (fail) {
        printHelpAndQuit(-1);
    }
    Map<String, Snapshot> snapshots = new TreeMap<String, Snapshot>();
    HashSet<String> snapshotNames = new HashSet<String>();
    snapshotNames.add(snapshotName);
    SpecificSnapshotFilter filter = new SpecificSnapshotFilter(snapshotNames);
    for (File directory : directories) {
        SnapshotUtil.retrieveSnapshotFiles(directory, snapshots, filter, false, SnapshotPathType.SNAP_PATH, CONSOLE_LOG);
    }
    if (snapshots.size() > 1) {
        System.err.println("Error: Found " + snapshots.size() + " snapshots with specified name");
        int ii = 0;
        for (Snapshot entry : snapshots.values()) {
            System.err.println("Snapshot " + ii + " taken " + new Date(entry.getInstanceId().getTimestamp()));
            System.err.println("Files: ");
            for (File digest : entry.m_digests) {
                System.err.println("\t" + digest.getPath());
            }
            for (Map.Entry<String, TableFiles> e2 : entry.m_tableFiles.entrySet()) {
                System.err.println("\t" + e2.getKey());
                for (File tableFile : e2.getValue().m_files) {
                    System.err.println("\t\t" + tableFile.getPath());
                }
            }
            ii++;
        }
        System.exit(-1);
    }
    if (snapshots.size() < 1) {
        System.err.println("Error: Did not find any snapshots with the specified name");
        System.exit(-1);
    }
    /*
         * Build a plan for what partitions to pull from which save file
         */
    final Snapshot snapshot = snapshots.values().iterator().next();
    Map<String, Map<File, Set<Integer>>> tableToFilesWithPartitions = new TreeMap<String, Map<File, Set<Integer>>>();
    for (String tableName : tables) {
        if (!snapshot.m_tableFiles.containsKey(tableName)) {
            System.err.println("Error: Snapshot does not contain table " + tableName);
            System.exit(-1);
        }
        TableFiles tableFiles = snapshot.m_tableFiles.get(tableName);
        if (!tableFiles.m_isReplicated) {
            TreeSet<Integer> partitionsIds = new TreeSet<Integer>();
            Map<File, Set<Integer>> partitionsFromFile = new TreeMap<File, Set<Integer>>();
            for (int ii = 0; ii < tableFiles.m_files.size(); ii++) {
                Set<Integer> validParititions = tableFiles.m_validPartitionIds.get(ii);
                TreeSet<Integer> partitionsToTake = new TreeSet<Integer>(validParititions);
                partitionsToTake.removeAll(partitionsIds);
                partitionsIds.addAll(validParititions);
                if (!partitionsToTake.isEmpty()) {
                    partitionsFromFile.put(tableFiles.m_files.get(ii), partitionsToTake);
                }
            }
            int totalPartitionCount = tableFiles.m_totalPartitionCounts.get(0);
            if (!((partitionsIds.size() == totalPartitionCount) && (partitionsIds.first() == 0) && (partitionsIds.last() == totalPartitionCount - 1))) {
                System.err.println("Error: Not all partitions present for table " + tableName);
                fail = true;
            } else {
                tableToFilesWithPartitions.put(tableName, partitionsFromFile);
            }
        } else {
            Map<File, Set<Integer>> partitionsFromFile = new TreeMap<File, Set<Integer>>();
            partitionsFromFile.put(tableFiles.m_files.get(0), null);
            tableToFilesWithPartitions.put(tableName, partitionsFromFile);
        }
    }
    if (fail) {
        System.exit(-1);
    }
    /*
         * For every output file that will be created attempt to instantiate and print an error
         * if the file already exists or couldn't be created.
         */
    for (Map.Entry<String, Map<File, Set<Integer>>> entry : tableToFilesWithPartitions.entrySet()) {
        String tableName = entry.getKey();
        File outfile = new File(outdir.getPath() + File.separator + tableName + "." + type.toLowerCase());
        try {
            if (!outfile.createNewFile()) {
                System.err.println("Error: Failed to create output file " + outfile.getPath() + " for table " + tableName + "\n File already exists");
                fail = true;
            }
        } catch (IOException e) {
            System.err.println(e.getMessage());
            System.err.println("Error: Failed to create output file " + outfile.getPath() + " for table " + tableName);
            fail = true;
        }
    }
    if (fail) {
        System.exit(-1);
    }
    /*
         * Actually convert the tables and write the data to the appropriate destination
         */
    for (Map.Entry<String, Map<File, Set<Integer>>> entry : tableToFilesWithPartitions.entrySet()) {
        String tableName = entry.getKey();
        File outfile = new File(outdir.getPath() + File.separator + tableName + "." + type.toLowerCase());
        Map<File, Set<Integer>> partitionsFromFile = entry.getValue();
        for (Map.Entry<File, Set<Integer>> e2 : partitionsFromFile.entrySet()) {
            File infile = e2.getKey();
            Set<Integer> partitionSet = e2.getValue();
            Integer[] partitions = null;
            if (partitionSet != null) {
                partitions = new Integer[partitionSet.size()];
                int ii = 0;
                for (Integer partition : partitionSet) {
                    partitions[ii++] = partition;
                }
            }
            try {
                CSVTableSaveFile.convertTableSaveFile(delimiter, partitions, outfile, infile);
            } catch (Exception e) {
                System.err.println(e.getMessage());
                System.err.println("Error: Failed to convert " + infile.getPath() + " to " + outfile.getPath());
            }
        }
    }
    if (fail) {
        System.exit(-1);
    }
}

Example 4 with Snapshot

use of org.voltdb.sysprocs.saverestore.SnapshotUtil.Snapshot in project voltdb by VoltDB.

the class SnapshotVerifier method verifySnapshots.

/**
     * Perform snapshot verification.
     * @param directories list of directories to search for snapshots
     * @param snapshotNames set of snapshot names/nonces to verify
     */
public static void verifySnapshots(final List<String> directories, final Set<String> snapshotNames, boolean expectHashinator) {
    FileFilter filter = new SnapshotFilter();
    if (!snapshotNames.isEmpty()) {
        filter = new SpecificSnapshotFilter(snapshotNames);
    }
    Map<String, Snapshot> snapshots = new HashMap<String, Snapshot>();
    for (String directory : directories) {
        SnapshotUtil.retrieveSnapshotFiles(new File(directory), snapshots, filter, true, SnapshotPathType.SNAP_PATH, CONSOLE_LOG);
    }
    if (snapshots.isEmpty()) {
        System.out.println("Snapshot corrupted");
        System.out.println("No files found");
    }
    for (Snapshot s : snapshots.values()) {
        System.out.println(SnapshotUtil.generateSnapshotReport(s.getTxnId(), s, expectHashinator).getSecond());
    }
}