Examples with SpecificSnapshotFilter org.voltdb.sysprocs.saverestore.SnapshotUtil.SpecificSnapshotFilter used on opensource projects

Example 1 with SpecificSnapshotFilter

use of org.voltdb.sysprocs.saverestore.SnapshotUtil.SpecificSnapshotFilter in project voltdb by VoltDB.

the class SnapshotConverter method main.

/**
     * @param args
     */
public static void main(String[] args) {
    String snapshotName = null;
    ArrayList<File> directories = new ArrayList<File>();
    ArrayList<String> tables = new ArrayList<String>();
    File outdir = null;
    String type = null;
    char delimiter = '\0';
    for (int ii = 0; ii < args.length; ii++) {
        String arg = args[ii];
        if (arg.equals("--help")) {
            printHelpAndQuit(0);
        } else if (arg.equals("--dir")) {
            if (args.length < ii + 1) {
                System.err.println("Error: Not enough args following --dirs");
                printHelpAndQuit(-1);
            }
            boolean invalidDir = false;
            String dir = args[ii + 1];
            ii++;
            File f = new File(dir);
            if (!f.exists()) {
                System.err.println("Error: " + dir + " does not exist");
                invalidDir = true;
            }
            if (!f.canRead()) {
                System.err.println("Error: " + dir + " does not have read permission set");
                invalidDir = true;
            }
            if (!f.canExecute()) {
                System.err.println("Error: " + dir + " does not have execute permission set");
                invalidDir = true;
            }
            directories.add(f);
            if (invalidDir) {
                System.exit(-1);
            }
        } else if (arg.equals("--timezone")) {
            if (args.length < ii + 1) {
                System.err.println("Error: Not enough args following --timezone");
                printHelpAndQuit(-1);
            }
            String tzId = args[ii + 1];
            ii++;
            VoltTableUtil.tz = TimeZone.getTimeZone(tzId);
        } else if (arg.equals("--table")) {
            if (args.length < ii + 1) {
                System.err.println("Error: Not enough args following --tables");
                printHelpAndQuit(-1);
            }
            tables.add(args[ii + 1].toUpperCase());
            ii++;
        } else if (arg.equals("--outdir")) {
            if (args.length < ii + 1) {
                System.err.println("Error: Not enough args following --outdir");
                printHelpAndQuit(-1);
            }
            boolean invalidDir = false;
            outdir = new File(args[ii + 1]);
            if (!outdir.exists()) {
                System.err.println("Error: " + outdir.getPath() + " does not exist");
                invalidDir = true;
            }
            if (!outdir.canRead()) {
                System.err.println("Error: " + outdir.getPath() + " does not have read permission set");
                invalidDir = true;
            }
            if (!outdir.canExecute()) {
                System.err.println("Error: " + outdir.getPath() + " does not have execute permission set");
                invalidDir = true;
            }
            if (!outdir.canWrite()) {
                System.err.println("Error: " + outdir.getPath() + " does not have write permission set");
                invalidDir = true;
            }
            if (invalidDir) {
                System.exit(-1);
            }
            ii++;
        } else if (arg.equals("--type")) {
            if (args.length < ii + 1) {
                System.err.println("Error: Not enough args following --type");
                printHelpAndQuit(-1);
            }
            type = args[ii + 1];
            if (type.equalsIgnoreCase("csv")) {
                delimiter = ',';
            } else if (type.equalsIgnoreCase("tsv")) {
                delimiter = '\t';
            } else {
                System.err.println("Error: --type must be one of CSV or TSV");
                printHelpAndQuit(-1);
            }
            ii++;
        } else {
            if (snapshotName != null) {
                System.err.println("Error: Multiple snapshots specified for conversion. First - " + snapshotName + " second " + args[ii]);
                printHelpAndQuit(-1);
            }
            snapshotName = args[ii];
        }
    }
    boolean fail = false;
    if (snapshotName == null) {
        System.err.println("Error: No --name specified");
        fail = true;
    }
    if (directories.isEmpty()) {
        directories.add(new File("."));
    }
    if (tables.isEmpty()) {
        System.err.println("Error: No --tables specified");
        fail = true;
    }
    if (outdir == null) {
        outdir = new File(".");
    }
    if (type == null) {
        System.err.println("Error: No --type specified");
        fail = true;
    }
    if (fail) {
        printHelpAndQuit(-1);
    }
    Map<String, Snapshot> snapshots = new TreeMap<String, Snapshot>();
    HashSet<String> snapshotNames = new HashSet<String>();
    snapshotNames.add(snapshotName);
    SpecificSnapshotFilter filter = new SpecificSnapshotFilter(snapshotNames);
    for (File directory : directories) {
        SnapshotUtil.retrieveSnapshotFiles(directory, snapshots, filter, false, SnapshotPathType.SNAP_PATH, CONSOLE_LOG);
    }
    if (snapshots.size() > 1) {
        System.err.println("Error: Found " + snapshots.size() + " snapshots with specified name");
        int ii = 0;
        for (Snapshot entry : snapshots.values()) {
            System.err.println("Snapshot " + ii + " taken " + new Date(entry.getInstanceId().getTimestamp()));
            System.err.println("Files: ");
            for (File digest : entry.m_digests) {
                System.err.println("\t" + digest.getPath());
            }
            for (Map.Entry<String, TableFiles> e2 : entry.m_tableFiles.entrySet()) {
                System.err.println("\t" + e2.getKey());
                for (File tableFile : e2.getValue().m_files) {
                    System.err.println("\t\t" + tableFile.getPath());
                }
            }
            ii++;
        }
        System.exit(-1);
    }
    if (snapshots.size() < 1) {
        System.err.println("Error: Did not find any snapshots with the specified name");
        System.exit(-1);
    }
    /*
         * Build a plan for what partitions to pull from which save file
         */
    final Snapshot snapshot = snapshots.values().iterator().next();
    Map<String, Map<File, Set<Integer>>> tableToFilesWithPartitions = new TreeMap<String, Map<File, Set<Integer>>>();
    for (String tableName : tables) {
        if (!snapshot.m_tableFiles.containsKey(tableName)) {
            System.err.println("Error: Snapshot does not contain table " + tableName);
            System.exit(-1);
        }
        TableFiles tableFiles = snapshot.m_tableFiles.get(tableName);
        if (!tableFiles.m_isReplicated) {
            TreeSet<Integer> partitionsIds = new TreeSet<Integer>();
            Map<File, Set<Integer>> partitionsFromFile = new TreeMap<File, Set<Integer>>();
            for (int ii = 0; ii < tableFiles.m_files.size(); ii++) {
                Set<Integer> validParititions = tableFiles.m_validPartitionIds.get(ii);
                TreeSet<Integer> partitionsToTake = new TreeSet<Integer>(validParititions);
                partitionsToTake.removeAll(partitionsIds);
                partitionsIds.addAll(validParititions);
                if (!partitionsToTake.isEmpty()) {
                    partitionsFromFile.put(tableFiles.m_files.get(ii), partitionsToTake);
                }
            }
            int totalPartitionCount = tableFiles.m_totalPartitionCounts.get(0);
            if (!((partitionsIds.size() == totalPartitionCount) && (partitionsIds.first() == 0) && (partitionsIds.last() == totalPartitionCount - 1))) {
                System.err.println("Error: Not all partitions present for table " + tableName);
                fail = true;
            } else {
                tableToFilesWithPartitions.put(tableName, partitionsFromFile);
            }
        } else {
            Map<File, Set<Integer>> partitionsFromFile = new TreeMap<File, Set<Integer>>();
            partitionsFromFile.put(tableFiles.m_files.get(0), null);
            tableToFilesWithPartitions.put(tableName, partitionsFromFile);
        }
    }
    if (fail) {
        System.exit(-1);
    }
    /*
         * For every output file that will be created attempt to instantiate and print an error
         * if the file already exists or couldn't be created.
         */
    for (Map.Entry<String, Map<File, Set<Integer>>> entry : tableToFilesWithPartitions.entrySet()) {
        String tableName = entry.getKey();
        File outfile = new File(outdir.getPath() + File.separator + tableName + "." + type.toLowerCase());
        try {
            if (!outfile.createNewFile()) {
                System.err.println("Error: Failed to create output file " + outfile.getPath() + " for table " + tableName + "\n File already exists");
                fail = true;
            }
        } catch (IOException e) {
            System.err.println(e.getMessage());
            System.err.println("Error: Failed to create output file " + outfile.getPath() + " for table " + tableName);
            fail = true;
        }
    }
    if (fail) {
        System.exit(-1);
    }
    /*
         * Actually convert the tables and write the data to the appropriate destination
         */
    for (Map.Entry<String, Map<File, Set<Integer>>> entry : tableToFilesWithPartitions.entrySet()) {
        String tableName = entry.getKey();
        File outfile = new File(outdir.getPath() + File.separator + tableName + "." + type.toLowerCase());
        Map<File, Set<Integer>> partitionsFromFile = entry.getValue();
        for (Map.Entry<File, Set<Integer>> e2 : partitionsFromFile.entrySet()) {
            File infile = e2.getKey();
            Set<Integer> partitionSet = e2.getValue();
            Integer[] partitions = null;
            if (partitionSet != null) {
                partitions = new Integer[partitionSet.size()];
                int ii = 0;
                for (Integer partition : partitionSet) {
                    partitions[ii++] = partition;
                }
            }
            try {
                CSVTableSaveFile.convertTableSaveFile(delimiter, partitions, outfile, infile);
            } catch (Exception e) {
                System.err.println(e.getMessage());
                System.err.println("Error: Failed to convert " + infile.getPath() + " to " + outfile.getPath());
            }
        }
    }
    if (fail) {
        System.exit(-1);
    }
}

Example 2 with SpecificSnapshotFilter

use of org.voltdb.sysprocs.saverestore.SnapshotUtil.SpecificSnapshotFilter in project voltdb by VoltDB.

the class SnapshotVerifier method verifySnapshots.

/**
     * Perform snapshot verification.
     * @param directories list of directories to search for snapshots
     * @param snapshotNames set of snapshot names/nonces to verify
     */
public static void verifySnapshots(final List<String> directories, final Set<String> snapshotNames, boolean expectHashinator) {
    FileFilter filter = new SnapshotFilter();
    if (!snapshotNames.isEmpty()) {
        filter = new SpecificSnapshotFilter(snapshotNames);
    }
    Map<String, Snapshot> snapshots = new HashMap<String, Snapshot>();
    for (String directory : directories) {
        SnapshotUtil.retrieveSnapshotFiles(new File(directory), snapshots, filter, true, SnapshotPathType.SNAP_PATH, CONSOLE_LOG);
    }
    if (snapshots.isEmpty()) {
        System.out.println("Snapshot corrupted");
        System.out.println("No files found");
    }
    for (Snapshot s : snapshots.values()) {
        System.out.println(SnapshotUtil.generateSnapshotReport(s.getTxnId(), s, expectHashinator).getSecond());
    }
}