Examples with BlobReferenceRetriever org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever used on opensource projects

Example 1 with BlobReferenceRetriever

use of org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever in project jackrabbit-oak by apache.

the class DataStoreCheckCommand method execute.

@Override
public void execute(String... args) throws Exception {
    OptionParser parser = new OptionParser();
    parser.allowsUnrecognizedOptions();
    String helpStr = "datastorecheck [--id] [--ref] [--consistency] [--store <path>|<mongo_uri>] " + "[--s3ds <s3ds_config>|--fds <fds_config>|--azureblobds <azureblobds_config>] [--dump <path>]";
    Closer closer = Closer.create();
    try {
        // Options for operations requested
        OptionSpecBuilder idOp = parser.accepts("id", "Get ids");
        OptionSpecBuilder refOp = parser.accepts("ref", "Get references");
        OptionSpecBuilder consistencyOp = parser.accepts("consistency", "Check consistency");
        // Node Store - needed for --ref, --consistency
        ArgumentAcceptingOptionSpec<String> store = parser.accepts("store", "Node Store").requiredIf(refOp, consistencyOp).withRequiredArg().ofType(String.class);
        // Optional argument to specify the dump path
        ArgumentAcceptingOptionSpec<String> dump = parser.accepts("dump", "Dump Path").withRequiredArg().ofType(String.class);
        // Optional argument to specify tracking
        ArgumentAcceptingOptionSpec<String> track = parser.accepts("track", "Local repository home folder").withRequiredArg().ofType(String.class);
        OptionSpec<?> help = parser.acceptsAll(asList("h", "?", "help"), "show help").forHelp();
        // Required rules (any one of --id, --ref, --consistency)
        idOp.requiredUnless(refOp, consistencyOp);
        refOp.requiredUnless(idOp, consistencyOp);
        consistencyOp.requiredUnless(idOp, refOp);
        OptionSet options = null;
        try {
            options = parser.parse(args);
        } catch (Exception e) {
            System.err.println(e);
            parser.printHelpOn(System.err);
            return;
        }
        if (options.has(help)) {
            parser.printHelpOn(System.out);
            return;
        }
        String dumpPath = JAVA_IO_TMPDIR.value();
        if (options.has(dump)) {
            dumpPath = options.valueOf(dump);
        }
        GarbageCollectableBlobStore blobStore = null;
        BlobReferenceRetriever marker = null;
        if (options.has(store)) {
            String source = options.valueOf(store);
            if (source.startsWith(MongoURI.MONGODB_PREFIX)) {
                MongoClientURI uri = new MongoClientURI(source);
                MongoClient client = new MongoClient(uri);
                DocumentNodeStore nodeStore = new DocumentMK.Builder().setMongoDB(client.getDB(uri.getDatabase())).getNodeStore();
                closer.register(Utils.asCloseable(nodeStore));
                blobStore = (GarbageCollectableBlobStore) nodeStore.getBlobStore();
                marker = new DocumentBlobReferenceRetriever(nodeStore);
            } else {
                marker = SegmentTarUtils.newBlobReferenceRetriever(source, closer);
            }
        }
        // Initialize S3/FileDataStore if configured
        GarbageCollectableBlobStore dataStore = Utils.bootstrapDataStore(args, closer);
        if (dataStore != null) {
            blobStore = dataStore;
        }
        // blob store still not initialized means configuration not supported
        if (blobStore == null) {
            System.err.println("Operation not defined for SegmentNodeStore without external datastore");
            parser.printHelpOn(System.err);
            return;
        }
        FileRegister register = new FileRegister(options);
        closer.register(register);
        if (options.has(idOp) || options.has(consistencyOp)) {
            File dumpFile = register.createFile(idOp, dumpPath);
            retrieveBlobIds(blobStore, dumpFile);
            // If track path specified copy the file to the location
            if (options.has(track)) {
                String trackPath = options.valueOf(track);
                File trackingFileParent = new File(FilenameUtils.concat(trackPath, "blobids"));
                File trackingFile = new File(trackingFileParent, "blob-" + String.valueOf(System.currentTimeMillis()) + ".gen");
                FileUtils.copyFile(dumpFile, trackingFile);
            }
        }
        if (options.has(refOp) || options.has(consistencyOp)) {
            retrieveBlobReferences(blobStore, marker, register.createFile(refOp, dumpPath));
        }
        if (options.has(consistencyOp)) {
            checkConsistency(register.get(idOp), register.get(refOp), register.createFile(consistencyOp, dumpPath));
        }
    } catch (Throwable t) {
        t.printStackTrace();
    } finally {
        closer.close();
    }
}

Example 2 with BlobReferenceRetriever

use of org.apache.jackrabbit.oak.plugins.blob.BlobReferenceRetriever in project jackrabbit-oak by apache.

the class DataStoreCheckCommand method checkDataStore.

static int checkDataStore(String... args) {
    OptionParser parser = new OptionParser();
    parser.allowsUnrecognizedOptions();
    String helpStr = "datastorecheck [--id] [--ref] [--consistency] [--store <path>|<mongo_uri>] " + "[--s3ds <s3ds_config>|--fds <fds_config>|--azureblobds <azureblobds_config>|--nods]" + " [--dump <path>] [--repoHome <repo_home>] [--track] [--verbose]";
    try (Closer closer = Closer.create()) {
        // Options for operations requested
        OptionSpecBuilder idOp = parser.accepts("id", "Get ids");
        OptionSpecBuilder refOp = parser.accepts("ref", "Get references");
        OptionSpecBuilder consistencyOp = parser.accepts("consistency", "Check consistency");
        // Node Store - needed for --ref, --consistency
        ArgumentAcceptingOptionSpec<String> store = parser.accepts("store", "Node Store").requiredIf(refOp, consistencyOp).withRequiredArg().ofType(String.class);
        // Optional argument to specify the dump path
        ArgumentAcceptingOptionSpec<String> dump = parser.accepts("dump", "Dump Path").withRequiredArg().ofType(String.class);
        // Optional argument to specify tracking
        OptionSpecBuilder trackOverride = parser.accepts("track", "Force override tracked ids");
        // Required argument for --consistency to specify tracking folder (otherwise can have inconsistencies)
        ArgumentAcceptingOptionSpec<String> repoHome = parser.accepts("repoHome", "Local repository home folder").requiredIf(trackOverride, consistencyOp).withRequiredArg().ofType(String.class);
        // Optional argument to specify tracking
        OptionSpecBuilder verbose = parser.accepts("verbose", "Output backend formatted ids/paths");
        OptionSpec<?> help = parser.acceptsAll(asList("h", "?", "help"), "show help").forHelp();
        // Required rules (any one of --id, --ref, --consistency)
        idOp.requiredUnless(refOp, consistencyOp);
        refOp.requiredUnless(idOp, consistencyOp);
        consistencyOp.requiredUnless(idOp, refOp);
        trackOverride.availableIf(idOp, consistencyOp);
        OptionSet options = null;
        try {
            options = parser.parse(args);
        } catch (Exception e) {
            System.err.println(e);
            System.err.println(Arrays.toString(args));
            System.err.println();
            System.err.println("Options :");
            parser.printHelpOn(System.err);
            return 1;
        }
        if (options.has(help)) {
            parser.printHelpOn(System.out);
            return 0;
        }
        String dumpPath = JAVA_IO_TMPDIR.value();
        if (options.has(dump)) {
            dumpPath = options.valueOf(dump);
        }
        GarbageCollectableBlobStore blobStore = null;
        BlobReferenceRetriever marker = null;
        NodeStore nodeStore = null;
        if (options.has(store)) {
            String source = options.valueOf(store);
            if (source.startsWith(MongoURI.MONGODB_PREFIX)) {
                MongoClientURI uri = new MongoClientURI(source);
                MongoClient client = new MongoClient(uri);
                DocumentNodeStore docNodeStore = newMongoDocumentNodeStoreBuilder().setMongoDB(client.getDB(uri.getDatabase())).build();
                closer.register(Utils.asCloseable(docNodeStore));
                blobStore = (GarbageCollectableBlobStore) docNodeStore.getBlobStore();
                marker = new DocumentBlobReferenceRetriever(docNodeStore);
                nodeStore = docNodeStore;
            } else {
                FileStore fileStore = fileStoreBuilder(new File(source)).withStrictVersionCheck(true).build();
                marker = new SegmentBlobReferenceRetriever(fileStore);
                closer.register(fileStore);
                nodeStore = SegmentNodeStoreBuilders.builder(fileStore).build();
            }
        }
        // Initialize S3/FileDataStore if configured
        String dsType = "";
        GarbageCollectableBlobStore dataStore = Utils.bootstrapDataStore(args, closer);
        if (dataStore != null) {
            dsType = getDSType(args);
            blobStore = dataStore;
        }
        // blob store still not initialized means configuration not supported
        if (blobStore == null) {
            System.err.println("Operation not defined for SegmentNodeStore without external datastore");
            parser.printHelpOn(System.err);
            return 1;
        }
        FileRegister register = new FileRegister(options);
        closer.register(register);
        if (options.has(idOp) || options.has(consistencyOp)) {
            File idTemp = createTempFile("ids", null);
            closer.register(new Closeable() {

                @Override
                public void close() throws IOException {
                    forceDelete(idTemp);
                }
            });
            File dumpFile = register.createFile(idOp, dumpPath);
            retrieveBlobIds(blobStore, idTemp);
            // If track path and track override specified copy the file to the location
            if (options.has(repoHome) && options.has(trackOverride)) {
                String trackPath = options.valueOf(repoHome);
                File trackingFileParent = new File(FilenameUtils.concat(trackPath, "blobids"));
                File trackingFile = new File(trackingFileParent, "blob-" + String.valueOf(System.currentTimeMillis()) + ".gen");
                FileUtils.copyFile(idTemp, trackingFile);
            }
            if (options.has(verbose)) {
                verboseIds(closer, dsType, idTemp, dumpFile);
            } else {
                FileUtils.copyFile(idTemp, dumpFile);
            }
        }
        if (options.has(refOp) || options.has(consistencyOp)) {
            if (options.has(verbose) && (nodeStore instanceof SegmentNodeStore || nodeStore instanceof org.apache.jackrabbit.oak.segment.SegmentNodeStore)) {
                NodeTraverser traverser = new NodeTraverser(nodeStore, dsType);
                closer.register(traverser);
                traverser.traverse();
                FileUtils.copyFile(traverser.references, register.createFile(refOp, dumpPath));
            } else {
                retrieveBlobReferences(blobStore, marker, register.createFile(refOp, dumpPath), dsType, options.has(verbose));
            }
        }
        if (options.has(consistencyOp)) {
            checkConsistency(register.get(idOp), register.get(refOp), register.createFile(consistencyOp, dumpPath), options.valueOf(repoHome), dsType);
        }
        return 0;
    } catch (Throwable t) {
        t.printStackTrace(System.err);
        return 1;
    }
}