Examples with GarbageCollectableBlobStore org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore used on opensource projects

Example 1 with GarbageCollectableBlobStore

use of org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore in project jackrabbit-oak by apache.

the class DocumentNodeStoreService method registerJMXBeans.

private void registerJMXBeans(final DocumentNodeStore store, DocumentMK.Builder mkBuilder) throws IOException {
    addRegistration(registerMBean(whiteboard, CacheStatsMBean.class, store.getNodeCacheStats(), CacheStatsMBean.TYPE, store.getNodeCacheStats().getName()));
    addRegistration(registerMBean(whiteboard, CacheStatsMBean.class, store.getNodeChildrenCacheStats(), CacheStatsMBean.TYPE, store.getNodeChildrenCacheStats().getName()));
    for (CacheStats cs : store.getDiffCacheStats()) {
        addRegistration(registerMBean(whiteboard, CacheStatsMBean.class, cs, CacheStatsMBean.TYPE, cs.getName()));
    }
    DocumentStore ds = store.getDocumentStore();
    if (ds.getCacheStats() != null) {
        for (CacheStats cacheStats : ds.getCacheStats()) {
            addRegistration(registerMBean(whiteboard, CacheStatsMBean.class, cacheStats, CacheStatsMBean.TYPE, cacheStats.getName()));
        }
    }
    addRegistration(registerMBean(whiteboard, CheckpointMBean.class, new DocumentCheckpointMBean(store), CheckpointMBean.TYPE, "Document node store checkpoint management"));
    addRegistration(registerMBean(whiteboard, DocumentNodeStoreMBean.class, store.getMBean(), DocumentNodeStoreMBean.TYPE, "Document node store management"));
    if (mkBuilder.getBlobStoreCacheStats() != null) {
        addRegistration(registerMBean(whiteboard, CacheStatsMBean.class, mkBuilder.getBlobStoreCacheStats(), CacheStatsMBean.TYPE, mkBuilder.getBlobStoreCacheStats().getName()));
    }
    if (mkBuilder.getDocumentStoreStatsCollector() instanceof DocumentStoreStatsMBean) {
        addRegistration(registerMBean(whiteboard, DocumentStoreStatsMBean.class, (DocumentStoreStatsMBean) mkBuilder.getDocumentStoreStatsCollector(), DocumentStoreStatsMBean.TYPE, "DocumentStore Statistics"));
    }
    // register persistent cache stats
    Map<CacheType, PersistentCacheStats> persistenceCacheStats = mkBuilder.getPersistenceCacheStats();
    for (PersistentCacheStats pcs : persistenceCacheStats.values()) {
        addRegistration(registerMBean(whiteboard, PersistentCacheStatsMBean.class, pcs, PersistentCacheStatsMBean.TYPE, pcs.getName()));
    }
    final long versionGcMaxAgeInSecs = toLong(prop(PROP_VER_GC_MAX_AGE), DEFAULT_VER_GC_MAX_AGE);
    final long blobGcMaxAgeInSecs = toLong(prop(PROP_BLOB_GC_MAX_AGE), DEFAULT_BLOB_GC_MAX_AGE);
    if (store.getBlobStore() instanceof GarbageCollectableBlobStore) {
        BlobGarbageCollector gc = store.createBlobGarbageCollector(blobGcMaxAgeInSecs, ClusterRepositoryInfo.getOrCreateId(nodeStore));
        addRegistration(registerMBean(whiteboard, BlobGCMBean.class, new BlobGC(gc, executor), BlobGCMBean.TYPE, "Document node store blob garbage collection"));
    }
    Runnable startGC = new Runnable() {

        @Override
        public void run() {
            try {
                store.getVersionGarbageCollector().gc(versionGcMaxAgeInSecs, TimeUnit.SECONDS);
            } catch (IOException e) {
                log.warn("Error occurred while executing the Version Garbage Collector", e);
            }
        }
    };
    Runnable cancelGC = new Runnable() {

        @Override
        public void run() {
            store.getVersionGarbageCollector().cancel();
        }
    };
    Supplier<String> status = new Supplier<String>() {

        @Override
        public String get() {
            return store.getVersionGarbageCollector().getStatus();
        }
    };
    RevisionGC revisionGC = new RevisionGC(startGC, cancelGC, status, executor);
    addRegistration(registerMBean(whiteboard, RevisionGCMBean.class, revisionGC, RevisionGCMBean.TYPE, "Document node store revision garbage collection"));
    BlobStoreStats blobStoreStats = mkBuilder.getBlobStoreStats();
    if (!customBlobStore && blobStoreStats != null) {
        addRegistration(registerMBean(whiteboard, BlobStoreStatsMBean.class, blobStoreStats, BlobStoreStatsMBean.TYPE, ds.getClass().getSimpleName()));
    }
    if (!mkBuilder.isBundlingDisabled()) {
        addRegistration(registerMBean(whiteboard, BackgroundObserverMBean.class, store.getBundlingConfigHandler().getMBean(), BackgroundObserverMBean.TYPE, "BundlingConfigObserver"));
    }
}

Example 2 with GarbageCollectableBlobStore

use of org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore in project jackrabbit-oak by apache.

the class DataStoreCheckCommand method execute.

@Override
public void execute(String... args) throws Exception {
    OptionParser parser = new OptionParser();
    parser.allowsUnrecognizedOptions();
    String helpStr = "datastorecheck [--id] [--ref] [--consistency] [--store <path>|<mongo_uri>] " + "[--s3ds <s3ds_config>|--fds <fds_config>|--azureblobds <azureblobds_config>] [--dump <path>]";
    Closer closer = Closer.create();
    try {
        // Options for operations requested
        OptionSpecBuilder idOp = parser.accepts("id", "Get ids");
        OptionSpecBuilder refOp = parser.accepts("ref", "Get references");
        OptionSpecBuilder consistencyOp = parser.accepts("consistency", "Check consistency");
        // Node Store - needed for --ref, --consistency
        ArgumentAcceptingOptionSpec<String> store = parser.accepts("store", "Node Store").requiredIf(refOp, consistencyOp).withRequiredArg().ofType(String.class);
        // Optional argument to specify the dump path
        ArgumentAcceptingOptionSpec<String> dump = parser.accepts("dump", "Dump Path").withRequiredArg().ofType(String.class);
        // Optional argument to specify tracking
        ArgumentAcceptingOptionSpec<String> track = parser.accepts("track", "Local repository home folder").withRequiredArg().ofType(String.class);
        OptionSpec<?> help = parser.acceptsAll(asList("h", "?", "help"), "show help").forHelp();
        // Required rules (any one of --id, --ref, --consistency)
        idOp.requiredUnless(refOp, consistencyOp);
        refOp.requiredUnless(idOp, consistencyOp);
        consistencyOp.requiredUnless(idOp, refOp);
        OptionSet options = null;
        try {
            options = parser.parse(args);
        } catch (Exception e) {
            System.err.println(e);
            parser.printHelpOn(System.err);
            return;
        }
        if (options.has(help)) {
            parser.printHelpOn(System.out);
            return;
        }
        String dumpPath = JAVA_IO_TMPDIR.value();
        if (options.has(dump)) {
            dumpPath = options.valueOf(dump);
        }
        GarbageCollectableBlobStore blobStore = null;
        BlobReferenceRetriever marker = null;
        if (options.has(store)) {
            String source = options.valueOf(store);
            if (source.startsWith(MongoURI.MONGODB_PREFIX)) {
                MongoClientURI uri = new MongoClientURI(source);
                MongoClient client = new MongoClient(uri);
                DocumentNodeStore nodeStore = new DocumentMK.Builder().setMongoDB(client.getDB(uri.getDatabase())).getNodeStore();
                closer.register(Utils.asCloseable(nodeStore));
                blobStore = (GarbageCollectableBlobStore) nodeStore.getBlobStore();
                marker = new DocumentBlobReferenceRetriever(nodeStore);
            } else {
                marker = SegmentTarUtils.newBlobReferenceRetriever(source, closer);
            }
        }
        // Initialize S3/FileDataStore if configured
        GarbageCollectableBlobStore dataStore = Utils.bootstrapDataStore(args, closer);
        if (dataStore != null) {
            blobStore = dataStore;
        }
        // blob store still not initialized means configuration not supported
        if (blobStore == null) {
            System.err.println("Operation not defined for SegmentNodeStore without external datastore");
            parser.printHelpOn(System.err);
            return;
        }
        FileRegister register = new FileRegister(options);
        closer.register(register);
        if (options.has(idOp) || options.has(consistencyOp)) {
            File dumpFile = register.createFile(idOp, dumpPath);
            retrieveBlobIds(blobStore, dumpFile);
            // If track path specified copy the file to the location
            if (options.has(track)) {
                String trackPath = options.valueOf(track);
                File trackingFileParent = new File(FilenameUtils.concat(trackPath, "blobids"));
                File trackingFile = new File(trackingFileParent, "blob-" + String.valueOf(System.currentTimeMillis()) + ".gen");
                FileUtils.copyFile(dumpFile, trackingFile);
            }
        }
        if (options.has(refOp) || options.has(consistencyOp)) {
            retrieveBlobReferences(blobStore, marker, register.createFile(refOp, dumpPath));
        }
        if (options.has(consistencyOp)) {
            checkConsistency(register.get(idOp), register.get(refOp), register.createFile(consistencyOp, dumpPath));
        }
    } catch (Throwable t) {
        t.printStackTrace();
    } finally {
        closer.close();
    }
}

Example 3 with GarbageCollectableBlobStore

use of org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore in project jackrabbit-oak by apache.

the class DataStoreTrackerGCTest method init.

public DataStoreState init(DocumentNodeStore s, int idStart) throws Exception {
    NodeBuilder a = s.getRoot().builder();
    int number = 10;
    int maxDeleted = 5;
    // track the number of the assets to be deleted
    List<Integer> processed = Lists.newArrayList();
    Random rand = new Random(47);
    for (int i = idStart; i < idStart + maxDeleted; i++) {
        int n = rand.nextInt(number);
        if (!processed.contains(idStart + n)) {
            processed.add(idStart + n);
        }
    }
    DataStoreState state = new DataStoreState();
    for (int i = idStart; i < idStart + number; i++) {
        Blob b = s.createBlob(randomStream(i, 16516));
        Iterator<String> idIter = ((GarbageCollectableBlobStore) s.getBlobStore()).resolveChunks(b.toString());
        while (idIter.hasNext()) {
            String chunk = idIter.next();
            state.blobsAdded.add(chunk);
            if (!processed.contains(i)) {
                state.blobsPresent.add(chunk);
            }
        }
        a.child("c" + i).setProperty("x", b);
        // Add a duplicated entry
        if (i == idStart) {
            a.child("cdup").setProperty("x", b);
        }
    }
    s.merge(a, INSTANCE, EMPTY);
    a = s.getRoot().builder();
    for (int id : processed) {
        a.child("c" + id).remove();
        s.merge(a, INSTANCE, EMPTY);
    }
    // minutes
    long maxAge = 10;
    // 1. Go past GC age and check no GC done as nothing deleted
    clock.waitUntil(clock.getTime() + MINUTES.toMillis(maxAge));
    VersionGarbageCollector vGC = s.getVersionGarbageCollector();
    VersionGarbageCollector.VersionGCStats stats = vGC.gc(0, MILLISECONDS);
    return state;
}

Example 4 with GarbageCollectableBlobStore

use of org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore in project jackrabbit-oak by apache.

the class DataStoreTrackerGCTest method addNodeSpecialChars.

private HashSet<String> addNodeSpecialChars(DocumentNodeStore ds) throws Exception {
    List<String> specialCharSets = Lists.newArrayList("q\\%22afdg\\%22", "a\nbcd", "a\n\rabcd", "012\\efg");
    HashSet<String> set = new HashSet<String>();
    NodeBuilder a = ds.getRoot().builder();
    int toBeDeleted = 0;
    for (int i = 0; i < specialCharSets.size(); i++) {
        Blob b = ds.createBlob(randomStream(i, 18432));
        NodeBuilder n = a.child("cspecial" + i);
        n.child(specialCharSets.get(i)).setProperty("x", b);
        Iterator<String> idIter = ((GarbageCollectableBlobStore) ds.getBlobStore()).resolveChunks(b.toString());
        List<String> ids = Lists.newArrayList(idIter);
        if (toBeDeleted != i) {
            set.addAll(ids);
        }
    }
    ds.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
    // Delete one node again
    a = ds.getRoot().builder();
    a.child("cspecial" + 0).remove();
    ds.merge(a, INSTANCE, EMPTY);
    // minutes
    long maxAge = 10;
    // 1. Go past GC age and check no GC done as nothing deleted
    clock.waitUntil(clock.getTime() + MINUTES.toMillis(maxAge));
    VersionGarbageCollector vGC = ds.getVersionGarbageCollector();
    VersionGarbageCollector.VersionGCStats stats = vGC.gc(0, MILLISECONDS);
    return set;
}

Example 5 with GarbageCollectableBlobStore

use of org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore in project jackrabbit-oak by apache.

the class MongoBlobGCTest method iterate.

protected Set<String> iterate() throws Exception {
    GarbageCollectableBlobStore store = (GarbageCollectableBlobStore) mk.getNodeStore().getBlobStore();
    Iterator<String> cur = store.getAllChunkIds(0);
    Set<String> existing = Sets.newHashSet();
    while (cur.hasNext()) {
        existing.add(cur.next());
    }
    return existing;
}