use of org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore in project jackrabbit-oak by apache.
the class MongoBlobGCTest method setUp.
public DataStoreState setUp(boolean deleteDirect, int count) throws Exception {
DocumentNodeStore s = mk.getNodeStore();
NodeBuilder a = s.getRoot().builder();
int number = count;
int maxDeleted = 5;
// track the number of the assets to be deleted
List<Integer> processed = Lists.newArrayList();
Random rand = new Random(47);
for (int i = 0; i < maxDeleted; i++) {
int n = rand.nextInt(number);
if (!processed.contains(n)) {
processed.add(n);
}
}
DataStoreState state = new DataStoreState();
for (int i = 0; i < number; i++) {
Blob b = s.createBlob(randomStream(i, 16516));
Iterator<String> idIter = ((GarbageCollectableBlobStore) s.getBlobStore()).resolveChunks(b.toString());
while (idIter.hasNext()) {
String chunk = idIter.next();
state.blobsAdded.add(chunk);
if (!processed.contains(i)) {
state.blobsPresent.add(chunk);
}
}
a.child("c" + i).setProperty("x", b);
// Add a duplicated entry
if (i == 0) {
a.child("cdup").setProperty("x", b);
}
}
s.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
if (deleteDirect) {
for (int id : processed) {
deleteFromMongo("c" + id);
}
} else {
a = s.getRoot().builder();
for (int id : processed) {
a.child("c" + id).remove();
s.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
}
// hours
long maxAge = 10;
// 1. Go past GC age and check no GC done as nothing deleted
clock.waitUntil(clock.getTime() + TimeUnit.MINUTES.toMillis(maxAge));
VersionGarbageCollector vGC = s.getVersionGarbageCollector();
VersionGCStats stats = vGC.gc(0, TimeUnit.MILLISECONDS);
Assert.assertEquals(processed.size(), stats.deletedDocGCCount);
}
return state;
}
use of org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore in project jackrabbit-oak by apache.
the class MongoBlobGCTest method addNodeSpecialChars.
private HashSet<String> addNodeSpecialChars() throws Exception {
List<String> specialCharSets = Lists.newArrayList("q\\%22afdg\\%22", "a\nbcd", "a\n\rabcd", "012\\efg");
DocumentNodeStore ds = mk.getNodeStore();
HashSet<String> set = new HashSet<String>();
NodeBuilder a = ds.getRoot().builder();
for (int i = 0; i < specialCharSets.size(); i++) {
Blob b = ds.createBlob(randomStream(i, 18432));
NodeBuilder n = a.child("cspecial");
n.child(specialCharSets.get(i)).setProperty("x", b);
Iterator<String> idIter = ((GarbageCollectableBlobStore) ds.getBlobStore()).resolveChunks(b.toString());
set.addAll(Lists.newArrayList(idIter));
}
ds.merge(a, EmptyHook.INSTANCE, CommitInfo.EMPTY);
return set;
}
use of org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore in project jackrabbit-oak by apache.
the class MongoBlobGCTest method consistencyCheckWithRenegadeDelete.
@Test
public void consistencyCheckWithRenegadeDelete() throws Exception {
DataStoreState state = setUp(true);
// Simulate faulty state by deleting some blobs directly
Random rand = new Random(87);
List<String> existing = Lists.newArrayList(state.blobsPresent);
GarbageCollectableBlobStore store = (GarbageCollectableBlobStore) mk.getNodeStore().getBlobStore();
long count = store.countDeleteChunks(ImmutableList.of(existing.get(rand.nextInt(existing.size()))), 0);
ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);
MarkSweepGarbageCollector gcObj = init(86400, executor);
long candidates = gcObj.checkConsistency();
assertEquals(1, executor.getTaskCount());
assertEquals(count, candidates);
}
use of org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore in project jackrabbit-oak by apache.
the class DataStoreCheckCommand method retrieveBlobReferences.
private static void retrieveBlobReferences(GarbageCollectableBlobStore blobStore, BlobReferenceRetriever marker, File marked) throws IOException {
final BufferedWriter writer = Files.newWriter(marked, Charsets.UTF_8);
final AtomicInteger count = new AtomicInteger();
boolean threw = true;
try {
final Joiner delimJoiner = Joiner.on(DELIM).skipNulls();
final GarbageCollectableBlobStore finalBlobStore = blobStore;
System.out.println("Starting dump of blob references");
Stopwatch watch = createStarted();
marker.collectReferences(new ReferenceCollector() {
@Override
public void addReference(String blobId, String nodeId) {
try {
Iterator<String> idIter = finalBlobStore.resolveChunks(blobId);
while (idIter.hasNext()) {
String id = delimJoiner.join(idIter.next(), nodeId);
count.getAndIncrement();
writeAsLine(writer, id, true);
}
} catch (Exception e) {
throw new RuntimeException("Error in retrieving references", e);
}
}
});
writer.flush();
sort(marked, new Comparator<String>() {
@Override
public int compare(String s1, String s2) {
return s1.split(DELIM)[0].compareTo(s2.split(DELIM)[0]);
}
});
System.out.println(count.get() + " blob references found");
System.out.println("Finished in " + watch.elapsed(TimeUnit.SECONDS) + " seconds");
threw = false;
} finally {
close(writer, threw);
}
}
use of org.apache.jackrabbit.oak.spi.blob.GarbageCollectableBlobStore in project jackrabbit-oak by apache.
the class Registrations method registerSegmentStore.
/**
* Configures and registers a new SegmentNodeStore instance together will
* all required components. Anything that must be disposed of (like
* registered services or MBeans) will be registered via the
* {@code registration} parameter.
*
* @param context An instance of {@link ComponentContext}.
* @param blobStore An instance of {@link BlobStore}. It can be
* {@code null}.
* @param statisticsProvider An instance of {@link StatisticsProvider}.
* @param closer An instance of {@link Closer}. It will be used
* to track every registered service or
* component.
* @param whiteboard An instance of {@link Whiteboard}. It will be
* used to register services in the OSGi
* framework.
* @param role The role of this component. It can be {@code
* null}.
* @param descriptors Determines if repository descriptors related to
* discovery services should be registered.
* @return A configured {@link SegmentNodeStore}, or {@code null} if the
* setup failed.
* @throws IOException In case an unrecoverable error occurs.
*/
static SegmentNodeStore registerSegmentStore(@Nonnull ComponentContext context, @Nullable BlobStore blobStore, @Nonnull StatisticsProvider statisticsProvider, @Nonnull Closer closer, @Nonnull Whiteboard whiteboard, @Nullable String role, boolean descriptors) throws IOException {
Configuration configuration = new Configuration(context, role);
Closeables closeables = new Closeables(closer);
Registrations registrations = new Registrations(whiteboard, role);
// Listen for GCMonitor services
GCMonitor gcMonitor = GCMonitor.EMPTY;
if (configuration.isPrimarySegmentStore()) {
GCMonitorTracker tracker = new GCMonitorTracker();
tracker.start(whiteboard);
closeables.add(tracker);
gcMonitor = tracker;
}
// Create the gc options
if (configuration.getCompactionGainThreshold() != null) {
log.warn("Detected deprecated flag 'compaction.gainThreshold'. " + "Please use 'compaction.sizeDeltaEstimation' instead and " + "'compaction.disableEstimation' to disable estimation.");
}
SegmentGCOptions gcOptions = new SegmentGCOptions(configuration.getPauseCompaction(), configuration.getRetryCount(), configuration.getForceCompactionTimeout()).setRetainedGenerations(configuration.getRetainedGenerations()).setGcSizeDeltaEstimation(configuration.getSizeDeltaEstimation()).setMemoryThreshold(configuration.getMemoryThreshold()).setEstimationDisabled(configuration.getDisableEstimation()).withGCNodeWriteMonitor(configuration.getGCProcessLog());
// Build the FileStore
FileStoreBuilder builder = fileStoreBuilder(configuration.getSegmentDirectory()).withSegmentCacheSize(configuration.getSegmentCacheSize()).withStringCacheSize(configuration.getStringCacheSize()).withTemplateCacheSize(configuration.getTemplateCacheSize()).withStringDeduplicationCacheSize(configuration.getStringDeduplicationCacheSize()).withTemplateDeduplicationCacheSize(configuration.getTemplateDeduplicationCacheSize()).withNodeDeduplicationCacheSize(configuration.getNodeDeduplicationCacheSize()).withMaxFileSize(configuration.getMaxFileSize()).withMemoryMapping(configuration.getMemoryMapping()).withGCMonitor(gcMonitor).withIOMonitor(new MetricsIOMonitor(statisticsProvider)).withStatisticsProvider(statisticsProvider).withGCOptions(gcOptions);
if (configuration.hasCustomBlobStore() && blobStore != null) {
log.info("Initializing SegmentNodeStore with BlobStore [{}]", blobStore);
builder.withBlobStore(blobStore);
}
if (configuration.isStandbyInstance()) {
builder.withSnfeListener(IGNORE_SNFE);
}
final FileStore store;
try {
store = builder.build();
} catch (InvalidFileStoreVersionException e) {
log.error("The storage format is not compatible with this version of Oak Segment Tar", e);
return null;
}
// store should be closed last
closeables.add(store);
// Listen for Executor services on the whiteboard
WhiteboardExecutor executor = new WhiteboardExecutor();
executor.start(whiteboard);
closeables.add(executor);
// Expose stats about the segment cache
CacheStatsMBean segmentCacheStats = store.getSegmentCacheStats();
closeables.add(registrations.registerMBean(CacheStatsMBean.class, segmentCacheStats, CacheStats.TYPE, segmentCacheStats.getName()));
// Expose stats about the string and template caches
CacheStatsMBean stringCacheStats = store.getStringCacheStats();
closeables.add(registrations.registerMBean(CacheStatsMBean.class, stringCacheStats, CacheStats.TYPE, stringCacheStats.getName()));
CacheStatsMBean templateCacheStats = store.getTemplateCacheStats();
closeables.add(registrations.registerMBean(CacheStatsMBean.class, templateCacheStats, CacheStats.TYPE, templateCacheStats.getName()));
WriterCacheManager cacheManager = builder.getCacheManager();
CacheStatsMBean stringDeduplicationCacheStats = cacheManager.getStringCacheStats();
if (stringDeduplicationCacheStats != null) {
closeables.add(registrations.registerMBean(CacheStatsMBean.class, stringDeduplicationCacheStats, CacheStats.TYPE, stringDeduplicationCacheStats.getName()));
}
CacheStatsMBean templateDeduplicationCacheStats = cacheManager.getTemplateCacheStats();
if (templateDeduplicationCacheStats != null) {
closeables.add(registrations.registerMBean(CacheStatsMBean.class, templateDeduplicationCacheStats, CacheStats.TYPE, templateDeduplicationCacheStats.getName()));
}
CacheStatsMBean nodeDeduplicationCacheStats = cacheManager.getNodeCacheStats();
if (nodeDeduplicationCacheStats != null) {
closeables.add(registrations.registerMBean(CacheStatsMBean.class, nodeDeduplicationCacheStats, CacheStats.TYPE, nodeDeduplicationCacheStats.getName()));
}
if (configuration.isPrimarySegmentStore()) {
final FileStoreGCMonitor monitor = new FileStoreGCMonitor(Clock.SIMPLE);
closeables.add(registrations.register(GCMonitor.class, monitor));
if (!configuration.isStandbyInstance()) {
closeables.add(registrations.registerMBean(SegmentRevisionGC.class, new SegmentRevisionGCMBean(store, gcOptions, monitor), SegmentRevisionGC.TYPE, "Segment node store revision garbage collection"));
}
Runnable cancelGC = new Runnable() {
@Override
public void run() {
store.cancelGC();
}
};
Supplier<String> statusMessage = new Supplier<String>() {
@Override
public String get() {
return monitor.getStatus();
}
};
closeables.add(registrations.registerMBean(RevisionGCMBean.class, new RevisionGC(store.getGCRunner(), cancelGC, statusMessage, executor), RevisionGCMBean.TYPE, "Revision garbage collection"));
}
// Expose statistics about the FileStore
closeables.add(registrations.registerMBean(FileStoreStatsMBean.class, store.getStats(), FileStoreStatsMBean.TYPE, "FileStore statistics"));
// register segment node store
SegmentNodeStore.SegmentNodeStoreBuilder segmentNodeStoreBuilder = SegmentNodeStoreBuilders.builder(store).withStatisticsProvider(statisticsProvider);
if (configuration.isStandbyInstance() || !configuration.isPrimarySegmentStore()) {
segmentNodeStoreBuilder.dispatchChanges(false);
}
SegmentNodeStore segmentNodeStore = segmentNodeStoreBuilder.build();
if (configuration.isPrimarySegmentStore()) {
ObserverTracker observerTracker = new ObserverTracker(segmentNodeStore);
observerTracker.start(context.getBundleContext());
closeables.add(observerTracker);
}
if (configuration.isPrimarySegmentStore()) {
closeables.add(registrations.registerMBean(CheckpointMBean.class, new SegmentCheckpointMBean(segmentNodeStore), CheckpointMBean.TYPE, "Segment node store checkpoint management"));
}
if (descriptors) {
// ensure a clusterId is initialized
// and expose it as 'oak.clusterid' repository descriptor
GenericDescriptors clusterIdDesc = new GenericDescriptors();
clusterIdDesc.put(ClusterRepositoryInfo.OAK_CLUSTERID_REPOSITORY_DESCRIPTOR_KEY, new SimpleValueFactory().createValue(getOrCreateId(segmentNodeStore)), true, false);
closeables.add(registrations.register(Descriptors.class, clusterIdDesc));
// Register "discovery lite" descriptors
closeables.add(registrations.register(Descriptors.class, new SegmentDiscoveryLiteDescriptors(segmentNodeStore)));
}
// If a shared data store register the repo id in the data store
if (configuration.isPrimarySegmentStore() && isShared(blobStore)) {
SharedDataStore sharedDataStore = (SharedDataStore) blobStore;
try {
sharedDataStore.addMetadataRecord(new ByteArrayInputStream(new byte[0]), SharedStoreRecordType.REPOSITORY.getNameFromId(getOrCreateId(segmentNodeStore)));
} catch (Exception e) {
throw new IOException("Could not register a unique repositoryId", e);
}
if (blobStore instanceof BlobTrackingStore) {
BlobTrackingStore trackingStore = (BlobTrackingStore) blobStore;
if (trackingStore.getTracker() != null) {
trackingStore.getTracker().close();
}
trackingStore.addTracker(new BlobIdTracker(configuration.getRepositoryHome(), getOrCreateId(segmentNodeStore), configuration.getBlobSnapshotInterval(), sharedDataStore));
}
}
if (configuration.isPrimarySegmentStore() && blobStore instanceof GarbageCollectableBlobStore) {
BlobGarbageCollector gc = new MarkSweepGarbageCollector(new SegmentBlobReferenceRetriever(store), (GarbageCollectableBlobStore) blobStore, executor, TimeUnit.SECONDS.toMillis(configuration.getBlobGcMaxAge()), getOrCreateId(segmentNodeStore));
closeables.add(registrations.registerMBean(BlobGCMBean.class, new BlobGC(gc, executor), BlobGCMBean.TYPE, "Segment node store blob garbage collection"));
}
// Expose an MBean for backup/restore operations
closeables.add(registrations.registerMBean(FileStoreBackupRestoreMBean.class, new FileStoreBackupRestoreImpl(segmentNodeStore, store.getRevisions(), store.getReader(), configuration.getBackupDirectory(), executor), FileStoreBackupRestoreMBean.TYPE, "Segment node store backup/restore"));
// Expose statistics about the SegmentNodeStore
closeables.add(registrations.registerMBean(SegmentNodeStoreStatsMBean.class, segmentNodeStore.getStats(), SegmentNodeStoreStatsMBean.TYPE, "SegmentNodeStore statistics"));
if (configuration.isPrimarySegmentStore()) {
log.info("Primary SegmentNodeStore initialized");
} else {
log.info("Secondary SegmentNodeStore initialized, role={}", role);
}
// Register a factory service to expose the FileStore
closeables.add(registrations.register(SegmentStoreProvider.class, new DefaultSegmentStoreProvider(store)));
if (configuration.isStandbyInstance()) {
return segmentNodeStore;
}
if (configuration.isPrimarySegmentStore()) {
Map<String, Object> props = new HashMap<String, Object>();
props.put(Constants.SERVICE_PID, SegmentNodeStore.class.getName());
props.put("oak.nodestore.description", new String[] { "nodeStoreType=segment" });
closeables.add(registrations.register(NodeStore.class, segmentNodeStore, props));
}
return segmentNodeStore;
}
Aggregations