use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext in project asterixdb by apache.
the class LSMRTreeWithAntiMatterTuples method merge.
@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
MergeOperation mergeOp = (MergeOperation) operation;
IIndexCursor cursor = mergeOp.getCursor();
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
search(opCtx, cursor, rtreeSearchPred);
// Bulk load the tuples from all on-disk RTrees into the new RTree.
LSMRTreeDiskComponent component = createDiskComponent(componentFactory, mergeOp.getTarget(), null, null, true);
ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false);
try {
while (cursor.hasNext()) {
cursor.next();
ITupleReference frameTuple = cursor.getTuple();
componentBulkLoader.add(frameTuple);
}
} finally {
cursor.close();
}
if (component.getLSMComponentFilter() != null) {
List<ITupleReference> filterTuples = new ArrayList<>();
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
}
getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
getFilterManager().writeFilter(component.getLSMComponentFilter(), component.getRTree());
}
componentBulkLoader.end();
return component;
}
use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext in project asterixdb by apache.
the class LSMRTree method merge.
@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
LSMRTreeMergeOperation mergeOp = (LSMRTreeMergeOperation) operation;
IIndexCursor cursor = mergeOp.getCursor();
ISearchPredicate rtreeSearchPred = new SearchPredicate(null, null);
ILSMIndexOperationContext opCtx = ((LSMRTreeSortedCursor) cursor).getOpCtx();
opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
search(opCtx, cursor, rtreeSearchPred);
LSMRTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
ILSMDiskComponentBulkLoader componentBulkLoader;
// lsmHarness.endSearch() is called once when the r-trees have been merged.
if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents.get(diskComponents.size() - 1)) {
// Keep the deleted tuples since the oldest disk component is not included in the merge operation
long numElements = 0L;
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
numElements += ((LSMRTreeDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter().getNumElements();
}
componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false);
LSMRTreeDeletedKeysBTreeMergeCursor btreeCursor = new LSMRTreeDeletedKeysBTreeMergeCursor(opCtx);
search(opCtx, btreeCursor, rtreeSearchPred);
try {
while (btreeCursor.hasNext()) {
btreeCursor.next();
ITupleReference tuple = btreeCursor.getTuple();
componentBulkLoader.delete(tuple);
}
} finally {
btreeCursor.close();
}
} else {
//no buddy-btree needed
componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, 0L, false, false);
}
//search old rtree components
try {
while (cursor.hasNext()) {
cursor.next();
ITupleReference frameTuple = cursor.getTuple();
componentBulkLoader.add(frameTuple);
}
} finally {
cursor.close();
}
if (mergedComponent.getLSMComponentFilter() != null) {
List<ITupleReference> filterTuples = new ArrayList<>();
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
}
getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getRTree());
}
componentBulkLoader.end();
return mergedComponent;
}
use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext in project asterixdb by apache.
the class LSMInvertedIndex method merge.
@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
LSMInvertedIndexMergeOperation mergeOp = (LSMInvertedIndexMergeOperation) operation;
IIndexCursor cursor = mergeOp.getCursor();
RangePredicate mergePred = new RangePredicate(null, null, true, true, null, null);
ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
// Scan diskInvertedIndexes ignoring the memoryInvertedIndex.
search(opCtx, cursor, mergePred);
// Create an inverted index instance.
LSMInvertedIndexDiskComponent component = createDiskInvIndexComponent(componentFactory, mergeOp.getTarget(), mergeOp.getDeletedKeysBTreeTarget(), mergeOp.getBloomFilterTarget(), true);
ILSMDiskComponentBulkLoader componentBulkLoader;
// lsmHarness.endSearch() is called once when the inverted indexes have been merged.
if (mergeOp.getMergingComponents().get(mergeOp.getMergingComponents().size() - 1) != diskComponents.get(diskComponents.size() - 1)) {
// Keep the deleted tuples since the oldest disk component is not included in the merge operation
LSMInvertedIndexDeletedKeysBTreeMergeCursor btreeCursor = new LSMInvertedIndexDeletedKeysBTreeMergeCursor(opCtx);
search(opCtx, btreeCursor, mergePred);
long numElements = 0L;
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
numElements += ((LSMInvertedIndexDiskComponent) mergeOp.getMergingComponents().get(i)).getBloomFilter().getNumElements();
}
componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, numElements, false, false);
try {
while (btreeCursor.hasNext()) {
btreeCursor.next();
ITupleReference tuple = btreeCursor.getTuple();
componentBulkLoader.delete(tuple);
}
} finally {
btreeCursor.close();
}
} else {
componentBulkLoader = createComponentBulkLoader(component, 1.0f, false, 0L, false, false);
}
try {
while (cursor.hasNext()) {
cursor.next();
ITupleReference tuple = cursor.getTuple();
componentBulkLoader.add(tuple);
}
} finally {
cursor.close();
}
if (component.getLSMComponentFilter() != null) {
List<ITupleReference> filterTuples = new ArrayList<>();
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
ITupleReference min = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple();
ITupleReference max = mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple();
if (min != null) {
filterTuples.add(min);
}
if (max != null) {
filterTuples.add(max);
}
}
getFilterManager().updateFilter(component.getLSMComponentFilter(), filterTuples);
getFilterManager().writeFilter(component.getLSMComponentFilter(), ((OnDiskInvertedIndex) component.getInvIndex()).getBTree());
}
componentBulkLoader.end();
return component;
}
use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext in project asterixdb by apache.
the class ExternalRTree method scheduleMerge.
// The only change the the schedule merge is the method used to create the
// opCtx. first line <- in schedule merge, we->
@Override
public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback) throws HyracksDataException {
ILSMIndexOperationContext rctx = createOpContext(NoOpOperationCallback.INSTANCE, -1);
rctx.setOperation(IndexOperation.MERGE);
List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
ITreeIndexCursor cursor = new LSMRTreeSortedCursor(rctx, linearizer, buddyBTreeFields);
LSMComponentFileReferences relMergeFileRefs = getMergeFileReferences((ILSMDiskComponent) mergingComponents.get(0), (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1));
ILSMIndexAccessor accessor = new LSMRTreeAccessor(getLsmHarness(), rctx, buddyBTreeFields);
// create the merge operation.
LSMRTreeMergeOperation mergeOp = new LSMRTreeMergeOperation(accessor, mergingComponents, cursor, relMergeFileRefs.getInsertIndexFileReference(), relMergeFileRefs.getDeleteIndexFileReference(), relMergeFileRefs.getBloomFilterFileReference(), callback, fileManager.getBaseDir());
ioScheduler.scheduleOperation(mergeOp);
}
use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext in project asterixdb by apache.
the class LSMBTree method merge.
@Override
public ILSMDiskComponent merge(ILSMIOOperation operation) throws HyracksDataException {
LSMBTreeMergeOperation mergeOp = (LSMBTreeMergeOperation) operation;
IIndexCursor cursor = mergeOp.getCursor();
RangePredicate rangePred = new RangePredicate(null, null, true, true, null, null);
ILSMIndexOperationContext opCtx = ((LSMIndexSearchCursor) cursor).getOpCtx();
opCtx.getComponentHolder().addAll(mergeOp.getMergingComponents());
search(opCtx, cursor, rangePred);
List<ILSMComponent> mergedComponents = mergeOp.getMergingComponents();
long numElements = 0L;
if (hasBloomFilter) {
//count elements in btree for creating Bloomfilter
for (int i = 0; i < mergedComponents.size(); ++i) {
numElements += ((LSMBTreeDiskComponent) mergedComponents.get(i)).getBloomFilter().getNumElements();
}
}
LSMBTreeDiskComponent mergedComponent = createDiskComponent(componentFactory, mergeOp.getTarget(), mergeOp.getBloomFilterTarget(), true);
ILSMDiskComponentBulkLoader componentBulkLoader = createComponentBulkLoader(mergedComponent, 1.0f, false, numElements, false, false);
try {
while (cursor.hasNext()) {
cursor.next();
ITupleReference frameTuple = cursor.getTuple();
componentBulkLoader.add(frameTuple);
}
} finally {
cursor.close();
}
if (mergedComponent.getLSMComponentFilter() != null) {
List<ITupleReference> filterTuples = new ArrayList<>();
for (int i = 0; i < mergeOp.getMergingComponents().size(); ++i) {
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMinTuple());
filterTuples.add(mergeOp.getMergingComponents().get(i).getLSMComponentFilter().getMaxTuple());
}
getFilterManager().updateFilter(mergedComponent.getLSMComponentFilter(), filterTuples);
getFilterManager().writeFilter(mergedComponent.getLSMComponentFilter(), mergedComponent.getBTree());
}
componentBulkLoader.end();
return mergedComponent;
}
Aggregations