Examples with ILSMDiskComponent org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent used on opensource projects

Example 21 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class LSMHarness method exitComponents.

private void exitComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, ILSMDiskComponent newComponent, boolean failedOperation) throws HyracksDataException {
    /**
         * FLUSH and MERGE operations should always exit the components
         * to notify waiting threads.
         */
    if (!ctx.isAccessingComponents() && opType != LSMOperationType.FLUSH && opType != LSMOperationType.MERGE) {
        return;
    }
    List<ILSMDiskComponent> inactiveDiskComponents = null;
    List<ILSMDiskComponent> inactiveDiskComponentsToBeDeleted = null;
    try {
        synchronized (opTracker) {
            try {
                /**
                     * [flow control]
                     * If merge operations are lagged according to the merge policy,
                     * flushing in-memory components are hold until the merge operation catches up.
                     * See PrefixMergePolicy.isMergeLagging() for more details.
                     */
                if (opType == LSMOperationType.FLUSH) {
                    while (mergePolicy.isMergeLagging(lsmIndex)) {
                        try {
                            opTracker.wait();
                        } catch (InterruptedException e) {
                        //ignore
                        }
                    }
                } else if (opType == LSMOperationType.MERGE) {
                    opTracker.notifyAll();
                }
                int i = 0;
                // First check if there is any action that is needed to be taken based on the state of each component.
                for (ILSMComponent c : ctx.getComponentHolder()) {
                    boolean isMutableComponent = i == 0 && c.getType() == LSMComponentType.MEMORY ? true : false;
                    c.threadExit(opType, failedOperation, isMutableComponent);
                    if (c.getType() == LSMComponentType.MEMORY) {
                        switch(c.getState()) {
                            case READABLE_UNWRITABLE:
                                if (isMutableComponent && (opType == LSMOperationType.MODIFICATION || opType == LSMOperationType.FORCE_MODIFICATION)) {
                                    lsmIndex.changeFlushStatusForCurrentMutableCompoent(true);
                                }
                                break;
                            case INACTIVE:
                                ((AbstractLSMMemoryComponent) c).reset();
                                // Notify all waiting threads whenever the mutable component's has change to inactive. This is important because
                                // even though we switched the mutable components, it is possible that the component that we just switched
                                // to is still busy flushing its data to disk. Thus, the notification that was issued upon scheduling the flush
                                // is not enough.
                                opTracker.notifyAll();
                                break;
                            default:
                                break;
                        }
                    } else {
                        switch(c.getState()) {
                            case INACTIVE:
                                lsmIndex.addInactiveDiskComponent((AbstractLSMDiskComponent) c);
                                break;
                            default:
                                break;
                        }
                    }
                    i++;
                }
                ctx.setAccessingComponents(false);
                // Then, perform any action that is needed to be taken based on the operation type.
                switch(opType) {
                    case FLUSH:
                        // newComponent is null if the flush op. was not performed.
                        if (newComponent != null) {
                            lsmIndex.addDiskComponent(newComponent);
                            if (replicationEnabled) {
                                componentsToBeReplicated.clear();
                                componentsToBeReplicated.add(newComponent);
                                triggerReplication(componentsToBeReplicated, false, opType);
                            }
                            mergePolicy.diskComponentAdded(lsmIndex, false);
                        }
                        break;
                    case MERGE:
                        // newComponent is null if the merge op. was not performed.
                        if (newComponent != null) {
                            lsmIndex.subsumeMergedComponents(newComponent, ctx.getComponentHolder());
                            if (replicationEnabled) {
                                componentsToBeReplicated.clear();
                                componentsToBeReplicated.add(newComponent);
                                triggerReplication(componentsToBeReplicated, false, opType);
                            }
                            mergePolicy.diskComponentAdded(lsmIndex, fullMergeIsRequested.get());
                        }
                        break;
                    default:
                        break;
                }
            } catch (Throwable e) {
                e.printStackTrace();
                throw e;
            } finally {
                if (failedOperation && (opType == LSMOperationType.MODIFICATION || opType == LSMOperationType.FORCE_MODIFICATION)) {
                    //When the operation failed, completeOperation() method must be called
                    //in order to decrement active operation count which was incremented in beforeOperation() method.
                    opTracker.completeOperation(lsmIndex, opType, ctx.getSearchOperationCallback(), ctx.getModificationCallback());
                } else {
                    opTracker.afterOperation(lsmIndex, opType, ctx.getSearchOperationCallback(), ctx.getModificationCallback());
                }
                /*
                     * = Inactive disk components lazy cleanup if any =
                     * Prepare to cleanup inactive diskComponents which were old merged components
                     * and not anymore accessed.
                     * This cleanup is done outside of optracker synchronized block.
                     */
                inactiveDiskComponents = lsmIndex.getInactiveDiskComponents();
                if (!inactiveDiskComponents.isEmpty()) {
                    for (ILSMDiskComponent inactiveComp : inactiveDiskComponents) {
                        if (inactiveComp.getFileReferenceCount() == 1) {
                            if (inactiveDiskComponentsToBeDeleted == null) {
                                inactiveDiskComponentsToBeDeleted = new LinkedList<>();
                            }
                            inactiveDiskComponentsToBeDeleted.add(inactiveComp);
                        }
                    }
                    if (inactiveDiskComponentsToBeDeleted != null) {
                        inactiveDiskComponents.removeAll(inactiveDiskComponentsToBeDeleted);
                    }
                }
            }
        }
    } finally {
        /*
             * cleanup inactive disk components if any
             */
        if (inactiveDiskComponentsToBeDeleted != null) {
            try {
                //schedule a replication job to delete these inactive disk components from replicas
                if (replicationEnabled) {
                    lsmIndex.scheduleReplication(null, inactiveDiskComponentsToBeDeleted, false, ReplicationOperation.DELETE, opType);
                }
                for (ILSMComponent c : inactiveDiskComponentsToBeDeleted) {
                    ((AbstractLSMDiskComponent) c).destroy();
                }
            } catch (Throwable e) {
                e.printStackTrace();
                throw e;
            }
        }
    }
}

Example 22 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class AbstractLSMIndex method scheduleMerge.

@Override
public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback) throws HyracksDataException {
    // merge must create a different op ctx
    AbstractLSMIndexOperationContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    opCtx.setOperation(IndexOperation.MERGE);
    List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
    ILSMDiskComponent firstComponent = (ILSMDiskComponent) mergingComponents.get(0);
    ILSMDiskComponent lastComponent = (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1);
    LSMComponentFileReferences mergeFileRefs = getMergeFileReferences(firstComponent, lastComponent);
    ILSMIOOperation mergeOp = createMergeOperation(opCtx, mergingComponents, mergeFileRefs, callback);
    ioScheduler.scheduleOperation(mergeOp);
}

Example 23 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class AbstractLSMIndex method loadDiskComponents.

protected void loadDiskComponents() throws HyracksDataException {
    diskComponents.clear();
    List<LSMComponentFileReferences> validFileReferences = fileManager.cleanupAndGetValidFiles();
    for (LSMComponentFileReferences lsmComonentFileReference : validFileReferences) {
        ILSMDiskComponent component = loadComponent(lsmComonentFileReference);
        diskComponents.add(component);
    }
}

Example 24 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class CorrelatedPrefixMergePolicyTest method mockIndex.

private IndexInfo mockIndex(boolean isPrimary, List<ILSMDiskComponentId> componentIDs, List<ILSMDiskComponentId> resultComponentIDs, int partition) throws HyracksDataException {
    List<ILSMDiskComponent> components = new ArrayList<>();
    for (ILSMDiskComponentId id : componentIDs) {
        ILSMDiskComponent component = Mockito.mock(ILSMDiskComponent.class);
        Mockito.when(component.getComponentId()).thenReturn(id);
        Mockito.when(component.getComponentSize()).thenReturn(DEFAULT_COMPONENT_SIZE);
        Mockito.when(component.getState()).thenReturn(ComponentState.READABLE_UNWRITABLE);
        components.add(component);
    }
    ILSMIndex index = Mockito.mock(ILSMIndex.class);
    Mockito.when(index.getImmutableComponents()).thenReturn(components);
    ILSMIndexAccessor accessor = Mockito.mock(ILSMIndexAccessor.class);
    Mockito.doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            List<ILSMDiskComponent> mergedComponents = invocation.getArgumentAt(1, List.class);
            mergedComponents.forEach(component -> {
                try {
                    resultComponentIDs.add(component.getComponentId());
                } catch (HyracksDataException e) {
                    e.printStackTrace();
                }
            });
            return null;
        }
    }).when(accessor).scheduleMerge(Mockito.any(ILSMIOOperationCallback.class), Mockito.anyListOf(ILSMDiskComponent.class));
    Mockito.when(index.createAccessor(Mockito.any(IModificationOperationCallback.class), Mockito.any(ISearchOperationCallback.class))).thenReturn(accessor);
    Mockito.when(index.isPrimaryIndex()).thenReturn(isPrimary);
    return new IndexInfo(index, DATASET_ID, 0, partition);
}

Example 25 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class ExternalBTreeWithBuddy method destroy.

@Override
public void destroy() throws HyracksDataException {
    if (isActive) {
        throw new HyracksDataException("Failed to destroy the index since it is activated.");
    }
    for (ILSMDiskComponent c : diskComponents) {
        destroyDiskComponent(c);
        // Remove from second list to avoid destroying twice
        secondDiskComponents.remove(c);
    }
    for (ILSMDiskComponent c : secondDiskComponents) {
        destroyDiskComponent(c);
    }
    diskComponents.clear();
    secondDiskComponents.clear();
    fileManager.deleteDirs();
    version = 0;
}