Examples with ILSMComponent org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent used on opensource projects

Example 36 with ILSMComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent in project asterixdb by apache.

the class LSMHarness method exitComponents.

private void exitComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, ILSMDiskComponent newComponent, boolean failedOperation) throws HyracksDataException {
    /**
         * FLUSH and MERGE operations should always exit the components
         * to notify waiting threads.
         */
    if (!ctx.isAccessingComponents() && opType != LSMOperationType.FLUSH && opType != LSMOperationType.MERGE) {
        return;
    }
    List<ILSMDiskComponent> inactiveDiskComponents = null;
    List<ILSMDiskComponent> inactiveDiskComponentsToBeDeleted = null;
    try {
        synchronized (opTracker) {
            try {
                /**
                     * [flow control]
                     * If merge operations are lagged according to the merge policy,
                     * flushing in-memory components are hold until the merge operation catches up.
                     * See PrefixMergePolicy.isMergeLagging() for more details.
                     */
                if (opType == LSMOperationType.FLUSH) {
                    while (mergePolicy.isMergeLagging(lsmIndex)) {
                        try {
                            opTracker.wait();
                        } catch (InterruptedException e) {
                        //ignore
                        }
                    }
                } else if (opType == LSMOperationType.MERGE) {
                    opTracker.notifyAll();
                }
                int i = 0;
                // First check if there is any action that is needed to be taken based on the state of each component.
                for (ILSMComponent c : ctx.getComponentHolder()) {
                    boolean isMutableComponent = i == 0 && c.getType() == LSMComponentType.MEMORY ? true : false;
                    c.threadExit(opType, failedOperation, isMutableComponent);
                    if (c.getType() == LSMComponentType.MEMORY) {
                        switch(c.getState()) {
                            case READABLE_UNWRITABLE:
                                if (isMutableComponent && (opType == LSMOperationType.MODIFICATION || opType == LSMOperationType.FORCE_MODIFICATION)) {
                                    lsmIndex.changeFlushStatusForCurrentMutableCompoent(true);
                                }
                                break;
                            case INACTIVE:
                                ((AbstractLSMMemoryComponent) c).reset();
                                // Notify all waiting threads whenever the mutable component's has change to inactive. This is important because
                                // even though we switched the mutable components, it is possible that the component that we just switched
                                // to is still busy flushing its data to disk. Thus, the notification that was issued upon scheduling the flush
                                // is not enough.
                                opTracker.notifyAll();
                                break;
                            default:
                                break;
                        }
                    } else {
                        switch(c.getState()) {
                            case INACTIVE:
                                lsmIndex.addInactiveDiskComponent((AbstractLSMDiskComponent) c);
                                break;
                            default:
                                break;
                        }
                    }
                    i++;
                }
                ctx.setAccessingComponents(false);
                // Then, perform any action that is needed to be taken based on the operation type.
                switch(opType) {
                    case FLUSH:
                        // newComponent is null if the flush op. was not performed.
                        if (newComponent != null) {
                            lsmIndex.addDiskComponent(newComponent);
                            if (replicationEnabled) {
                                componentsToBeReplicated.clear();
                                componentsToBeReplicated.add(newComponent);
                                triggerReplication(componentsToBeReplicated, false, opType);
                            }
                            mergePolicy.diskComponentAdded(lsmIndex, false);
                        }
                        break;
                    case MERGE:
                        // newComponent is null if the merge op. was not performed.
                        if (newComponent != null) {
                            lsmIndex.subsumeMergedComponents(newComponent, ctx.getComponentHolder());
                            if (replicationEnabled) {
                                componentsToBeReplicated.clear();
                                componentsToBeReplicated.add(newComponent);
                                triggerReplication(componentsToBeReplicated, false, opType);
                            }
                            mergePolicy.diskComponentAdded(lsmIndex, fullMergeIsRequested.get());
                        }
                        break;
                    default:
                        break;
                }
            } catch (Throwable e) {
                e.printStackTrace();
                throw e;
            } finally {
                if (failedOperation && (opType == LSMOperationType.MODIFICATION || opType == LSMOperationType.FORCE_MODIFICATION)) {
                    //When the operation failed, completeOperation() method must be called
                    //in order to decrement active operation count which was incremented in beforeOperation() method.
                    opTracker.completeOperation(lsmIndex, opType, ctx.getSearchOperationCallback(), ctx.getModificationCallback());
                } else {
                    opTracker.afterOperation(lsmIndex, opType, ctx.getSearchOperationCallback(), ctx.getModificationCallback());
                }
                /*
                     * = Inactive disk components lazy cleanup if any =
                     * Prepare to cleanup inactive diskComponents which were old merged components
                     * and not anymore accessed.
                     * This cleanup is done outside of optracker synchronized block.
                     */
                inactiveDiskComponents = lsmIndex.getInactiveDiskComponents();
                if (!inactiveDiskComponents.isEmpty()) {
                    for (ILSMDiskComponent inactiveComp : inactiveDiskComponents) {
                        if (inactiveComp.getFileReferenceCount() == 1) {
                            if (inactiveDiskComponentsToBeDeleted == null) {
                                inactiveDiskComponentsToBeDeleted = new LinkedList<>();
                            }
                            inactiveDiskComponentsToBeDeleted.add(inactiveComp);
                        }
                    }
                    if (inactiveDiskComponentsToBeDeleted != null) {
                        inactiveDiskComponents.removeAll(inactiveDiskComponentsToBeDeleted);
                    }
                }
            }
        }
    } finally {
        /*
             * cleanup inactive disk components if any
             */
        if (inactiveDiskComponentsToBeDeleted != null) {
            try {
                //schedule a replication job to delete these inactive disk components from replicas
                if (replicationEnabled) {
                    lsmIndex.scheduleReplication(null, inactiveDiskComponentsToBeDeleted, false, ReplicationOperation.DELETE, opType);
                }
                for (ILSMComponent c : inactiveDiskComponentsToBeDeleted) {
                    ((AbstractLSMDiskComponent) c).destroy();
                }
            } catch (Throwable e) {
                e.printStackTrace();
                throw e;
            }
        }
    }
}

Example 37 with ILSMComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent in project asterixdb by apache.

the class AbstractLSMIndex method scheduleMerge.

@Override
public void scheduleMerge(ILSMIndexOperationContext ctx, ILSMIOOperationCallback callback) throws HyracksDataException {
    // merge must create a different op ctx
    AbstractLSMIndexOperationContext opCtx = createOpContext(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
    opCtx.setOperation(IndexOperation.MERGE);
    List<ILSMComponent> mergingComponents = ctx.getComponentHolder();
    ILSMDiskComponent firstComponent = (ILSMDiskComponent) mergingComponents.get(0);
    ILSMDiskComponent lastComponent = (ILSMDiskComponent) mergingComponents.get(mergingComponents.size() - 1);
    LSMComponentFileReferences mergeFileRefs = getMergeFileReferences(firstComponent, lastComponent);
    ILSMIOOperation mergeOp = createMergeOperation(opCtx, mergingComponents, mergeFileRefs, callback);
    ioScheduler.scheduleOperation(mergeOp);
}

Example 38 with ILSMComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent in project asterixdb by apache.

the class AbstractLSMIndex method scheduleReplication.

@Override
public void scheduleReplication(ILSMIndexOperationContext ctx, List<ILSMDiskComponent> lsmComponents, boolean bulkload, ReplicationOperation operation, LSMOperationType opType) throws HyracksDataException {
    //get set of files to be replicated for this component
    Set<String> componentFiles = new HashSet<>();
    //get set of files to be replicated for each component
    for (ILSMComponent lsmComponent : lsmComponents) {
        componentFiles.addAll(getLSMComponentPhysicalFiles(lsmComponent));
    }
    ReplicationExecutionType executionType;
    if (bulkload) {
        executionType = ReplicationExecutionType.SYNC;
    } else {
        executionType = ReplicationExecutionType.ASYNC;
    }
    //create replication job and submit it
    LSMIndexReplicationJob job = new LSMIndexReplicationJob(this, ctx, componentFiles, operation, executionType, opType);
    try {
        diskBufferCache.getIOReplicationManager().submitJob(job);
    } catch (IOException e) {
        throw HyracksDataException.create(e);
    }
}