Examples with ILSMDiskComponent org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent used on opensource projects

Example 16 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class ExternalIndexHarness method exitComponents.

private void exitComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, ILSMDiskComponent newComponent, boolean failedOperation) throws HyracksDataException {
    /**
         * FLUSH and MERGE operations should always exit the components
         * to notify waiting threads.
         */
    if (!ctx.isAccessingComponents() && opType != LSMOperationType.FLUSH && opType != LSMOperationType.MERGE) {
        return;
    }
    synchronized (opTracker) {
        try {
            // First check if there is any action that is needed to be taken based on the state of each component.
            for (ILSMComponent c : ctx.getComponentHolder()) {
                c.threadExit(opType, failedOperation, false);
                switch(c.getState()) {
                    case INACTIVE:
                        if (replicationEnabled) {
                            componentsToBeReplicated.clear();
                            componentsToBeReplicated.add((ILSMDiskComponent) c);
                            lsmIndex.scheduleReplication(null, componentsToBeReplicated, false, ReplicationOperation.DELETE, opType);
                        }
                        ((ILSMDiskComponent) c).destroy();
                        break;
                    default:
                        break;
                }
            }
            ctx.setAccessingComponents(false);
            // Then, perform any action that is needed to be taken based on the operation type.
            switch(opType) {
                case MERGE:
                    // newComponent is null if the merge op. was not performed.
                    if (newComponent != null) {
                        beforeSubsumeMergedComponents(newComponent, ctx.getComponentHolder());
                        lsmIndex.subsumeMergedComponents(newComponent, ctx.getComponentHolder());
                        if (replicationEnabled) {
                            componentsToBeReplicated.clear();
                            componentsToBeReplicated.add(newComponent);
                            triggerReplication(componentsToBeReplicated, false, opType);
                        }
                        mergePolicy.diskComponentAdded(lsmIndex, fullMergeIsRequested.get());
                    }
                    break;
                default:
                    break;
            }
        } finally {
            opTracker.afterOperation(lsmIndex, opType, ctx.getSearchOperationCallback(), ctx.getModificationCallback());
        }
    }
}

Example 17 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class ExternalIndexHarness method merge.

@Override
public void merge(ILSMIndexOperationContext ctx, ILSMIOOperation operation) throws HyracksDataException {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Started a merge operation for index: " + lsmIndex + " ...");
    }
    ILSMDiskComponent newComponent = null;
    try {
        newComponent = lsmIndex.merge(operation);
        operation.getCallback().afterOperation(LSMOperationType.MERGE, ctx.getComponentHolder(), newComponent);
        lsmIndex.markAsValid(newComponent);
    } finally {
        exitComponents(ctx, LSMOperationType.MERGE, newComponent, false);
        operation.getCallback().afterFinalize(LSMOperationType.MERGE, newComponent);
    }
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Finished the merge operation for index: " + lsmIndex);
    }
}

Example 18 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class ConstantMergePolicy method diskComponentAdded.

@Override
public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException {
    List<ILSMDiskComponent> immutableComponents = index.getImmutableComponents();
    if (!areComponentsMergable(immutableComponents)) {
        return;
    }
    if (fullMergeIsRequested) {
        ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        accessor.scheduleFullMerge(index.getIOOperationCallback());
    } else if (immutableComponents.size() >= numComponents) {
        ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        accessor.scheduleMerge(index.getIOOperationCallback(), immutableComponents);
    }
}

Example 19 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class ConstantMergePolicy method isMergeLagging.

@Override
public boolean isMergeLagging(ILSMIndex index) throws HyracksDataException {
    // see PrefixMergePolicy.isMergeLagging() for the rationale behind this code.
    /**
         * case 1.
         * if totalImmutableCommponentCount < threshold,
         * merge operation is not lagged ==> return false.
         * case 2.
         * if a) totalImmutableCommponentCount >= threshold && b) there is an ongoing merge,
         * merge operation is lagged. ==> return true.
         * case 3. *SPECIAL CASE*
         * if a) totalImmutableCommponentCount >= threshold && b) there is *NO* ongoing merge,
         * merge operation is lagged. ==> *schedule a merge operation* and then return true.
         * This is a special case that requires to schedule a merge operation.
         * Otherwise, all flush operations will be hung.
         * This case can happen in a following situation:
         * The system may crash when
         * condition 1) the mergableImmutableCommponentCount >= threshold and
         * condition 2) merge operation is going on.
         * After the system is recovered, still condition 1) is true.
         * If there are flush operations in the same dataset partition after the recovery,
         * all these flush operations may not proceed since there is no ongoing merge and
         * there will be no new merge either in this situation.
         */
    List<ILSMDiskComponent> immutableComponents = index.getImmutableComponents();
    int totalImmutableComponentCount = immutableComponents.size();
    // [case 1]
    if (totalImmutableComponentCount < numComponents) {
        return false;
    }
    boolean isMergeOngoing = isMergeOngoing(immutableComponents);
    // here, implicitly (totalImmutableComponentCount >= numComponents) is true by passing case 1.
    if (isMergeOngoing) {
        // [case 2]
        return true;
    } else {
        // schedule a merge operation after making sure that all components are mergable
        if (!areComponentsMergable(immutableComponents)) {
            throw new IllegalStateException();
        }
        ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        accessor.scheduleMerge(index.getIOOperationCallback(), immutableComponents);
        return true;
    }
}

Example 20 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class LSMHarness method flush.

@Override
public void flush(ILSMIndexOperationContext ctx, ILSMIOOperation operation) throws HyracksDataException {
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Started a flush operation for index: " + lsmIndex + " ...");
    }
    ILSMDiskComponent newComponent = null;
    try {
        newComponent = lsmIndex.flush(operation);
        operation.getCallback().afterOperation(LSMOperationType.FLUSH, null, newComponent);
        lsmIndex.markAsValid(newComponent);
    } catch (Throwable e) {
        e.printStackTrace();
        throw e;
    } finally {
        exitComponents(ctx, LSMOperationType.FLUSH, newComponent, false);
        operation.getCallback().afterFinalize(LSMOperationType.FLUSH, newComponent);
    }
    if (LOGGER.isLoggable(Level.INFO)) {
        LOGGER.info("Finished the flush operation for index: " + lsmIndex);
    }
}