Examples with ILSMComponent org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent used on opensource projects

Example 6 with ILSMComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent in project asterixdb by apache.

the class LSMHarness method getAndEnterComponents.

protected boolean getAndEnterComponents(ILSMIndexOperationContext ctx, LSMOperationType opType, boolean isTryOperation) throws HyracksDataException {
    validateOperationEnterComponentsState(ctx);
    synchronized (opTracker) {
        while (true) {
            lsmIndex.getOperationalComponents(ctx);
            // Before entering the components, prune those corner cases that indeed should not proceed.
            switch(opType) {
                case FLUSH:
                    ILSMComponent flushingComponent = ctx.getComponentHolder().get(0);
                    if (!((AbstractLSMMemoryComponent) flushingComponent).isModified()) {
                        //since the component is empty, set its state back to READABLE_WRITABLE
                        if (((AbstractLSMIndex) lsmIndex).getCurrentMutableComponentState() == ComponentState.READABLE_UNWRITABLE) {
                            ((AbstractLSMIndex) lsmIndex).setCurrentMutableComponentState(ComponentState.READABLE_WRITABLE);
                            opTracker.notifyAll();
                        }
                        return false;
                    }
                    if (((AbstractLSMMemoryComponent) flushingComponent).getWriterCount() > 0) {
                        /*
                             * This case is a case where even though FLUSH log was flushed to disk and scheduleFlush is triggered,
                             * the current in-memory component (whose state was changed to READABLE_WRITABLE (RW)
                             * from READABLE_UNWRITABLE(RU) before FLUSH log was written to log tail (which is memory buffer of log file)
                             * and then the state was changed back to RW (as shown in the following scenario)) can have writers
                             * based on the current code base/design.
                             * Thus, the writer count of the component may be greater than 0.
                             * if this happens, intead of throwing exception, scheduleFlush() deal with this situation by not flushing
                             * the component.
                             * Please see issue 884 for more detail information:
                             * https://code.google.com/p/asterixdb/issues/detail?id=884&q=owner%3Akisskys%40gmail.com&colspec=ID%20Type%20Status%20Priority%20Milestone%20Owner%20Summary%20ETA%20Severity
                             *
                             */
                        return false;
                    }
                    break;
                case MERGE:
                    if (ctx.getComponentHolder().size() < 2) {
                        // There is only a single component. There is nothing to merge.
                        return false;
                    }
                default:
                    break;
            }
            if (enterComponents(ctx, opType)) {
                return true;
            } else if (isTryOperation) {
                return false;
            }
            try {
                // Flush and merge operations should never reach this wait call, because they are always try operations.
                // If they fail to enter the components, then it means that there are an ongoing flush/merge operation on
                // the same components, so they should not proceed.
                opTracker.wait();
            } catch (InterruptedException e) {
                throw new HyracksDataException(e);
            }
        }
    }
}

Example 7 with ILSMComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent in project asterixdb by apache.

the class PrefixMergePolicy method getMergableImmutableComponentCount.

/**
     * This method returns the number of mergable components among the given list
     * of immutable components that are ordered from the latest component to order ones. A caller
     * need to make sure the order in the list.
     *
     * @param immutableComponents
     * @return the number of mergable component
     */
private int getMergableImmutableComponentCount(List<ILSMDiskComponent> immutableComponents) {
    int count = 0;
    for (ILSMComponent c : immutableComponents) {
        long componentSize = ((ILSMDiskComponent) c).getComponentSize();
        //stop when the first non-mergable component is found.
        if (c.getState() != ComponentState.READABLE_UNWRITABLE || componentSize > maxMergableComponentSize) {
            break;
        }
        ++count;
    }
    return count;
}

Example 8 with ILSMComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent in project asterixdb by apache.

the class PrefixMergePolicy method scheduleMerge.

/**
     * schedule a merge operation according to this prefix merge policy
     *
     * @param index
     * @return true if merge is scheduled, false otherwise.
     * @throws HyracksDataException
     * @throws IndexException
     */
private boolean scheduleMerge(final ILSMIndex index) throws HyracksDataException {
    // 1.  Look at the candidate components for merging in oldest-first order.  If one exists, identify the prefix of the sequence of
    // all such components for which the sum of their sizes exceeds MaxMrgCompSz.  Schedule a merge of those components into a new component.
    // 2.  If a merge from 1 doesn't happen, see if the set of candidate components for merging exceeds MaxTolCompCnt.  If so, schedule
    // a merge all of the current candidates into a new single component.
    List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents());
    // Reverse the components order so that we look at components from oldest to newest.
    Collections.reverse(immutableComponents);
    long totalSize = 0;
    int startIndex = -1;
    for (int i = 0; i < immutableComponents.size(); i++) {
        ILSMComponent c = immutableComponents.get(i);
        long componentSize = ((ILSMDiskComponent) c).getComponentSize();
        if (componentSize > maxMergableComponentSize) {
            startIndex = i;
            totalSize = 0;
            continue;
        }
        totalSize += componentSize;
        boolean isLastComponent = i + 1 == immutableComponents.size() ? true : false;
        if (totalSize > maxMergableComponentSize || (isLastComponent && i - startIndex >= maxToleranceComponentCount)) {
            List<ILSMDiskComponent> mergableComponents = new ArrayList<>();
            for (int j = startIndex + 1; j <= i; j++) {
                mergableComponents.add(immutableComponents.get(j));
            }
            // Reverse the components order back to its original order
            Collections.reverse(mergableComponents);
            ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
            accessor.scheduleMerge(index.getIOOperationCallback(), mergableComponents);
            return true;
        }
    }
    return false;
}

Example 9 with ILSMComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent in project asterixdb by apache.

the class CorrelatedPrefixMergePolicy method scheduleMerge.

private boolean scheduleMerge(ILSMIndex index) throws HyracksDataException {
    List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents());
    Collections.reverse(immutableComponents);
    long totalSize = 0;
    int startIndex = -1;
    int numComponents = immutableComponents.size();
    for (int i = 0; i < numComponents; i++) {
        ILSMComponent c = immutableComponents.get(i);
        long componentSize = ((ILSMDiskComponent) c).getComponentSize();
        if (componentSize > maxMergableComponentSize || ((ILSMDiskComponent) c).getComponentId().notFound()) {
            startIndex = i;
            totalSize = 0;
            continue;
        }
        totalSize += componentSize;
        boolean isLastComponent = i + 1 == numComponents ? true : false;
        if (totalSize > maxMergableComponentSize || (isLastComponent && i - startIndex >= maxToleranceComponentCount)) {
            //merge disk components from startIndex+1 to i
            long minID = Long.MAX_VALUE;
            long maxID = Long.MIN_VALUE;
            for (int j = startIndex + 1; j <= i; j++) {
                ILSMDiskComponentId id = immutableComponents.get(j).getComponentId();
                if (minID > id.getMinId()) {
                    minID = id.getMinId();
                }
                if (maxID < id.getMaxId()) {
                    maxID = id.getMaxId();
                }
            }
            Set<IndexInfo> indexInfos = datasetLifecycleManager.getDatasetInfo(datasetId).getDatsetIndexInfos();
            int partition = getIndexPartition(index, indexInfos);
            triggerScheduledMerge(minID, maxID, indexInfos.stream().filter(info -> info.getPartition() == partition).collect(Collectors.toSet()));
            return true;
        }
    }
    return false;
}

Example 10 with ILSMComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMComponent in project asterixdb by apache.

the class CorrelatedPrefixMergePolicy method getMergableImmutableComponentCount.

/**
     * This method returns the number of mergable components among the given list
     * of immutable components that are ordered from the latest component to order ones. A caller
     * need to make sure the order in the list.
     *
     * @param immutableComponents
     * @return the number of mergable component
     * @throws HyracksDataException
     */
private int getMergableImmutableComponentCount(List<ILSMDiskComponent> immutableComponents) throws HyracksDataException {
    int count = 0;
    for (ILSMComponent c : immutableComponents) {
        long componentSize = ((ILSMDiskComponent) c).getComponentSize();
        //stop when the first non-mergable component is found.
        if (c.getState() != ComponentState.READABLE_UNWRITABLE || componentSize > maxMergableComponentSize || ((ILSMDiskComponent) c).getComponentId().notFound()) {
            break;
        }
        ++count;
    }
    return count;
}