Examples with ILSMDiskComponent org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent used on opensource projects

Example 6 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class PrefixMergePolicy method scheduleMerge.

/**
     * schedule a merge operation according to this prefix merge policy
     *
     * @param index
     * @return true if merge is scheduled, false otherwise.
     * @throws HyracksDataException
     * @throws IndexException
     */
private boolean scheduleMerge(final ILSMIndex index) throws HyracksDataException {
    // 1.  Look at the candidate components for merging in oldest-first order.  If one exists, identify the prefix of the sequence of
    // all such components for which the sum of their sizes exceeds MaxMrgCompSz.  Schedule a merge of those components into a new component.
    // 2.  If a merge from 1 doesn't happen, see if the set of candidate components for merging exceeds MaxTolCompCnt.  If so, schedule
    // a merge all of the current candidates into a new single component.
    List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents());
    // Reverse the components order so that we look at components from oldest to newest.
    Collections.reverse(immutableComponents);
    long totalSize = 0;
    int startIndex = -1;
    for (int i = 0; i < immutableComponents.size(); i++) {
        ILSMComponent c = immutableComponents.get(i);
        long componentSize = ((ILSMDiskComponent) c).getComponentSize();
        if (componentSize > maxMergableComponentSize) {
            startIndex = i;
            totalSize = 0;
            continue;
        }
        totalSize += componentSize;
        boolean isLastComponent = i + 1 == immutableComponents.size() ? true : false;
        if (totalSize > maxMergableComponentSize || (isLastComponent && i - startIndex >= maxToleranceComponentCount)) {
            List<ILSMDiskComponent> mergableComponents = new ArrayList<>();
            for (int j = startIndex + 1; j <= i; j++) {
                mergableComponents.add(immutableComponents.get(j));
            }
            // Reverse the components order back to its original order
            Collections.reverse(mergableComponents);
            ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
            accessor.scheduleMerge(index.getIOOperationCallback(), mergableComponents);
            return true;
        }
    }
    return false;
}

Example 7 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class PrefixMergePolicy method diskComponentAdded.

@Override
public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException {
    ArrayList<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents());
    if (!areComponentsReadableWritableState(immutableComponents)) {
        return;
    }
    if (fullMergeIsRequested) {
        ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        accessor.scheduleFullMerge(index.getIOOperationCallback());
        return;
    }
    scheduleMerge(index);
}

Example 8 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class CorrelatedPrefixMergePolicy method scheduleMerge.

private boolean scheduleMerge(ILSMIndex index) throws HyracksDataException {
    List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents());
    Collections.reverse(immutableComponents);
    long totalSize = 0;
    int startIndex = -1;
    int numComponents = immutableComponents.size();
    for (int i = 0; i < numComponents; i++) {
        ILSMComponent c = immutableComponents.get(i);
        long componentSize = ((ILSMDiskComponent) c).getComponentSize();
        if (componentSize > maxMergableComponentSize || ((ILSMDiskComponent) c).getComponentId().notFound()) {
            startIndex = i;
            totalSize = 0;
            continue;
        }
        totalSize += componentSize;
        boolean isLastComponent = i + 1 == numComponents ? true : false;
        if (totalSize > maxMergableComponentSize || (isLastComponent && i - startIndex >= maxToleranceComponentCount)) {
            //merge disk components from startIndex+1 to i
            long minID = Long.MAX_VALUE;
            long maxID = Long.MIN_VALUE;
            for (int j = startIndex + 1; j <= i; j++) {
                ILSMDiskComponentId id = immutableComponents.get(j).getComponentId();
                if (minID > id.getMinId()) {
                    minID = id.getMinId();
                }
                if (maxID < id.getMaxId()) {
                    maxID = id.getMaxId();
                }
            }
            Set<IndexInfo> indexInfos = datasetLifecycleManager.getDatasetInfo(datasetId).getDatsetIndexInfos();
            int partition = getIndexPartition(index, indexInfos);
            triggerScheduledMerge(minID, maxID, indexInfos.stream().filter(info -> info.getPartition() == partition).collect(Collectors.toSet()));
            return true;
        }
    }
    return false;
}

Example 9 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class CorrelatedPrefixMergePolicy method getMergableImmutableComponentCount.

/**
     * This method returns the number of mergable components among the given list
     * of immutable components that are ordered from the latest component to order ones. A caller
     * need to make sure the order in the list.
     *
     * @param immutableComponents
     * @return the number of mergable component
     * @throws HyracksDataException
     */
private int getMergableImmutableComponentCount(List<ILSMDiskComponent> immutableComponents) throws HyracksDataException {
    int count = 0;
    for (ILSMComponent c : immutableComponents) {
        long componentSize = ((ILSMDiskComponent) c).getComponentSize();
        //stop when the first non-mergable component is found.
        if (c.getState() != ComponentState.READABLE_UNWRITABLE || componentSize > maxMergableComponentSize || ((ILSMDiskComponent) c).getComponentId().notFound()) {
            break;
        }
        ++count;
    }
    return count;
}

Example 10 with ILSMDiskComponent

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent in project asterixdb by apache.

the class CorrelatedPrefixMergePolicy method diskComponentAdded.

@Override
public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException {
    if (fullMergeIsRequested) {
        //full merge request is handled by each index separately, since it is possible that
        //when a primary index wants to send full merge requests for all secondaries,
        //one secondary index is being merged and the request cannot be scheduled
        List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents());
        if (!areComponentsReadableUnwritableState(immutableComponents)) {
            return;
        }
        ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        accessor.scheduleFullMerge(index.getIOOperationCallback());
        return;
    }
    if (!index.isPrimaryIndex()) {
        return;
    }
    List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents());
    if (!areComponentsReadableUnwritableState(immutableComponents)) {
        return;
    }
    scheduleMerge(index);
}