Examples with ILSMDiskComponentId org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentId used on opensource projects

Example 1 with ILSMDiskComponentId

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentId in project asterixdb by apache.

the class CorrelatedPrefixMergePolicy method scheduleMerge.

private boolean scheduleMerge(ILSMIndex index) throws HyracksDataException {
    List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents());
    Collections.reverse(immutableComponents);
    long totalSize = 0;
    int startIndex = -1;
    int numComponents = immutableComponents.size();
    for (int i = 0; i < numComponents; i++) {
        ILSMComponent c = immutableComponents.get(i);
        long componentSize = ((ILSMDiskComponent) c).getComponentSize();
        if (componentSize > maxMergableComponentSize || ((ILSMDiskComponent) c).getComponentId().notFound()) {
            startIndex = i;
            totalSize = 0;
            continue;
        }
        totalSize += componentSize;
        boolean isLastComponent = i + 1 == numComponents ? true : false;
        if (totalSize > maxMergableComponentSize || (isLastComponent && i - startIndex >= maxToleranceComponentCount)) {
            //merge disk components from startIndex+1 to i
            long minID = Long.MAX_VALUE;
            long maxID = Long.MIN_VALUE;
            for (int j = startIndex + 1; j <= i; j++) {
                ILSMDiskComponentId id = immutableComponents.get(j).getComponentId();
                if (minID > id.getMinId()) {
                    minID = id.getMinId();
                }
                if (maxID < id.getMaxId()) {
                    maxID = id.getMaxId();
                }
            }
            Set<IndexInfo> indexInfos = datasetLifecycleManager.getDatasetInfo(datasetId).getDatsetIndexInfos();
            int partition = getIndexPartition(index, indexInfos);
            triggerScheduledMerge(minID, maxID, indexInfos.stream().filter(info -> info.getPartition() == partition).collect(Collectors.toSet()));
            return true;
        }
    }
    return false;
}

Example 2 with ILSMDiskComponentId

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentId in project asterixdb by apache.

the class CorrelatedPrefixMergePolicy method triggerScheduledMerge.

/**
     * Submit merge requests for all disk components within [minID, maxID]
     * of all indexes of a given dataset in the given partition
     *
     * @param minID
     * @param maxID
     * @param partition
     * @param indexInfos
     * @throws HyracksDataException
     */
private void triggerScheduledMerge(long minID, long maxID, Set<IndexInfo> indexInfos) throws HyracksDataException {
    for (IndexInfo info : indexInfos) {
        ILSMIndex lsmIndex = info.getIndex();
        List<ILSMDiskComponent> immutableComponents = new ArrayList<>(lsmIndex.getImmutableComponents());
        if (isMergeOngoing(immutableComponents)) {
            continue;
        }
        List<ILSMDiskComponent> mergableComponents = new ArrayList<>();
        for (ILSMDiskComponent component : immutableComponents) {
            ILSMDiskComponentId id = component.getComponentId();
            if (!id.notFound()) {
                if (id.getMinId() >= minID && id.getMaxId() <= maxID) {
                    mergableComponents.add(component);
                }
                if (id.getMaxId() < minID) {
                    //if the component.maxID < minID, we can safely skip the rest disk components in the list
                    break;
                }
            }
        }
        ILSMIndexAccessor accessor = lsmIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        accessor.scheduleMerge(lsmIndex.getIOOperationCallback(), mergableComponents);
    }
}

Example 3 with ILSMDiskComponentId

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentId in project asterixdb by apache.

the class AbstractLSMIOOperationCallback method getComponentId.

private ILSMDiskComponentId getComponentId(List<ILSMComponent> oldComponents) throws HyracksDataException {
    if (oldComponents == null) {
        //if oldComponents == null, then getComponentLSN would treat it as a flush operation,
        //and return the LSN for the flushed component
        long id = getComponentLSN(null);
        if (id == 0) {
            logger.log(Level.WARNING, "Flushing a memory component without setting the LSN");
            id = ILSMDiskComponentId.NOT_FOUND;
        }
        return new LSMDiskComponentId(id, id);
    } else {
        long minId = Long.MAX_VALUE;
        long maxId = Long.MIN_VALUE;
        for (ILSMComponent oldComponent : oldComponents) {
            ILSMDiskComponentId oldComponentId = ((ILSMDiskComponent) oldComponent).getComponentId();
            if (oldComponentId.getMinId() < minId) {
                minId = oldComponentId.getMinId();
            }
            if (oldComponentId.getMaxId() > maxId) {
                maxId = oldComponentId.getMaxId();
            }
        }
        return new LSMDiskComponentId(minId, maxId);
    }
}

Example 4 with ILSMDiskComponentId

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentId in project asterixdb by apache.

the class AbstractLSMIOOperationCallback method putComponentIdIntoMetadata.

private void putComponentIdIntoMetadata(ILSMDiskComponent component, List<ILSMComponent> oldComponents) throws HyracksDataException {
    DiskComponentMetadata metadata = component.getMetadata();
    ILSMDiskComponentId componentId = getComponentId(oldComponents);
    metadata.put(ILSMDiskComponentId.COMPONENT_ID_MIN_KEY, LongPointable.FACTORY.createPointable(componentId.getMinId()));
    metadata.put(ILSMDiskComponentId.COMPONENT_ID_MAX_KEY, LongPointable.FACTORY.createPointable(componentId.getMaxId()));
}

Example 5 with ILSMDiskComponentId

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponentId in project asterixdb by apache.

the class CorrelatedPrefixMergePolicyTest method testBasic.

@Test
public void testBasic() {
    try {
        List<ILSMDiskComponentId> componentIDs = Arrays.asList(new LSMDiskComponentId(5, 5), new LSMDiskComponentId(4, 4), new LSMDiskComponentId(3, 3), new LSMDiskComponentId(2, 2), new LSMDiskComponentId(1, 1));
        List<ILSMDiskComponentId> resultPrimaryIDs = new ArrayList<>();
        IndexInfo primary = mockIndex(true, componentIDs, resultPrimaryIDs, 0);
        List<ILSMDiskComponentId> resultSecondaryIDs = new ArrayList<>();
        IndexInfo secondary = mockIndex(false, componentIDs, resultSecondaryIDs, 0);
        ILSMMergePolicy policy = mockMergePolicy(primary, secondary);
        policy.diskComponentAdded(secondary.getIndex(), false);
        Assert.assertTrue(resultPrimaryIDs.isEmpty());
        Assert.assertTrue(resultSecondaryIDs.isEmpty());
        policy.diskComponentAdded(primary.getIndex(), false);
        Assert.assertEquals(Arrays.asList(new LSMDiskComponentId(4, 4), new LSMDiskComponentId(3, 3), new LSMDiskComponentId(2, 2), new LSMDiskComponentId(1, 1)), resultPrimaryIDs);
        Assert.assertEquals(Arrays.asList(new LSMDiskComponentId(4, 4), new LSMDiskComponentId(3, 3), new LSMDiskComponentId(2, 2), new LSMDiskComponentId(1, 1)), resultSecondaryIDs);
    } catch (HyracksDataException e) {
        Assert.fail(e.getMessage());
    }
}