Examples with ILSMIndexAccessor org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor used on opensource projects

Example 11 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class CorrelatedPrefixMergePolicy method diskComponentAdded.

@Override
public void diskComponentAdded(final ILSMIndex index, boolean fullMergeIsRequested) throws HyracksDataException {
    if (fullMergeIsRequested) {
        //full merge request is handled by each index separately, since it is possible that
        //when a primary index wants to send full merge requests for all secondaries,
        //one secondary index is being merged and the request cannot be scheduled
        List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents());
        if (!areComponentsReadableUnwritableState(immutableComponents)) {
            return;
        }
        ILSMIndexAccessor accessor = index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        accessor.scheduleFullMerge(index.getIOOperationCallback());
        return;
    }
    if (!index.isPrimaryIndex()) {
        return;
    }
    List<ILSMDiskComponent> immutableComponents = new ArrayList<>(index.getImmutableComponents());
    if (!areComponentsReadableUnwritableState(immutableComponents)) {
        return;
    }
    scheduleMerge(index);
}

Example 12 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class CorrelatedPrefixMergePolicy method triggerScheduledMerge.

/**
     * Submit merge requests for all disk components within [minID, maxID]
     * of all indexes of a given dataset in the given partition
     *
     * @param minID
     * @param maxID
     * @param partition
     * @param indexInfos
     * @throws HyracksDataException
     */
private void triggerScheduledMerge(long minID, long maxID, Set<IndexInfo> indexInfos) throws HyracksDataException {
    for (IndexInfo info : indexInfos) {
        ILSMIndex lsmIndex = info.getIndex();
        List<ILSMDiskComponent> immutableComponents = new ArrayList<>(lsmIndex.getImmutableComponents());
        if (isMergeOngoing(immutableComponents)) {
            continue;
        }
        List<ILSMDiskComponent> mergableComponents = new ArrayList<>();
        for (ILSMDiskComponent component : immutableComponents) {
            ILSMDiskComponentId id = component.getComponentId();
            if (!id.notFound()) {
                if (id.getMinId() >= minID && id.getMaxId() <= maxID) {
                    mergableComponents.add(component);
                }
                if (id.getMaxId() < minID) {
                    //if the component.maxID < minID, we can safely skip the rest disk components in the list
                    break;
                }
            }
        }
        ILSMIndexAccessor accessor = lsmIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        accessor.scheduleMerge(lsmIndex.getIOOperationCallback(), mergableComponents);
    }
}

Example 13 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class LSMInsertDeleteOperatorNodePushable method nextFrame.

@Override
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
    currentTupleIdx = 0;
    lastFlushedTupleIdx = 0;
    flushedPartialTuples = false;
    accessor.reset(buffer);
    ILSMIndexAccessor lsmAccessor = (ILSMIndexAccessor) indexAccessor;
    int tupleCount = accessor.getTupleCount();
    try {
        for (; i < tupleCount; i++, currentTupleIdx++) {
            if (tupleFilter != null) {
                frameTuple.reset(accessor, i);
                if (!tupleFilter.accept(frameTuple)) {
                    continue;
                }
            }
            tuple.reset(accessor, i);
            switch(op) {
                case INSERT:
                    if (i == 0 && isPrimary) {
                        lsmAccessor.insert(tuple);
                    } else {
                        lsmAccessor.forceInsert(tuple);
                    }
                    break;
                case DELETE:
                    if (i == 0 && isPrimary) {
                        lsmAccessor.delete(tuple);
                    } else {
                        lsmAccessor.forceDelete(tuple);
                    }
                    break;
                default:
                    {
                        throw HyracksDataException.create(ErrorCode.INVALID_OPERATOR_OPERATION, op.toString(), LSMInsertDeleteOperatorNodePushable.class.getSimpleName());
                    }
            }
        }
    } catch (HyracksDataException e) {
        if (e.getErrorCode() == ErrorCode.INVALID_OPERATOR_OPERATION) {
            throw e;
        } else {
            throw HyracksDataException.create(ErrorCode.ERROR_PROCESSING_TUPLE, e, i);
        }
    } catch (Exception e) {
        throw HyracksDataException.create(ErrorCode.ERROR_PROCESSING_TUPLE, e, i);
    }
    writeBuffer.ensureFrameSize(buffer.capacity());
    if (flushedPartialTuples) {
        flushPartialFrame();
    } else {
        FrameUtils.copyAndFlip(buffer, writeBuffer.getBuffer());
        FrameUtils.flushFrame(writeBuffer.getBuffer(), writer);
    }
    i = 0;
}

Example 14 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class DatasetLifecycleManager method flushDatasetOpenIndexes.

/*
     * This method can only be called asynchronously safely if we're sure no modify operation will take place until the flush is scheduled
     */
private void flushDatasetOpenIndexes(DatasetInfo dsInfo, boolean asyncFlush) throws HyracksDataException {
    if (!dsInfo.isExternal() && dsInfo.isDurable()) {
        synchronized (logRecord) {
            TransactionUtil.formFlushLogRecord(logRecord, dsInfo.getDatasetID(), null, logManager.getNodeId(), dsInfo.getIndexes().size());
            try {
                logManager.log(logRecord);
            } catch (ACIDException e) {
                throw new HyracksDataException("could not write flush log while closing dataset", e);
            }
            try {
                //notification will come from LogPage class (notifyFlushTerminator)
                logRecord.wait();
            } catch (InterruptedException e) {
                throw new HyracksDataException(e);
            }
        }
        for (IndexInfo iInfo : dsInfo.getIndexes().values()) {
            //update resource lsn
            AbstractLSMIOOperationCallback ioOpCallback = (AbstractLSMIOOperationCallback) iInfo.getIndex().getIOOperationCallback();
            ioOpCallback.updateLastLSN(logRecord.getLSN());
        }
    }
    if (asyncFlush) {
        for (IndexInfo iInfo : dsInfo.getIndexes().values()) {
            ILSMIndexAccessor accessor = iInfo.getIndex().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
            accessor.scheduleFlush(iInfo.getIndex().getIOOperationCallback());
        }
    } else {
        for (IndexInfo iInfo : dsInfo.getIndexes().values()) {
            // TODO: This is not efficient since we flush the indexes sequentially.
            // Think of a way to allow submitting the flush requests concurrently. We don't do them concurrently because this
            // may lead to a deadlock scenario between the DatasetLifeCycleManager and the PrimaryIndexOperationTracker.
            flushAndWaitForIO(dsInfo, iInfo);
        }
    }
}

Example 15 with ILSMIndexAccessor

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor in project asterixdb by apache.

the class DatasetLifecycleManager method flushAndWaitForIO.

private static void flushAndWaitForIO(DatasetInfo dsInfo, IndexInfo iInfo) throws HyracksDataException {
    if (iInfo.isOpen()) {
        ILSMIndexAccessor accessor = iInfo.getIndex().createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        accessor.scheduleFlush(iInfo.getIndex().getIOOperationCallback());
    }
    // Wait for the above flush op.
    synchronized (dsInfo) {
        while (dsInfo.getNumActiveIOOps() > 0) {
            try {
                //notification will come from DatasetInfo class (undeclareActiveIOOperation)
                dsInfo.wait();
            } catch (InterruptedException e) {
                throw new HyracksDataException(e);
            }
        }
    }
}