Examples with ILSMIndex org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex used on opensource projects

Example 6 with ILSMIndex

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex in project asterixdb by apache.

the class PrimaryIndexOperationTracker method flushIfRequested.

public void flushIfRequested() throws HyracksDataException {
    // If we need a flush, and this is the last completing operation, then schedule the flush,
    // or if there is a flush scheduled by the checkpoint (flushOnExit), then schedule it
    boolean needsFlush = false;
    Set<ILSMIndex> indexes = dsInfo.getDatasetIndexes();
    if (!flushOnExit) {
        for (ILSMIndex lsmIndex : indexes) {
            if (lsmIndex.hasFlushRequestForCurrentMutableComponent()) {
                needsFlush = true;
                break;
            }
        }
    }
    if (needsFlush || flushOnExit) {
        //Make the current mutable components READABLE_UNWRITABLE to stop coming modify operations from entering them until the current flush is scheduled.
        for (ILSMIndex lsmIndex : indexes) {
            ILSMOperationTracker opTracker = lsmIndex.getOperationTracker();
            synchronized (opTracker) {
                ILSMMemoryComponent memComponent = lsmIndex.getCurrentMemoryComponent();
                if (memComponent.getState() == ComponentState.READABLE_WRITABLE && memComponent.isModified()) {
                    memComponent.setState(ComponentState.READABLE_UNWRITABLE);
                }
            }
        }
        LogRecord logRecord = new LogRecord();
        flushOnExit = false;
        if (dsInfo.isDurable()) {
            /**
                 * Generate a FLUSH log.
                 * Flush will be triggered when the log is written to disk by LogFlusher.
                 */
            TransactionUtil.formFlushLogRecord(logRecord, datasetID, this, logManager.getNodeId(), dsInfo.getDatasetIndexes().size());
            try {
                logManager.log(logRecord);
            } catch (ACIDException e) {
                throw new HyracksDataException("could not write flush log", e);
            }
            flushLogCreated = true;
        } else {
            //trigger flush for temporary indexes without generating a FLUSH log.
            triggerScheduleFlush(logRecord);
        }
    }
}

Example 7 with ILSMIndex

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex in project asterixdb by apache.

the class PrimaryIndexOperationTracker method triggerScheduleFlush.

//This method is called sequentially by LogPage.notifyFlushTerminator in the sequence flushes were scheduled.
public synchronized void triggerScheduleFlush(LogRecord logRecord) throws HyracksDataException {
    for (ILSMIndex lsmIndex : dsInfo.getDatasetIndexes()) {
        //get resource
        ILSMIndexAccessor accessor = lsmIndex.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        //update resource lsn
        AbstractLSMIOOperationCallback ioOpCallback = (AbstractLSMIOOperationCallback) lsmIndex.getIOOperationCallback();
        ioOpCallback.updateLastLSN(logRecord.getLSN());
        //schedule flush after update
        accessor.scheduleFlush(lsmIndex.getIOOperationCallback());
    }
    flushLogCreated = false;
}

Example 8 with ILSMIndex

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex in project asterixdb by apache.

the class RecoveryManager method startRecoveryRedoPhase.

private synchronized void startRecoveryRedoPhase(Set<Integer> partitions, ILogReader logReader, long lowWaterMarkLSN, Set<Integer> winnerJobSet) throws IOException, ACIDException {
    int redoCount = 0;
    int jobId = -1;
    long resourceId;
    long maxDiskLastLsn;
    long lsn = -1;
    ILSMIndex index = null;
    LocalResource localResource = null;
    DatasetLocalResource localResourceMetadata = null;
    boolean foundWinner = false;
    JobEntityCommits jobEntityWinners = null;
    IAppRuntimeContextProvider appRuntimeContext = txnSubsystem.getAsterixAppRuntimeContextProvider();
    IDatasetLifecycleManager datasetLifecycleManager = appRuntimeContext.getDatasetLifecycleManager();
    Map<Long, LocalResource> resourcesMap = localResourceRepository.loadAndGetAllResources();
    Map<Long, Long> resourceId2MaxLSNMap = new HashMap<>();
    TxnId tempKeyTxnId = new TxnId(-1, -1, -1, null, -1, false);
    ILogRecord logRecord = null;
    try {
        logReader.initializeScan(lowWaterMarkLSN);
        logRecord = logReader.next();
        while (logRecord != null) {
            if (IS_DEBUG_MODE) {
                LOGGER.info(logRecord.getLogRecordForDisplay());
            }
            lsn = logRecord.getLSN();
            jobId = logRecord.getJobId();
            foundWinner = false;
            switch(logRecord.getLogType()) {
                case LogType.UPDATE:
                    if (partitions.contains(logRecord.getResourcePartition())) {
                        if (winnerJobSet.contains(jobId)) {
                            foundWinner = true;
                        } else if (jobId2WinnerEntitiesMap.containsKey(jobId)) {
                            jobEntityWinners = jobId2WinnerEntitiesMap.get(jobId);
                            tempKeyTxnId.setTxnId(jobId, logRecord.getDatasetId(), logRecord.getPKHashValue(), logRecord.getPKValue(), logRecord.getPKValueSize());
                            if (jobEntityWinners.containsEntityCommitForTxnId(lsn, tempKeyTxnId)) {
                                foundWinner = true;
                            }
                        }
                        if (foundWinner) {
                            resourceId = logRecord.getResourceId();
                            localResource = resourcesMap.get(resourceId);
                            /*******************************************************************
                                 * [Notice]
                                 * -> Issue
                                 * Delete index may cause a problem during redo.
                                 * The index operation to be redone couldn't be redone because the corresponding index
                                 * may not exist in NC due to the possible index drop DDL operation.
                                 * -> Approach
                                 * Avoid the problem during redo.
                                 * More specifically, the problem will be detected when the localResource of
                                 * the corresponding index is retrieved, which will end up with 'null'.
                                 * If null is returned, then just go and process the next
                                 * log record.
                                 *******************************************************************/
                            if (localResource == null) {
                                LOGGER.log(Level.WARNING, "resource was not found for resource id " + resourceId);
                                logRecord = logReader.next();
                                continue;
                            }
                            /*******************************************************************/
                            //get index instance from IndexLifeCycleManager
                            //if index is not registered into IndexLifeCycleManager,
                            //create the index using LocalMetadata stored in LocalResourceRepository
                            //get partition path in this node
                            localResourceMetadata = (DatasetLocalResource) localResource.getResource();
                            index = (ILSMIndex) datasetLifecycleManager.get(localResource.getPath());
                            if (index == null) {
                                //#. create index instance and register to indexLifeCycleManager
                                index = (ILSMIndex) localResourceMetadata.createInstance(serviceCtx);
                                datasetLifecycleManager.register(localResource.getPath(), index);
                                datasetLifecycleManager.open(localResource.getPath());
                                //#. get maxDiskLastLSN
                                ILSMIndex lsmIndex = index;
                                try {
                                    maxDiskLastLsn = ((AbstractLSMIOOperationCallback) lsmIndex.getIOOperationCallback()).getComponentLSN(lsmIndex.getImmutableComponents());
                                } catch (HyracksDataException e) {
                                    datasetLifecycleManager.close(localResource.getPath());
                                    throw e;
                                }
                                //#. set resourceId and maxDiskLastLSN to the map
                                resourceId2MaxLSNMap.put(resourceId, maxDiskLastLsn);
                            } else {
                                maxDiskLastLsn = resourceId2MaxLSNMap.get(resourceId);
                            }
                            if (lsn > maxDiskLastLsn) {
                                redo(logRecord, datasetLifecycleManager);
                                redoCount++;
                            }
                        }
                    }
                    break;
                case LogType.JOB_COMMIT:
                case LogType.ENTITY_COMMIT:
                case LogType.ABORT:
                case LogType.FLUSH:
                case LogType.WAIT:
                case LogType.MARKER:
                    //do nothing
                    break;
                default:
                    throw new ACIDException("Unsupported LogType: " + logRecord.getLogType());
            }
            logRecord = logReader.next();
        }
        LOGGER.info("Logs REDO phase completed. Redo logs count: " + redoCount);
    } finally {
        //close all indexes
        Set<Long> resourceIdList = resourceId2MaxLSNMap.keySet();
        for (long r : resourceIdList) {
            datasetLifecycleManager.close(resourcesMap.get(r).getPath());
        }
    }
}

Example 9 with ILSMIndex

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex in project asterixdb by apache.

the class RecoveryManager method undo.

private static void undo(ILogRecord logRecord, IDatasetLifecycleManager datasetLifecycleManager) {
    try {
        ILSMIndex index = (ILSMIndex) datasetLifecycleManager.getIndex(logRecord.getDatasetId(), logRecord.getResourceId());
        ILSMIndexAccessor indexAccessor = index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.INSERT_BYTE) {
            indexAccessor.forceDelete(logRecord.getNewValue());
        } else if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.DELETE_BYTE) {
            indexAccessor.forceInsert(logRecord.getOldValue());
        } else if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.UPSERT_BYTE) {
            // undo, upsert the old value if found, otherwise, physical delete
            if (logRecord.getOldValue() == null) {
                indexAccessor.forcePhysicalDelete(logRecord.getNewValue());
            } else {
                indexAccessor.forceUpsert(logRecord.getOldValue());
            }
        } else {
            throw new IllegalStateException("Unsupported OperationType: " + logRecord.getNewOp());
        }
    } catch (Exception e) {
        throw new IllegalStateException("Failed to undo", e);
    }
}

Example 10 with ILSMIndex

use of org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex in project asterixdb by apache.

the class RecoveryManager method redo.

private static void redo(ILogRecord logRecord, IDatasetLifecycleManager datasetLifecycleManager) {
    try {
        int datasetId = logRecord.getDatasetId();
        long resourceId = logRecord.getResourceId();
        ILSMIndex index = (ILSMIndex) datasetLifecycleManager.getIndex(datasetId, resourceId);
        ILSMIndexAccessor indexAccessor = index.createAccessor(NoOpOperationCallback.INSTANCE, NoOpOperationCallback.INSTANCE);
        if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.INSERT_BYTE) {
            indexAccessor.forceInsert(logRecord.getNewValue());
        } else if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.DELETE_BYTE) {
            indexAccessor.forceDelete(logRecord.getNewValue());
        } else if (logRecord.getNewOp() == AbstractIndexModificationOperationCallback.UPSERT_BYTE) {
            // redo, upsert the new value
            indexAccessor.forceUpsert(logRecord.getNewValue());
        } else {
            throw new IllegalStateException("Unsupported OperationType: " + logRecord.getNewOp());
        }
    } catch (Exception e) {
        throw new IllegalStateException("Failed to redo", e);
    }
}