Examples with CdcrUpdateLog org.apache.solr.update.CdcrUpdateLog used on opensource projects

Example 1 with CdcrUpdateLog

use of org.apache.solr.update.CdcrUpdateLog in project lucene-solr by apache.

the class ReplicationHandler method getTlogFileList.

/**
   * Retrieves the list of tlog files associated to a commit point.
   */
List<Map<String, Object>> getTlogFileList(IndexCommit commit) throws IOException {
    long maxVersion = this.getMaxVersion(commit);
    CdcrUpdateLog ulog = (CdcrUpdateLog) core.getUpdateHandler().getUpdateLog();
    String[] logList = ulog.getLogList(new File(ulog.getLogDir()));
    List<Map<String, Object>> tlogFiles = new ArrayList<>();
    for (String fileName : logList) {
        // filter out tlogs that are older than the current index commit generation, so that the list of tlog files is
        // in synch with the latest index commit point
        long startVersion = Math.abs(Long.parseLong(fileName.substring(fileName.lastIndexOf('.') + 1)));
        if (startVersion < maxVersion) {
            Map<String, Object> fileMeta = new HashMap<>();
            fileMeta.put(NAME, fileName);
            fileMeta.put(SIZE, new File(ulog.getLogDir(), fileName).length());
            tlogFiles.add(fileMeta);
        }
    }
    return tlogFiles;
}

Example 2 with CdcrUpdateLog

use of org.apache.solr.update.CdcrUpdateLog in project lucene-solr by apache.

the class IndexFetcher method moveTlogFiles.

/**
   * <p>
   *   Copy all the tlog files from the temp tlog dir to the actual tlog dir, and reset
   *   the {@link UpdateLog}. The copy will try to preserve the original tlog directory
   *   if the copy fails.
   * </p>
   * <p>
   *   This assumes that the tlog files transferred from the leader are in synch with the
   *   index files transferred from the leader. The reset of the update log relies on the version
   *   of the latest operations found in the tlog files. If the tlogs are ahead of the latest commit
   *   point, it will not copy all the needed buffered updates for the replay and it will miss
   *   some operations.
   * </p>
   */
private boolean moveTlogFiles(File tmpTlogDir) {
    UpdateLog ulog = solrCore.getUpdateHandler().getUpdateLog();
    VersionInfo vinfo = ulog.getVersionInfo();
    // block updates until the new update log is initialised
    vinfo.blockUpdates();
    try {
        // reset the update log before copying the new tlog directory
        CdcrUpdateLog.BufferedUpdates bufferedUpdates = ((CdcrUpdateLog) ulog).resetForRecovery();
        // try to move the temp tlog files to the tlog directory
        if (!copyTmpTlogFiles2Tlog(tmpTlogDir))
            return false;
        // reinitialise the update log and copy the buffered updates
        if (bufferedUpdates.tlog != null) {
            // map file path to its new backup location
            File parentDir = FileSystems.getDefault().getPath(solrCore.getUpdateHandler().getUpdateLog().getLogDir()).getParent().toFile();
            File backupTlogDir = new File(parentDir, tmpTlogDir.getName());
            bufferedUpdates.tlog = new File(backupTlogDir, bufferedUpdates.tlog.getName());
        }
        // init the update log with the new set of tlog files, and copy the buffered updates
        ((CdcrUpdateLog) ulog).initForRecovery(bufferedUpdates.tlog, bufferedUpdates.offset);
    } catch (Exception e) {
        LOG.error("Unable to copy tlog files", e);
        return false;
    } finally {
        vinfo.unblockUpdates();
    }
    return true;
}

Example 3 with CdcrUpdateLog

use of org.apache.solr.update.CdcrUpdateLog in project lucene-solr by apache.

the class CdcrBufferManager method stateUpdate.

/**
   * This method is synchronised as it can both be called by the leaderStateManager and the bufferStateManager.
   */
@Override
public synchronized void stateUpdate() {
    CdcrUpdateLog ulog = (CdcrUpdateLog) core.getUpdateHandler().getUpdateLog();
    // If I am not the leader, I should always buffer my updates
    if (!leaderStateManager.amILeader()) {
        ulog.enableBuffer();
        return;
    } else // If I am the leader, I should buffer my updates only if buffer is enabled
    if (bufferStateManager.getState().equals(CdcrParams.BufferState.ENABLED)) {
        ulog.enableBuffer();
        return;
    }
    // otherwise, disable the buffer
    ulog.disableBuffer();
}

Example 4 with CdcrUpdateLog

use of org.apache.solr.update.CdcrUpdateLog in project lucene-solr by apache.

the class CdcrRequestHandler method handleLastProcessedVersionAction.

/**
   * <p>
   * We have to take care of four cases:
   * <ul>
   * <li>Replication & Buffering</li>
   * <li>Replication & No Buffering</li>
   * <li>No Replication & Buffering</li>
   * <li>No Replication & No Buffering</li>
   * </ul>
   * In the first three cases, at least one log reader should have been initialised. We should take the lowest
   * last processed version across all the initialised readers. In the last case, there isn't a log reader
   * initialised. We should instantiate one and get the version of the first entries.
   * </p>
   */
private void handleLastProcessedVersionAction(SolrQueryRequest req, SolrQueryResponse rsp) {
    String collectionName = core.getCoreDescriptor().getCloudDescriptor().getCollectionName();
    String shard = core.getCoreDescriptor().getCloudDescriptor().getShardId();
    if (!leaderStateManager.amILeader()) {
        log.warn("Action {} sent to non-leader replica @ {}:{}", CdcrParams.CdcrAction.LASTPROCESSEDVERSION, collectionName, shard);
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Action " + CdcrParams.CdcrAction.LASTPROCESSEDVERSION + " sent to non-leader replica");
    }
    // take care of the first three cases
    // first check the log readers from the replicator states
    long lastProcessedVersion = Long.MAX_VALUE;
    for (CdcrReplicatorState state : replicatorManager.getReplicatorStates()) {
        long version = Long.MAX_VALUE;
        if (state.getLogReader() != null) {
            version = state.getLogReader().getLastVersion();
        }
        lastProcessedVersion = Math.min(lastProcessedVersion, version);
    }
    // next check the log reader of the buffer
    CdcrUpdateLog.CdcrLogReader bufferLogReader = ((CdcrUpdateLog) core.getUpdateHandler().getUpdateLog()).getBufferToggle();
    if (bufferLogReader != null) {
        lastProcessedVersion = Math.min(lastProcessedVersion, bufferLogReader.getLastVersion());
    }
    // the fourth case: no cdc replication, no buffering: all readers were null
    if (processStateManager.getState().equals(CdcrParams.ProcessState.STOPPED) && bufferStateManager.getState().equals(CdcrParams.BufferState.DISABLED)) {
        CdcrUpdateLog.CdcrLogReader logReader = ((CdcrUpdateLog) core.getUpdateHandler().getUpdateLog()).newLogReader();
        try {
            // let the reader initialize lastVersion
            logReader.next();
            lastProcessedVersion = Math.min(lastProcessedVersion, logReader.getLastVersion());
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error while fetching the last processed version", e);
        } catch (IOException e) {
            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Error while fetching the last processed version", e);
        } finally {
            logReader.close();
        }
    }
    log.debug("Returning the lowest last processed version {}  @ {}:{}", lastProcessedVersion, collectionName, shard);
    rsp.add(CdcrParams.LAST_PROCESSED_VERSION, lastProcessedVersion);
}

Example 5 with CdcrUpdateLog

use of org.apache.solr.update.CdcrUpdateLog in project lucene-solr by apache.

the class ReplicationHandler method getFileList.

@SuppressWarnings("unchecked")
private void getFileList(SolrParams solrParams, SolrQueryResponse rsp) {
    String v = solrParams.get(GENERATION);
    if (v == null) {
        rsp.add("status", "no index generation specified");
        return;
    }
    long gen = Long.parseLong(v);
    IndexCommit commit = core.getDeletionPolicy().getCommitPoint(gen);
    //System.out.println("ask for files for gen:" + commit.getGeneration() + core.getCoreDescriptor().getCoreContainer().getZkController().getNodeName());
    if (commit == null) {
        rsp.add("status", "invalid index generation");
        return;
    }
    // reserve the indexcommit for sometime
    core.getDeletionPolicy().setReserveDuration(gen, reserveCommitDuration);
    List<Map<String, Object>> result = new ArrayList<>();
    Directory dir = null;
    try {
        dir = core.getDirectoryFactory().get(core.getNewIndexDir(), DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
        SegmentInfos infos = SegmentInfos.readCommit(dir, commit.getSegmentsFileName());
        for (SegmentCommitInfo commitInfo : infos) {
            for (String file : commitInfo.files()) {
                Map<String, Object> fileMeta = new HashMap<>();
                fileMeta.put(NAME, file);
                fileMeta.put(SIZE, dir.fileLength(file));
                try (final IndexInput in = dir.openInput(file, IOContext.READONCE)) {
                    try {
                        long checksum = CodecUtil.retrieveChecksum(in);
                        fileMeta.put(CHECKSUM, checksum);
                    } catch (Exception e) {
                        LOG.warn("Could not read checksum from index file: " + file, e);
                    }
                }
                result.add(fileMeta);
            }
        }
        // add the segments_N file
        Map<String, Object> fileMeta = new HashMap<>();
        fileMeta.put(NAME, infos.getSegmentsFileName());
        fileMeta.put(SIZE, dir.fileLength(infos.getSegmentsFileName()));
        if (infos.getId() != null) {
            try (final IndexInput in = dir.openInput(infos.getSegmentsFileName(), IOContext.READONCE)) {
                try {
                    fileMeta.put(CHECKSUM, CodecUtil.retrieveChecksum(in));
                } catch (Exception e) {
                    LOG.warn("Could not read checksum from index file: " + infos.getSegmentsFileName(), e);
                }
            }
        }
        result.add(fileMeta);
    } catch (IOException e) {
        rsp.add("status", "unable to get file names for given index generation");
        rsp.add(EXCEPTION, e);
        LOG.error("Unable to get file names for indexCommit generation: " + gen, e);
    } finally {
        if (dir != null) {
            try {
                core.getDirectoryFactory().release(dir);
            } catch (IOException e) {
                SolrException.log(LOG, "Could not release directory after fetching file list", e);
            }
        }
    }
    rsp.add(CMD_GET_FILE_LIST, result);
    // fetch list of tlog files only if cdcr is activated
    if (solrParams.getBool(TLOG_FILES, true) && core.getUpdateHandler().getUpdateLog() != null && core.getUpdateHandler().getUpdateLog() instanceof CdcrUpdateLog) {
        try {
            List<Map<String, Object>> tlogfiles = getTlogFileList(commit);
            LOG.info("Adding tlog files to list: " + tlogfiles);
            rsp.add(TLOG_FILES, tlogfiles);
        } catch (IOException e) {
            rsp.add("status", "unable to get tlog file names for given index generation");
            rsp.add(EXCEPTION, e);
            LOG.error("Unable to get tlog file names for indexCommit generation: " + gen, e);
        }
    }
    if (confFileNameAlias.size() < 1 || core.getCoreContainer().isZooKeeperAware())
        return;
    LOG.debug("Adding config files to list: " + includeConfFiles);
    //if configuration files need to be included get their details
    rsp.add(CONF_FILES, getConfFileInfoFromCache(confFileNameAlias, confFileInfoCache));
}