Examples with OpTime com.eightkdata.mongowp.OpTime used on opensource projects

Example 1 with OpTime

use of com.eightkdata.mongowp.OpTime in project torodb by torodb.

the class DefaultOplogApplierService method createFetcher.

private OplogFetcher createFetcher() {
    OpTime lastAppliedOptime;
    long lastAppliedHash;
    try (ReadOplogTransaction oplogReadTrans = oplogManager.createReadTransaction()) {
        lastAppliedOptime = oplogReadTrans.getLastAppliedOptime();
        lastAppliedHash = oplogReadTrans.getLastAppliedHash();
    }
    return replFilters.filterOplogFetcher(oplogFetcherFactory.createFetcher(lastAppliedHash, lastAppliedOptime));
}

Example 2 with OpTime

use of com.eightkdata.mongowp.OpTime in project torodb by torodb.

the class TopologyCoordinator method lookForSyncSource.

/**
   * Looks for an optimal sync source to replicate from.
   *
   * The first attempt, we ignore those nodes with slave delay higher than our own, hidden nodes,
   * and nodes that are excessively lagged. The second attempt includes such nodes, in case those
   * are the only ones we can reach. This loop attempts to set 'closestIndex'.
   *
   * @param now              the current time
   * @param lastOpAppliedOp  the last OpTime this node has apply
   * @param onlyOptimal      if true, slaves with more delay than ourselve, hidden nodes or
   *                         excessively lagged nodes are ignored
   * @param oldestSyncOpTime the oldest optime considered not excessively lagged. Only used if
   *                         onlyOptimal is true.
   * @return the new optimal sync source, which is not {@link Optional#isPresent() present} if no
   *         one can be chosen
   */
private Optional<MemberConfig> lookForSyncSource(Instant now, Optional<OpTime> lastOpAppliedOp, boolean onlyOptimal, OpTime oldestSyncOpTime) {
    OpTime lastOpApplied = lastOpAppliedOp.orElse(OpTime.EPOCH);
    Stream<MemberHeartbeatData> hbCandidateStream = _hbdata.stream().filter(MemberHeartbeatData::isUp).filter(hbData -> hbData.getState().isReadable()).filter(hbData -> hbData.getOpTime().isAfter(lastOpApplied));
    if (onlyOptimal) {
        hbCandidateStream = hbCandidateStream.filter(hbData -> hbData.getOpTime().isEqualOrAfter(oldestSyncOpTime));
    }
    Stream<MemberConfig> mcCandidateStream = hbCandidateStream.map(this::getMemberConfig).filter(mc -> !isBlacklistedMember(mc, now));
    if (onlyOptimal) {
        mcCandidateStream = mcCandidateStream.filter(mc -> !mc.isHidden()).filter(mc -> mc.getSlaveDelay() < slaveDelaySecs);
    }
    //If there are several candidates, the one whose ping is lower is returned
    return mcCandidateStream.reduce((MemberConfig cand1, MemberConfig cand2) -> {
        long ping1 = getPing(cand1.getHostAndPort());
        long ping2 = getPing(cand2.getHostAndPort());
        if (ping1 < ping2) {
            return cand1;
        }
        return cand2;
    });
}

Example 3 with OpTime

use of com.eightkdata.mongowp.OpTime in project torodb by torodb.

the class TopologyCoordinator method chooseNewSyncSource.

/**
   * Chooses and sets a new sync source, based on our current knowledge of the world.
   *
   * @return the new sync source or {@link Optional#empty()} if we cannot calculate a new sync
   *         source yet
   */
@Nonnull
Optional<HostAndPort> chooseNewSyncSource(Instant now, Optional<OpTime> lastOpApplied) {
    // if we have a target we've requested to sync from, use it
    if (_forceSyncSourceIndex != -1) {
        assert _forceSyncSourceIndex < _rsConfig.getMembers().size();
        HostAndPort syncSource = _rsConfig.getMembers().get(_forceSyncSourceIndex).getHostAndPort();
        _syncSource = Optional.of(syncSource);
        _forceSyncSourceIndex = -1;
        String msg = "syncing from: " + syncSource + " by request";
        LOGGER.info(msg);
        return _syncSource;
    }
    // wait for 2N pings before choosing a sync target
    if (_hbdata == null) {
        //we dont have a repl config yet
        assert _rsConfig == null;
        return Optional.empty();
    }
    int needMorePings = _hbdata.size() * 2 - getTotalPings();
    if (needMorePings > 0) {
        LOGGER.info("Waiting for {}  pings from other members before syncing", needMorePings);
        _syncSource = Optional.empty();
        return _syncSource;
    }
    // If we are only allowed to sync from the primary, set that
    if (!_rsConfig.isChainingAllowed()) {
        if (_currentPrimaryIndex == -1) {
            LOGGER.warn("Cannot select sync source because chaining is not allowed and primary " + "is unknown/down");
            _syncSource = Optional.empty();
            return _syncSource;
        } else if (isBlacklistedMember(getCurrentPrimaryMember(), now)) {
            LOGGER.warn("Cannot select sync source because chaining is not allowed and " + "primary is not currently accepting our updates");
            _syncSource = Optional.empty();
            return _syncSource;
        } else {
            HostAndPort syncSource = _rsConfig.getMembers().get(_currentPrimaryIndex).getHostAndPort();
            _syncSource = Optional.of(syncSource);
            String msg = "syncing from primary: " + syncSource;
            LOGGER.info(msg);
            return _syncSource;
        }
    }
    // find the member with the lowest ping time that is ahead of me
    // Find primary's oplog time. Reject sync candidates that are more than
    // maxSyncSourceLagSecs seconds behind.
    OpTime primaryOpTime;
    if (_currentPrimaryIndex != -1) {
        primaryOpTime = _hbdata.get(_currentPrimaryIndex).getOpTime();
        assert primaryOpTime != null;
    } else {
        // choose a time that will exclude no candidates, since we don't see a primary
        primaryOpTime = OpTime.ofSeconds(_maxSyncSourceLagSecs);
    }
    if (primaryOpTime.getSecs() < _maxSyncSourceLagSecs) {
        // erh - I think this means there was just a new election
        // and we don't yet know the new primary's optime
        primaryOpTime = OpTime.ofSeconds(_maxSyncSourceLagSecs);
    }
    OpTime oldestSyncOpTime = OpTime.ofSeconds(primaryOpTime.getSecs() - _maxSyncSourceLagSecs);
    Optional<MemberConfig> newSyncSourceMember = lookForSyncSource(now, lastOpApplied, true, oldestSyncOpTime);
    if (!newSyncSourceMember.isPresent()) {
        newSyncSourceMember = lookForSyncSource(now, lastOpApplied, false, oldestSyncOpTime);
    }
    if (!newSyncSourceMember.isPresent()) {
        // Did not find any members to sync from
        String msg = "could not find member to sync from";
        // Only log when we had a valid sync source before
        if (_syncSource.isPresent()) {
            LOGGER.info(msg);
        }
        _syncSource = Optional.empty();
        return _syncSource;
    } else {
        _syncSource = Optional.of(newSyncSourceMember.get().getHostAndPort());
        LOGGER.info("syncing from: {}", _syncSource.get());
        return _syncSource;
    }
}

Example 4 with OpTime

use of com.eightkdata.mongowp.OpTime in project torodb by torodb.

the class TopologyCoordinator method shouldChangeSyncSource.

/**
   * Determines if a new sync source should be chosen, if a better candidate sync source is
   * available.
   *
   * It returns true if there exists a viable sync source member other than our current source,
   * whose oplog has reached an optime greater than the max sync source lag later than current
   * source's. It can return true in other scenarios (like if {@link #setForceSyncSourceIndex(int) }
   * has been called or if we don't have a current sync source.
   *
   * @param now is used to skip over currently blacklisted sync sources.
   * @return
   */
boolean shouldChangeSyncSource(HostAndPort currentSource, Instant now) {
    // If the user requested a sync source change, return true.
    if (_forceSyncSourceIndex != -1) {
        return true;
    }
    OptionalInt currentMemberIndex = _rsConfig.findMemberIndexByHostAndPort(currentSource);
    if (!currentMemberIndex.isPresent()) {
        return true;
    }
    assert _hbdata.get(currentMemberIndex.getAsInt()) != null;
    OpTime currentOpTime = _hbdata.get(currentMemberIndex.getAsInt()).getOpTime();
    if (currentOpTime == null) {
        // change.
        return false;
    }
    long currentSecs = currentOpTime.getSecs();
    long goalSecs = currentSecs + _maxSyncSourceLagSecs;
    for (int i = 0; i < _hbdata.size(); i++) {
        MemberHeartbeatData it = _hbdata.get(i);
        MemberConfig candidateConfig = _rsConfig.getMembers().get(i);
        OpTime itOpTime = it.getOpTime();
        if (itOpTime != null && it.isUp() && it.getState().isReadable() && !isBlacklistedMember(candidateConfig, now) && goalSecs < itOpTime.getSecs()) {
            LOGGER.info("changing sync target because current sync target's most recent OpTime " + "is {}  which is more than {} seconds behind member {} whose most recent " + "OpTime is {} ", currentOpTime, _maxSyncSourceLagSecs, candidateConfig.getHostAndPort(), itOpTime);
            return true;
        }
    }
    return false;
}

Example 5 with OpTime

use of com.eightkdata.mongowp.OpTime in project torodb by torodb.

the class OplogManager method loadState.

@Locked(exclusive = true)
private void loadState() throws OplogManagerPersistException {
    try {
        retrier.retry(() -> {
            try (ReadOnlyMongodTransaction transaction = connection.openReadOnlyTransaction()) {
                Status<FindResult> status = transaction.execute(new Request(OPLOG_DB, null, true, null), FindCommand.INSTANCE, new FindArgument.Builder().setCollection(OPLOG_COL).setSlaveOk(true).build());
                if (!status.isOk()) {
                    throw new RetrierAbortException(new MongoException(status));
                }
                Iterator<BsonDocument> batch = status.getResult().getCursor().getFirstBatch();
                if (!batch.hasNext()) {
                    lastAppliedHash = 0;
                    lastAppliedOpTime = OpTime.EPOCH;
                } else {
                    BsonDocument doc = batch.next();
                    BsonDocument subDoc = BsonReaderTool.getDocument(doc, KEY);
                    lastAppliedHash = BsonReaderTool.getLong(subDoc, "hash");
                    long optimeAsLong = BsonReaderTool.getLong(subDoc, "optime_i");
                    BsonDateTime optimeAsDateTime = DefaultBsonValues.newDateTime(optimeAsLong);
                    lastAppliedOpTime = new OpTime(TimestampToDateTime.toTimestamp(optimeAsDateTime, DefaultBsonValues::newTimestamp), BsonReaderTool.getLong(subDoc, "optime_t"));
                }
                notifyLastAppliedOpTimeChange();
                return Empty.getInstance();
            }
        }, Hint.INFREQUENT_ROLLBACK);
    } catch (RetrierGiveUpException ex) {
        throw new OplogManagerPersistException(ex);
    }
}