Search in sources :

Example 11 with OplogOperation

use of com.eightkdata.mongowp.server.api.oplog.OplogOperation in project torodb by torodb.

the class SimpleAnalyzedOplogBatchExecutorTest method testVisit_SingleOp_OplogApplyingEx.

@Test
public void testVisit_SingleOp_OplogApplyingEx() throws Exception {
    //GIVEN
    OplogOperation operation = mock(OplogOperation.class);
    SingleOpAnalyzedOplogBatch batch = new SingleOpAnalyzedOplogBatch(operation);
    ApplierContext applierContext = new ApplierContext.Builder().setReapplying(true).setUpdatesAsUpserts(true).build();
    Timer timer = mock(Timer.class);
    Context context = mock(Context.class);
    given(metrics.getSingleOpTimer(operation)).willReturn(timer);
    given(timer.time()).willReturn(context);
    doThrow(new OplogApplyingException(new MongoException(ErrorCode.BAD_VALUE))).when(executor).execute(operation, applierContext);
    //WHEN
    try {
        executor.visit(batch, applierContext);
        fail("An exception was expected");
    } catch (RetrierGiveUpException | RetrierAbortException ignore) {
    }
    //THEN
    then(metrics).should().getSingleOpTimer(operation);
    then(timer).should().time();
    then(executor).should(times(1)).execute(operation, applierContext);
}
Also used : ApplierContext(com.torodb.mongodb.repl.oplogreplier.ApplierContext) Context(com.codahale.metrics.Timer.Context) OplogApplyingException(com.torodb.mongodb.repl.oplogreplier.OplogOperationApplier.OplogApplyingException) MongoException(com.eightkdata.mongowp.exceptions.MongoException) Timer(com.codahale.metrics.Timer) RetrierAbortException(com.torodb.core.retrier.RetrierAbortException) OplogOperation(com.eightkdata.mongowp.server.api.oplog.OplogOperation) RetrierGiveUpException(com.torodb.core.retrier.RetrierGiveUpException) ApplierContext(com.torodb.mongodb.repl.oplogreplier.ApplierContext) Test(org.junit.Test)

Example 12 with OplogOperation

use of com.eightkdata.mongowp.server.api.oplog.OplogOperation in project torodb by torodb.

the class RecoveryService method initialSync.

private boolean initialSync() throws TryAgainException, FatalErrorException {
    /*
     * 1. store that data is inconsistent 2. decide a sync source 3. lastRemoteOptime1 = get the
     * last optime of the sync source 4. clone all databases except local 5. lastRemoteOptime2 = get
     * the last optime of the sync source 6. apply remote oplog from lastRemoteOptime1 to
     * lastRemoteOptime2 7. lastRemoteOptime3 = get the last optime of the sync source 8. apply
     * remote oplog from lastRemoteOptime2 to lastRemoteOptime3 9. rebuild indexes 10. store
     * lastRemoteOptime3 as the last applied operation optime 11. store that data is consistent 12.
     * change replication state to SECONDARY
     */
    //TODO: Support fastsync (used to restore a node by copying the data from other up-to-date node)
    LOGGER.info("Starting initial sync");
    callback.setConsistentState(false);
    HostAndPort syncSource;
    try {
        syncSource = syncSourceProvider.newSyncSource();
        LOGGER.info("Using node " + syncSource + " to replicate from");
    } catch (NoSyncSourceFoundException ex) {
        throw new TryAgainException("No sync source");
    }
    MongoClient remoteClient;
    try {
        remoteClient = remoteClientFactory.createClient(syncSource);
    } catch (UnreachableMongoServerException ex) {
        throw new TryAgainException(ex);
    }
    try {
        LOGGER.debug("Remote client obtained");
        MongoConnection remoteConnection = remoteClient.openConnection();
        try (OplogReader reader = oplogReaderProvider.newReader(remoteConnection)) {
            OplogOperation lastClonedOp = reader.getLastOp();
            OpTime lastRemoteOptime1 = lastClonedOp.getOpTime();
            try (WriteOplogTransaction oplogTransaction = oplogManager.createWriteTransaction()) {
                LOGGER.info("Remote database cloning started");
                oplogTransaction.truncate();
                LOGGER.info("Local databases dropping started");
                Status<?> status = dropDatabases();
                if (!status.isOk()) {
                    throw new TryAgainException("Error while trying to drop collections: " + status);
                }
                LOGGER.info("Local databases dropping finished");
                if (!isRunning()) {
                    LOGGER.warn("Recovery stopped before it can finish");
                    return false;
                }
                LOGGER.info("Remote database cloning started");
                cloneDatabases(remoteClient);
                LOGGER.info("Remote database cloning finished");
                oplogTransaction.forceNewValue(lastClonedOp.getHash(), lastClonedOp.getOpTime());
            }
            if (!isRunning()) {
                LOGGER.warn("Recovery stopped before it can finish");
                return false;
            }
            TorodServer torodServer = server.getTorodServer();
            try (TorodConnection connection = torodServer.openConnection();
                SharedWriteTorodTransaction trans = connection.openWriteTransaction(false)) {
                OpTime lastRemoteOptime2 = reader.getLastOp().getOpTime();
                LOGGER.info("First oplog application started");
                applyOplog(reader, lastRemoteOptime1, lastRemoteOptime2);
                trans.commit();
                LOGGER.info("First oplog application finished");
                if (!isRunning()) {
                    LOGGER.warn("Recovery stopped before it can finish");
                    return false;
                }
                OplogOperation lastOperation = reader.getLastOp();
                OpTime lastRemoteOptime3 = lastOperation.getOpTime();
                LOGGER.info("Second oplog application started");
                applyOplog(reader, lastRemoteOptime2, lastRemoteOptime3);
                trans.commit();
                LOGGER.info("Second oplog application finished");
                if (!isRunning()) {
                    LOGGER.warn("Recovery stopped before it can finish");
                    return false;
                }
                LOGGER.info("Index rebuild started");
                rebuildIndexes();
                trans.commit();
                LOGGER.info("Index rebuild finished");
                if (!isRunning()) {
                    LOGGER.warn("Recovery stopped before it can finish");
                    return false;
                }
                trans.commit();
            }
        } catch (OplogStartMissingException ex) {
            throw new TryAgainException(ex);
        } catch (OplogOperationUnsupported ex) {
            throw new TryAgainException(ex);
        } catch (MongoException | RollbackException ex) {
            throw new TryAgainException(ex);
        } catch (OplogManagerPersistException ex) {
            throw new FatalErrorException();
        } catch (UserException ex) {
            throw new FatalErrorException(ex);
        }
        callback.setConsistentState(true);
        LOGGER.info("Initial sync finished");
    } finally {
        remoteClient.close();
    }
    return true;
}
Also used : OplogManagerPersistException(com.torodb.mongodb.repl.OplogManager.OplogManagerPersistException) SharedWriteTorodTransaction(com.torodb.torod.SharedWriteTorodTransaction) MongoException(com.eightkdata.mongowp.exceptions.MongoException) OplogOperationUnsupported(com.eightkdata.mongowp.exceptions.OplogOperationUnsupported) UnreachableMongoServerException(com.eightkdata.mongowp.client.core.UnreachableMongoServerException) OplogStartMissingException(com.eightkdata.mongowp.exceptions.OplogStartMissingException) RollbackException(com.torodb.core.transaction.RollbackException) OpTime(com.eightkdata.mongowp.OpTime) WriteOplogTransaction(com.torodb.mongodb.repl.OplogManager.WriteOplogTransaction) HostAndPort(com.google.common.net.HostAndPort) MongoClient(com.eightkdata.mongowp.client.core.MongoClient) TorodServer(com.torodb.torod.TorodServer) UserException(com.torodb.core.exceptions.user.UserException) NoSyncSourceFoundException(com.torodb.mongodb.repl.exceptions.NoSyncSourceFoundException) OplogOperation(com.eightkdata.mongowp.server.api.oplog.OplogOperation) MongoConnection(com.eightkdata.mongowp.client.core.MongoConnection) TorodConnection(com.torodb.torod.TorodConnection)

Example 13 with OplogOperation

use of com.eightkdata.mongowp.server.api.oplog.OplogOperation in project torodb by torodb.

the class RecoveryService method applyOplog.

/**
   * Applies all the oplog operations stored on the remote server whose optime is higher than
   * <em>from</em> but lower or equal than <em>to</em>.
   *
   * @param myOplog
   * @param remoteOplog
   * @param to
   * @param from
   */
private void applyOplog(OplogReader remoteOplog, OpTime from, OpTime to) throws TryAgainException, MongoException, FatalErrorException {
    MongoCursor<OplogOperation> oplogCursor = remoteOplog.between(from, true, to, true);
    if (!oplogCursor.hasNext()) {
        throw new OplogStartMissingException(remoteOplog.getSyncSource());
    }
    OplogOperation firstOp = oplogCursor.next();
    if (!firstOp.getOpTime().equals(from)) {
        throw new TryAgainException("Remote oplog does not cointain our last operation");
    }
    OplogFetcher fetcher = new LimitedOplogFetcher(oplogCursor);
    ApplierContext context = new ApplierContext.Builder().setReapplying(true).setUpdatesAsUpserts(true).build();
    try {
        oplogApplier.apply(fetcher, context).waitUntilFinished();
    } catch (StopReplicationException | RollbackReplicationException | CancellationException | UnexpectedOplogApplierException ex) {
        throw new FatalErrorException(ex);
    }
    OpTime lastAppliedOptime;
    try (ReadOplogTransaction oplogTrans = oplogManager.createReadTransaction()) {
        lastAppliedOptime = oplogTrans.getLastAppliedOptime();
    }
    if (!lastAppliedOptime.equals(to)) {
        LOGGER.warn("Unexpected optime for last operation to apply. " + "Expected " + to + ", but " + lastAppliedOptime + " found");
    }
}
Also used : OplogFetcher(com.torodb.mongodb.repl.oplogreplier.fetcher.OplogFetcher) LimitedOplogFetcher(com.torodb.mongodb.repl.oplogreplier.fetcher.LimitedOplogFetcher) ReadOplogTransaction(com.torodb.mongodb.repl.OplogManager.ReadOplogTransaction) OplogStartMissingException(com.eightkdata.mongowp.exceptions.OplogStartMissingException) RollbackReplicationException(com.torodb.mongodb.repl.oplogreplier.RollbackReplicationException) OpTime(com.eightkdata.mongowp.OpTime) ApplierContext(com.torodb.mongodb.repl.oplogreplier.ApplierContext) LimitedOplogFetcher(com.torodb.mongodb.repl.oplogreplier.fetcher.LimitedOplogFetcher) CancellationException(java.util.concurrent.CancellationException) UnexpectedOplogApplierException(com.torodb.mongodb.repl.oplogreplier.OplogApplier.UnexpectedOplogApplierException) OplogOperation(com.eightkdata.mongowp.server.api.oplog.OplogOperation) StopReplicationException(com.torodb.mongodb.repl.oplogreplier.StopReplicationException)

Example 14 with OplogOperation

use of com.eightkdata.mongowp.server.api.oplog.OplogOperation in project torodb by torodb.

the class ReplSyncFetcher method fetch.

/**
   *
   * @param reader
   * @return true iff rollback is needed
   * @throws com.torodb.torod.mongodb.repl.ReplSyncFetcher.StopFetchException
   * @throws com.torodb.torod.mongodb.repl.ReplSyncFetcher.RestartFetchException
   */
private boolean fetch(OplogReader reader) throws StopFetchException, RestartFetchException {
    try {
        MongoCursor<OplogOperation> cursor = reader.queryGte(lastFetchedOpTime);
        Batch<OplogOperation> batch = cursor.fetchBatch();
        postBatchChecks(reader, cursor, batch);
        try {
            if (isRollbackNeeded(reader, batch, lastFetchedOpTime, lastFetchedHash)) {
                return true;
            }
            while (fetchIterationCanContinue()) {
                if (!batch.hasNext()) {
                    preBatchChecks(batch);
                    batch = cursor.fetchBatch();
                    postBatchChecks(reader, cursor, batch);
                    continue;
                }
                if (batch.hasNext()) {
                    OplogOperation nextOp = batch.next();
                    assert nextOp != null;
                    boolean delivered = false;
                    while (!delivered) {
                        try {
                            LOGGER.debug("Delivered op: {}", nextOp);
                            callback.deliver(nextOp);
                            delivered = true;
                            opsReadCounter++;
                        } catch (InterruptedException ex) {
                            LOGGER.warn(serviceName() + " interrupted while a " + "message was being to deliver. Retrying", ex);
                        }
                    }
                    lastFetchedHash = nextOp.getHash();
                    lastFetchedOpTime = nextOp.getOpTime();
                    metrics.getLastOpTimeFetched().setValue(lastFetchedOpTime.toString());
                }
            }
        } finally {
            cursor.close();
        }
    } catch (MongoException ex) {
        throw new RestartFetchException();
    }
    return false;
}
Also used : MongoException(com.eightkdata.mongowp.exceptions.MongoException) OplogOperation(com.eightkdata.mongowp.server.api.oplog.OplogOperation)

Example 15 with OplogOperation

use of com.eightkdata.mongowp.server.api.oplog.OplogOperation in project torodb by torodb.

the class SimpleAnalyzedOplogBatchExecutorTest method testVisit_SingleOp_Success.

@Test
public void testVisit_SingleOp_Success() throws Exception {
    //GIVEN
    OplogOperation operation = mock(OplogOperation.class);
    SingleOpAnalyzedOplogBatch batch = new SingleOpAnalyzedOplogBatch(operation);
    ApplierContext applierContext = new ApplierContext.Builder().setReapplying(true).setUpdatesAsUpserts(true).build();
    Timer timer = mock(Timer.class);
    Context context = mock(Context.class);
    given(metrics.getSingleOpTimer(operation)).willReturn(timer);
    given(timer.time()).willReturn(context);
    doNothing().when(executor).execute(any(OplogOperation.class), any());
    //WHEN
    OplogOperation result = executor.visit(batch, applierContext);
    //THEN
    then(metrics).should().getSingleOpTimer(operation);
    then(timer).should().time();
    then(context).should().close();
    then(executor).should(times(1)).execute(operation, applierContext);
    assertEquals(operation, result);
}
Also used : ApplierContext(com.torodb.mongodb.repl.oplogreplier.ApplierContext) Context(com.codahale.metrics.Timer.Context) Timer(com.codahale.metrics.Timer) OplogOperation(com.eightkdata.mongowp.server.api.oplog.OplogOperation) ApplierContext(com.torodb.mongodb.repl.oplogreplier.ApplierContext) Test(org.junit.Test)

Aggregations

OplogOperation (com.eightkdata.mongowp.server.api.oplog.OplogOperation)23 ApplierContext (com.torodb.mongodb.repl.oplogreplier.ApplierContext)10 Test (org.junit.Test)10 Context (com.codahale.metrics.Timer.Context)8 Timer (com.codahale.metrics.Timer)7 RetrierGiveUpException (com.torodb.core.retrier.RetrierGiveUpException)7 RollbackException (com.torodb.core.transaction.RollbackException)6 MongoException (com.eightkdata.mongowp.exceptions.MongoException)5 RetrierAbortException (com.torodb.core.retrier.RetrierAbortException)5 InsertOplogOperation (com.eightkdata.mongowp.server.api.oplog.InsertOplogOperation)4 OplogBatch (com.torodb.mongodb.repl.oplogreplier.OplogBatch)4 OpTime (com.eightkdata.mongowp.OpTime)3 UserException (com.torodb.core.exceptions.user.UserException)3 OplogManagerPersistException (com.torodb.mongodb.repl.OplogManager.OplogManagerPersistException)3 WriteOplogTransaction (com.torodb.mongodb.repl.OplogManager.WriteOplogTransaction)3 OplogStartMissingException (com.eightkdata.mongowp.exceptions.OplogStartMissingException)2 FinishedOplogBatch (com.torodb.mongodb.repl.oplogreplier.FinishedOplogBatch)2 NormalOplogBatch (com.torodb.mongodb.repl.oplogreplier.NormalOplogBatch)2 NotReadyForMoreOplogBatch (com.torodb.mongodb.repl.oplogreplier.NotReadyForMoreOplogBatch)2 OplogFetcher (com.torodb.mongodb.repl.oplogreplier.fetcher.OplogFetcher)2