Examples with OplogBatch com.torodb.mongodb.repl.oplogreplier.OplogBatch used on opensource projects

Example 1 with OplogBatch

use of com.torodb.mongodb.repl.oplogreplier.OplogBatch in project torodb by torodb.

the class ContinuousOplogFetcher method fetch.

@Override
public OplogBatch fetch() throws StopReplicationException, RollbackReplicationException {
    if (state.isClosed()) {
        return FinishedOplogBatch.getInstance();
    }
    try {
        return retrier.retry(() -> {
            try {
                if (state.isClosed()) {
                    return FinishedOplogBatch.getInstance();
                }
                state.prepareToFetch();
                MongoCursor<OplogOperation> cursor = state.getLastUsedMongoCursor();
                Batch<OplogOperation> batch = cursor.tryFetchBatch();
                if (batch == null || !batch.hasNext()) {
                    Thread.sleep(1000);
                    batch = cursor.tryFetchBatch();
                    if (batch == null || !batch.hasNext()) {
                        return NotReadyForMoreOplogBatch.getInstance();
                    }
                }
                List<OplogOperation> fetchedOps = null;
                long fetchTime = 0;
                /*
           * As we already modify the cursor by fetching the batch, we cannot retry the whole block
           * (as the cursor would be reused and the previous batch will be discarted).
           *
           * Then, if we leave the following try section with an error, we need to discard the
           * cursor, so the next iteration starts from the last batch we returned. On the other
           * hand, if we finished successfully, then we need to update the state.
           */
                boolean successful = false;
                try {
                    fetchedOps = batch.asList();
                    fetchTime = batch.getFetchTime();
                    postBatchChecks(cursor, fetchedOps);
                    OplogBatch result = new NormalOplogBatch(fetchedOps, true);
                    successful = true;
                    return result;
                } finally {
                    if (!successful) {
                        cursor.close();
                    } else {
                        assert fetchedOps != null;
                        assert fetchTime != 0;
                        state.updateState(fetchedOps, fetchTime);
                    }
                }
            } catch (RestartFetchException ex) {
                //lets choose a new reader
                state.discardReader();
                //and then try again
                throw new RollbackException(ex);
            } catch (DeadCursorException ex) {
                //lets retry the whole block with the same reader
                throw new RollbackException(ex);
            } catch (StopReplicationException | RollbackReplicationException ex) {
                //do not try again
                throw new RetrierAbortException(ex);
            } catch (MongoServerException ex) {
                //TODO: Fix this violation on the abstraction!
                LOGGER.debug("Found an unwrapped MongodbServerException");
                //lets choose a new reader
                state.discardReader();
                //rollback and hopefully use another member
                throw new RollbackException(ex);
            } catch (MongoException | MongoRuntimeException ex) {
                LOGGER.warn("Catched an error while reading the remote " + "oplog: {}", ex.getLocalizedMessage());
                //lets choose a new reader
                state.discardReader();
                //rollback and hopefully use another member
                throw new RollbackException(ex);
            }
        }, Hint.CRITICAL, Hint.TIME_SENSIBLE);
    } catch (RetrierGiveUpException ex) {
        this.close();
        throw new StopReplicationException("Stopping replication after several attepts to " + "fetch the remote oplog", ex);
    } catch (RetrierAbortException ex) {
        this.close();
        Throwable cause = ex.getCause();
        if (cause != null) {
            if (cause instanceof StopReplicationException) {
                throw (StopReplicationException) cause;
            }
            if (cause instanceof RollbackReplicationException) {
                throw (RollbackReplicationException) cause;
            }
        }
        throw new StopReplicationException("Stopping replication after a unknown abort " + "exception", ex);
    }
}

Example 2 with OplogBatch

use of com.torodb.mongodb.repl.oplogreplier.OplogBatch in project torodb by torodb.

the class FilteredOplogFetcher method fetch.

@Override
public OplogBatch fetch() throws StopReplicationException, RollbackReplicationException {
    OplogBatch batch = delegate.fetch();
    List<OplogOperation> filteredBatch = batch.getOps().stream().filter(filter).collect(Collectors.toList());
    if (filteredBatch.isEmpty()) {
        if (batch.isLastOne()) {
            return FinishedOplogBatch.getInstance();
        } else {
            return NotReadyForMoreOplogBatch.getInstance();
        }
    } else {
        if (batch.isLastOne()) {
            throw new AssertionError("Batchs produced by a finished oplog fetcher cannot " + "contain ops");
        }
        return new NormalOplogBatch(filteredBatch, batch.isReadyForMore());
    }
}

Example 3 with OplogBatch

use of com.torodb.mongodb.repl.oplogreplier.OplogBatch in project torodb by torodb.

the class ContinuousOplogFetcherTest method testMediumOplog.

@Test
public void testMediumOplog() throws Exception {
    int oplogSize = 1000;
    List<OplogOperation> oplog = createInsertStream(this::createSimpleInsert).limit(oplogSize).collect(Collectors.toList());
    oplogSupplier = () -> oplog;
    OplogOperation firstOp = oplog.get(0);
    ContinuousOplogFetcher fetcher = factory.createFetcher(firstOp.getHash(), firstOp.getOpTime());
    List<OplogOperation> recivedOplog = new ArrayList<>(oplogSize);
    OplogBatch batch = null;
    while (batch == null || !(batch.isLastOne() || batch.isReadyForMore())) {
        batch = fetcher.fetch();
        recivedOplog.addAll(batch.getOps());
    }
    assertEquals("Unexpected number of oplog entries fetched: ", oplog.size() - 1, recivedOplog.size());
    assertEquals(oplog.subList(1, oplog.size()), recivedOplog);
}

Example 4 with OplogBatch

use of com.torodb.mongodb.repl.oplogreplier.OplogBatch in project torodb by torodb.

the class ContinuousOplogFetcherTest method testBigOplog.

@Test
public void testBigOplog() throws Exception {
    int oplogSize = 100000;
    List<OplogOperation> oplog = createInsertStream(this::createSimpleInsert).limit(oplogSize).collect(Collectors.toList());
    oplogSupplier = () -> oplog;
    OplogOperation firstOp = oplog.get(0);
    ContinuousOplogFetcher fetcher = factory.createFetcher(firstOp.getHash(), firstOp.getOpTime());
    List<OplogOperation> recivedOplog = new ArrayList<>(oplogSize);
    OplogBatch batch = null;
    while (batch == null || !(batch.isLastOne() || !batch.isReadyForMore())) {
        batch = fetcher.fetch();
        recivedOplog.addAll(batch.getOps());
    }
    assertEquals("Unexpected number of oplog entries fetched: ", oplog.size() - 1, recivedOplog.size());
    assertEquals(oplog.subList(1, oplog.size()), recivedOplog);
}

Example 5 with OplogBatch

use of com.torodb.mongodb.repl.oplogreplier.OplogBatch in project torodb by torodb.

the class ContinuousOplogFetcherTest method testShortOplog.

@Test
public void testShortOplog() throws Exception {
    int oplogSize = 2;
    List<OplogOperation> oplog = createInsertStream(this::createSimpleInsert).limit(oplogSize).collect(Collectors.toList());
    oplogSupplier = () -> oplog;
    OplogOperation firstOp = oplog.get(0);
    ContinuousOplogFetcher fetcher = factory.createFetcher(firstOp.getHash(), firstOp.getOpTime());
    List<OplogOperation> recivedOplog = new ArrayList<>(oplogSize);
    OplogBatch batch = null;
    while (batch == null || !(batch.isLastOne() || batch.isReadyForMore())) {
        batch = fetcher.fetch();
        recivedOplog.addAll(batch.getOps());
    }
    assertEquals("Unexpected number of oplog entries fetched: ", oplog.size() - 1, recivedOplog.size());
    assertEquals(oplog.subList(1, oplog.size()), recivedOplog);
}