Examples with OplogOperation com.eightkdata.mongowp.server.api.oplog.OplogOperation used on opensource projects

Example 16 with OplogOperation

use of com.eightkdata.mongowp.server.api.oplog.OplogOperation in project torodb by torodb.

the class SimpleAnalyzedOplogBatchExecutorTest method testVisit_SingleOp_UserEx.

@Test
public void testVisit_SingleOp_UserEx() throws Exception {
    //GIVEN
    OplogOperation operation = mock(OplogOperation.class);
    SingleOpAnalyzedOplogBatch batch = new SingleOpAnalyzedOplogBatch(operation);
    ApplierContext applierContext = new ApplierContext.Builder().setReapplying(true).setUpdatesAsUpserts(true).build();
    Timer timer = mock(Timer.class);
    Context context = mock(Context.class);
    given(metrics.getSingleOpTimer(operation)).willReturn(timer);
    given(timer.time()).willReturn(context);
    doThrow(new DatabaseNotFoundException("test")).when(executor).execute(operation, applierContext);
    //WHEN
    try {
        executor.visit(batch, applierContext);
        fail("An exception was expected");
    } catch (RetrierGiveUpException | RetrierAbortException ignore) {
    }
    //THEN
    then(metrics).should().getSingleOpTimer(operation);
    then(timer).should().time();
    then(executor).should(times(1)).execute(operation, applierContext);
}

Example 17 with OplogOperation

use of com.eightkdata.mongowp.server.api.oplog.OplogOperation in project torodb by torodb.

the class DefaultOplogApplier method createBatcherFlow.

/**
   * Creates a flow that batches and analyze a input of {@link AnalyzedOplogBatch remote jobs}.
   *
   * This flow tries to accummulate several remote jobs into a bigger one and does not emit until:
   * <ul>
   * <li>A maximum number of operations are batched</li>
   * <li>Or a maximum time has happen since the last emit</li>
   * <li>Or the recived job is not {@link AnalyzedOplogBatch#isReadyForMore()}</li>
   * </ul>
   *
   */
private Flow<OplogBatch, AnalyzedStreamElement, NotUsed> createBatcherFlow(ApplierContext context) {
    Predicate<OplogBatch> finishBatchPredicate = (OplogBatch rawBatch) -> !rawBatch.isReadyForMore();
    ToIntFunction<OplogBatch> costFunction = (rawBatch) -> rawBatch.count();
    Supplier<RawStreamElement> zeroFun = () -> RawStreamElement.INITIAL_ELEMENT;
    BiFunction<RawStreamElement, OplogBatch, RawStreamElement> acumFun = (streamElem, newBatch) -> streamElem.concat(newBatch);
    BatchAnalyzer batchAnalyzer = batchAnalyzerFactory.createBatchAnalyzer(context);
    return Flow.of(OplogBatch.class).via(new BatchFlow<>(batchLimits.maxSize, batchLimits.maxPeriod, finishBatchPredicate, costFunction, zeroFun, acumFun)).filter(rawElem -> rawElem.rawBatch != null && !rawElem.rawBatch.isEmpty()).map(rawElem -> {
        List<OplogOperation> rawOps = rawElem.rawBatch.getOps();
        List<AnalyzedOplogBatch> analyzed = batchAnalyzer.apply(rawOps);
        return new AnalyzedStreamElement(rawElem, analyzed);
    });
}

Example 18 with OplogOperation

use of com.eightkdata.mongowp.server.api.oplog.OplogOperation in project torodb by torodb.

the class ReplSyncApplier method runProtected.

@Override
protected void runProtected() {
    runThread = Thread.currentThread();
    /*
     * TODO: In general, the replication context can be set as not reaplying. But it is not frequent
     * but possible to stop the replication after some oplog ops have been apply but not marked as
     * executed on the oplog manager. For that reason, all oplog ops betwen the last operation that
     * have been marked as applyed and the current last operation on the remote oplog must be
     * executed as replying operations. As it is not possible to do that yet, we have to always
     * apply operations as replying to be safe.
     */
    ApplierContext applierContext = new ApplierContext.Builder().setReapplying(true).setUpdatesAsUpserts(true).build();
    while (isRunning()) {
        OplogOperation lastOperation = null;
        ExclusiveWriteMongodTransaction trans = connection.openExclusiveWriteTransaction();
        try (ExclusiveWriteMongodTransaction transaction = trans) {
            try {
                for (OplogOperation opToApply : callback.takeOps()) {
                    lastOperation = opToApply;
                    LOGGER.trace("Executing {}", opToApply);
                    try {
                        boolean done = false;
                        while (!done) {
                            try {
                                oplogOpApplier.apply(opToApply, transaction, applierContext);
                                transaction.commit();
                                done = true;
                            } catch (RollbackException ex) {
                                LOGGER.debug("Recived a rollback exception while applying an oplog op", ex);
                            }
                        }
                    } catch (OplogApplyingException ex) {
                        if (!callback.failedToApply(opToApply, ex)) {
                            LOGGER.error(serviceName() + " stopped because one operation " + "cannot be executed", ex);
                            break;
                        }
                    } catch (UserException ex) {
                        if (callback.failedToApply(opToApply, ex)) {
                            LOGGER.error(serviceName() + " stopped because one operation " + "cannot be executed", ex);
                            break;
                        }
                    } catch (Throwable ex) {
                        if (callback.failedToApply(opToApply, ex)) {
                            LOGGER.error(serviceName() + " stopped because " + "an unknown error", ex);
                            break;
                        }
                    }
                    callback.markAsApplied(opToApply);
                }
            } catch (InterruptedException ex) {
                LOGGER.debug("Interrupted applier thread while applying an operator");
            }
        }
        if (lastOperation != null) {
            try (WriteOplogTransaction oplogTransaction = oplogManager.createWriteTransaction()) {
                oplogTransaction.addOperation(lastOperation);
            } catch (OplogManagerPersistException ex) {
                if (callback.failedToApply(lastOperation, ex)) {
                    LOGGER.error(serviceName() + " stopped because " + "the last applied operation couldn't " + "be persisted", ex);
                    break;
                }
            }
        }
    }
}

Example 19 with OplogOperation

use of com.eightkdata.mongowp.server.api.oplog.OplogOperation in project torodb by torodb.

the class BatchAnalyzer method apply.

@Override
public List<AnalyzedOplogBatch> apply(List<OplogOperation> oplogOps) {
    List<AnalyzedOplogBatch> result = new ArrayList<>();
    int fromExcluded = -1;
    for (int i = 0; i < oplogOps.size(); i++) {
        OplogOperation op = oplogOps.get(i);
        switch(op.getType()) {
            case DB:
            case NOOP:
                LOGGER.debug("Ignoring operation {}", op);
                break;
            case DB_CMD:
                addParallelToBatch(oplogOps, fromExcluded, i, result);
                fromExcluded = i;
                result.add(new SingleOpAnalyzedOplogBatch((DbCmdOplogOperation) op));
                break;
            case DELETE:
            case INSERT:
            case UPDATE:
                {
                    //CUD operations on system collection must be addressed sequentially
                    if (SYSTEM_COLLECTIONS.contains(((CollectionOplogOperation) op).getCollection())) {
                        addParallelToBatch(oplogOps, fromExcluded, i, result);
                        fromExcluded = i;
                        result.add(new SingleOpAnalyzedOplogBatch(op));
                    }
                    break;
                }
            default:
                throw new AssertionError("Found an unknown oplog operation " + op);
        }
    }
    addParallelToBatch(oplogOps, fromExcluded, oplogOps.size(), result);
    return result;
}

Example 20 with OplogOperation

use of com.eightkdata.mongowp.server.api.oplog.OplogOperation in project torodb by torodb.

the class SimpleAnalyzedOplogBatchExecutor method visit.

@Override
public OplogOperation visit(CudAnalyzedOplogBatch batch, ApplierContext arg) throws RetrierGiveUpException {
    metrics.getCudBatchSize().update(batch.getOriginalBatch().size());
    try (Context context = metrics.getCudBatchTimer().time()) {
        try {
            execute(batch, arg);
        } catch (UserException | NamespaceJobExecutionException ex) {
            throw new RetrierGiveUpException("Unexpected exception while replying", ex);
        } catch (RollbackException ex) {
            ApplierContext retryingReplingContext = new ApplierContext.Builder().setReapplying(true).setUpdatesAsUpserts(true).build();
            retrier.retry(() -> {
                try {
                    execute(batch, retryingReplingContext);
                    return Empty.getInstance();
                } catch (UserException | NamespaceJobExecutionException ex2) {
                    throw new RetrierAbortException("Unexpected user exception while applying " + "the batch " + batch, ex2);
                }
            }, Hint.CRITICAL, Hint.TIME_SENSIBLE);
        }
    }
    List<OplogOperation> originalBatch = batch.getOriginalBatch();
    return originalBatch.get(originalBatch.size() - 1);
}