Examples with AnalyzedOplogBatch com.torodb.mongodb.repl.oplogreplier.batch.AnalyzedOplogBatch used on opensource projects

Example 1 with AnalyzedOplogBatch

use of com.torodb.mongodb.repl.oplogreplier.batch.AnalyzedOplogBatch in project torodb by torodb.

the class DefaultOplogApplier method createBatcherFlow.

/**
   * Creates a flow that batches and analyze a input of {@link AnalyzedOplogBatch remote jobs}.
   *
   * This flow tries to accummulate several remote jobs into a bigger one and does not emit until:
   * <ul>
   * <li>A maximum number of operations are batched</li>
   * <li>Or a maximum time has happen since the last emit</li>
   * <li>Or the recived job is not {@link AnalyzedOplogBatch#isReadyForMore()}</li>
   * </ul>
   *
   */
private Flow<OplogBatch, AnalyzedStreamElement, NotUsed> createBatcherFlow(ApplierContext context) {
    Predicate<OplogBatch> finishBatchPredicate = (OplogBatch rawBatch) -> !rawBatch.isReadyForMore();
    ToIntFunction<OplogBatch> costFunction = (rawBatch) -> rawBatch.count();
    Supplier<RawStreamElement> zeroFun = () -> RawStreamElement.INITIAL_ELEMENT;
    BiFunction<RawStreamElement, OplogBatch, RawStreamElement> acumFun = (streamElem, newBatch) -> streamElem.concat(newBatch);
    BatchAnalyzer batchAnalyzer = batchAnalyzerFactory.createBatchAnalyzer(context);
    return Flow.of(OplogBatch.class).via(new BatchFlow<>(batchLimits.maxSize, batchLimits.maxPeriod, finishBatchPredicate, costFunction, zeroFun, acumFun)).filter(rawElem -> rawElem.rawBatch != null && !rawElem.rawBatch.isEmpty()).map(rawElem -> {
        List<OplogOperation> rawOps = rawElem.rawBatch.getOps();
        List<AnalyzedOplogBatch> analyzed = batchAnalyzer.apply(rawOps);
        return new AnalyzedStreamElement(rawElem, analyzed);
    });
}

Example 2 with AnalyzedOplogBatch

use of com.torodb.mongodb.repl.oplogreplier.batch.AnalyzedOplogBatch in project torodb by torodb.

the class DefaultOplogApplier method apply.

@Override
public ApplyingJob apply(OplogFetcher fetcher, ApplierContext applierContext) {
    Materializer materializer = ActorMaterializer.create(actorSystem);
    RunnableGraph<Pair<UniqueKillSwitch, CompletionStage<Done>>> graph = createOplogSource(fetcher).async().via(createBatcherFlow(applierContext)).viaMat(KillSwitches.single(), Keep.right()).async().map(analyzedElem -> {
        for (AnalyzedOplogBatch analyzedOplogBatch : analyzedElem.analyzedBatch) {
            batchExecutor.apply(analyzedOplogBatch, applierContext);
        }
        return analyzedElem;
    }).map(this::metricExecution).toMat(Sink.foreach(this::storeLastAppliedOp), (killSwitch, completionStage) -> new Pair<>(killSwitch, completionStage));
    Pair<UniqueKillSwitch, CompletionStage<Done>> pair = graph.run(materializer);
    UniqueKillSwitch killSwitch = pair.first();
    CompletableFuture<Empty> whenComplete = pair.second().toCompletableFuture().thenApply(done -> Empty.getInstance()).whenComplete((done, t) -> {
        fetcher.close();
        if (done != null) {
            LOGGER.trace("Oplog replication stream finished normally");
        } else {
            Throwable cause;
            if (t instanceof CompletionException) {
                cause = t.getCause();
            } else {
                cause = t;
            }
            //the completable future has been cancelled
            if (cause instanceof CancellationException) {
                LOGGER.debug("Oplog replication stream has been cancelled");
                killSwitch.shutdown();
            } else {
                //in this case the exception should came from the stream
                cause = Throwables.getRootCause(cause);
                LOGGER.error("Oplog replication stream finished exceptionally: " + cause.getLocalizedMessage(), cause);
                //the stream should be finished exceptionally, but just in case we
                //notify the kill switch to stop the stream.
                killSwitch.shutdown();
            }
        }
    });
    return new DefaultApplyingJob(killSwitch, whenComplete);
}