Examples with BatchFlow com.torodb.concurrent.akka.BatchFlow used on opensource projects

Example 1 with BatchFlow

use of com.torodb.concurrent.akka.BatchFlow in project torodb by torodb.

the class DefaultOplogApplier method createBatcherFlow.

/**
   * Creates a flow that batches and analyze a input of {@link AnalyzedOplogBatch remote jobs}.
   *
   * This flow tries to accummulate several remote jobs into a bigger one and does not emit until:
   * <ul>
   * <li>A maximum number of operations are batched</li>
   * <li>Or a maximum time has happen since the last emit</li>
   * <li>Or the recived job is not {@link AnalyzedOplogBatch#isReadyForMore()}</li>
   * </ul>
   *
   */
private Flow<OplogBatch, AnalyzedStreamElement, NotUsed> createBatcherFlow(ApplierContext context) {
    Predicate<OplogBatch> finishBatchPredicate = (OplogBatch rawBatch) -> !rawBatch.isReadyForMore();
    ToIntFunction<OplogBatch> costFunction = (rawBatch) -> rawBatch.count();
    Supplier<RawStreamElement> zeroFun = () -> RawStreamElement.INITIAL_ELEMENT;
    BiFunction<RawStreamElement, OplogBatch, RawStreamElement> acumFun = (streamElem, newBatch) -> streamElem.concat(newBatch);
    BatchAnalyzer batchAnalyzer = batchAnalyzerFactory.createBatchAnalyzer(context);
    return Flow.of(OplogBatch.class).via(new BatchFlow<>(batchLimits.maxSize, batchLimits.maxPeriod, finishBatchPredicate, costFunction, zeroFun, acumFun)).filter(rawElem -> rawElem.rawBatch != null && !rawElem.rawBatch.isEmpty()).map(rawElem -> {
        List<OplogOperation> rawOps = rawElem.rawBatch.getOps();
        List<AnalyzedOplogBatch> analyzed = batchAnalyzer.apply(rawOps);
        return new AnalyzedStreamElement(rawElem, analyzed);
    });
}