Search in sources :

Example 1 with Computation

use of io.mantisrx.runtime.computation.Computation in project mantis by Netflix.

the class StageExecutors method executeMantisGroups.

@SuppressWarnings("unchecked")
private static <K, T, R> Observable<Observable<R>> executeMantisGroups(Observable<Observable<MantisGroup<K, T>>> go, final Computation computation, final Context context, final long groupTakeUntil) {
    logger.info("initializing {}", computation.getClass().getCanonicalName());
    computation.init(context);
    // from groups to observable
    final Func2<Context, Observable<MantisGroup<K, T>>, Observable<R>> c = (Func2<Context, Observable<MantisGroup<K, T>>, Observable<R>>) computation;
    return go.lift(new MonitorOperator<>("worker_stage_outer")).map((Func1<Observable<MantisGroup<K, T>>, Observable<R>>) group -> c.call(context, group.lift(new MonitorOperator<>("worker_stage_inner_input"))).lift(new MonitorOperator("worker_stage_inner_output")));
}
Also used : Context(io.mantisrx.runtime.Context) Index(io.mantisrx.runtime.source.Index) StageConfig(io.mantisrx.runtime.StageConfig) MantisRxSingleThreadScheduler(io.mantisrx.runtime.scheduler.MantisRxSingleThreadScheduler) MetricsRegistry(io.mantisrx.common.metrics.MetricsRegistry) LoggerFactory(org.slf4j.LoggerFactory) Action1(rx.functions.Action1) Observable(rx.Observable) Computation(io.mantisrx.runtime.computation.Computation) ServiceRegistry(io.mantisrx.server.core.ServiceRegistry) Func1(rx.functions.Func1) Func2(rx.functions.Func2) GroupToGroup(io.mantisrx.runtime.GroupToGroup) GroupedObservableUtils(io.reactivx.mantis.operators.GroupedObservableUtils) Schedulers(rx.schedulers.Schedulers) GroupToScalar(io.mantisrx.runtime.GroupToScalar) RxMetrics(io.reactivex.mantis.remote.observable.RxMetrics) KeyToScalar(io.mantisrx.runtime.KeyToScalar) Metrics(io.mantisrx.common.metrics.Metrics) Counter(io.mantisrx.common.metrics.Counter) Groups(io.mantisrx.runtime.Groups) Logger(org.slf4j.Logger) RxThreadFactory(rx.internal.util.RxThreadFactory) ScalarToKey(io.mantisrx.runtime.ScalarToKey) SourceHolder(io.mantisrx.runtime.SourceHolder) KeyToKey(io.mantisrx.runtime.KeyToKey) Context(io.mantisrx.runtime.Context) STAGE_CONCURRENCY(io.mantisrx.runtime.parameter.ParameterUtils.STAGE_CONCURRENCY) TimeUnit(java.util.concurrent.TimeUnit) GroupedObservable(rx.observables.GroupedObservable) ScalarToScalar(io.mantisrx.runtime.ScalarToScalar) Action0(rx.functions.Action0) ScalarToGroup(io.mantisrx.runtime.ScalarToGroup) SinkHolder(io.mantisrx.runtime.SinkHolder) MantisMarker(io.mantisrx.runtime.markers.MantisMarker) MantisGroup(io.mantisrx.common.MantisGroup) MonitorOperator(io.mantisrx.common.metrics.rx.MonitorOperator) MonitorOperator(io.mantisrx.common.metrics.rx.MonitorOperator) MantisGroup(io.mantisrx.common.MantisGroup) Observable(rx.Observable) GroupedObservable(rx.observables.GroupedObservable) Func2(rx.functions.Func2)

Example 2 with Computation

use of io.mantisrx.runtime.computation.Computation in project mantis by Netflix.

the class StageExecutors method executeMantisGroupsInParallel.

/**
 * @param go
 * @param computation
 *
 * @return untyped to support multiple callers return types
 */
@SuppressWarnings("unchecked")
private static <K, T, R> Observable<Observable<R>> executeMantisGroupsInParallel(Observable<Observable<MantisGroup<K, T>>> go, Computation computation, final Context context, final boolean applyTimeoutToInners, final long timeout, final int concurrency) {
    logger.info("initializing {}", computation.getClass().getCanonicalName());
    computation.init(context);
    // from groups to observable
    final Func2<Context, Observable<MantisGroup<K, T>>, Observable<R>> c = (Func2<Context, Observable<MantisGroup<K, T>>, Observable<R>>) computation;
    if (concurrency == StageConfig.DEFAULT_STAGE_CONCURRENCY) {
        return go.lift(new MonitorOperator<>("worker_stage_outer")).map(observable -> c.call(context, observable.observeOn(Schedulers.computation()).lift(new MonitorOperator<>("worker_stage_inner_input"))).lift(new MonitorOperator<>("worker_stage_inner_output")));
    } else {
        final MantisRxSingleThreadScheduler[] mantisRxSingleThreadSchedulers = new MantisRxSingleThreadScheduler[concurrency];
        RxThreadFactory rxThreadFactory = new RxThreadFactory("MantisRxSingleThreadScheduler-");
        logger.info("creating {} Mantis threads", concurrency);
        for (int i = 0; i < concurrency; i++) {
            mantisRxSingleThreadSchedulers[i] = new MantisRxSingleThreadScheduler(rxThreadFactory);
        }
        return go.lift(new MonitorOperator<>("worker_stage_outer")).map(observable -> observable.groupBy(e -> Math.abs(e.getKeyValue().hashCode()) % concurrency).flatMap(gbo -> c.call(context, gbo.observeOn(mantisRxSingleThreadSchedulers[gbo.getKey().intValue()]).lift(new MonitorOperator<MantisGroup<K, T>>("worker_stage_inner_input"))).lift(new MonitorOperator<R>("worker_stage_inner_output"))));
    }
}
Also used : Context(io.mantisrx.runtime.Context) Index(io.mantisrx.runtime.source.Index) StageConfig(io.mantisrx.runtime.StageConfig) MantisRxSingleThreadScheduler(io.mantisrx.runtime.scheduler.MantisRxSingleThreadScheduler) MetricsRegistry(io.mantisrx.common.metrics.MetricsRegistry) LoggerFactory(org.slf4j.LoggerFactory) Action1(rx.functions.Action1) Observable(rx.Observable) Computation(io.mantisrx.runtime.computation.Computation) ServiceRegistry(io.mantisrx.server.core.ServiceRegistry) Func1(rx.functions.Func1) Func2(rx.functions.Func2) GroupToGroup(io.mantisrx.runtime.GroupToGroup) GroupedObservableUtils(io.reactivx.mantis.operators.GroupedObservableUtils) Schedulers(rx.schedulers.Schedulers) GroupToScalar(io.mantisrx.runtime.GroupToScalar) RxMetrics(io.reactivex.mantis.remote.observable.RxMetrics) KeyToScalar(io.mantisrx.runtime.KeyToScalar) Metrics(io.mantisrx.common.metrics.Metrics) Counter(io.mantisrx.common.metrics.Counter) Groups(io.mantisrx.runtime.Groups) Logger(org.slf4j.Logger) RxThreadFactory(rx.internal.util.RxThreadFactory) ScalarToKey(io.mantisrx.runtime.ScalarToKey) SourceHolder(io.mantisrx.runtime.SourceHolder) KeyToKey(io.mantisrx.runtime.KeyToKey) Context(io.mantisrx.runtime.Context) STAGE_CONCURRENCY(io.mantisrx.runtime.parameter.ParameterUtils.STAGE_CONCURRENCY) TimeUnit(java.util.concurrent.TimeUnit) GroupedObservable(rx.observables.GroupedObservable) ScalarToScalar(io.mantisrx.runtime.ScalarToScalar) Action0(rx.functions.Action0) ScalarToGroup(io.mantisrx.runtime.ScalarToGroup) SinkHolder(io.mantisrx.runtime.SinkHolder) MantisMarker(io.mantisrx.runtime.markers.MantisMarker) MantisGroup(io.mantisrx.common.MantisGroup) MonitorOperator(io.mantisrx.common.metrics.rx.MonitorOperator) MantisRxSingleThreadScheduler(io.mantisrx.runtime.scheduler.MantisRxSingleThreadScheduler) MantisGroup(io.mantisrx.common.MantisGroup) Observable(rx.Observable) GroupedObservable(rx.observables.GroupedObservable) MonitorOperator(io.mantisrx.common.metrics.rx.MonitorOperator) RxThreadFactory(rx.internal.util.RxThreadFactory) Func2(rx.functions.Func2)

Example 3 with Computation

use of io.mantisrx.runtime.computation.Computation in project mantis by Netflix.

the class StageExecutors method executeGroupsInParallel.

@SuppressWarnings("unchecked")
private static <K, T, R> Observable<Observable<R>> executeGroupsInParallel(Observable<GroupedObservable<K, T>> go, final Computation computation, final Context context, final long groupTakeUntil) {
    logger.info("initializing {}", computation.getClass().getCanonicalName());
    computation.init(context);
    // from groups to observable
    final Func2<Context, GroupedObservable<K, T>, Observable<R>> c = (Func2<Context, GroupedObservable<K, T>, Observable<R>>) computation;
    return go.lift(new MonitorOperator<>("worker_stage_outer")).map((Func1<GroupedObservable<K, T>, Observable<R>>) group -> c.call(context, GroupedObservableUtils.createGroupedObservable(group.getKey(), group.doOnUnsubscribe(() -> {
        if (groupsExpiredCounter != null)
            groupsExpiredCounter.increment();
    }).timeout(groupTakeUntil, TimeUnit.SECONDS, (Observable<? extends T>) Observable.empty()).subscribeOn(Schedulers.computation()).lift(new MonitorOperator<T>("worker_stage_inner_input")))).lift(new MonitorOperator("worker_stage_inner_output")));
}
Also used : Context(io.mantisrx.runtime.Context) Index(io.mantisrx.runtime.source.Index) StageConfig(io.mantisrx.runtime.StageConfig) MantisRxSingleThreadScheduler(io.mantisrx.runtime.scheduler.MantisRxSingleThreadScheduler) MetricsRegistry(io.mantisrx.common.metrics.MetricsRegistry) LoggerFactory(org.slf4j.LoggerFactory) Action1(rx.functions.Action1) Observable(rx.Observable) Computation(io.mantisrx.runtime.computation.Computation) ServiceRegistry(io.mantisrx.server.core.ServiceRegistry) Func1(rx.functions.Func1) Func2(rx.functions.Func2) GroupToGroup(io.mantisrx.runtime.GroupToGroup) GroupedObservableUtils(io.reactivx.mantis.operators.GroupedObservableUtils) Schedulers(rx.schedulers.Schedulers) GroupToScalar(io.mantisrx.runtime.GroupToScalar) RxMetrics(io.reactivex.mantis.remote.observable.RxMetrics) KeyToScalar(io.mantisrx.runtime.KeyToScalar) Metrics(io.mantisrx.common.metrics.Metrics) Counter(io.mantisrx.common.metrics.Counter) Groups(io.mantisrx.runtime.Groups) Logger(org.slf4j.Logger) RxThreadFactory(rx.internal.util.RxThreadFactory) ScalarToKey(io.mantisrx.runtime.ScalarToKey) SourceHolder(io.mantisrx.runtime.SourceHolder) KeyToKey(io.mantisrx.runtime.KeyToKey) Context(io.mantisrx.runtime.Context) STAGE_CONCURRENCY(io.mantisrx.runtime.parameter.ParameterUtils.STAGE_CONCURRENCY) TimeUnit(java.util.concurrent.TimeUnit) GroupedObservable(rx.observables.GroupedObservable) ScalarToScalar(io.mantisrx.runtime.ScalarToScalar) Action0(rx.functions.Action0) ScalarToGroup(io.mantisrx.runtime.ScalarToGroup) SinkHolder(io.mantisrx.runtime.SinkHolder) MantisMarker(io.mantisrx.runtime.markers.MantisMarker) MantisGroup(io.mantisrx.common.MantisGroup) MonitorOperator(io.mantisrx.common.metrics.rx.MonitorOperator) MonitorOperator(io.mantisrx.common.metrics.rx.MonitorOperator) Observable(rx.Observable) GroupedObservable(rx.observables.GroupedObservable) GroupedObservable(rx.observables.GroupedObservable) Func2(rx.functions.Func2)

Example 4 with Computation

use of io.mantisrx.runtime.computation.Computation in project mantis by Netflix.

the class StageExecutors method executeInnersInParallel.

/**
 * @param oo
 * @param computation
 *
 * @return untyped to support multiple callers return types
 */
@SuppressWarnings("unchecked")
private static <T, R> Observable<Observable<R>> executeInnersInParallel(Observable<Observable<T>> oo, Computation computation, final Context context, final boolean applyTimeoutToInners, final long timeout, final int concurrency) {
    logger.info("initializing {}", computation.getClass().getCanonicalName());
    computation.init(context);
    // from groups to observable
    final Func2<Context, Observable<T>, Observable<R>> c = (Func2<Context, Observable<T>, Observable<R>>) computation;
    if (concurrency == StageConfig.DEFAULT_STAGE_CONCURRENCY) {
        return oo.lift(new MonitorOperator<>("worker_stage_outer")).map(observable -> c.call(context, observable.observeOn(Schedulers.computation()).lift(new MonitorOperator<T>("worker_stage_inner_input"))).lift(new MonitorOperator<R>("worker_stage_inner_output")));
    } else {
        final MantisRxSingleThreadScheduler[] mantisRxSingleThreadSchedulers = new MantisRxSingleThreadScheduler[concurrency];
        RxThreadFactory rxThreadFactory = new RxThreadFactory("MantisRxSingleThreadScheduler-");
        logger.info("creating {} Mantis threads", concurrency);
        for (int i = 0; i < concurrency; i++) {
            mantisRxSingleThreadSchedulers[i] = new MantisRxSingleThreadScheduler(rxThreadFactory);
        }
        return oo.lift(new MonitorOperator<>("worker_stage_outer")).map(observable -> observable.groupBy(e -> System.nanoTime() % concurrency).flatMap(go -> c.call(context, go.observeOn(mantisRxSingleThreadSchedulers[go.getKey().intValue()]).lift(new MonitorOperator<>("worker_stage_inner_input"))).lift(new MonitorOperator<>("worker_stage_inner_output"))));
    }
}
Also used : Context(io.mantisrx.runtime.Context) Index(io.mantisrx.runtime.source.Index) StageConfig(io.mantisrx.runtime.StageConfig) MantisRxSingleThreadScheduler(io.mantisrx.runtime.scheduler.MantisRxSingleThreadScheduler) MetricsRegistry(io.mantisrx.common.metrics.MetricsRegistry) LoggerFactory(org.slf4j.LoggerFactory) Action1(rx.functions.Action1) Observable(rx.Observable) Computation(io.mantisrx.runtime.computation.Computation) ServiceRegistry(io.mantisrx.server.core.ServiceRegistry) Func1(rx.functions.Func1) Func2(rx.functions.Func2) GroupToGroup(io.mantisrx.runtime.GroupToGroup) GroupedObservableUtils(io.reactivx.mantis.operators.GroupedObservableUtils) Schedulers(rx.schedulers.Schedulers) GroupToScalar(io.mantisrx.runtime.GroupToScalar) RxMetrics(io.reactivex.mantis.remote.observable.RxMetrics) KeyToScalar(io.mantisrx.runtime.KeyToScalar) Metrics(io.mantisrx.common.metrics.Metrics) Counter(io.mantisrx.common.metrics.Counter) Groups(io.mantisrx.runtime.Groups) Logger(org.slf4j.Logger) RxThreadFactory(rx.internal.util.RxThreadFactory) ScalarToKey(io.mantisrx.runtime.ScalarToKey) SourceHolder(io.mantisrx.runtime.SourceHolder) KeyToKey(io.mantisrx.runtime.KeyToKey) Context(io.mantisrx.runtime.Context) STAGE_CONCURRENCY(io.mantisrx.runtime.parameter.ParameterUtils.STAGE_CONCURRENCY) TimeUnit(java.util.concurrent.TimeUnit) GroupedObservable(rx.observables.GroupedObservable) ScalarToScalar(io.mantisrx.runtime.ScalarToScalar) Action0(rx.functions.Action0) ScalarToGroup(io.mantisrx.runtime.ScalarToGroup) SinkHolder(io.mantisrx.runtime.SinkHolder) MantisMarker(io.mantisrx.runtime.markers.MantisMarker) MantisGroup(io.mantisrx.common.MantisGroup) MonitorOperator(io.mantisrx.common.metrics.rx.MonitorOperator) MantisRxSingleThreadScheduler(io.mantisrx.runtime.scheduler.MantisRxSingleThreadScheduler) Observable(rx.Observable) GroupedObservable(rx.observables.GroupedObservable) MonitorOperator(io.mantisrx.common.metrics.rx.MonitorOperator) RxThreadFactory(rx.internal.util.RxThreadFactory) Func2(rx.functions.Func2)

Aggregations

MantisGroup (io.mantisrx.common.MantisGroup)4 Counter (io.mantisrx.common.metrics.Counter)4 Metrics (io.mantisrx.common.metrics.Metrics)4 MetricsRegistry (io.mantisrx.common.metrics.MetricsRegistry)4 MonitorOperator (io.mantisrx.common.metrics.rx.MonitorOperator)4 Context (io.mantisrx.runtime.Context)4 GroupToGroup (io.mantisrx.runtime.GroupToGroup)4 GroupToScalar (io.mantisrx.runtime.GroupToScalar)4 Groups (io.mantisrx.runtime.Groups)4 KeyToKey (io.mantisrx.runtime.KeyToKey)4 KeyToScalar (io.mantisrx.runtime.KeyToScalar)4 ScalarToGroup (io.mantisrx.runtime.ScalarToGroup)4 ScalarToKey (io.mantisrx.runtime.ScalarToKey)4 ScalarToScalar (io.mantisrx.runtime.ScalarToScalar)4 SinkHolder (io.mantisrx.runtime.SinkHolder)4 SourceHolder (io.mantisrx.runtime.SourceHolder)4 StageConfig (io.mantisrx.runtime.StageConfig)4 Computation (io.mantisrx.runtime.computation.Computation)4 MantisMarker (io.mantisrx.runtime.markers.MantisMarker)4 STAGE_CONCURRENCY (io.mantisrx.runtime.parameter.ParameterUtils.STAGE_CONCURRENCY)4