use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast by hazelcast.
the class PartitionedProcessorTransform method flatMapUsingServiceAsyncBatchedPartitionedTransform.
public static <S, T, K, R> PartitionedProcessorTransform<T, K> flatMapUsingServiceAsyncBatchedPartitionedTransform(@Nonnull Transform upstream, @Nonnull String operationName, @Nonnull ServiceFactory<?, S> serviceFactory, int maxConcurrentOps, int maxBatchSize, @Nonnull BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<Traverser<R>>> flatMapAsyncFn, @Nonnull FunctionEx<? super T, ? extends K> partitionKeyFn) {
String name = operationName + "UsingPartitionedServiceAsync";
ProcessorSupplier supplier = flatMapUsingServiceAsyncBatchedP(serviceFactory, maxConcurrentOps, maxBatchSize, flatMapAsyncFn);
ProcessorMetaSupplier metaSupplier = ProcessorMetaSupplier.of(getPreferredLP(serviceFactory), serviceFactory.permission(), supplier);
return new PartitionedProcessorTransform<>(name, upstream, metaSupplier, partitionKeyFn);
}
use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast by hazelcast.
the class ProcessorTransform method flatMapUsingServiceAsyncBatchedTransform.
public static <S, T, R> ProcessorTransform flatMapUsingServiceAsyncBatchedTransform(@Nonnull Transform upstream, @Nonnull String operationName, @Nonnull ServiceFactory<?, S> serviceFactory, int maxConcurrentOps, int maxBatchSize, @Nonnull BiFunctionEx<? super S, ? super List<T>, ? extends CompletableFuture<Traverser<R>>> flatMapAsyncFn) {
String name = operationName + "UsingServiceAsyncBatched";
ProcessorSupplier supplier = flatMapUsingServiceAsyncBatchedP(serviceFactory, maxConcurrentOps, maxBatchSize, flatMapAsyncFn);
ProcessorMetaSupplier metaSupplier = ProcessorMetaSupplier.of(getPreferredLP(serviceFactory), serviceFactory.permission(), supplier);
return new ProcessorTransform(name, upstream, metaSupplier);
}
use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast by hazelcast.
the class StreamSourceTransform method addToDag.
@Override
public void addToDag(Planner p, Context context) {
if (emitsWatermarks || eventTimePolicy == null) {
// Reached when the source either emits both JetEvents and watermarks
// or neither. In these cases we don't have to insert watermarks.
final ProcessorMetaSupplier metaSupplier = metaSupplierFn.apply(eventTimePolicy != null ? eventTimePolicy : noEventTime());
determineLocalParallelism(metaSupplier.preferredLocalParallelism(), context, false);
p.addVertex(this, name(), determinedLocalParallelism(), metaSupplier);
} else {
// ------------
// | sourceP |
// ------------
// |
// isolated
// v
// -------------
// | insertWmP |
// -------------
String v1name = name();
final ProcessorMetaSupplier metaSupplier = metaSupplierFn.apply(eventTimePolicy);
determineLocalParallelism(metaSupplier.preferredLocalParallelism(), context, false);
Vertex v1 = p.dag.newVertex(v1name, metaSupplier).localParallelism(determinedLocalParallelism());
PlannerVertex pv2 = p.addVertex(this, v1name + "-add-timestamps", determinedLocalParallelism(), insertWatermarksP(eventTimePolicy));
p.dag.edge(between(v1, pv2.v).isolated());
}
}
use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast by hazelcast.
the class TestAbstractSqlConnector method fullScanReader.
@Nonnull
@Override
public Vertex fullScanReader(@Nonnull DAG dag, @Nonnull Table table_, @Nullable Expression<Boolean> predicate, @Nonnull List<Expression<?>> projection, @Nullable FunctionEx<ExpressionEvalContext, EventTimePolicy<JetSqlRow>> eventTimePolicyProvider) {
TestTable table = (TestTable) table_;
List<Object[]> rows = table.rows;
boolean streaming = table.streaming;
FunctionEx<Context, TestDataGenerator> createContextFn = ctx -> {
ExpressionEvalContext evalContext = ExpressionEvalContext.from(ctx);
EventTimePolicy<JetSqlRow> eventTimePolicy = eventTimePolicyProvider == null ? EventTimePolicy.noEventTime() : eventTimePolicyProvider.apply(evalContext);
return new TestDataGenerator(rows, predicate, projection, evalContext, eventTimePolicy, streaming);
};
ProcessorMetaSupplier pms = createProcessorSupplier(createContextFn);
return dag.newUniqueVertex(table.toString(), pms);
}
use of com.hazelcast.jet.core.ProcessorMetaSupplier in project hazelcast by hazelcast.
the class TestAllTypesSqlConnector method fullScanReader.
@Nonnull
@Override
public Vertex fullScanReader(@Nonnull DAG dag, @Nonnull Table table, @Nullable Expression<Boolean> predicate, @Nonnull List<Expression<?>> projection, @Nullable FunctionEx<ExpressionEvalContext, EventTimePolicy<JetSqlRow>> eventTimePolicyProvider) {
if (eventTimePolicyProvider != null) {
throw QueryException.error("Ordering function are not supported for " + TYPE_NAME + " mappings");
}
BatchSource<JetSqlRow> source = SourceBuilder.batch("batch", ExpressionEvalContext::from).<JetSqlRow>fillBufferFn((ctx, buf) -> {
JetSqlRow row = ExpressionUtil.evaluate(predicate, projection, VALUES, ctx);
if (row != null) {
buf.add(row);
}
buf.close();
}).build();
ProcessorMetaSupplier pms = ((BatchSourceTransform<JetSqlRow>) source).metaSupplier;
return dag.newUniqueVertex(table.toString(), pms);
}
Aggregations