use of com.hazelcast.jet.core.EventTimePolicy in project hazelcast by hazelcast.
the class JmsSourceBuilder method build.
/**
* Creates and returns the JMS {@link StreamSource} with the supplied
* components and the projection function {@code projectionFn}.
* <p>
* The given function must be stateless.
*
* @param projectionFn the function which creates output object from each
* message
* @param <T> the type of the items the source emits
*/
@Nonnull
public <T> StreamSource<T> build(@Nonnull FunctionEx<? super Message, ? extends T> projectionFn) {
String usernameLocal = username;
String passwordLocal = password;
String destinationLocal = destinationName;
ProcessingGuarantee maxGuaranteeLocal = maxGuarantee;
@SuppressWarnings("UnnecessaryLocalVariable") boolean isTopicLocal = isTopic;
if (connectionFn == null) {
connectionFn = factory -> requireNonNull(usernameLocal != null || passwordLocal != null ? factory.createConnection(usernameLocal, passwordLocal) : factory.createConnection());
}
if (consumerFn == null) {
checkNotNull(destinationLocal, "neither consumerFn nor destinationName set");
consumerFn = session -> session.createConsumer(isTopicLocal ? session.createTopic(destinationLocal) : session.createQueue(destinationLocal));
if (isTopic) {
// the user didn't specify a custom consumerFn and we know we're using a non-durable consumer
// for a topic - there's no point in using any guarantee, see `maxGuarantee`
maxGuaranteeLocal = ProcessingGuarantee.NONE;
}
}
ProcessingGuarantee maxGuaranteeFinal = maxGuaranteeLocal;
FunctionEx<? super ConnectionFactory, ? extends Connection> connectionFnLocal = connectionFn;
@SuppressWarnings("UnnecessaryLocalVariable") SupplierEx<? extends ConnectionFactory> factorySupplierLocal = factorySupplier;
SupplierEx<? extends Connection> newConnectionFn = () -> connectionFnLocal.apply(factorySupplierLocal.get());
FunctionEx<? super Session, ? extends MessageConsumer> consumerFnLocal = consumerFn;
boolean isSharedConsumerLocal = isSharedConsumer;
FunctionEx<? super Message, ?> messageIdFnLocal = messageIdFn;
FunctionEx<EventTimePolicy<? super T>, ProcessorMetaSupplier> metaSupplierFactory = policy -> isTopicLocal ? streamJmsTopicP(destinationLocal, isSharedConsumerLocal, maxGuaranteeFinal, policy, newConnectionFn, consumerFnLocal, messageIdFnLocal, projectionFn) : streamJmsQueueP(destinationLocal, maxGuaranteeFinal, policy, newConnectionFn, consumerFnLocal, messageIdFnLocal, projectionFn);
return Sources.streamFromProcessorWithWatermarks(sourceName(), true, metaSupplierFactory);
}
use of com.hazelcast.jet.core.EventTimePolicy in project hazelcast by hazelcast.
the class TestAbstractSqlConnector method fullScanReader.
@Nonnull
@Override
public Vertex fullScanReader(@Nonnull DAG dag, @Nonnull Table table_, @Nullable Expression<Boolean> predicate, @Nonnull List<Expression<?>> projection, @Nullable FunctionEx<ExpressionEvalContext, EventTimePolicy<JetSqlRow>> eventTimePolicyProvider) {
TestTable table = (TestTable) table_;
List<Object[]> rows = table.rows;
boolean streaming = table.streaming;
FunctionEx<Context, TestDataGenerator> createContextFn = ctx -> {
ExpressionEvalContext evalContext = ExpressionEvalContext.from(ctx);
EventTimePolicy<JetSqlRow> eventTimePolicy = eventTimePolicyProvider == null ? EventTimePolicy.noEventTime() : eventTimePolicyProvider.apply(evalContext);
return new TestDataGenerator(rows, predicate, projection, evalContext, eventTimePolicy, streaming);
};
ProcessorMetaSupplier pms = createProcessorSupplier(createContextFn);
return dag.newUniqueVertex(table.toString(), pms);
}
use of com.hazelcast.jet.core.EventTimePolicy in project hazelcast by hazelcast.
the class TestAllTypesSqlConnector method fullScanReader.
@Nonnull
@Override
public Vertex fullScanReader(@Nonnull DAG dag, @Nonnull Table table, @Nullable Expression<Boolean> predicate, @Nonnull List<Expression<?>> projection, @Nullable FunctionEx<ExpressionEvalContext, EventTimePolicy<JetSqlRow>> eventTimePolicyProvider) {
if (eventTimePolicyProvider != null) {
throw QueryException.error("Ordering function are not supported for " + TYPE_NAME + " mappings");
}
BatchSource<JetSqlRow> source = SourceBuilder.batch("batch", ExpressionEvalContext::from).<JetSqlRow>fillBufferFn((ctx, buf) -> {
JetSqlRow row = ExpressionUtil.evaluate(predicate, projection, VALUES, ctx);
if (row != null) {
buf.add(row);
}
buf.close();
}).build();
ProcessorMetaSupplier pms = ((BatchSourceTransform<JetSqlRow>) source).metaSupplier;
return dag.newUniqueVertex(table.toString(), pms);
}
use of com.hazelcast.jet.core.EventTimePolicy in project hazelcast by hazelcast.
the class Planner method createDag.
@SuppressWarnings("rawtypes")
DAG createDag(Context context) {
pipeline.makeNamesUnique();
Map<Transform, List<Transform>> adjacencyMap = pipeline.adjacencyMap();
validateNoLeakage(adjacencyMap);
checkTopologicalSort(adjacencyMap.entrySet());
// Find the greatest common denominator of all frame lengths
// appearing in the pipeline
long frameSizeGcd = Util.gcd(adjacencyMap.keySet().stream().map(Transform::preferredWatermarkStride).filter(frameSize -> frameSize > 0).mapToLong(i -> i).toArray());
if (frameSizeGcd == 0) {
// even if there are no window aggregations, we want the watermarks for latency debugging
frameSizeGcd = MAXIMUM_WATERMARK_GAP;
}
if (frameSizeGcd > MAXIMUM_WATERMARK_GAP) {
frameSizeGcd = Util.gcd(frameSizeGcd, MAXIMUM_WATERMARK_GAP);
}
LoggingUtil.logFine(LOGGER, "Watermarks in the pipeline will be throttled to %d", frameSizeGcd);
// Update watermark throttling frame length on all transforms with the determined length
for (Transform transform : adjacencyMap.keySet()) {
if (transform instanceof StreamSourceTransform) {
StreamSourceTransform t = (StreamSourceTransform) transform;
EventTimePolicy policy = t.getEventTimePolicy();
if (policy != null) {
t.setEventTimePolicy(withFrameSize(policy, frameSizeGcd));
}
} else if (transform instanceof TimestampTransform) {
TimestampTransform t = (TimestampTransform) transform;
t.setEventTimePolicy(withFrameSize(t.getEventTimePolicy(), frameSizeGcd));
}
}
// fuse subsequent map/filter/flatMap transforms into one
Map<Transform, List<Transform>> originalParents = new HashMap<>();
List<Transform> transforms = new ArrayList<>(adjacencyMap.keySet());
for (int i = 0; i < transforms.size(); i++) {
Transform transform = transforms.get(i);
List<Transform> chain = findFusableChain(transform, adjacencyMap);
if (chain == null) {
continue;
}
// remove transforms in the chain and replace the parent with a fused transform
transforms.removeAll(chain.subList(1, chain.size()));
Transform fused = fuseFlatMapTransforms(chain);
transforms.set(i, fused);
Transform lastInChain = chain.get(chain.size() - 1);
for (Transform downstream : adjacencyMap.get(lastInChain)) {
originalParents.put(downstream, new ArrayList<>(downstream.upstream()));
downstream.upstream().replaceAll(p -> p == lastInChain ? fused : p);
}
}
for (Transform transform : transforms) {
transform.addToDag(this, context);
}
// restore original parents
for (Entry<Transform, List<Transform>> en : originalParents.entrySet()) {
List<Transform> upstream = en.getKey().upstream();
for (int i = 0; i < upstream.size(); i++) {
en.getKey().upstream().set(i, en.getValue().get(i));
}
}
return dag;
}
Aggregations