use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.
the class Planner method fuseFlatMapTransforms.
@SuppressWarnings("rawtypes")
private static Transform fuseFlatMapTransforms(List<Transform> chain) {
assert chain.size() > 1 : "chain.size()=" + chain.size();
assert chain.get(0).upstream().size() == 1;
int lastFlatMap = 0;
FunctionEx<Object, Traverser> flatMapFn = null;
for (int i = 0; i < chain.size(); i++) {
if (chain.get(i) instanceof FlatMapTransform) {
FunctionEx<Object, Traverser> function = ((FlatMapTransform) chain.get(i)).flatMapFn();
FunctionEx<Object, Object> inputMapFn = mergeMapFunctions(chain.subList(lastFlatMap, i));
if (inputMapFn != null) {
flatMapFn = flatMapFn == null ? (Object t) -> {
Object mappedValue = inputMapFn.apply(t);
return mappedValue != null ? function.apply(mappedValue) : Traversers.empty();
} : flatMapFn.andThen(r -> r.map(inputMapFn).flatMap(function));
} else {
flatMapFn = flatMapFn == null ? function::apply : flatMapFn.andThen(r -> r.flatMap(function));
}
lastFlatMap = i + 1;
}
}
FunctionEx trailingMapFn = mergeMapFunctions(chain.subList(lastFlatMap, chain.size()));
String name = chain.stream().map(Transform::name).collect(Collectors.joining(", ", "fused(", ")"));
Transform fused;
if (flatMapFn == null) {
fused = new MapTransform(name, chain.get(0).upstream().get(0), trailingMapFn);
} else {
if (trailingMapFn != null) {
flatMapFn = flatMapFn.andThen(t -> t.map(trailingMapFn));
}
fused = new FlatMapTransform(name, chain.get(0).upstream().get(0), flatMapFn);
}
// if the first stage of the chain is rebalanced, then we set
// the rebalance flag of the created fused stage. Only consider
// the case when first element of the chain is rebalanced
// because there isn't any other case. If any stage in the
// middle includes rebalance, then those stages are not fused
// by findFusableChain().
fused.setRebalanceInput(0, chain.get(0).shouldRebalanceInput(0));
return fused;
}
use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.
the class ElasticSourceP method init.
@Override
protected void init(@Nonnull Context context) throws Exception {
super.init(context);
logger = context.logger();
logger.fine("init");
client = configuration.clientFn().get();
SearchRequest sr = configuration.searchRequestFn().get();
sr.scroll(configuration.scrollKeepAlive());
if (configuration.isSlicingEnabled()) {
if (configuration.isCoLocatedReadingEnabled()) {
int sliceId = context.localProcessorIndex();
int totalSlices = context.localParallelism();
if (totalSlices > 1) {
logger.fine("Slice id=" + sliceId + ", max=" + totalSlices);
sr.source().slice(new SliceBuilder(sliceId, totalSlices));
}
} else {
int sliceId = context.globalProcessorIndex();
int totalSlices = context.totalParallelism();
if (totalSlices > 1) {
logger.fine("Slice id=" + sliceId + ", max=" + totalSlices);
sr.source().slice(new SliceBuilder(sliceId, totalSlices));
}
}
}
if (configuration.isCoLocatedReadingEnabled()) {
logger.fine("Assigned shards: " + shards);
if (shards.isEmpty()) {
traverser = Traversers.empty();
return;
}
Node node = createLocalElasticNode();
client.getLowLevelClient().setNodes(singleton(node));
String preference = "_shards:" + shards.stream().map(shard -> String.valueOf(shard.getShard())).collect(joining(",")) + "|_only_local";
sr.preference(preference);
}
scrollTraverser = new ElasticScrollTraverser(configuration, client, sr, logger);
traverser = scrollTraverser.map(configuration.mapToItemFn());
}
use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.
the class CreateDagVisitor method onSlidingWindow.
public Vertex onSlidingWindow(SlidingWindowPhysicalRel rel) {
int orderingFieldIndex = rel.orderingFieldIndex();
FunctionEx<ExpressionEvalContext, SlidingWindowPolicy> windowPolicySupplier = rel.windowPolicyProvider();
// this vertex is used only if there's no aggregation by a window bound
Vertex vertex = dag.newUniqueVertex("Sliding-Window", flatMapUsingServiceP(ServiceFactories.nonSharedService(ctx -> {
ExpressionEvalContext evalContext = ExpressionEvalContext.from(ctx);
SlidingWindowPolicy windowPolicy = windowPolicySupplier.apply(evalContext);
return row -> WindowUtils.addWindowBounds(row, orderingFieldIndex, windowPolicy);
}), (BiFunctionEx<Function<JetSqlRow, Traverser<JetSqlRow>>, JetSqlRow, Traverser<JetSqlRow>>) Function::apply));
connectInput(rel.getInput(), vertex, null);
return vertex;
}
use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.
the class StreamKafkaP method saveToSnapshot.
@Override
public boolean saveToSnapshot() {
if (!emitFromTraverser(traverser)) {
return false;
}
if (snapshotTraverser == null) {
Stream<Entry<BroadcastKey<?>, ?>> snapshotStream = offsets.entrySet().stream().flatMap(entry -> IntStream.range(0, entry.getValue().length).filter(partition -> entry.getValue()[partition] >= 0).mapToObj(partition -> {
TopicPartition key = new TopicPartition(entry.getKey(), partition);
long offset = entry.getValue()[partition];
long watermark = eventTimeMapper.getWatermark(currentAssignment.get(key));
return entry(broadcastKey(key), new long[] { offset, watermark });
}));
snapshotTraverser = traverseStream(snapshotStream).onFirstNull(() -> {
snapshotTraverser = null;
if (getLogger().isFinestEnabled()) {
getLogger().finest("Finished saving snapshot." + " Saved offsets: " + offsets() + ", Saved watermarks: " + watermarks());
}
});
if (processorIndex == 0) {
Entry<BroadcastKey<?>, ?> partitionCountsItem = entry(broadcastKey(PARTITION_COUNTS_SNAPSHOT_KEY), topics.stream().collect(Collectors.toMap(topic -> topic, topic -> offsets.get(topic).length)));
snapshotTraverser = snapshotTraverser.append(partitionCountsItem);
}
}
return emitFromTraverserToSnapshot(snapshotTraverser);
}
use of com.hazelcast.jet.Traverser in project hazelcast by hazelcast.
the class AppendableTraverserTest method test_flatMapperUsage.
@Test
public void test_flatMapperUsage() {
// an instance of AppendableTraverser is repeatedly returned
// from a flatMap function
Traverser tt = Traversers.traverseItems(10, 20).flatMap(item -> {
assertTrue(t.isEmpty());
t.append(item);
t.append(item + 1);
return t;
});
assertEquals(10, tt.next());
assertEquals(11, tt.next());
assertEquals(20, tt.next());
assertEquals(21, tt.next());
assertNull(tt.next());
}
Aggregations