Examples with Tag com.hazelcast.jet.datamodel.Tag used on opensource projects

Example 1 with Tag

use of com.hazelcast.jet.datamodel.Tag in project hazelcast-jet by hazelcast.

the class AggregateTransform_IntegrationTest method test_aggregate3_with_aggBuilder.

@Test
public void test_aggregate3_with_aggBuilder() {
    // Given
    JetInstance instance = createJetMember();
    IListJet<Integer> list = instance.getList("list");
    list.add(1);
    list.add(2);
    list.add(3);
    IListJet<String> list1 = instance.getList("list1");
    list1.add("a");
    list1.add("b");
    list1.add("c");
    IListJet<Double> list2 = instance.getList("list2");
    list2.add(6.0d);
    list2.add(7.0d);
    list2.add(8.0d);
    // When
    Pipeline p = Pipeline.create();
    BatchStage<Integer> stage0 = p.drawFrom(Sources.list("list"));
    BatchStage<String> stage1 = p.drawFrom(Sources.list("list1"));
    BatchStage<Double> stage2 = p.drawFrom(Sources.list("list2"));
    AggregateBuilder<Integer> builder = stage0.aggregateBuilder();
    Tag<Integer> tag0 = builder.tag0();
    Tag<String> tag1 = builder.add(stage1);
    Tag<Double> tag2 = builder.add(stage2);
    BatchStage<ThreeBags> resultStage = builder.build(AggregateOperation.withCreate(ThreeBags::threeBags).andAccumulate(tag0, (acc, item0) -> acc.bag0().add(item0)).andAccumulate(tag1, (acc, item1) -> acc.bag1().add(item1)).andAccumulate(tag2, (acc, item2) -> acc.bag2().add(item2)).andCombine(ThreeBags::combineWith).andDeduct(ThreeBags::deduct).andFinish(ThreeBags::finish));
    resultStage.drainTo(Sinks.list("sink"));
    instance.newJob(p).join();
    // Then
    ThreeBags threeBags = (ThreeBags) instance.getHazelcastInstance().getList("sink").iterator().next();
    assertNotNull(threeBags);
    sort(threeBags);
    assertEquals(threeBags(list, list1, list2), threeBags);
}

Example 2 with Tag

use of com.hazelcast.jet.datamodel.Tag in project hazelcast-jet by hazelcast.

the class WindowAggregateTransform_IntegrationTest method test_aggregate2_with_aggregateBuilder.

@Test
public void test_aggregate2_with_aggregateBuilder() {
    IMap<Long, String> map = instance.getMap("source");
    // key is timestamp
    map.put(0L, "foo");
    map.put(2L, "baz");
    map.put(10L, "flush-item");
    IMap<Long, String> map2 = instance.getMap("source1");
    // key is timestamp
    map2.put(0L, "faa");
    map2.put(2L, "buu");
    map2.put(10L, "flush-item");
    Pipeline p = Pipeline.create();
    StreamStage<Entry<Long, String>> stage1 = p.drawFrom(Sources.<Long, String>mapJournal("source1", START_FROM_OLDEST));
    stage1.addTimestamps(Entry::getKey, 0);
    WindowAggregateBuilder<Entry<Long, String>> b = p.drawFrom(Sources.<Long, String>mapJournal("source", START_FROM_OLDEST)).addTimestamps(Entry::getKey, 0).window(WindowDefinition.tumbling(2)).aggregateBuilder();
    Tag<Entry<Long, String>> tag0 = b.tag0();
    Tag<Entry<Long, String>> tag1 = b.add(stage1);
    b.build(AggregateOperation.withCreate(TwoBags::twoBags).andAccumulate(tag0, (acc, item0) -> acc.bag0().add(item0)).andAccumulate(tag1, (acc, item1) -> acc.bag1().add(item1)).andCombine(TwoBags::combineWith).andDeduct(TwoBags::deduct).andFinish(TwoBags::finish)).peek().drainTo(Sinks.list("sink"));
    instance.newJob(p);
    assertTrueEventually(() -> assertEquals(listToString(asList(new TimestampedItem<>(2, TwoBags.twoBags(asList(entry(0L, "foo")), asList(entry(0L, "faa")))), new TimestampedItem<>(4, TwoBags.twoBags(asList(entry(2L, "baz")), asList(entry(2L, "buu")))))), listToString(instance.getHazelcastInstance().getList("sink"))), 5);
}

Example 3 with Tag

use of com.hazelcast.jet.datamodel.Tag in project hazelcast-jet by hazelcast.

the class HashJoinTransform method addToDag.

// ---------           ----------           ----------
// | primary |         | joined-1 |         | joined-2 |
// ---------           ----------           ----------
// |                   |                     |
// |              distributed          distributed
// |               broadcast            broadcast
// |                   v                     v
// |             -------------         -------------
// |            | collector-1 |       | collector-2 |
// |             -------------         -------------
// |                   |                     |
// |                 local                 local
// local             broadcast             broadcast
// unicast           prioritized           prioritized
// ordinal 0           ordinal 1             ordinal 2
// \                   |                     |
// ----------------\  |   /----------------/
// v  v  v
// --------
// | joiner |
// --------
@Override
@SuppressWarnings("unchecked")
public void addToDag(Planner p) {
    String namePrefix = p.uniqueVertexName(this.name(), "");
    PlannerVertex primary = p.xform2vertex.get(this.upstream().get(0));
    List keyFns = this.clauses.stream().map(JoinClause::leftKeyFn).collect(toList());
    List<Tag> tags = this.tags;
    DistributedBiFunction mapToOutputBiFn = this.mapToOutputBiFn;
    DistributedTriFunction mapToOutputTriFn = this.mapToOutputTriFn;
    Vertex joiner = p.addVertex(this, namePrefix + "-joiner", localParallelism(), () -> new HashJoinP<>(keyFns, tags, mapToOutputBiFn, mapToOutputTriFn)).v;
    p.dag.edge(from(primary.v, primary.nextAvailableOrdinal()).to(joiner, 0));
    String collectorName = namePrefix + "-collector";
    int collectorOrdinal = 1;
    for (Transform fromTransform : tailList(this.upstream())) {
        PlannerVertex fromPv = p.xform2vertex.get(fromTransform);
        JoinClause<?, ?, ?, ?> clause = this.clauses.get(collectorOrdinal - 1);
        DistributedFunction<Object, Object> getKeyFn = (DistributedFunction<Object, Object>) clause.rightKeyFn();
        DistributedFunction<Object, Object> projectFn = (DistributedFunction<Object, Object>) clause.rightProjectFn();
        Vertex collector = p.dag.newVertex(collectorName + collectorOrdinal, () -> new HashJoinCollectP(getKeyFn, projectFn));
        collector.localParallelism(1);
        p.dag.edge(from(fromPv.v, fromPv.nextAvailableOrdinal()).to(collector, 0).distributed().broadcast());
        p.dag.edge(from(collector, 0).to(joiner, collectorOrdinal).broadcast().priority(-1));
        collectorOrdinal++;
    }
}

Example 4 with Tag

use of com.hazelcast.jet.datamodel.Tag in project hazelcast-jet-reference-manual by hazelcast.

the class BuildComputation method s14.

static void s14() {
    // tag::s14[]
    Pipeline p = Pipeline.create();
    p.<Tweet>drawFrom(Sources.mapJournal("tweets", mapPutEvents(), mapEventNewValue(), START_FROM_CURRENT)).flatMap(tweet -> traverseArray(tweet.text().toLowerCase().split("\\W+")).map(word -> new TweetWord(tweet.timestamp(), word))).filter(tweetWord -> !tweetWord.word().isEmpty()).addTimestamps(TweetWord::timestamp, TimeUnit.SECONDS.toMillis(5)).window(sliding(MINUTES.toMillis(1), SECONDS.toMillis(1))).groupingKey(TweetWord::word).aggregate(counting()).drainTo(Sinks.list("result"));
// end::s14[]
}

Example 5 with Tag

use of com.hazelcast.jet.datamodel.Tag in project hazelcast-jet-reference-manual by hazelcast.

the class BuildComputation method s5.

static void s5() {
    // tag::s5[]
    Pipeline p = Pipeline.create();
    p.drawFrom(Sources.<String>list("text")).flatMap(line -> traverseArray(line.toLowerCase().split("\\W+"))).filter(word -> !word.isEmpty()).aggregate(counting()).drainTo(Sinks.list("result"));
// end::s5[]
}