Examples with BroadcastKey com.hazelcast.jet.core.BroadcastKey used on opensource projects

Example 6 with BroadcastKey

use of com.hazelcast.jet.core.BroadcastKey in project hazelcast by hazelcast.

the class StreamKafkaP method restoreFromSnapshot.

@Override
public void restoreFromSnapshot(@Nonnull Object key0, @Nonnull Object value) {
    @SuppressWarnings("unchecked") Object key = ((BroadcastKey<Object>) key0).key();
    if (PARTITION_COUNTS_SNAPSHOT_KEY.equals(key)) {
        @SuppressWarnings("unchecked") Map<String, Integer> partitionCounts = (Map<String, Integer>) value;
        for (Entry<String, Integer> partitionCountEntry : partitionCounts.entrySet()) {
            String topicName = partitionCountEntry.getKey();
            int partitionCount = partitionCountEntry.getValue();
            int topicIndex = topics.indexOf(topicName);
            assert topicIndex >= 0;
            handleNewPartitions(topicIndex, partitionCount, true);
        }
    } else {
        TopicPartition topicPartition = (TopicPartition) key;
        long[] value1 = (long[]) value;
        long offset = value1[0];
        long watermark = value1[1];
        if (!offsets.containsKey(topicPartition.topic())) {
            getLogger().warning("Offset for topic '" + topicPartition.topic() + "' is restored from the snapshot, but the topic is not supposed to be read, ignoring");
            return;
        }
        int topicIndex = topics.indexOf(topicPartition.topic());
        assert topicIndex >= 0;
        handleNewPartitions(topicIndex, topicPartition.partition() + 1, true);
        if (!handledByThisProcessor(topicIndex, topicPartition.partition())) {
            return;
        }
        long[] topicOffsets = offsets.get(topicPartition.topic());
        assert topicOffsets[topicPartition.partition()] < 0 : "duplicate offset for topicPartition '" + topicPartition + "' restored, offset1=" + topicOffsets[topicPartition.partition()] + ", offset2=" + offset;
        topicOffsets[topicPartition.partition()] = offset;
        consumer.seek(topicPartition, offset + 1);
        Integer partitionIndex = currentAssignment.get(topicPartition);
        assert partitionIndex != null;
        eventTimeMapper.restoreWatermark(partitionIndex, watermark);
    }
}

Example 7 with BroadcastKey

use of com.hazelcast.jet.core.BroadcastKey in project hazelcast by hazelcast.

the class StreamKafkaP method saveToSnapshot.

@Override
public boolean saveToSnapshot() {
    if (!emitFromTraverser(traverser)) {
        return false;
    }
    if (snapshotTraverser == null) {
        Stream<Entry<BroadcastKey<?>, ?>> snapshotStream = offsets.entrySet().stream().flatMap(entry -> IntStream.range(0, entry.getValue().length).filter(partition -> entry.getValue()[partition] >= 0).mapToObj(partition -> {
            TopicPartition key = new TopicPartition(entry.getKey(), partition);
            long offset = entry.getValue()[partition];
            long watermark = eventTimeMapper.getWatermark(currentAssignment.get(key));
            return entry(broadcastKey(key), new long[] { offset, watermark });
        }));
        snapshotTraverser = traverseStream(snapshotStream).onFirstNull(() -> {
            snapshotTraverser = null;
            if (getLogger().isFinestEnabled()) {
                getLogger().finest("Finished saving snapshot." + " Saved offsets: " + offsets() + ", Saved watermarks: " + watermarks());
            }
        });
        if (processorIndex == 0) {
            Entry<BroadcastKey<?>, ?> partitionCountsItem = entry(broadcastKey(PARTITION_COUNTS_SNAPSHOT_KEY), topics.stream().collect(Collectors.toMap(topic -> topic, topic -> offsets.get(topic).length)));
            snapshotTraverser = snapshotTraverser.append(partitionCountsItem);
        }
    }
    return emitFromTraverserToSnapshot(snapshotTraverser);
}

Example 8 with BroadcastKey

use of com.hazelcast.jet.core.BroadcastKey in project hazelcast-jet by hazelcast.

the class SessionWindowP method restoreFromSnapshot.

@Override
@SuppressWarnings("unchecked")
protected void restoreFromSnapshot(@Nonnull Object key, @Nonnull Object value) {
    if (key instanceof BroadcastKey) {
        BroadcastKey bcastKey = (BroadcastKey) key;
        if (!Keys.CURRENT_WATERMARK.equals(bcastKey.key())) {
            throw new JetException("Unexpected broadcast key: " + bcastKey.key());
        }
        long newCurrentWatermark = (long) value;
        assert processingGuarantee != EXACTLY_ONCE || minRestoredCurrentWatermark == Long.MAX_VALUE || minRestoredCurrentWatermark == newCurrentWatermark : "different values for currentWatermark restored, before=" + minRestoredCurrentWatermark + ", new=" + newCurrentWatermark;
        minRestoredCurrentWatermark = Math.min(newCurrentWatermark, minRestoredCurrentWatermark);
        return;
    }
    keyToWindows.put((K) key, (Windows) value);
}

Example 9 with BroadcastKey

use of com.hazelcast.jet.core.BroadcastKey in project hazelcast-jet by hazelcast.

the class StreamKafkaP method restoreFromSnapshot.

@Override
public void restoreFromSnapshot(@Nonnull Object key, @Nonnull Object value) {
    TopicPartition topicPartition = ((BroadcastKey<TopicPartition>) key).key();
    long[] value1 = (long[]) value;
    long offset = value1[0];
    long watermark = value1[1];
    long[] topicOffsets = offsets.get(topicPartition.topic());
    if (topicOffsets == null) {
        getLogger().severe("Offset for topic '" + topicPartition.topic() + "' is present in snapshot, but the topic is not supposed to be read");
        return;
    }
    if (topicPartition.partition() >= topicOffsets.length) {
        getLogger().severe("Offset for partition '" + topicPartition + "' is present in snapshot," + " but that topic currently has only " + topicOffsets.length + " partitions");
    }
    Integer partitionIndex = currentAssignment.get(topicPartition);
    if (partitionIndex != null) {
        assert topicOffsets[topicPartition.partition()] < 0 : "duplicate offset for topicPartition '" + topicPartition + "' restored, offset1=" + topicOffsets[topicPartition.partition()] + ", offset2=" + offset;
        topicOffsets[topicPartition.partition()] = offset;
        consumer.seek(topicPartition, offset + 1);
        watermarkSourceUtil.restoreWatermark(partitionIndex, watermark);
    }
}

Example 10 with BroadcastKey

use of com.hazelcast.jet.core.BroadcastKey in project hazelcast-jet by hazelcast.

the class StreamKafkaP method saveToSnapshot.

@Override
public boolean saveToSnapshot() {
    if (!emitFromTraverser(traverser)) {
        return false;
    }
    if (snapshotTraverser == null) {
        Stream<Entry<BroadcastKey<TopicPartition>, long[]>> snapshotStream = offsets.entrySet().stream().flatMap(entry -> IntStream.range(0, entry.getValue().length).filter(partition -> entry.getValue()[partition] >= 0).mapToObj(partition -> {
            TopicPartition key = new TopicPartition(entry.getKey(), partition);
            long offset = entry.getValue()[partition];
            long watermark = watermarkSourceUtil.getWatermark(currentAssignment.get(key));
            return entry(broadcastKey(key), new long[] { offset, watermark });
        }));
        snapshotTraverser = traverseStream(snapshotStream).onFirstNull(() -> snapshotTraverser = null);
    }
    return emitFromTraverserToSnapshot(snapshotTraverser);
}