Examples with BroadcastKey com.hazelcast.jet.core.BroadcastKey used on opensource projects

Example 1 with BroadcastKey

use of com.hazelcast.jet.core.BroadcastKey in project hazelcast-jet by hazelcast.

the class SlidingWindowP method restoreFromSnapshot.

@Override
@SuppressWarnings("unchecked")
protected void restoreFromSnapshot(@Nonnull Object key, @Nonnull Object value) {
    if (key instanceof BroadcastKey) {
        BroadcastKey bcastKey = (BroadcastKey) key;
        if (!Keys.NEXT_WIN_TO_EMIT.equals(bcastKey.key())) {
            throw new JetException("Unexpected broadcast key: " + bcastKey.key());
        }
        long newNextWinToEmit = (long) value;
        assert processingGuarantee != EXACTLY_ONCE || minRestoredNextWinToEmit == Long.MAX_VALUE || minRestoredNextWinToEmit == newNextWinToEmit : "different values for nextWinToEmit restored, before=" + minRestoredNextWinToEmit + ", new=" + newNextWinToEmit;
        minRestoredNextWinToEmit = Math.min(newNextWinToEmit, minRestoredNextWinToEmit);
        return;
    }
    SnapshotKey k = (SnapshotKey) key;
    if (tsToKeyToAcc.computeIfAbsent(k.timestamp, x -> new HashMap<>()).put((K) k.key, (A) value) != null) {
        throw new JetException("Duplicate key in snapshot: " + k);
    }
    topTs = max(topTs, k.timestamp);
}

Example 2 with BroadcastKey

use of com.hazelcast.jet.core.BroadcastKey in project hazelcast-jet by hazelcast.

the class MasterContext method rewriteDagWithSnapshotRestore.

private void rewriteDagWithSnapshotRestore(DAG dag, long snapshotId) {
    logger.info(jobIdString() + ": restoring state from snapshotId=" + snapshotId);
    for (Vertex vertex : dag) {
        // items with keys of type BroadcastKey need to be broadcast to all processors
        DistributedFunction<Entry<Object, Object>, ?> projection = (Entry<Object, Object> e) -> (e.getKey() instanceof BroadcastKey) ? new BroadcastEntry<>(e) : e;
        // We add the vertex even in case when the map is empty: this ensures, that
        // Processor.finishSnapshotRestore() method is always called on all vertices in
        // a job which is restored from a snapshot.
        String mapName = snapshotDataMapName(jobId, snapshotId, vertex.getName());
        Vertex readSnapshotVertex = dag.newVertex("__snapshot_read." + vertex.getName(), readMapP(mapName));
        // We need a separate mapping vertex and can't use readMapP's projectionFn:
        // the projection will cause key/value deserialization on partition thread, which doesn't have job's
        // class loader. If the key/value uses a custom object, it will fail. For example, StreamKafkaP uses
        // TopicPartition as the key or a custom processor can use custom key.
        Vertex mapSnapshotVertex = dag.newVertex("__snapshot_map_." + vertex.getName(), mapP(projection));
        readSnapshotVertex.localParallelism(vertex.getLocalParallelism());
        mapSnapshotVertex.localParallelism(vertex.getLocalParallelism());
        int destOrdinal = dag.getInboundEdges(vertex.getName()).size();
        dag.edge(between(readSnapshotVertex, mapSnapshotVertex).isolated()).edge(new SnapshotRestoreEdge(mapSnapshotVertex, vertex, destOrdinal));
    }
}

Example 3 with BroadcastKey

use of com.hazelcast.jet.core.BroadcastKey in project hazelcast by hazelcast.

the class SlidingWindowP method restoreFromSnapshot.

@Override
@SuppressWarnings("unchecked")
protected void restoreFromSnapshot(@Nonnull Object key, @Nonnull Object value) {
    if (key instanceof BroadcastKey) {
        BroadcastKey bcastKey = (BroadcastKey) key;
        if (!Keys.NEXT_WIN_TO_EMIT.equals(bcastKey.key())) {
            throw new JetException("Unexpected broadcast key: " + bcastKey.key());
        }
        long newNextWinToEmit = (long) value;
        assert processingGuarantee != EXACTLY_ONCE || minRestoredNextWinToEmit == Long.MAX_VALUE || minRestoredNextWinToEmit == newNextWinToEmit : "different values for nextWinToEmit restored, before=" + minRestoredNextWinToEmit + ", new=" + newNextWinToEmit;
        minRestoredNextWinToEmit = Math.min(newNextWinToEmit, minRestoredNextWinToEmit);
        return;
    }
    SnapshotKey k = (SnapshotKey) key;
    // align frame timestamp to our frame - they can be misaligned
    // if the slide step was changed in the updated DAG
    long higherFrameTs = winPolicy.higherFrameTs(k.timestamp - 1);
    if (higherFrameTs != k.timestamp) {
        if (!badFrameRestored) {
            badFrameRestored = true;
            getLogger().warning("Frames in the state do not match the current frame size: they were likely " + "saved for a different window slide step or a different offset. The window results will " + "probably be incorrect until all restored frames are emitted.");
        }
    }
    minRestoredFrameTs = Math.min(higherFrameTs, minRestoredFrameTs);
    tsToKeyToAcc.computeIfAbsent(higherFrameTs, createMapPerTsFunction).merge((K) k.key, (A) value, (o, n) -> {
        if (!badFrameRestored) {
            throw new JetException("Duplicate key in snapshot: " + k);
        }
        if (combineFn == null) {
            throw new JetException("AggregateOperation.combineFn required for merging restored frames");
        }
        combineFn.accept(o, n);
        totalKeysInFrames.inc(-1);
        return o;
    });
    totalKeysInFrames.inc();
    topTs = max(topTs, higherFrameTs);
}

Example 4 with BroadcastKey

use of com.hazelcast.jet.core.BroadcastKey in project hazelcast by hazelcast.

the class ExplodeSnapshotP method traverser.

/* We can't close the BufferObjectDataInput cleanly. We close it when the returned traverser is fully iterated,
    but the caller might not fully iterate it and we have no opportunity to close it.
    On the other hand, the returned object doesn't hold any resources, so relying on the GC is sufficient.
    See #19799 */
@SuppressWarnings("squid:S2095")
private Traverser<Object> traverser(byte[] data) {
    BufferObjectDataInput in = serializationService.createObjectDataInput(data);
    return () -> uncheckCall(() -> {
        Object key = in.readObject();
        if (key == SnapshotDataValueTerminator.INSTANCE) {
            return null;
        }
        Object value = in.readObject();
        return key instanceof BroadcastKey ? new BroadcastEntry(key, value) : entry(key, value);
    });
}

Example 5 with BroadcastKey

use of com.hazelcast.jet.core.BroadcastKey in project hazelcast by hazelcast.

the class SessionWindowP method restoreFromSnapshot.

@Override
@SuppressWarnings("unchecked")
protected void restoreFromSnapshot(@Nonnull Object key, @Nonnull Object value) {
    if (key instanceof BroadcastKey) {
        BroadcastKey bcastKey = (BroadcastKey) key;
        if (!Keys.CURRENT_WATERMARK.equals(bcastKey.key())) {
            throw new JetException("Unexpected broadcast key: " + bcastKey.key());
        }
        long newCurrentWatermark = (long) value;
        assert processingGuarantee != EXACTLY_ONCE || minRestoredCurrentWatermark == Long.MAX_VALUE || minRestoredCurrentWatermark == newCurrentWatermark : "different values for currentWatermark restored, before=" + minRestoredCurrentWatermark + ", new=" + newCurrentWatermark;
        minRestoredCurrentWatermark = Math.min(newCurrentWatermark, minRestoredCurrentWatermark);
        return;
    }
    if (keyToWindows.put((K) key, (Windows) value) != null) {
        throw new JetException("Duplicate key in snapshot: " + key);
    }
}