Examples with StreamElement org.apache.flink.streaming.runtime.streamrecord.StreamElement used on opensource projects

Example 21 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class DemultiplexingRecordDeserializerTest method write.

private Buffer write(BufferBuilder bufferBuilder, StreamElement... elements) throws IOException {
    try (BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer()) {
        DataOutputSerializer output = new DataOutputSerializer(128);
        final SerializationDelegate<StreamElement> delegate = new SerializationDelegate<>(new StreamElementSerializer<>(LongSerializer.INSTANCE));
        for (StreamElement element : elements) {
            delegate.setInstance(element);
            bufferBuilder.appendAndCommit(RecordWriter.serializeRecord(output, delegate));
        }
        return bufferConsumer.build();
    }
}

Example 22 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class CommitterOperatorTest method testStateRestore.

@Test
void testStateRestore() throws Exception {
    final OneInputStreamOperatorTestHarness<CommittableMessage<String>, CommittableMessage<String>> testHarness = createTestHarness(new TestSink.RetryOnceCommitter());
    testHarness.open();
    final CommittableSummary<String> committableSummary = new CommittableSummary<>(1, 1, 0L, 1, 1, 0);
    testHarness.processElement(new StreamRecord<>(committableSummary));
    final CommittableWithLineage<String> first = new CommittableWithLineage<>("1", 0L, 1);
    testHarness.processElement(new StreamRecord<>(first));
    final OperatorSubtaskState snapshot = testHarness.snapshot(0L, 2L);
    // Trigger first checkpoint but committer needs retry
    testHarness.notifyOfCompletedCheckpoint(0);
    assertThat(testHarness.getOutput()).isEmpty();
    testHarness.close();
    final ForwardingCommitter committer = new ForwardingCommitter();
    final OneInputStreamOperatorTestHarness<CommittableMessage<String>, CommittableMessage<String>> restored = createTestHarness(committer);
    restored.initializeState(snapshot);
    restored.open();
    // Previous committables are immediately committed if possible
    final List<StreamElement> output = fromOutput(restored.getOutput());
    assertThat(output).hasSize(2);
    assertThat(committer.getSuccessfulCommits()).isEqualTo(1);
    SinkV2Assertions.assertThat(toCommittableSummary(output.get(0))).hasFailedCommittables(committableSummary.getNumberOfFailedCommittables()).hasOverallCommittables(committableSummary.getNumberOfCommittables()).hasPendingCommittables(0);
    SinkV2Assertions.assertThat(toCommittableWithLinage(output.get(1))).isEqualTo(new CommittableWithLineage<>(first.getCommittable(), 1L, 0));
    restored.close();
}

Example 23 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class AsyncWaitOperator method snapshotState.

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
    super.snapshotState(context);
    ListState<StreamElement> partitionableState = getOperatorStateBackend().getListState(new ListStateDescriptor<>(STATE_NAME, inStreamElementSerializer));
    partitionableState.clear();
    try {
        partitionableState.addAll(queue.values());
    } catch (Exception e) {
        partitionableState.clear();
        throw new Exception("Could not add stream element queue entries to operator state " + "backend of operator " + getOperatorName() + '.', e);
    }
}

Example 24 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class OrderedStreamElementQueueTest method testCompletionOrder.

/**
 * Tests that only the head element is pulled from the ordered queue if it has been completed.
 */
@Test
public void testCompletionOrder() {
    final OrderedStreamElementQueue<Integer> queue = new OrderedStreamElementQueue<>(4);
    ResultFuture<Integer> entry1 = putSuccessfully(queue, new StreamRecord<>(1, 0L));
    ResultFuture<Integer> entry2 = putSuccessfully(queue, new StreamRecord<>(2, 1L));
    putSuccessfully(queue, new Watermark(2L));
    ResultFuture<Integer> entry4 = putSuccessfully(queue, new StreamRecord<>(3, 3L));
    Assert.assertEquals(Collections.emptyList(), popCompleted(queue));
    Assert.assertEquals(4, queue.size());
    Assert.assertFalse(queue.isEmpty());
    entry2.complete(Collections.singleton(11));
    entry4.complete(Collections.singleton(13));
    Assert.assertEquals(Collections.emptyList(), popCompleted(queue));
    Assert.assertEquals(4, queue.size());
    Assert.assertFalse(queue.isEmpty());
    entry1.complete(Collections.singleton(10));
    List<StreamElement> expected = Arrays.asList(new StreamRecord<>(10, 0L), new StreamRecord<>(11, 1L), new Watermark(2L), new StreamRecord<>(13, 3L));
    Assert.assertEquals(expected, popCompleted(queue));
    Assert.assertEquals(0, queue.size());
    Assert.assertTrue(queue.isEmpty());
}

Example 25 with StreamElement

use of org.apache.flink.streaming.runtime.streamrecord.StreamElement in project flink by apache.

the class BatchMultipleInputStreamOperatorTest method createMultipleInputStreamOperator.

/**
 * Create a BatchMultipleInputStreamOperator which contains the following sub-graph.
 *
 * <pre>
 *
 * source1  source2
 *   |        |
 *  agg1     agg2
 * (b) \     / (p)
 *      join1   source3
 *     (p) \     / (b)
 *         join2
 * (b) is the build side of the join, (p) is the probe side of the join.
 * </pre>
 */
private TestingBatchMultipleInputStreamOperator createMultipleInputStreamOperator() throws Exception {
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    Transformation<RowData> source1 = createSource(env, "source1");
    Transformation<RowData> source2 = createSource(env, "source2");
    Transformation<RowData> source3 = createSource(env, "source3");
    OneInputTransformation<RowData, RowData> agg1 = createOneInputTransform(source1, "agg1", new TestingOneInputStreamOperator(true), InternalTypeInfo.of(RowType.of(DataTypes.STRING().getLogicalType())));
    OneInputTransformation<RowData, RowData> agg2 = createOneInputTransform(source2, "agg2", new TestingOneInputStreamOperator(true), InternalTypeInfo.of(RowType.of(DataTypes.STRING().getLogicalType())));
    TwoInputTransformation<RowData, RowData, RowData> join1 = createTwoInputTransform(agg1, agg2, "join1", new TestingTwoInputStreamOperator(true), InternalTypeInfo.of(RowType.of(DataTypes.STRING().getLogicalType())));
    TwoInputTransformation<RowData, RowData, RowData> join2 = createTwoInputTransform(join1, source3, "join2", new TestingTwoInputStreamOperator(true), InternalTypeInfo.of(RowType.of(DataTypes.STRING().getLogicalType())));
    TableOperatorWrapperGenerator generator = new TableOperatorWrapperGenerator(Arrays.asList(source1, source2, source3), join2, new int[] { 1, 2, 0 });
    generator.generate();
    List<Pair<Transformation<?>, InputSpec>> inputTransformAndInputSpecPairs = generator.getInputTransformAndInputSpecPairs();
    List<StreamElement> outputData = new ArrayList<>();
    return new TestingBatchMultipleInputStreamOperator(createStreamOperatorParameters(new TestingOutput(outputData)), inputTransformAndInputSpecPairs.stream().map(Pair::getValue).collect(Collectors.toList()), generator.getHeadWrappers(), generator.getTailWrapper(), outputData);
}