Examples with DataInputView org.apache.flink.core.memory.DataInputView used on opensource projects

Example 26 with DataInputView

use of org.apache.flink.core.memory.DataInputView in project flink by apache.

the class StateInitializationContextImplTest method getKeyedStateStreams.

@Test
public void getKeyedStateStreams() throws Exception {
    int readKeyGroupCount = 0;
    for (KeyGroupStatePartitionStreamProvider stateStreamProvider : initializationContext.getRawKeyedStateInputs()) {
        Assert.assertNotNull(stateStreamProvider);
        try (InputStream is = stateStreamProvider.getStream()) {
            DataInputView div = new DataInputViewStreamWrapper(is);
            int val = div.readInt();
            ++readKeyGroupCount;
            Assert.assertEquals(stateStreamProvider.getKeyGroupId(), val);
        }
    }
    Assert.assertEquals(writtenKeyGroups, readKeyGroupCount);
}

Example 27 with DataInputView

use of org.apache.flink.core.memory.DataInputView in project flink by apache.

the class StateInitializationContextImplTest method getOperatorStateStreams.

@Test
public void getOperatorStateStreams() throws Exception {
    int i = 0;
    int s = 0;
    for (StatePartitionStreamProvider streamProvider : initializationContext.getRawOperatorStateInputs()) {
        if (0 == i % 4) {
            ++i;
        }
        Assert.assertNotNull(streamProvider);
        try (InputStream is = streamProvider.getStream()) {
            DataInputView div = new DataInputViewStreamWrapper(is);
            int val = div.readInt();
            Assert.assertEquals(i * NUM_HANDLES + s, val);
        }
        ++s;
        if (s == i % 4) {
            s = 0;
            ++i;
        }
    }
}

Example 28 with DataInputView

use of org.apache.flink.core.memory.DataInputView in project flink by apache.

the class FailingCollectionSource method run.

@Override
public void run(SourceContext<T> ctx) throws Exception {
    ByteArrayInputStream bais = new ByteArrayInputStream(elementsSerialized);
    final DataInputView input = new DataInputViewStreamWrapper(bais);
    // if we are restored from a checkpoint and need to skip elements, skip them now.
    int toSkip = numElementsToSkip;
    if (toSkip > 0) {
        try {
            while (toSkip > 0) {
                serializer.deserialize(input);
                toSkip--;
            }
        } catch (Exception e) {
            throw new IOException("Failed to deserialize an element from the source. " + "If you are using user-defined serialization (Value and Writable types), check the " + "serialization functions.\nSerializer is " + serializer);
        }
        this.numElementsEmitted = this.numElementsToSkip;
    }
    while (isRunning && numElementsEmitted < numElements) {
        if (!failedBefore) {
            // delay a bit, if we have not failed before
            Thread.sleep(1);
            if (numSuccessfulCheckpoints >= 1 && lastCheckpointedEmittedNum >= 1) {
                // cause a failure if we have not failed before and have a completed checkpoint
                // and have processed at least one element
                failedBefore = true;
                throw new Exception("Artificial Failure");
            }
        }
        if (failedBefore || numElementsEmitted < failureAfterNumElements) {
            // the function failed before, or we are in the elements before the failure
            T next;
            try {
                next = serializer.deserialize(input);
            } catch (Exception e) {
                throw new IOException("Failed to deserialize an element from the source. " + "If you are using user-defined serialization (Value and Writable types), check the " + "serialization functions.\nSerializer is " + serializer);
            }
            synchronized (ctx.getCheckpointLock()) {
                ctx.collect(next);
                numElementsEmitted++;
            }
        } else {
            // if our work is done, delay a bit to prevent busy waiting
            Thread.sleep(1);
        }
    }
}