use of org.apache.flink.core.memory.DataInputView in project flink by apache.
the class StateInitializationContextImplTest method getKeyedStateStreams.
@Test
public void getKeyedStateStreams() throws Exception {
int readKeyGroupCount = 0;
for (KeyGroupStatePartitionStreamProvider stateStreamProvider : initializationContext.getRawKeyedStateInputs()) {
Assert.assertNotNull(stateStreamProvider);
try (InputStream is = stateStreamProvider.getStream()) {
DataInputView div = new DataInputViewStreamWrapper(is);
int val = div.readInt();
++readKeyGroupCount;
Assert.assertEquals(stateStreamProvider.getKeyGroupId(), val);
}
}
Assert.assertEquals(writtenKeyGroups, readKeyGroupCount);
}
use of org.apache.flink.core.memory.DataInputView in project flink by apache.
the class StateInitializationContextImplTest method getOperatorStateStreams.
@Test
public void getOperatorStateStreams() throws Exception {
int i = 0;
int s = 0;
for (StatePartitionStreamProvider streamProvider : initializationContext.getRawOperatorStateInputs()) {
if (0 == i % 4) {
++i;
}
Assert.assertNotNull(streamProvider);
try (InputStream is = streamProvider.getStream()) {
DataInputView div = new DataInputViewStreamWrapper(is);
int val = div.readInt();
Assert.assertEquals(i * NUM_HANDLES + s, val);
}
++s;
if (s == i % 4) {
s = 0;
++i;
}
}
}
use of org.apache.flink.core.memory.DataInputView in project flink by apache.
the class FailingCollectionSource method run.
@Override
public void run(SourceContext<T> ctx) throws Exception {
ByteArrayInputStream bais = new ByteArrayInputStream(elementsSerialized);
final DataInputView input = new DataInputViewStreamWrapper(bais);
// if we are restored from a checkpoint and need to skip elements, skip them now.
int toSkip = numElementsToSkip;
if (toSkip > 0) {
try {
while (toSkip > 0) {
serializer.deserialize(input);
toSkip--;
}
} catch (Exception e) {
throw new IOException("Failed to deserialize an element from the source. " + "If you are using user-defined serialization (Value and Writable types), check the " + "serialization functions.\nSerializer is " + serializer);
}
this.numElementsEmitted = this.numElementsToSkip;
}
while (isRunning && numElementsEmitted < numElements) {
if (!failedBefore) {
// delay a bit, if we have not failed before
Thread.sleep(1);
if (numSuccessfulCheckpoints >= 1 && lastCheckpointedEmittedNum >= 1) {
// cause a failure if we have not failed before and have a completed checkpoint
// and have processed at least one element
failedBefore = true;
throw new Exception("Artificial Failure");
}
}
if (failedBefore || numElementsEmitted < failureAfterNumElements) {
// the function failed before, or we are in the elements before the failure
T next;
try {
next = serializer.deserialize(input);
} catch (Exception e) {
throw new IOException("Failed to deserialize an element from the source. " + "If you are using user-defined serialization (Value and Writable types), check the " + "serialization functions.\nSerializer is " + serializer);
}
synchronized (ctx.getCheckpointLock()) {
ctx.collect(next);
numElementsEmitted++;
}
} else {
// if our work is done, delay a bit to prevent busy waiting
Thread.sleep(1);
}
}
}
Aggregations