Examples with DataOutputView org.apache.flink.core.memory.DataOutputView used on opensource projects

Example 1 with DataOutputView

use of org.apache.flink.core.memory.DataOutputView in project flink by apache.

the class OperatorStateOutputCheckpointStreamTest method writeAllTestKeyGroups.

private OperatorStateHandle writeAllTestKeyGroups(OperatorStateCheckpointOutputStream stream, int numPartitions) throws Exception {
    DataOutputView dov = new DataOutputViewStreamWrapper(stream);
    for (int i = 0; i < numPartitions; ++i) {
        Assert.assertEquals(i, stream.getNumberOfPartitions());
        stream.startNewPartition();
        dov.writeInt(i);
    }
    return stream.closeAndGetHandle();
}

Example 2 with DataOutputView

use of org.apache.flink.core.memory.DataOutputView in project flink by apache.

the class KeyedStateCheckpointOutputStreamTest method writeAllTestKeyGroups.

private KeyGroupsStateHandle writeAllTestKeyGroups(KeyedStateCheckpointOutputStream stream, KeyGroupRange keyRange) throws Exception {
    DataOutputView dov = new DataOutputViewStreamWrapper(stream);
    for (int kg : keyRange) {
        stream.startNewKeyGroup(kg);
        dov.writeInt(kg);
    }
    return stream.closeAndGetHandle();
}

Example 3 with DataOutputView

use of org.apache.flink.core.memory.DataOutputView in project flink by apache.

the class StateInitializationContextImplTest method setUp.

@Before
public void setUp() throws Exception {
    this.writtenKeyGroups = 0;
    this.writtenOperatorStates = new HashSet<>();
    this.closableRegistry = new CloseableRegistry();
    OperatorStateStore stateStore = mock(OperatorStateStore.class);
    ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos(64);
    List<KeyGroupsStateHandle> keyGroupsStateHandles = new ArrayList<>(NUM_HANDLES);
    int prev = 0;
    for (int i = 0; i < NUM_HANDLES; ++i) {
        out.reset();
        int size = i % 4;
        int end = prev + size;
        DataOutputView dov = new DataOutputViewStreamWrapper(out);
        KeyGroupRangeOffsets offsets = new KeyGroupRangeOffsets(i == 9 ? KeyGroupRange.EMPTY_KEY_GROUP_RANGE : new KeyGroupRange(prev, end));
        prev = end + 1;
        for (int kg : offsets.getKeyGroupRange()) {
            offsets.setKeyGroupOffset(kg, out.getPosition());
            dov.writeInt(kg);
            ++writtenKeyGroups;
        }
        KeyGroupsStateHandle handle = new KeyGroupsStateHandle(offsets, new ByteStateHandleCloseChecking("kg-" + i, out.toByteArray()));
        keyGroupsStateHandles.add(handle);
    }
    List<OperatorStateHandle> operatorStateHandles = new ArrayList<>(NUM_HANDLES);
    for (int i = 0; i < NUM_HANDLES; ++i) {
        int size = i % 4;
        out.reset();
        DataOutputView dov = new DataOutputViewStreamWrapper(out);
        LongArrayList offsets = new LongArrayList(size);
        for (int s = 0; s < size; ++s) {
            offsets.add(out.getPosition());
            int val = i * NUM_HANDLES + s;
            dov.writeInt(val);
            writtenOperatorStates.add(val);
        }
        Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>();
        offsetsMap.put(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, new OperatorStateHandle.StateMetaInfo(offsets.toArray(), OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
        OperatorStateHandle operatorStateHandle = new OperatorStateHandle(offsetsMap, new ByteStateHandleCloseChecking("os-" + i, out.toByteArray()));
        operatorStateHandles.add(operatorStateHandle);
    }
    this.initializationContext = new StateInitializationContextImpl(true, stateStore, mock(KeyedStateStore.class), keyGroupsStateHandles, operatorStateHandles, closableRegistry);
}

Example 4 with DataOutputView

use of org.apache.flink.core.memory.DataOutputView in project flink by apache.

the class OutputEmitterTest method testWrongKeyClass.

@Test
public void testWrongKeyClass() {
    // Test for IntValue
    @SuppressWarnings({ "unchecked", "rawtypes" }) final TypeComparator<Record> doubleComp = new RecordComparatorFactory(new int[] { 0 }, new Class[] { DoubleValue.class }).createComparator();
    final ChannelSelector<SerializationDelegate<Record>> oe1 = new OutputEmitter<Record>(ShipStrategyType.PARTITION_HASH, doubleComp);
    final SerializationDelegate<Record> delegate = new SerializationDelegate<Record>(new RecordSerializerFactory().getSerializer());
    ;
    Record rec = null;
    try {
        PipedInputStream pipedInput = new PipedInputStream(1024 * 1024);
        DataInputView in = new DataInputViewStreamWrapper(pipedInput);
        DataOutputView out = new DataOutputViewStreamWrapper(new PipedOutputStream(pipedInput));
        rec = new Record(1);
        rec.setField(0, new IntValue());
        rec.write(out);
        rec = new Record();
        rec.read(in);
    } catch (IOException e) {
        fail("Test erroneous");
    }
    try {
        delegate.setInstance(rec);
        oe1.selectChannels(delegate, 100);
    } catch (DeserializationException re) {
        return;
    }
    Assert.fail("Expected a NullKeyFieldException.");
}

Example 5 with DataOutputView

use of org.apache.flink.core.memory.DataOutputView in project flink by apache.

the class AbstractIterativeTask method createWorksetUpdateOutputCollector.

// -----------------------------------------------------------------------------------------------------------------
// Iteration State Update Handling
// -----------------------------------------------------------------------------------------------------------------
/**
	 * Creates a new {@link WorksetUpdateOutputCollector}.
	 * <p>
	 * This collector is used by {@link IterationIntermediateTask} or {@link IterationTailTask} to update the
	 * workset.
	 * <p>
	 * If a non-null delegate is given, the new {@link Collector} will write to the solution set and also call
	 * collect(T) of the delegate.
	 *
	 * @param delegate null -OR- the delegate on which to call collect() by the newly created collector
	 * @return a new {@link WorksetUpdateOutputCollector}
	 */
protected Collector<OT> createWorksetUpdateOutputCollector(Collector<OT> delegate) {
    DataOutputView outputView = worksetBackChannel.getWriteEnd();
    TypeSerializer<OT> serializer = getOutputSerializer();
    return new WorksetUpdateOutputCollector<OT>(outputView, serializer, delegate);
}