Examples with DataOutputViewStreamWrapper org.apache.flink.core.memory.DataOutputViewStreamWrapper used on opensource projects

Example 1 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class SavepointV1SerializerTest method testSerializeDeserializeV1.

/**
	 * Test serialization of {@link SavepointV1} instance.
	 */
@Test
public void testSerializeDeserializeV1() throws Exception {
    Random r = new Random(42);
    for (int i = 0; i < 100; ++i) {
        SavepointV1 expected = new SavepointV1(i + 123123, SavepointV1Test.createTaskStates(1 + r.nextInt(64), 1 + r.nextInt(64)));
        SavepointV1Serializer serializer = SavepointV1Serializer.INSTANCE;
        // Serialize
        ByteArrayOutputStreamWithPos baos = new ByteArrayOutputStreamWithPos();
        serializer.serialize(expected, new DataOutputViewStreamWrapper(baos));
        byte[] bytes = baos.toByteArray();
        // Deserialize
        ByteArrayInputStream bais = new ByteArrayInputStream(bytes);
        Savepoint actual = serializer.deserialize(new DataInputViewStreamWrapper(bais), Thread.currentThread().getContextClassLoader());
        assertEquals(expected, actual);
    }
}

Example 2 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class HeapListState method getSerializedValue.

@Override
public byte[] getSerializedValue(K key, N namespace) throws Exception {
    Preconditions.checkState(namespace != null, "No namespace given.");
    Preconditions.checkState(key != null, "No key given.");
    ArrayList<V> result = stateTable.get(key, namespace);
    if (result == null) {
        return null;
    }
    TypeSerializer<V> serializer = stateDesc.getElementSerializer();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputViewStreamWrapper view = new DataOutputViewStreamWrapper(baos);
    // write the same as RocksDB writes lists, with one ',' separator
    for (int i = 0; i < result.size(); i++) {
        serializer.serialize(result.get(i), view);
        if (i < result.size() - 1) {
            view.writeByte(',');
        }
    }
    view.flush();
    return baos.toByteArray();
}

Example 3 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project flink by apache.

the class SharedBuffer method writeObject.

private void writeObject(ObjectOutputStream oos) throws IOException {
    DataOutputViewStreamWrapper target = new DataOutputViewStreamWrapper(oos);
    Map<SharedBufferEntry<K, V>, Integer> entryIDs = new HashMap<>();
    int totalEdges = 0;
    int entryCounter = 0;
    oos.defaultWriteObject();
    // number of pages
    oos.writeInt(pages.size());
    for (Map.Entry<K, SharedBufferPage<K, V>> pageEntry : pages.entrySet()) {
        SharedBufferPage<K, V> page = pageEntry.getValue();
        // key for the current page
        oos.writeObject(page.getKey());
        // number of page entries
        oos.writeInt(page.entries.size());
        for (Map.Entry<ValueTimeWrapper<V>, SharedBufferEntry<K, V>> sharedBufferEntry : page.entries.entrySet()) {
            // serialize the sharedBufferEntry
            SharedBufferEntry<K, V> sharedBuffer = sharedBufferEntry.getValue();
            // assign id to the sharedBufferEntry for the future serialization of the previous
            // relation
            entryIDs.put(sharedBuffer, entryCounter++);
            ValueTimeWrapper<V> valueTimeWrapper = sharedBuffer.getValueTime();
            valueSerializer.serialize(valueTimeWrapper.value, target);
            oos.writeLong(valueTimeWrapper.getTimestamp());
            int edges = sharedBuffer.edges.size();
            totalEdges += edges;
            oos.writeInt(sharedBuffer.referenceCounter);
        }
    }
    // write the edges between the shared buffer entries
    oos.writeInt(totalEdges);
    for (Map.Entry<K, SharedBufferPage<K, V>> pageEntry : pages.entrySet()) {
        SharedBufferPage<K, V> page = pageEntry.getValue();
        for (Map.Entry<ValueTimeWrapper<V>, SharedBufferEntry<K, V>> sharedBufferEntry : page.entries.entrySet()) {
            SharedBufferEntry<K, V> sharedBuffer = sharedBufferEntry.getValue();
            if (!entryIDs.containsKey(sharedBuffer)) {
                throw new RuntimeException("Could not find id for entry: " + sharedBuffer);
            } else {
                int id = entryIDs.get(sharedBuffer);
                for (SharedBufferEdge<K, V> edge : sharedBuffer.edges) {
                    // of the source and target SharedBufferEntry
                    if (edge.target != null) {
                        if (!entryIDs.containsKey(edge.getTarget())) {
                            throw new RuntimeException("Could not find id for entry: " + edge.getTarget());
                        } else {
                            int targetId = entryIDs.get(edge.getTarget());
                            oos.writeInt(id);
                            oos.writeInt(targetId);
                            oos.writeObject(edge.version);
                        }
                    } else {
                        oos.writeInt(id);
                        oos.writeInt(-1);
                        oos.writeObject(edge.version);
                    }
                }
            }
        }
    }
}

Example 4 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project beam by apache.

the class DedupingOperator method snapshotState.

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
    // copy from AbstractStreamOperator
    if (getKeyedStateBackend() != null) {
        KeyedStateCheckpointOutputStream out;
        try {
            out = context.getRawKeyedOperatorStateOutput();
        } catch (Exception exception) {
            throw new Exception("Could not open raw keyed operator state stream for " + getOperatorName() + '.', exception);
        }
        try {
            KeyGroupsList allKeyGroups = out.getKeyGroupList();
            for (int keyGroupIdx : allKeyGroups) {
                out.startNewKeyGroup(keyGroupIdx);
                DataOutputViewStreamWrapper dov = new DataOutputViewStreamWrapper(out);
                // if (this instanceof KeyGroupCheckpointedOperator)
                snapshotKeyGroupState(keyGroupIdx, dov);
            }
        } catch (Exception exception) {
            throw new Exception("Could not write timer service of " + getOperatorName() + " to checkpoint state stream.", exception);
        } finally {
            try {
                out.close();
            } catch (Exception closeException) {
                LOG.warn("Could not close raw keyed operator state stream for {}. This " + "might have prevented deleting some state data.", getOperatorName(), closeException);
            }
        }
    }
}

Example 5 with DataOutputViewStreamWrapper

use of org.apache.flink.core.memory.DataOutputViewStreamWrapper in project beam by apache.

the class DoFnOperator method snapshotState.

@Override
public void snapshotState(StateSnapshotContext context) throws Exception {
    // copy from AbstractStreamOperator
    if (getKeyedStateBackend() != null) {
        KeyedStateCheckpointOutputStream out;
        try {
            out = context.getRawKeyedOperatorStateOutput();
        } catch (Exception exception) {
            throw new Exception("Could not open raw keyed operator state stream for " + getOperatorName() + '.', exception);
        }
        try {
            KeyGroupsList allKeyGroups = out.getKeyGroupList();
            for (int keyGroupIdx : allKeyGroups) {
                out.startNewKeyGroup(keyGroupIdx);
                DataOutputViewStreamWrapper dov = new DataOutputViewStreamWrapper(out);
                // if (this instanceof KeyGroupCheckpointedOperator)
                snapshotKeyGroupState(keyGroupIdx, dov);
                // Maybe this is a normal DoFn that has no timerService
                if (keyCoder != null) {
                    timerService.snapshotTimersForKeyGroup(dov, keyGroupIdx);
                }
            }
        } catch (Exception exception) {
            throw new Exception("Could not write timer service of " + getOperatorName() + " to checkpoint state stream.", exception);
        } finally {
            try {
                out.close();
            } catch (Exception closeException) {
                LOG.warn("Could not close raw keyed operator state stream for {}. This " + "might have prevented deleting some state data.", getOperatorName(), closeException);
            }
        }
    }
}