Examples with ByteStreamStateHandle org.apache.flink.runtime.state.memory.ByteStreamStateHandle used on opensource projects

Example 1 with ByteStreamStateHandle

use of org.apache.flink.runtime.state.memory.ByteStreamStateHandle in project flink by apache.

the class CheckpointCoordinatorTest method generateChainedPartitionableStateHandle.

private static ChainedStateHandle<OperatorStateHandle> generateChainedPartitionableStateHandle(Map<String, List<? extends Serializable>> states) throws IOException {
    List<List<? extends Serializable>> namedStateSerializables = new ArrayList<>(states.size());
    for (Map.Entry<String, List<? extends Serializable>> entry : states.entrySet()) {
        namedStateSerializables.add(entry.getValue());
    }
    Tuple2<byte[], List<long[]>> serializationWithOffsets = serializeTogetherAndTrackOffsets(namedStateSerializables);
    Map<String, OperatorStateHandle.StateMetaInfo> offsetsMap = new HashMap<>(states.size());
    int idx = 0;
    for (Map.Entry<String, List<? extends Serializable>> entry : states.entrySet()) {
        offsetsMap.put(entry.getKey(), new OperatorStateHandle.StateMetaInfo(serializationWithOffsets.f1.get(idx), OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
        ++idx;
    }
    ByteStreamStateHandle streamStateHandle = new TestByteStreamStateHandleDeepCompare(String.valueOf(UUID.randomUUID()), serializationWithOffsets.f0);
    OperatorStateHandle operatorStateHandle = new OperatorStateHandle(offsetsMap, streamStateHandle);
    return ChainedStateHandle.wrapSingleHandle(operatorStateHandle);
}

Example 2 with ByteStreamStateHandle

use of org.apache.flink.runtime.state.memory.ByteStreamStateHandle in project flink by apache.

the class MigrationV0ToV1Test method testSavepointMigrationV0ToV1.

/**
	 * Simple test of savepoint methods.
	 */
@Test
public void testSavepointMigrationV0ToV1() throws Exception {
    String target = tmp.getRoot().getAbsolutePath();
    assertEquals(0, tmp.getRoot().listFiles().length);
    long checkpointId = ThreadLocalRandom.current().nextLong(Integer.MAX_VALUE);
    int numTaskStates = 4;
    int numSubtaskStates = 16;
    Collection<org.apache.flink.migration.runtime.checkpoint.TaskState> expected = createTaskStatesOld(numTaskStates, numSubtaskStates);
    SavepointV0 savepoint = new SavepointV0(checkpointId, expected);
    assertEquals(SavepointV0.VERSION, savepoint.getVersion());
    assertEquals(checkpointId, savepoint.getCheckpointId());
    assertEquals(expected, savepoint.getOldTaskStates());
    assertFalse(savepoint.getOldTaskStates().isEmpty());
    Exception latestException = null;
    Path path = null;
    FSDataOutputStream fdos = null;
    FileSystem fs = null;
    try {
        // Try to create a FS output stream
        for (int attempt = 0; attempt < 10; attempt++) {
            path = new Path(target, FileUtils.getRandomFilename("savepoint-"));
            if (fs == null) {
                fs = FileSystem.get(path.toUri());
            }
            try {
                fdos = fs.create(path, false);
                break;
            } catch (Exception e) {
                latestException = e;
            }
        }
        if (fdos == null) {
            throw new IOException("Failed to create file output stream at " + path, latestException);
        }
        try (DataOutputStream dos = new DataOutputStream(fdos)) {
            dos.writeInt(SavepointStore.MAGIC_NUMBER);
            dos.writeInt(savepoint.getVersion());
            SavepointV0Serializer.INSTANCE.serializeOld(savepoint, dos);
        }
        ClassLoader cl = Thread.currentThread().getContextClassLoader();
        Savepoint sp = SavepointStore.loadSavepoint(path.toString(), cl);
        int t = 0;
        for (TaskState taskState : sp.getTaskStates()) {
            for (int p = 0; p < taskState.getParallelism(); ++p) {
                SubtaskState subtaskState = taskState.getState(p);
                ChainedStateHandle<StreamStateHandle> legacyOperatorState = subtaskState.getLegacyOperatorState();
                for (int c = 0; c < legacyOperatorState.getLength(); ++c) {
                    StreamStateHandle stateHandle = legacyOperatorState.get(c);
                    try (InputStream is = stateHandle.openInputStream()) {
                        Tuple4<Integer, Integer, Integer, Integer> expTestState = new Tuple4<>(0, t, p, c);
                        Tuple4<Integer, Integer, Integer, Integer> actTestState;
                        //check function state
                        if (p % 4 != 0) {
                            assertEquals(1, is.read());
                            actTestState = InstantiationUtil.deserializeObject(is, cl);
                            assertEquals(expTestState, actTestState);
                        } else {
                            assertEquals(0, is.read());
                        }
                        //check operator state
                        expTestState.f0 = 1;
                        actTestState = InstantiationUtil.deserializeObject(is, cl);
                        assertEquals(expTestState, actTestState);
                    }
                }
                //check keyed state
                KeyGroupsStateHandle keyGroupsStateHandle = subtaskState.getManagedKeyedState();
                if (t % 3 != 0) {
                    assertEquals(1, keyGroupsStateHandle.getNumberOfKeyGroups());
                    assertEquals(p, keyGroupsStateHandle.getGroupRangeOffsets().getKeyGroupRange().getStartKeyGroup());
                    ByteStreamStateHandle stateHandle = (ByteStreamStateHandle) keyGroupsStateHandle.getDelegateStateHandle();
                    HashMap<String, KvStateSnapshot<?, ?, ?, ?>> testKeyedState = MigrationInstantiationUtil.deserializeObject(stateHandle.getData(), cl);
                    assertEquals(2, testKeyedState.size());
                    for (KvStateSnapshot<?, ?, ?, ?> snapshot : testKeyedState.values()) {
                        MemValueState.Snapshot<?, ?, ?> castedSnapshot = (MemValueState.Snapshot<?, ?, ?>) snapshot;
                        byte[] data = castedSnapshot.getData();
                        assertEquals(t, data[0]);
                        assertEquals(p, data[1]);
                    }
                } else {
                    assertEquals(null, keyGroupsStateHandle);
                }
            }
            ++t;
        }
        savepoint.dispose();
    } finally {
        // Dispose
        SavepointStore.removeSavepointFile(path.toString());
    }
}

Example 3 with ByteStreamStateHandle

use of org.apache.flink.runtime.state.memory.ByteStreamStateHandle in project flink by apache.

the class FileStateBackendTest method testStateOutputStream.

@Test
public void testStateOutputStream() {
    URI basePath = randomHdfsFileUri();
    try {
        FsStateBackend backend = CommonTestUtils.createCopySerializable(new FsStateBackend(basePath, 15));
        JobID jobId = new JobID();
        CheckpointStreamFactory streamFactory = backend.createStreamFactory(jobId, "test_op");
        // we know how FsCheckpointStreamFactory is implemented so we know where it
        // will store checkpoints
        Path checkpointPath = new Path(new Path(basePath), jobId.toString());
        byte[] state1 = new byte[1274673];
        byte[] state2 = new byte[1];
        byte[] state3 = new byte[0];
        byte[] state4 = new byte[177];
        Random rnd = new Random();
        rnd.nextBytes(state1);
        rnd.nextBytes(state2);
        rnd.nextBytes(state3);
        rnd.nextBytes(state4);
        long checkpointId = 97231523452L;
        CheckpointStreamFactory.CheckpointStateOutputStream stream1 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        CheckpointStreamFactory.CheckpointStateOutputStream stream2 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        CheckpointStreamFactory.CheckpointStateOutputStream stream3 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        stream1.write(state1);
        stream2.write(state2);
        stream3.write(state3);
        FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
        ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
        ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();
        // use with try-with-resources
        FileStateHandle handle4;
        try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis())) {
            stream4.write(state4);
            handle4 = (FileStateHandle) stream4.closeAndGetHandle();
        }
        // close before accessing handle
        CheckpointStreamFactory.CheckpointStateOutputStream stream5 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        stream5.write(state4);
        stream5.close();
        try {
            stream5.closeAndGetHandle();
            fail();
        } catch (IOException e) {
        // uh-huh
        }
        validateBytesInStream(handle1.openInputStream(), state1);
        handle1.discardState();
        assertFalse(isDirectoryEmpty(checkpointPath));
        ensureFileDeleted(handle1.getFilePath());
        validateBytesInStream(handle2.openInputStream(), state2);
        handle2.discardState();
        // stream 3 has zero bytes, so it should not return anything
        assertNull(handle3);
        validateBytesInStream(handle4.openInputStream(), state4);
        handle4.discardState();
        assertTrue(isDirectoryEmpty(checkpointPath));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 4 with ByteStreamStateHandle

use of org.apache.flink.runtime.state.memory.ByteStreamStateHandle in project flink by apache.

the class SavepointV1Serializer method deserializeStreamStateHandle.

private static StreamStateHandle deserializeStreamStateHandle(DataInputStream dis) throws IOException {
    int type = dis.read();
    if (NULL_HANDLE == type) {
        return null;
    } else if (FILE_STREAM_STATE_HANDLE == type) {
        long size = dis.readLong();
        String pathString = dis.readUTF();
        return new FileStateHandle(new Path(pathString), size);
    } else if (BYTE_STREAM_STATE_HANDLE == type) {
        String handleName = dis.readUTF();
        int numBytes = dis.readInt();
        byte[] data = new byte[numBytes];
        dis.readFully(data);
        return new ByteStreamStateHandle(handleName, data);
    } else {
        throw new IOException("Unknown implementation of StreamStateHandle, code: " + type);
    }
}

Example 5 with ByteStreamStateHandle

use of org.apache.flink.runtime.state.memory.ByteStreamStateHandle in project flink by apache.

the class SavepointV1Serializer method serializeStreamStateHandle.

private static void serializeStreamStateHandle(StreamStateHandle stateHandle, DataOutputStream dos) throws IOException {
    if (stateHandle == null) {
        dos.writeByte(NULL_HANDLE);
    } else if (stateHandle instanceof FileStateHandle) {
        dos.writeByte(FILE_STREAM_STATE_HANDLE);
        FileStateHandle fileStateHandle = (FileStateHandle) stateHandle;
        dos.writeLong(stateHandle.getStateSize());
        dos.writeUTF(fileStateHandle.getFilePath().toString());
    } else if (stateHandle instanceof ByteStreamStateHandle) {
        dos.writeByte(BYTE_STREAM_STATE_HANDLE);
        ByteStreamStateHandle byteStreamStateHandle = (ByteStreamStateHandle) stateHandle;
        dos.writeUTF(byteStreamStateHandle.getHandleName());
        byte[] internalData = byteStreamStateHandle.getData();
        dos.writeInt(internalData.length);
        dos.write(byteStreamStateHandle.getData());
    } else {
        throw new IOException("Unknown implementation of StreamStateHandle: " + stateHandle.getClass());
    }
    dos.flush();
}