Examples with ByteStreamStateHandle org.apache.flink.runtime.state.memory.ByteStreamStateHandle used on opensource projects

Example 6 with ByteStreamStateHandle

use of org.apache.flink.runtime.state.memory.ByteStreamStateHandle in project flink by apache.

the class CheckpointCoordinatorTest method generateKeyGroupState.

public static KeyGroupsStateHandle generateKeyGroupState(KeyGroupRange keyGroupRange, List<? extends Serializable> states) throws IOException {
    Preconditions.checkArgument(keyGroupRange.getNumberOfKeyGroups() == states.size());
    Tuple2<byte[], List<long[]>> serializedDataWithOffsets = serializeTogetherAndTrackOffsets(Collections.<List<? extends Serializable>>singletonList(states));
    KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange, serializedDataWithOffsets.f1.get(0));
    ByteStreamStateHandle allSerializedStatesHandle = new TestByteStreamStateHandleDeepCompare(String.valueOf(UUID.randomUUID()), serializedDataWithOffsets.f0);
    KeyGroupsStateHandle keyGroupsStateHandle = new KeyGroupsStateHandle(keyGroupRangeOffsets, allSerializedStatesHandle);
    return keyGroupsStateHandle;
}

Example 7 with ByteStreamStateHandle

use of org.apache.flink.runtime.state.memory.ByteStreamStateHandle in project flink by apache.

the class CheckpointCoordinatorTest method testReplicateModeStateHandle.

@Test
public void testReplicateModeStateHandle() {
    Map<String, OperatorStateHandle.StateMetaInfo> metaInfoMap = new HashMap<>(1);
    metaInfoMap.put("t-1", new OperatorStateHandle.StateMetaInfo(new long[] { 0, 23 }, OperatorStateHandle.Mode.BROADCAST));
    metaInfoMap.put("t-2", new OperatorStateHandle.StateMetaInfo(new long[] { 42, 64 }, OperatorStateHandle.Mode.BROADCAST));
    metaInfoMap.put("t-3", new OperatorStateHandle.StateMetaInfo(new long[] { 72, 83 }, OperatorStateHandle.Mode.SPLIT_DISTRIBUTE));
    OperatorStateHandle osh = new OperatorStateHandle(metaInfoMap, new ByteStreamStateHandle("test", new byte[100]));
    OperatorStateRepartitioner repartitioner = RoundRobinOperatorStateRepartitioner.INSTANCE;
    List<Collection<OperatorStateHandle>> repartitionedStates = repartitioner.repartitionState(Collections.singletonList(osh), 3);
    Map<String, Integer> checkCounts = new HashMap<>(3);
    for (Collection<OperatorStateHandle> operatorStateHandles : repartitionedStates) {
        for (OperatorStateHandle operatorStateHandle : operatorStateHandles) {
            for (Map.Entry<String, OperatorStateHandle.StateMetaInfo> stateNameToMetaInfo : operatorStateHandle.getStateNameToPartitionOffsets().entrySet()) {
                String stateName = stateNameToMetaInfo.getKey();
                Integer count = checkCounts.get(stateName);
                if (null == count) {
                    checkCounts.put(stateName, 1);
                } else {
                    checkCounts.put(stateName, 1 + count);
                }
                OperatorStateHandle.StateMetaInfo stateMetaInfo = stateNameToMetaInfo.getValue();
                if (OperatorStateHandle.Mode.SPLIT_DISTRIBUTE.equals(stateMetaInfo.getDistributionMode())) {
                    Assert.assertEquals(1, stateNameToMetaInfo.getValue().getOffsets().length);
                } else {
                    Assert.assertEquals(2, stateNameToMetaInfo.getValue().getOffsets().length);
                }
            }
        }
    }
    Assert.assertEquals(3, checkCounts.size());
    Assert.assertEquals(3, checkCounts.get("t-1").intValue());
    Assert.assertEquals(3, checkCounts.get("t-2").intValue());
    Assert.assertEquals(2, checkCounts.get("t-3").intValue());
}

Example 8 with ByteStreamStateHandle

use of org.apache.flink.runtime.state.memory.ByteStreamStateHandle in project flink by apache.

the class FsCheckpointStateOutputStreamTest method runTest.

private void runTest(int numBytes, int bufferSize, int threshold, boolean expectFile) throws Exception {
    FsCheckpointStreamFactory.CheckpointStateOutputStream stream = new FsCheckpointStreamFactory.FsCheckpointStateOutputStream(TEMP_DIR_PATH, FileSystem.getLocalFileSystem(), bufferSize, threshold);
    Random rnd = new Random();
    byte[] original = new byte[numBytes];
    byte[] bytes = new byte[original.length];
    rnd.nextBytes(original);
    System.arraycopy(original, 0, bytes, 0, original.length);
    // the test writes a mixture of writing individual bytes and byte arrays
    int pos = 0;
    while (pos < bytes.length) {
        boolean single = rnd.nextBoolean();
        if (single) {
            stream.write(bytes[pos++]);
        } else {
            int num = rnd.nextInt(Math.min(10, bytes.length - pos));
            stream.write(bytes, pos, num);
            pos += num;
        }
    }
    StreamStateHandle handle = stream.closeAndGetHandle();
    if (expectFile) {
        assertTrue(handle instanceof FileStateHandle);
    } else {
        assertTrue(handle instanceof ByteStreamStateHandle);
    }
    // make sure the writing process did not alter the original byte array
    assertArrayEquals(original, bytes);
    try (InputStream inStream = handle.openInputStream()) {
        byte[] validation = new byte[bytes.length];
        DataInputStream dataInputStream = new DataInputStream(inStream);
        dataInputStream.readFully(validation);
        assertArrayEquals(bytes, validation);
    }
    handle.discardState();
}

Example 9 with ByteStreamStateHandle

use of org.apache.flink.runtime.state.memory.ByteStreamStateHandle in project flink by apache.

the class FileStateBackendTest method testStateOutputStream.

@Test
public void testStateOutputStream() throws IOException {
    File basePath = tempFolder.newFolder().getAbsoluteFile();
    try {
        // the state backend has a very low in-mem state threshold (15 bytes)
        FsStateBackend backend = CommonTestUtils.createCopySerializable(new FsStateBackend(basePath.toURI(), 15));
        JobID jobId = new JobID();
        // we know how FsCheckpointStreamFactory is implemented so we know where it
        // will store checkpoints
        File checkpointPath = new File(basePath.getAbsolutePath(), jobId.toString());
        CheckpointStreamFactory streamFactory = backend.createStreamFactory(jobId, "test_op");
        byte[] state1 = new byte[1274673];
        byte[] state2 = new byte[1];
        byte[] state3 = new byte[0];
        byte[] state4 = new byte[177];
        Random rnd = new Random();
        rnd.nextBytes(state1);
        rnd.nextBytes(state2);
        rnd.nextBytes(state3);
        rnd.nextBytes(state4);
        long checkpointId = 97231523452L;
        CheckpointStreamFactory.CheckpointStateOutputStream stream1 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        CheckpointStreamFactory.CheckpointStateOutputStream stream2 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        CheckpointStreamFactory.CheckpointStateOutputStream stream3 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        stream1.write(state1);
        stream2.write(state2);
        stream3.write(state3);
        FileStateHandle handle1 = (FileStateHandle) stream1.closeAndGetHandle();
        ByteStreamStateHandle handle2 = (ByteStreamStateHandle) stream2.closeAndGetHandle();
        ByteStreamStateHandle handle3 = (ByteStreamStateHandle) stream3.closeAndGetHandle();
        // use with try-with-resources
        StreamStateHandle handle4;
        try (CheckpointStreamFactory.CheckpointStateOutputStream stream4 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis())) {
            stream4.write(state4);
            handle4 = stream4.closeAndGetHandle();
        }
        // close before accessing handle
        CheckpointStreamFactory.CheckpointStateOutputStream stream5 = streamFactory.createCheckpointStateOutputStream(checkpointId, System.currentTimeMillis());
        stream5.write(state4);
        stream5.close();
        try {
            stream5.closeAndGetHandle();
            fail();
        } catch (IOException e) {
        // uh-huh
        }
        validateBytesInStream(handle1.openInputStream(), state1);
        handle1.discardState();
        assertFalse(isDirectoryEmpty(basePath));
        ensureLocalFileDeleted(handle1.getFilePath());
        validateBytesInStream(handle2.openInputStream(), state2);
        handle2.discardState();
        // nothing was written to the stream, so it will return nothing
        assertNull(handle3);
        validateBytesInStream(handle4.openInputStream(), state4);
        handle4.discardState();
        assertTrue(isDirectoryEmpty(checkpointPath));
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 10 with ByteStreamStateHandle

use of org.apache.flink.runtime.state.memory.ByteStreamStateHandle in project flink by apache.

the class MultiStreamStateHandleTest method setup.

@Before
public void setup() {
    random = new Random(0x42);
    testData = new byte[TEST_DATA_LENGTH];
    for (int i = 0; i < testData.length; ++i) {
        testData[i] = (byte) i;
    }
    int idx = 0;
    streamStateHandles = new ArrayList<>();
    while (idx < testData.length) {
        int len = random.nextInt(5);
        byte[] sub = Arrays.copyOfRange(testData, idx, idx + len);
        streamStateHandles.add(new ByteStreamStateHandle(String.valueOf(idx), sub));
        idx += len;
    }
}