Examples with DataInputView org.apache.flink.core.memory.DataInputView used on opensource projects

Example 6 with DataInputView

use of org.apache.flink.core.memory.DataInputView in project flink by apache.

the class RocksDBIncrementalRestoreOperation method readMetaData.

/**
 * Reads Flink's state meta data file from the state handle.
 */
private KeyedBackendSerializationProxy<K> readMetaData(StreamStateHandle metaStateHandle) throws Exception {
    InputStream inputStream = null;
    try {
        inputStream = metaStateHandle.openInputStream();
        cancelStreamRegistry.registerCloseable(inputStream);
        DataInputView in = new DataInputViewStreamWrapper(inputStream);
        return readMetaData(in);
    } finally {
        if (cancelStreamRegistry.unregisterCloseable(inputStream)) {
            inputStream.close();
        }
    }
}

Example 7 with DataInputView

use of org.apache.flink.core.memory.DataInputView in project flink by apache.

the class StateInitializationContextImplTest method close.

@Test
public void close() throws Exception {
    int count = 0;
    int stopCount = NUM_HANDLES / 2;
    boolean isClosed = false;
    try {
        for (KeyGroupStatePartitionStreamProvider stateStreamProvider : initializationContext.getRawKeyedStateInputs()) {
            Assert.assertNotNull(stateStreamProvider);
            if (count == stopCount) {
                closableRegistry.close();
                isClosed = true;
            }
            try (InputStream is = stateStreamProvider.getStream()) {
                DataInputView div = new DataInputViewStreamWrapper(is);
                try {
                    int val = div.readInt();
                    Assert.assertEquals(stateStreamProvider.getKeyGroupId(), val);
                    if (isClosed) {
                        Assert.fail("Close was ignored: stream");
                    }
                    ++count;
                } catch (IOException ioex) {
                    if (!isClosed) {
                        throw ioex;
                    }
                }
            }
        }
        Assert.fail("Close was ignored: registry");
    } catch (IOException iex) {
        Assert.assertTrue(isClosed);
        Assert.assertEquals(stopCount, count);
    }
}

Example 8 with DataInputView

use of org.apache.flink.core.memory.DataInputView in project flink by apache.

the class StateInitializationContextImplTest method getOperatorStateStore.

@Test
public void getOperatorStateStore() throws Exception {
    Set<Integer> readStatesCount = new HashSet<>();
    for (StatePartitionStreamProvider statePartitionStreamProvider : initializationContext.getRawOperatorStateInputs()) {
        Assert.assertNotNull(statePartitionStreamProvider);
        try (InputStream is = statePartitionStreamProvider.getStream()) {
            DataInputView div = new DataInputViewStreamWrapper(is);
            Assert.assertTrue(readStatesCount.add(div.readInt()));
        }
    }
    Assert.assertEquals(writtenOperatorStates, readStatesCount);
}

Example 9 with DataInputView

use of org.apache.flink.core.memory.DataInputView in project flink by apache.

the class SpillingBufferTest method testWriteReadInMemory.

// --------------------------------------------------------------------------------------------
@Test
public void testWriteReadInMemory() throws Exception {
    final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager, new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
    // write a number of pairs
    final Tuple2<Integer, String> rec = new Tuple2<>();
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
        generator.next(rec);
        serializer.serialize(rec, outView);
    }
    // create the reader input view
    DataInputView inView = outView.flip();
    generator.reset();
    // notifyNonEmpty and re-generate all records and compare them
    final Tuple2<Integer, String> readRec = new Tuple2<>();
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
        generator.next(rec);
        serializer.deserialize(readRec, inView);
        int k1 = rec.f0;
        String v1 = rec.f1;
        int k2 = readRec.f0;
        String v2 = readRec.f1;
        Assert.assertTrue("The re-generated and the notifyNonEmpty record do not match.", k1 == k2 && v1.equals(v2));
    }
    // re-notifyNonEmpty the data
    inView = outView.flip();
    generator.reset();
    // notifyNonEmpty and re-generate all records and compare them
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
        generator.next(rec);
        serializer.deserialize(readRec, inView);
        int k1 = rec.f0;
        String v1 = rec.f1;
        int k2 = readRec.f0;
        String v2 = readRec.f1;
        Assert.assertTrue("The re-generated and the notifyNonEmpty record do not match.", k1 == k2 && v1.equals(v2));
    }
    this.memoryManager.release(outView.close());
    this.memoryManager.release(memory);
}

Example 10 with DataInputView

use of org.apache.flink.core.memory.DataInputView in project flink by apache.

the class SpillingBufferTest method testWriteReadExternal.

// --------------------------------------------------------------------------------------------
@Test
public void testWriteReadExternal() throws Exception {
    final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager, new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
    // write a number of pairs
    final Tuple2<Integer, String> rec = new Tuple2<>();
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
        generator.next(rec);
        serializer.serialize(rec, outView);
    }
    // create the reader input view
    DataInputView inView = outView.flip();
    generator.reset();
    // notifyNonEmpty and re-generate all records and compare them
    final Tuple2<Integer, String> readRec = new Tuple2<>();
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
        generator.next(rec);
        serializer.deserialize(readRec, inView);
        int k1 = rec.f0;
        String v1 = rec.f1;
        int k2 = readRec.f0;
        String v2 = readRec.f1;
        Assert.assertTrue("The re-generated and the notifyNonEmpty record do not match.", k1 == k2 && v1.equals(v2));
    }
    // re-notifyNonEmpty the data
    inView = outView.flip();
    generator.reset();
    // notifyNonEmpty and re-generate all records and compare them
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
        generator.next(rec);
        serializer.deserialize(readRec, inView);
        int k1 = rec.f0;
        String v1 = rec.f1;
        int k2 = readRec.f0;
        String v2 = readRec.f1;
        Assert.assertTrue("The re-generated and the notifyNonEmpty record do not match.", k1 == k2 && v1.equals(v2));
    }
    this.memoryManager.release(outView.close());
    this.memoryManager.release(memory);
}