Examples with MemorySegment org.apache.flink.core.memory.MemorySegment used on opensource projects

Example 11 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class SpillingBufferTest method testWriteReadExternal.

// --------------------------------------------------------------------------------------------
@Test
public void testWriteReadExternal() throws Exception {
    final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager, new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
    // write a number of pairs
    final Tuple2<Integer, String> rec = new Tuple2<>();
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
        generator.next(rec);
        serializer.serialize(rec, outView);
    }
    // create the reader input view
    DataInputView inView = outView.flip();
    generator.reset();
    // notifyNonEmpty and re-generate all records and compare them
    final Tuple2<Integer, String> readRec = new Tuple2<>();
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
        generator.next(rec);
        serializer.deserialize(readRec, inView);
        int k1 = rec.f0;
        String v1 = rec.f1;
        int k2 = readRec.f0;
        String v2 = readRec.f1;
        Assert.assertTrue("The re-generated and the notifyNonEmpty record do not match.", k1 == k2 && v1.equals(v2));
    }
    // re-notifyNonEmpty the data
    inView = outView.flip();
    generator.reset();
    // notifyNonEmpty and re-generate all records and compare them
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
        generator.next(rec);
        serializer.deserialize(readRec, inView);
        int k1 = rec.f0;
        String v1 = rec.f1;
        int k2 = readRec.f0;
        String v2 = readRec.f1;
        Assert.assertTrue("The re-generated and the notifyNonEmpty record do not match.", k1 == k2 && v1.equals(v2));
    }
    this.memoryManager.release(outView.close());
    this.memoryManager.release(memory);
}

Example 12 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class BufferFileWriterFileSegmentReaderTest method fillBufferWithAscendingNumbers.

public static int fillBufferWithAscendingNumbers(Buffer buffer, int currentNumber) {
    MemorySegment segment = buffer.getMemorySegment();
    final int size = buffer.getSize();
    for (int i = 0; i < size; i += 4) {
        segment.putInt(i, currentNumber++);
    }
    return currentNumber;
}

Example 13 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class BufferFileWriterReaderTest method verifyBufferFilledWithAscendingNumbers.

private int verifyBufferFilledWithAscendingNumbers(Buffer buffer, int currentNumber) {
    MemorySegment segment = buffer.getMemorySegment();
    final int size = buffer.getSize();
    for (int i = 0; i < size; i += 4) {
        if (segment.getInt(i) != currentNumber++) {
            throw new IllegalStateException("Read unexpected number from buffer.");
        }
    }
    return currentNumber;
}

Example 14 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class IOManagerAsyncTest method channelReadWriteOneSegment.

// ------------------------------------------------------------------------
//                           Test Methods
// ------------------------------------------------------------------------
@Test
public void channelReadWriteOneSegment() {
    final int NUM_IOS = 1111;
    try {
        final FileIOChannel.ID channelID = this.ioManager.createChannel();
        final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channelID);
        MemorySegment memSeg = MemorySegmentFactory.allocateUnpooledSegment(32 * 1024);
        for (int i = 0; i < NUM_IOS; i++) {
            for (int pos = 0; pos < memSeg.size(); pos += 4) {
                memSeg.putInt(pos, i);
            }
            writer.writeBlock(memSeg);
            memSeg = writer.getNextReturnedBlock();
        }
        writer.close();
        final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channelID);
        for (int i = 0; i < NUM_IOS; i++) {
            reader.readBlock(memSeg);
            memSeg = reader.getNextReturnedBlock();
            for (int pos = 0; pos < memSeg.size(); pos += 4) {
                if (memSeg.getInt(pos) != i) {
                    fail("Read memory segment contains invalid data.");
                }
            }
        }
        reader.closeAndDelete();
    } catch (Exception ex) {
        ex.printStackTrace();
        fail("Test encountered an exception: " + ex.getMessage());
    }
}

Example 15 with MemorySegment

use of org.apache.flink.core.memory.MemorySegment in project flink by apache.

the class IOManagerAsyncTest method channelReadWriteMultipleSegments.

@Test
public void channelReadWriteMultipleSegments() {
    final int NUM_IOS = 1111;
    final int NUM_SEGS = 16;
    try {
        final List<MemorySegment> memSegs = new ArrayList<MemorySegment>();
        for (int i = 0; i < NUM_SEGS; i++) {
            memSegs.add(MemorySegmentFactory.allocateUnpooledSegment(32 * 1024));
        }
        final FileIOChannel.ID channelID = this.ioManager.createChannel();
        final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channelID);
        for (int i = 0; i < NUM_IOS; i++) {
            final MemorySegment memSeg = memSegs.isEmpty() ? writer.getNextReturnedBlock() : memSegs.remove(memSegs.size() - 1);
            for (int pos = 0; pos < memSeg.size(); pos += 4) {
                memSeg.putInt(pos, i);
            }
            writer.writeBlock(memSeg);
        }
        writer.close();
        // get back the memory
        while (memSegs.size() < NUM_SEGS) {
            memSegs.add(writer.getNextReturnedBlock());
        }
        final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(channelID);
        while (!memSegs.isEmpty()) {
            reader.readBlock(memSegs.remove(0));
        }
        for (int i = 0; i < NUM_IOS; i++) {
            final MemorySegment memSeg = reader.getNextReturnedBlock();
            for (int pos = 0; pos < memSeg.size(); pos += 4) {
                if (memSeg.getInt(pos) != i) {
                    fail("Read memory segment contains invalid data.");
                }
            }
            reader.readBlock(memSeg);
        }
        reader.closeAndDelete();
        // get back the memory
        while (memSegs.size() < NUM_SEGS) {
            memSegs.add(reader.getNextReturnedBlock());
        }
    } catch (Exception ex) {
        ex.printStackTrace();
        fail("TEst encountered an exception: " + ex.getMessage());
    }
}