Examples with ListMemorySegmentSource org.apache.flink.runtime.memory.ListMemorySegmentSource used on opensource projects

Example 1 with ListMemorySegmentSource

use of org.apache.flink.runtime.memory.ListMemorySegmentSource in project flink by apache.

the class SpillingBufferTest method testWriteReadInMemory.

// --------------------------------------------------------------------------------------------
@Test
public void testWriteReadInMemory() throws Exception {
    final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager, new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
    // write a number of pairs
    final Tuple2<Integer, String> rec = new Tuple2<>();
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
        generator.next(rec);
        serializer.serialize(rec, outView);
    }
    // create the reader input view
    DataInputView inView = outView.flip();
    generator.reset();
    // notifyNonEmpty and re-generate all records and compare them
    final Tuple2<Integer, String> readRec = new Tuple2<>();
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
        generator.next(rec);
        serializer.deserialize(readRec, inView);
        int k1 = rec.f0;
        String v1 = rec.f1;
        int k2 = readRec.f0;
        String v2 = readRec.f1;
        Assert.assertTrue("The re-generated and the notifyNonEmpty record do not match.", k1 == k2 && v1.equals(v2));
    }
    // re-notifyNonEmpty the data
    inView = outView.flip();
    generator.reset();
    // notifyNonEmpty and re-generate all records and compare them
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
        generator.next(rec);
        serializer.deserialize(readRec, inView);
        int k1 = rec.f0;
        String v1 = rec.f1;
        int k2 = readRec.f0;
        String v2 = readRec.f1;
        Assert.assertTrue("The re-generated and the notifyNonEmpty record do not match.", k1 == k2 && v1.equals(v2));
    }
    this.memoryManager.release(outView.close());
    this.memoryManager.release(memory);
}

Example 2 with ListMemorySegmentSource

use of org.apache.flink.runtime.memory.ListMemorySegmentSource in project flink by apache.

the class SpillingBufferTest method testWriteReadExternal.

// --------------------------------------------------------------------------------------------
@Test
public void testWriteReadExternal() throws Exception {
    final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager, new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
    // write a number of pairs
    final Tuple2<Integer, String> rec = new Tuple2<>();
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
        generator.next(rec);
        serializer.serialize(rec, outView);
    }
    // create the reader input view
    DataInputView inView = outView.flip();
    generator.reset();
    // notifyNonEmpty and re-generate all records and compare them
    final Tuple2<Integer, String> readRec = new Tuple2<>();
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
        generator.next(rec);
        serializer.deserialize(readRec, inView);
        int k1 = rec.f0;
        String v1 = rec.f1;
        int k2 = readRec.f0;
        String v2 = readRec.f1;
        Assert.assertTrue("The re-generated and the notifyNonEmpty record do not match.", k1 == k2 && v1.equals(v2));
    }
    // re-notifyNonEmpty the data
    inView = outView.flip();
    generator.reset();
    // notifyNonEmpty and re-generate all records and compare them
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
        generator.next(rec);
        serializer.deserialize(readRec, inView);
        int k1 = rec.f0;
        String v1 = rec.f1;
        int k2 = readRec.f0;
        String v2 = readRec.f1;
        Assert.assertTrue("The re-generated and the notifyNonEmpty record do not match.", k1 == k2 && v1.equals(v2));
    }
    this.memoryManager.release(outView.close());
    this.memoryManager.release(memory);
}

Example 3 with ListMemorySegmentSource

use of org.apache.flink.runtime.memory.ListMemorySegmentSource in project flink by apache.

the class CompactingHashTable method createPartitions.

// --------------------------------------------------------------------------------------------
//  Setup and Tear Down of Structures
// --------------------------------------------------------------------------------------------
private void createPartitions(int numPartitions) {
    this.partitions.clear();
    ListMemorySegmentSource memSource = new ListMemorySegmentSource(this.availableMemory);
    for (int i = 0; i < numPartitions; i++) {
        this.partitions.add(new InMemoryPartition<T>(this.buildSideSerializer, i, memSource, this.segmentSize, pageSizeInBits));
    }
    this.compactionMemory = new InMemoryPartition<T>(this.buildSideSerializer, -1, memSource, this.segmentSize, pageSizeInBits);
}

Example 4 with ListMemorySegmentSource

use of org.apache.flink.runtime.memory.ListMemorySegmentSource in project flink by apache.

the class SpillingBufferTest method testWriteReadTooMuchInMemory.

@Test
public void testWriteReadTooMuchInMemory() throws Exception {
    final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager, new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
    // write a number of pairs
    final Tuple2<Integer, String> rec = new Tuple2<>();
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
        generator.next(rec);
        serializer.serialize(rec, outView);
    }
    // create the reader input view
    DataInputView inView = outView.flip();
    generator.reset();
    // notifyNonEmpty and re-generate all records and compare them
    final Tuple2<Integer, String> readRec = new Tuple2<>();
    try {
        for (int i = 0; i < NUM_PAIRS_INMEM + 1; i++) {
            generator.next(rec);
            serializer.deserialize(readRec, inView);
            int k1 = rec.f0;
            String v1 = rec.f1;
            int k2 = readRec.f0;
            String v2 = readRec.f1;
            Assert.assertTrue("The re-generated and the notifyNonEmpty record do not match.", k1 == k2 && v1.equals(v2));
        }
        Assert.fail("Read too much, expected EOFException.");
    } catch (EOFException eofex) {
    // expected
    }
    // re-notifyNonEmpty the data
    inView = outView.flip();
    generator.reset();
    // notifyNonEmpty and re-generate all records and compare them
    for (int i = 0; i < NUM_PAIRS_INMEM; i++) {
        generator.next(rec);
        serializer.deserialize(readRec, inView);
        int k1 = rec.f0;
        String v1 = rec.f1;
        int k2 = readRec.f0;
        String v2 = readRec.f1;
        Assert.assertTrue("The re-generated and the notifyNonEmpty record do not match.", k1 == k2 && v1.equals(v2));
    }
    this.memoryManager.release(outView.close());
    this.memoryManager.release(memory);
}

Example 5 with ListMemorySegmentSource

use of org.apache.flink.runtime.memory.ListMemorySegmentSource in project flink by apache.

the class SpillingBufferTest method testWriteReadTooMuchExternal.

@Test
public void testWriteReadTooMuchExternal() throws Exception {
    final TestData.TupleGenerator generator = new TestData.TupleGenerator(SEED, KEY_MAX, VALUE_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
    final TypeSerializer<Tuple2<Integer, String>> serializer = TestData.getIntStringTupleSerializer();
    // create the writer output view
    final ArrayList<MemorySegment> memory = new ArrayList<MemorySegment>(NUM_MEMORY_SEGMENTS);
    this.memoryManager.allocatePages(this.parentTask, memory, NUM_MEMORY_SEGMENTS);
    final SpillingBuffer outView = new SpillingBuffer(this.ioManager, new ListMemorySegmentSource(memory), this.memoryManager.getPageSize());
    // write a number of pairs
    final Tuple2<Integer, String> rec = new Tuple2<>();
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
        generator.next(rec);
        serializer.serialize(rec, outView);
    }
    // create the reader input view
    DataInputView inView = outView.flip();
    generator.reset();
    // notifyNonEmpty and re-generate all records and compare them
    final Tuple2<Integer, String> readRec = new Tuple2<>();
    try {
        for (int i = 0; i < NUM_PAIRS_EXTERNAL + 1; i++) {
            generator.next(rec);
            serializer.deserialize(readRec, inView);
            int k1 = rec.f0;
            String v1 = rec.f1;
            int k2 = readRec.f0;
            String v2 = readRec.f1;
            Assert.assertTrue("The re-generated and the notifyNonEmpty record do not match.", k1 == k2 && v1.equals(v2));
        }
        Assert.fail("Read too much, expected EOFException.");
    } catch (EOFException eofex) {
    // expected
    }
    // re-notifyNonEmpty the data
    inView = outView.flip();
    generator.reset();
    // notifyNonEmpty and re-generate all records and compare them
    for (int i = 0; i < NUM_PAIRS_EXTERNAL; i++) {
        generator.next(rec);
        serializer.deserialize(readRec, inView);
        int k1 = rec.f0;
        String v1 = rec.f1;
        int k2 = readRec.f0;
        String v2 = readRec.f1;
        Assert.assertTrue("The re-generated and the notifyNonEmpty record do not match.", k1 == k2 && v1.equals(v2));
    }
    this.memoryManager.release(outView.close());
    this.memoryManager.release(memory);
}