Examples with DummyInvokable org.apache.flink.runtime.operators.testutils.DummyInvokable used on opensource projects

Example 1 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class MemoryManagerLazyAllocationTest method allocateAllSingle.

@Test
public void allocateAllSingle() {
    try {
        final AbstractInvokable mockInvoke = new DummyInvokable();
        List<MemorySegment> segments = new ArrayList<MemorySegment>();
        try {
            for (int i = 0; i < NUM_PAGES; i++) {
                segments.add(this.memoryManager.allocatePages(mockInvoke, 1).get(0));
            }
        } catch (MemoryAllocationException e) {
            fail("Unable to allocate memory");
        }
        for (MemorySegment seg : segments) {
            this.memoryManager.release(seg);
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 2 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class MemoryManagerTest method allocateAllMulti.

@Test
public void allocateAllMulti() {
    try {
        final AbstractInvokable mockInvoke = new DummyInvokable();
        final List<MemorySegment> segments = new ArrayList<MemorySegment>();
        try {
            for (int i = 0; i < NUM_PAGES / 2; i++) {
                segments.addAll(this.memoryManager.allocatePages(mockInvoke, 2));
            }
        } catch (MemoryAllocationException e) {
            Assert.fail("Unable to allocate memory");
        }
        this.memoryManager.release(segments);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 3 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class MemorySegmentSimpleTest method setUp.

@Before
public void setUp() throws Exception {
    try {
        this.manager = new MemoryManager(MANAGED_MEMORY_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
        this.segment = manager.allocatePages(new DummyInvokable(), 1).get(0);
        this.random = new Random(RANDOM_SEED);
    } catch (Exception e) {
        e.printStackTrace();
        fail("Test setup failed.");
    }
}

Example 4 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class IOManagerITCase method parallelChannelsTest.

// ------------------------------------------------------------------------
/**
	 * This test instantiates multiple channels and writes to them in parallel and re-reads the data in 
	 * parallel. It is designed to check the ability of the IO manager to correctly handle multiple threads.
	 */
@Test
@SuppressWarnings("unchecked")
public void parallelChannelsTest() throws Exception {
    final Random rnd = new Random(SEED);
    final AbstractInvokable memOwner = new DummyInvokable();
    FileIOChannel.ID[] ids = new FileIOChannel.ID[NUM_CHANNELS];
    BlockChannelWriter<MemorySegment>[] writers = new BlockChannelWriter[NUM_CHANNELS];
    BlockChannelReader<MemorySegment>[] readers = new BlockChannelReader[NUM_CHANNELS];
    ChannelWriterOutputView[] outs = new ChannelWriterOutputView[NUM_CHANNELS];
    ChannelReaderInputView[] ins = new ChannelReaderInputView[NUM_CHANNELS];
    int[] writingCounters = new int[NUM_CHANNELS];
    int[] readingCounters = new int[NUM_CHANNELS];
    // instantiate the channels and writers
    for (int i = 0; i < NUM_CHANNELS; i++) {
        ids[i] = this.ioManager.createChannel();
        writers[i] = this.ioManager.createBlockChannelWriter(ids[i]);
        List<MemorySegment> memSegs = this.memoryManager.allocatePages(memOwner, rnd.nextInt(MAXIMUM_NUMBER_OF_SEGMENTS_PER_CHANNEL - 1) + 1);
        outs[i] = new ChannelWriterOutputView(writers[i], memSegs, this.memoryManager.getPageSize());
    }
    Value val = new Value();
    for (int i = 0; i < NUMBERS_TO_BE_WRITTEN; i++) {
        int channel = skewedSample(rnd, NUM_CHANNELS - 1);
        val.value = String.valueOf(writingCounters[channel]++);
        val.write(outs[channel]);
    }
    // close all writers
    for (int i = 0; i < NUM_CHANNELS; i++) {
        this.memoryManager.release(outs[i].close());
    }
    outs = null;
    writers = null;
    // instantiate the readers for sequential read
    for (int i = 0; i < NUM_CHANNELS; i++) {
        List<MemorySegment> memSegs = this.memoryManager.allocatePages(memOwner, rnd.nextInt(MAXIMUM_NUMBER_OF_SEGMENTS_PER_CHANNEL - 1) + 1);
        final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(ids[i]);
        final ChannelReaderInputView in = new ChannelReaderInputView(reader, memSegs, false);
        int nextVal = 0;
        try {
            while (true) {
                val.read(in);
                int intValue = 0;
                try {
                    intValue = Integer.parseInt(val.value);
                } catch (NumberFormatException nfex) {
                    Assert.fail("Invalid value read from reader. Valid decimal number expected.");
                }
                Assert.assertEquals("Written and read values do not match during sequential read.", nextVal, intValue);
                nextVal++;
            }
        } catch (EOFException eofex) {
        // expected
        }
        Assert.assertEquals("NUmber of written numbers differs from number of read numbers.", writingCounters[i], nextVal);
        this.memoryManager.release(in.close());
    }
    // instantiate the readers
    for (int i = 0; i < NUM_CHANNELS; i++) {
        List<MemorySegment> memSegs = this.memoryManager.allocatePages(memOwner, rnd.nextInt(MAXIMUM_NUMBER_OF_SEGMENTS_PER_CHANNEL - 1) + 1);
        readers[i] = this.ioManager.createBlockChannelReader(ids[i]);
        ins[i] = new ChannelReaderInputView(readers[i], memSegs, false);
    }
    // read a lot of values in a mixed order from the channels
    for (int i = 0; i < NUMBERS_TO_BE_WRITTEN; i++) {
        while (true) {
            final int channel = skewedSample(rnd, NUM_CHANNELS - 1);
            if (ins[channel] != null) {
                try {
                    val.read(ins[channel]);
                    int intValue;
                    try {
                        intValue = Integer.parseInt(val.value);
                    } catch (NumberFormatException nfex) {
                        Assert.fail("Invalid value read from reader. Valid decimal number expected.");
                        return;
                    }
                    Assert.assertEquals("Written and read values do not match.", readingCounters[channel]++, intValue);
                    break;
                } catch (EOFException eofex) {
                    this.memoryManager.release(ins[channel].close());
                    ins[channel] = null;
                }
            }
        }
    }
    // close all readers
    for (int i = 0; i < NUM_CHANNELS; i++) {
        if (ins[i] != null) {
            this.memoryManager.release(ins[i].close());
        }
        readers[i].closeAndDelete();
    }
    ins = null;
    readers = null;
    // check that files are deleted
    for (int i = 0; i < NUM_CHANNELS; i++) {
        File f = new File(ids[i].getPath());
        Assert.assertFalse("Channel file has not been deleted.", f.exists());
    }
}

Example 5 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class FileChannelStreamsITCase method testReadTooMany.

@Test
public void testReadTooMany() {
    try {
        final List<MemorySegment> memory = memManager.allocatePages(new DummyInvokable(), NUM_MEMORY_SEGMENTS);
        final PairGenerator generator = new PairGenerator(SEED, KEY_MAX, VALUE_SHORT_LENGTH, KeyMode.RANDOM, ValueMode.RANDOM_LENGTH);
        final FileIOChannel.ID channel = this.ioManager.createChannel();
        // create the writer output view
        final BlockChannelWriter<MemorySegment> writer = this.ioManager.createBlockChannelWriter(channel);
        final FileChannelOutputView outView = new FileChannelOutputView(writer, memManager, memory, MEMORY_PAGE_SIZE);
        // write a number of pairs
        Pair pair = new Pair();
        for (int i = 0; i < NUM_PAIRS_SHORT; i++) {
            generator.next(pair);
            pair.write(outView);
        }
        outView.close();
        // create the reader input view
        List<MemorySegment> readMemory = memManager.allocatePages(new DummyInvokable(), NUM_MEMORY_SEGMENTS);
        final BlockChannelReader<MemorySegment> reader = ioManager.createBlockChannelReader(channel);
        final FileChannelInputView inView = new FileChannelInputView(reader, memManager, readMemory, outView.getBytesInLatestSegment());
        generator.reset();
        // read and re-generate all records and compare them
        try {
            Pair readPair = new Pair();
            for (int i = 0; i < NUM_PAIRS_SHORT + 1; i++) {
                generator.next(pair);
                readPair.read(inView);
                assertEquals("The re-generated and the read record do not match.", pair, readPair);
            }
            fail("Expected an EOFException which did not occur.");
        } catch (EOFException eofex) {
        // expected
        }
        inView.close();
        reader.deleteChannel();
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}