Examples with AbstractInvokable org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable used on opensource projects

Example 1 with AbstractInvokable

use of org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable in project flink by apache.

the class MemoryManagerLazyAllocationTest method allocateAllSingle.

@Test
public void allocateAllSingle() {
    try {
        final AbstractInvokable mockInvoke = new DummyInvokable();
        List<MemorySegment> segments = new ArrayList<MemorySegment>();
        try {
            for (int i = 0; i < NUM_PAGES; i++) {
                segments.add(this.memoryManager.allocatePages(mockInvoke, 1).get(0));
            }
        } catch (MemoryAllocationException e) {
            fail("Unable to allocate memory");
        }
        for (MemorySegment seg : segments) {
            this.memoryManager.release(seg);
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 2 with AbstractInvokable

use of org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable in project flink by apache.

the class MemoryManagerTest method allocateAllMulti.

@Test
public void allocateAllMulti() {
    try {
        final AbstractInvokable mockInvoke = new DummyInvokable();
        final List<MemorySegment> segments = new ArrayList<MemorySegment>();
        try {
            for (int i = 0; i < NUM_PAGES / 2; i++) {
                segments.addAll(this.memoryManager.allocatePages(mockInvoke, 2));
            }
        } catch (MemoryAllocationException e) {
            Assert.fail("Unable to allocate memory");
        }
        this.memoryManager.release(segments);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 3 with AbstractInvokable

use of org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable in project flink by apache.

the class IOManagerITCase method parallelChannelsTest.

// ------------------------------------------------------------------------
/**
	 * This test instantiates multiple channels and writes to them in parallel and re-reads the data in 
	 * parallel. It is designed to check the ability of the IO manager to correctly handle multiple threads.
	 */
@Test
@SuppressWarnings("unchecked")
public void parallelChannelsTest() throws Exception {
    final Random rnd = new Random(SEED);
    final AbstractInvokable memOwner = new DummyInvokable();
    FileIOChannel.ID[] ids = new FileIOChannel.ID[NUM_CHANNELS];
    BlockChannelWriter<MemorySegment>[] writers = new BlockChannelWriter[NUM_CHANNELS];
    BlockChannelReader<MemorySegment>[] readers = new BlockChannelReader[NUM_CHANNELS];
    ChannelWriterOutputView[] outs = new ChannelWriterOutputView[NUM_CHANNELS];
    ChannelReaderInputView[] ins = new ChannelReaderInputView[NUM_CHANNELS];
    int[] writingCounters = new int[NUM_CHANNELS];
    int[] readingCounters = new int[NUM_CHANNELS];
    // instantiate the channels and writers
    for (int i = 0; i < NUM_CHANNELS; i++) {
        ids[i] = this.ioManager.createChannel();
        writers[i] = this.ioManager.createBlockChannelWriter(ids[i]);
        List<MemorySegment> memSegs = this.memoryManager.allocatePages(memOwner, rnd.nextInt(MAXIMUM_NUMBER_OF_SEGMENTS_PER_CHANNEL - 1) + 1);
        outs[i] = new ChannelWriterOutputView(writers[i], memSegs, this.memoryManager.getPageSize());
    }
    Value val = new Value();
    for (int i = 0; i < NUMBERS_TO_BE_WRITTEN; i++) {
        int channel = skewedSample(rnd, NUM_CHANNELS - 1);
        val.value = String.valueOf(writingCounters[channel]++);
        val.write(outs[channel]);
    }
    // close all writers
    for (int i = 0; i < NUM_CHANNELS; i++) {
        this.memoryManager.release(outs[i].close());
    }
    outs = null;
    writers = null;
    // instantiate the readers for sequential read
    for (int i = 0; i < NUM_CHANNELS; i++) {
        List<MemorySegment> memSegs = this.memoryManager.allocatePages(memOwner, rnd.nextInt(MAXIMUM_NUMBER_OF_SEGMENTS_PER_CHANNEL - 1) + 1);
        final BlockChannelReader<MemorySegment> reader = this.ioManager.createBlockChannelReader(ids[i]);
        final ChannelReaderInputView in = new ChannelReaderInputView(reader, memSegs, false);
        int nextVal = 0;
        try {
            while (true) {
                val.read(in);
                int intValue = 0;
                try {
                    intValue = Integer.parseInt(val.value);
                } catch (NumberFormatException nfex) {
                    Assert.fail("Invalid value read from reader. Valid decimal number expected.");
                }
                Assert.assertEquals("Written and read values do not match during sequential read.", nextVal, intValue);
                nextVal++;
            }
        } catch (EOFException eofex) {
        // expected
        }
        Assert.assertEquals("NUmber of written numbers differs from number of read numbers.", writingCounters[i], nextVal);
        this.memoryManager.release(in.close());
    }
    // instantiate the readers
    for (int i = 0; i < NUM_CHANNELS; i++) {
        List<MemorySegment> memSegs = this.memoryManager.allocatePages(memOwner, rnd.nextInt(MAXIMUM_NUMBER_OF_SEGMENTS_PER_CHANNEL - 1) + 1);
        readers[i] = this.ioManager.createBlockChannelReader(ids[i]);
        ins[i] = new ChannelReaderInputView(readers[i], memSegs, false);
    }
    // read a lot of values in a mixed order from the channels
    for (int i = 0; i < NUMBERS_TO_BE_WRITTEN; i++) {
        while (true) {
            final int channel = skewedSample(rnd, NUM_CHANNELS - 1);
            if (ins[channel] != null) {
                try {
                    val.read(ins[channel]);
                    int intValue;
                    try {
                        intValue = Integer.parseInt(val.value);
                    } catch (NumberFormatException nfex) {
                        Assert.fail("Invalid value read from reader. Valid decimal number expected.");
                        return;
                    }
                    Assert.assertEquals("Written and read values do not match.", readingCounters[channel]++, intValue);
                    break;
                } catch (EOFException eofex) {
                    this.memoryManager.release(ins[channel].close());
                    ins[channel] = null;
                }
            }
        }
    }
    // close all readers
    for (int i = 0; i < NUM_CHANNELS; i++) {
        if (ins[i] != null) {
            this.memoryManager.release(ins[i].close());
        }
        readers[i].closeAndDelete();
    }
    ins = null;
    readers = null;
    // check that files are deleted
    for (int i = 0; i < NUM_CHANNELS; i++) {
        File f = new File(ids[i].getPath());
        Assert.assertFalse("Channel file has not been deleted.", f.exists());
    }
}

Example 4 with AbstractInvokable

use of org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable in project flink by apache.

the class Task method notifyCheckpointComplete.

public void notifyCheckpointComplete(final long checkpointID) {
    AbstractInvokable invokable = this.invokable;
    if (executionState == ExecutionState.RUNNING && invokable != null) {
        if (invokable instanceof StatefulTask) {
            // build a local closure
            final StatefulTask statefulTask = (StatefulTask) invokable;
            final String taskName = taskNameWithSubtask;
            Runnable runnable = new Runnable() {

                @Override
                public void run() {
                    try {
                        statefulTask.notifyCheckpointComplete(checkpointID);
                    } catch (Throwable t) {
                        if (getExecutionState() == ExecutionState.RUNNING) {
                            // fail task if checkpoint confirmation failed.
                            failExternally(new RuntimeException("Error while confirming checkpoint", t));
                        }
                    }
                }
            };
            executeAsyncCallRunnable(runnable, "Checkpoint Confirmation for " + taskName);
        } else {
            LOG.error("Task received a checkpoint commit notification, but is not a checkpoint committing task - {}.", taskNameWithSubtask);
        }
    } else {
        LOG.debug("Ignoring checkpoint commit notification for non-running task {}.", taskNameWithSubtask);
    }
}

Example 5 with AbstractInvokable

use of org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable in project flink by apache.

the class NonReusingBlockResettableIteratorTest method testTwelveFoldBufferedBlockResettableIterator.

@Test
public void testTwelveFoldBufferedBlockResettableIterator() throws Exception {
    final AbstractInvokable memOwner = new DummyInvokable();
    // create the resettable Iterator
    final NonReusingBlockResettableIterator<Record> iterator = new NonReusingBlockResettableIterator<Record>(this.memman, this.reader, this.serializer, 12, memOwner);
    // open the iterator
    iterator.open();
    // now test walking through the iterator
    int lower = 0;
    int upper = 0;
    do {
        lower = upper;
        upper = lower;
        // find the upper bound
        while (iterator.hasNext()) {
            Record target = iterator.next();
            int val = target.getField(0, IntValue.class).getValue();
            Assert.assertEquals(upper++, val);
        }
        // now reset the buffer a few times
        for (int i = 0; i < 5; ++i) {
            iterator.reset();
            int count = 0;
            while (iterator.hasNext()) {
                Record target = iterator.next();
                int val = target.getField(0, IntValue.class).getValue();
                Assert.assertEquals(lower + (count++), val);
            }
            Assert.assertEquals(upper - lower, count);
        }
    } while (iterator.nextBlock());
    Assert.assertEquals(NUM_VALUES, upper);
    // close the iterator
    iterator.close();
}