Examples with AbstractInvokable org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable used on opensource projects

Example 21 with AbstractInvokable

use of org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable in project flink by apache.

the class Task method triggerCheckpointBarrier.

// ------------------------------------------------------------------------
//  Notifications on the invokable
// ------------------------------------------------------------------------
/**
	 * Calls the invokable to trigger a checkpoint, if the invokable implements the interface
	 * {@link StatefulTask}.
	 * 
	 * @param checkpointID The ID identifying the checkpoint.
	 * @param checkpointTimestamp The timestamp associated with the checkpoint.
	 * @param checkpointOptions Options for performing this checkpoint.
	 */
public void triggerCheckpointBarrier(final long checkpointID, long checkpointTimestamp, final CheckpointOptions checkpointOptions) {
    final AbstractInvokable invokable = this.invokable;
    final CheckpointMetaData checkpointMetaData = new CheckpointMetaData(checkpointID, checkpointTimestamp);
    if (executionState == ExecutionState.RUNNING && invokable != null) {
        if (invokable instanceof StatefulTask) {
            // build a local closure
            final StatefulTask statefulTask = (StatefulTask) invokable;
            final String taskName = taskNameWithSubtask;
            Runnable runnable = new Runnable() {

                @Override
                public void run() {
                    // activate safety net for checkpointing thread
                    LOG.debug("Creating FileSystem stream leak safety net for {}", Thread.currentThread().getName());
                    FileSystemSafetyNet.initializeSafetyNetForThread();
                    try {
                        boolean success = statefulTask.triggerCheckpoint(checkpointMetaData, checkpointOptions);
                        if (!success) {
                            checkpointResponder.declineCheckpoint(getJobID(), getExecutionId(), checkpointID, new CheckpointDeclineTaskNotReadyException(taskName));
                        }
                    } catch (Throwable t) {
                        if (getExecutionState() == ExecutionState.RUNNING) {
                            failExternally(new Exception("Error while triggering checkpoint " + checkpointID + " for " + taskNameWithSubtask, t));
                        } else {
                            LOG.debug("Encountered error while triggering checkpoint {} for " + "{} ({}) while being not in state running.", checkpointID, taskNameWithSubtask, executionId, t);
                        }
                    } finally {
                        // close and de-activate safety net for checkpointing thread
                        LOG.debug("Ensuring all FileSystem streams are closed for {}", Thread.currentThread().getName());
                        FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread();
                    }
                }
            };
            executeAsyncCallRunnable(runnable, String.format("Checkpoint Trigger for %s (%s).", taskNameWithSubtask, executionId));
        } else {
            checkpointResponder.declineCheckpoint(jobId, executionId, checkpointID, new CheckpointDeclineTaskNotCheckpointingException(taskNameWithSubtask));
            LOG.error("Task received a checkpoint request, but is not a checkpointing task - {} ({}).", taskNameWithSubtask, executionId);
        }
    } else {
        LOG.debug("Declining checkpoint request for non-running task {} ({}).", taskNameWithSubtask, executionId);
        // send back a message that we did not do the checkpoint
        checkpointResponder.declineCheckpoint(jobId, executionId, checkpointID, new CheckpointDeclineTaskNotReadyException(taskNameWithSubtask));
    }
}

Example 22 with AbstractInvokable

use of org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable in project flink by apache.

the class LargeRecordHandlerTest method testRecordHandlerSingleKey.

@Test
public void testRecordHandlerSingleKey() {
    final IOManager ioMan = new IOManagerAsync();
    final int PAGE_SIZE = 4 * 1024;
    final int NUM_PAGES = 24;
    final int NUM_RECORDS = 25000;
    try {
        final MemoryManager memMan = new MemoryManager(NUM_PAGES * PAGE_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
        final AbstractInvokable owner = new DummyInvokable();
        final List<MemorySegment> initialMemory = memMan.allocatePages(owner, 6);
        final List<MemorySegment> sortMemory = memMan.allocatePages(owner, NUM_PAGES - 6);
        final TupleTypeInfo<Tuple2<Long, String>> typeInfo = (TupleTypeInfo<Tuple2<Long, String>>) TypeInfoParser.<Tuple2<Long, String>>parse("Tuple2<Long, String>");
        final TypeSerializer<Tuple2<Long, String>> serializer = typeInfo.createSerializer(new ExecutionConfig());
        final TypeComparator<Tuple2<Long, String>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
        LargeRecordHandler<Tuple2<Long, String>> handler = new LargeRecordHandler<Tuple2<Long, String>>(serializer, comparator, ioMan, memMan, initialMemory, owner, 128);
        assertFalse(handler.hasData());
        // add the test data
        Random rnd = new Random();
        for (int i = 0; i < NUM_RECORDS; i++) {
            long val = rnd.nextLong();
            handler.addRecord(new Tuple2<Long, String>(val, String.valueOf(val)));
            assertTrue(handler.hasData());
        }
        MutableObjectIterator<Tuple2<Long, String>> sorted = handler.finishWriteAndSortKeys(sortMemory);
        try {
            handler.addRecord(new Tuple2<Long, String>(92L, "peter pepper"));
            fail("should throw an exception");
        } catch (IllegalStateException e) {
        // expected
        }
        Tuple2<Long, String> previous = null;
        Tuple2<Long, String> next;
        while ((next = sorted.next(null)) != null) {
            // key and value must be equal
            assertTrue(next.f0.equals(Long.parseLong(next.f1)));
            // order must be correct
            if (previous != null) {
                assertTrue(previous.f0 <= next.f0);
            }
            previous = next;
        }
        handler.close();
        assertFalse(handler.hasData());
        handler.close();
        try {
            handler.addRecord(new Tuple2<Long, String>(92L, "peter pepper"));
            fail("should throw an exception");
        } catch (IllegalStateException e) {
        // expected
        }
        assertTrue(memMan.verifyEmpty());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        ioMan.shutdown();
    }
}

Example 23 with AbstractInvokable

use of org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable in project flink by apache.

the class LargeRecordHandlerTest method testEmptyRecordHandler.

@Test
public void testEmptyRecordHandler() {
    final IOManager ioMan = new IOManagerAsync();
    final int PAGE_SIZE = 4 * 1024;
    final int NUM_PAGES = 50;
    try {
        final MemoryManager memMan = new MemoryManager(NUM_PAGES * PAGE_SIZE, 1, PAGE_SIZE, MemoryType.HEAP, true);
        final AbstractInvokable owner = new DummyInvokable();
        final List<MemorySegment> memory = memMan.allocatePages(owner, NUM_PAGES);
        final TupleTypeInfo<Tuple2<Long, String>> typeInfo = (TupleTypeInfo<Tuple2<Long, String>>) TypeInfoParser.<Tuple2<Long, String>>parse("Tuple2<Long, String>");
        final TypeSerializer<Tuple2<Long, String>> serializer = typeInfo.createSerializer(new ExecutionConfig());
        final TypeComparator<Tuple2<Long, String>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
        LargeRecordHandler<Tuple2<Long, String>> handler = new LargeRecordHandler<Tuple2<Long, String>>(serializer, comparator, ioMan, memMan, memory, owner, 128);
        assertFalse(handler.hasData());
        handler.close();
        assertFalse(handler.hasData());
        handler.close();
        try {
            handler.addRecord(new Tuple2<Long, String>(92L, "peter pepper"));
            fail("should throw an exception");
        } catch (IllegalStateException e) {
        // expected
        }
        assertTrue(memMan.verifyEmpty());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    } finally {
        ioMan.shutdown();
    }
}

Example 24 with AbstractInvokable

use of org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable in project flink by apache.

the class ReusingBlockResettableIteratorTest method testDoubleBufferedBlockResettableIterator.

@Test
public void testDoubleBufferedBlockResettableIterator() throws Exception {
    final AbstractInvokable memOwner = new DummyInvokable();
    // create the resettable Iterator
    final ReusingBlockResettableIterator<Record> iterator = new ReusingBlockResettableIterator<Record>(this.memman, this.reader, this.serializer, 2, memOwner);
    // open the iterator
    iterator.open();
    // now test walking through the iterator
    int lower = 0;
    int upper = 0;
    do {
        lower = upper;
        upper = lower;
        // find the upper bound
        while (iterator.hasNext()) {
            Record target = iterator.next();
            int val = target.getField(0, IntValue.class).getValue();
            Assert.assertEquals(upper++, val);
        }
        // now reset the buffer a few times
        for (int i = 0; i < 5; ++i) {
            iterator.reset();
            int count = 0;
            while (iterator.hasNext()) {
                Record target = iterator.next();
                int val = target.getField(0, IntValue.class).getValue();
                Assert.assertEquals(lower + (count++), val);
            }
            Assert.assertEquals(upper - lower, count);
        }
    } while (iterator.nextBlock());
    Assert.assertEquals(NUM_VALUES, upper);
    // close the iterator
    iterator.close();
}

Example 25 with AbstractInvokable

use of org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable in project flink by apache.

the class SpillingResettableMutableObjectIteratorTest method testResettableIterator.

/**
	 * Tests the resettable iterator with too little memory, so that the data
	 * has to be written to disk.
	 */
@Test
public void testResettableIterator() {
    try {
        final AbstractInvokable memOwner = new DummyInvokable();
        // create the resettable Iterator
        SpillingResettableMutableObjectIterator<Record> iterator = new SpillingResettableMutableObjectIterator<Record>(this.reader, this.serializer, this.memman, this.ioman, 2, memOwner);
        // open the iterator
        iterator.open();
        // now test walking through the iterator
        int count = 0;
        Record target = new Record();
        while ((target = iterator.next(target)) != null) {
            Assert.assertEquals("In initial run, element " + count + " does not match expected value!", count++, target.getField(0, IntValue.class).getValue());
        }
        Assert.assertEquals("Too few elements were deserialzied in initial run!", NUM_TESTRECORDS, count);
        // test resetting the iterator a few times
        for (int j = 0; j < 10; ++j) {
            count = 0;
            iterator.reset();
            target = new Record();
            // now we should get the same results
            while ((target = iterator.next(target)) != null) {
                Assert.assertEquals("After reset nr. " + j + 1 + " element " + count + " does not match expected value!", count++, target.getField(0, IntValue.class).getValue());
            }
            Assert.assertEquals("Too few elements were deserialzied after reset nr. " + j + 1 + "!", NUM_TESTRECORDS, count);
        }
        // close the iterator
        iterator.close();
    } catch (Exception ex) {
        ex.printStackTrace();
        Assert.fail("Test encountered an exception.");
    }
}