Examples with RunnableWithException org.apache.flink.util.function.RunnableWithException used on opensource projects

Example 1 with RunnableWithException

use of org.apache.flink.util.function.RunnableWithException in project flink by apache.

the class ContinuousFileReaderOperator method cleanUp.

private void cleanUp() throws Exception {
    LOG.debug("cleanup, state={}", state);
    RunnableWithException[] runClose = { () -> sourceContext.close(), () -> format.close(), () -> {
        if (this.format instanceof RichInputFormat) {
            ((RichInputFormat<?, ?>) this.format).closeInputFormat();
        }
    } };
    Exception firstException = null;
    for (RunnableWithException r : runClose) {
        try {
            r.run();
        } catch (Exception e) {
            firstException = ExceptionUtils.firstOrSuppressed(e, firstException);
        }
    }
    currentSplit = null;
    if (firstException != null) {
        throw firstException;
    }
}

Example 2 with RunnableWithException

use of org.apache.flink.util.function.RunnableWithException in project flink by apache.

the class StreamTaskTest method testProcessWithUnAvailableOutput.

@Test
public void testProcessWithUnAvailableOutput() throws Exception {
    final long sleepTimeOutsideMail = 42;
    final long sleepTimeInsideMail = 44;
    @Nullable WaitingThread waitingThread = null;
    try (final MockEnvironment environment = setupEnvironment(true, false)) {
        final int numberOfProcessCalls = 10;
        final AvailabilityTestInputProcessor inputProcessor = new AvailabilityTestInputProcessor(numberOfProcessCalls);
        final StreamTask task = new MockStreamTaskBuilder(environment).setStreamInputProcessor(inputProcessor).build();
        final MailboxExecutor executor = task.mailboxProcessor.getMainMailboxExecutor();
        TaskIOMetricGroup ioMetricGroup = task.getEnvironment().getMetricGroup().getIOMetricGroup();
        final RunnableWithException completeFutureTask = () -> {
            assertEquals(1, inputProcessor.currentNumProcessCalls);
            assertFalse(task.mailboxProcessor.isDefaultActionAvailable());
            environment.getWriter(1).getAvailableFuture().complete(null);
        };
        waitingThread = new WaitingThread(executor, completeFutureTask, sleepTimeInsideMail, sleepTimeOutsideMail, ioMetricGroup.getSoftBackPressuredTimePerSecond());
        // Make sure WaitingThread is started after Task starts processing.
        executor.submit(waitingThread::start, "This task will submit another task to execute after processing input once.");
        long startTs = System.currentTimeMillis();
        task.invoke();
        long totalDuration = System.currentTimeMillis() - startTs;
        assertThat(ioMetricGroup.getSoftBackPressuredTimePerSecond().getCount(), greaterThanOrEqualTo(sleepTimeOutsideMail));
        assertThat(ioMetricGroup.getSoftBackPressuredTimePerSecond().getCount(), Matchers.lessThanOrEqualTo(totalDuration - sleepTimeInsideMail));
        assertThat(ioMetricGroup.getIdleTimeMsPerSecond().getCount(), is(0L));
        assertEquals(numberOfProcessCalls, inputProcessor.currentNumProcessCalls);
    } finally {
        if (waitingThread != null) {
            waitingThread.join();
        }
    }
}

Example 3 with RunnableWithException

use of org.apache.flink.util.function.RunnableWithException in project flink by apache.

the class ContinuousFileProcessingTest method testFileReadingOperatorWithIngestionTime.

@Test
public void testFileReadingOperatorWithIngestionTime() throws Exception {
    String testBasePath = hdfsURI + "/" + UUID.randomUUID() + "/";
    Set<org.apache.hadoop.fs.Path> filesCreated = new HashSet<>();
    Map<Integer, String> expectedFileContents = new HashMap<>();
    Map<String, Long> modTimes = new HashMap<>();
    for (int i = 0; i < NO_OF_FILES; i++) {
        Tuple2<org.apache.hadoop.fs.Path, String> file = createFileAndFillWithData(testBasePath, "file", i, "This is test line.");
        filesCreated.add(file.f0);
        modTimes.put(file.f0.getName(), hdfs.getFileStatus(file.f0).getModificationTime());
        expectedFileContents.put(i, file.f1);
    }
    TextInputFormat format = new TextInputFormat(new Path(testBasePath));
    final long watermarkInterval = 10;
    final OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> tester = createHarness(format);
    SteppingMailboxProcessor localMailbox = createLocalMailbox(tester);
    tester.getExecutionConfig().setAutoWatermarkInterval(watermarkInterval);
    tester.setTimeCharacteristic(TimeCharacteristic.IngestionTime);
    tester.open();
    Assert.assertEquals(TimeCharacteristic.IngestionTime, tester.getTimeCharacteristic());
    tester.setProcessingTime(201);
    // test that watermarks are correctly emitted
    ConcurrentLinkedQueue<Object> output = tester.getOutput();
    while (output.isEmpty()) {
        localMailbox.runMailboxStep();
    }
    Assert.assertTrue(output.toString(), output.peek() instanceof Watermark);
    Assert.assertEquals(200, ((Watermark) output.poll()).getTimestamp());
    tester.setProcessingTime(301);
    Assert.assertTrue(output.peek() instanceof Watermark);
    Assert.assertEquals(300, ((Watermark) output.poll()).getTimestamp());
    tester.setProcessingTime(401);
    Assert.assertTrue(output.peek() instanceof Watermark);
    Assert.assertEquals(400, ((Watermark) output.poll()).getTimestamp());
    tester.setProcessingTime(501);
    Assert.assertTrue(output.peek() instanceof Watermark);
    Assert.assertEquals(500, ((Watermark) output.poll()).getTimestamp());
    Assert.assertTrue(output.isEmpty());
    // create the necessary splits for the test
    FileInputSplit[] splits = format.createInputSplits(tester.getExecutionConfig().getParallelism());
    // and feed them to the operator
    Map<Integer, List<String>> actualFileContents = new HashMap<>();
    long lastSeenWatermark = Long.MIN_VALUE;
    // counter for the lines read from the splits
    int lineCounter = 0;
    int watermarkCounter = 0;
    for (FileInputSplit split : splits) {
        // set the next "current processing time".
        long nextTimestamp = tester.getProcessingTime() + watermarkInterval;
        tester.setProcessingTime(nextTimestamp);
        // send the next split to be read and wait until it is fully read, the +1 is for the
        // watermark.
        RunnableWithException runnableWithException = () -> tester.processElement(new StreamRecord<>(new TimestampedFileInputSplit(modTimes.get(split.getPath().getName()), split.getSplitNumber(), split.getPath(), split.getStart(), split.getLength(), split.getHostnames())));
        runnableWithException.run();
        // BUT THIS IS JUST FOR THIS TEST
        while (tester.getOutput().isEmpty() || tester.getOutput().size() != (LINES_PER_FILE + 1)) {
            localMailbox.runMailboxStep();
        }
        // verify that the results are the expected
        for (Object line : tester.getOutput()) {
            if (line instanceof StreamRecord) {
                @SuppressWarnings("unchecked") StreamRecord<String> element = (StreamRecord<String>) line;
                lineCounter++;
                Assert.assertEquals(nextTimestamp, element.getTimestamp());
                int fileIdx = Character.getNumericValue(element.getValue().charAt(0));
                List<String> content = actualFileContents.get(fileIdx);
                if (content == null) {
                    content = new ArrayList<>();
                    actualFileContents.put(fileIdx, content);
                }
                content.add(element.getValue() + "\n");
            } else if (line instanceof Watermark) {
                long watermark = ((Watermark) line).getTimestamp();
                Assert.assertEquals(nextTimestamp - (nextTimestamp % watermarkInterval), watermark);
                Assert.assertTrue(watermark > lastSeenWatermark);
                watermarkCounter++;
                lastSeenWatermark = watermark;
            } else {
                Assert.fail("Unknown element in the list.");
            }
        }
        // clean the output to be ready for the next split
        tester.getOutput().clear();
    }
    // now we are processing one split after the other,
    // so all the elements must be here by now.
    Assert.assertEquals(NO_OF_FILES * LINES_PER_FILE, lineCounter);
    // because we expect one watermark per split.
    Assert.assertEquals(splits.length, watermarkCounter);
    // then close the reader gracefully so that the Long.MAX watermark is emitted
    synchronized (tester.getCheckpointLock()) {
        tester.close();
    }
    for (org.apache.hadoop.fs.Path file : filesCreated) {
        hdfs.delete(file, false);
    }
    // check if the last element is the LongMax watermark (by now this must be the only element)
    Assert.assertEquals(1, tester.getOutput().size());
    Assert.assertTrue(tester.getOutput().peek() instanceof Watermark);
    Assert.assertEquals(Long.MAX_VALUE, ((Watermark) tester.getOutput().poll()).getTimestamp());
    // check if the elements are the expected ones.
    Assert.assertEquals(expectedFileContents.size(), actualFileContents.size());
    for (Integer fileIdx : expectedFileContents.keySet()) {
        Assert.assertTrue("file" + fileIdx + " not found", actualFileContents.keySet().contains(fileIdx));
        List<String> cntnt = actualFileContents.get(fileIdx);
        Collections.sort(cntnt, new Comparator<String>() {

            @Override
            public int compare(String o1, String o2) {
                return getLineNo(o1) - getLineNo(o2);
            }
        });
        StringBuilder cntntStr = new StringBuilder();
        for (String line : cntnt) {
            cntntStr.append(line);
        }
        Assert.assertEquals(expectedFileContents.get(fileIdx), cntntStr.toString());
    }
}

Example 4 with RunnableWithException

use of org.apache.flink.util.function.RunnableWithException in project flink by apache.

the class StreamTaskTest method testProcessWithUnAvailableInput.

@Test
public void testProcessWithUnAvailableInput() throws Exception {
    final long sleepTimeOutsideMail = 42;
    final long sleepTimeInsideMail = 44;
    @Nullable WaitingThread waitingThread = null;
    try (final MockEnvironment environment = setupEnvironment(true, true)) {
        final UnAvailableTestInputProcessor inputProcessor = new UnAvailableTestInputProcessor();
        final StreamTask task = new MockStreamTaskBuilder(environment).setStreamInputProcessor(inputProcessor).build();
        TaskIOMetricGroup ioMetricGroup = task.getEnvironment().getMetricGroup().getIOMetricGroup();
        final MailboxExecutor executor = task.mailboxProcessor.getMainMailboxExecutor();
        final RunnableWithException completeFutureTask = () -> {
            inputProcessor.availabilityProvider.getUnavailableToResetAvailable().complete(null);
        };
        waitingThread = new WaitingThread(executor, completeFutureTask, sleepTimeInsideMail, sleepTimeOutsideMail, ioMetricGroup.getIdleTimeMsPerSecond());
        // Make sure WaitingThread is started after Task starts processing.
        executor.submit(waitingThread::start, "Start WaitingThread after Task starts processing input.");
        SystemClock clock = SystemClock.getInstance();
        long startTs = clock.absoluteTimeMillis();
        task.invoke();
        long totalDuration = clock.absoluteTimeMillis() - startTs;
        assertThat(ioMetricGroup.getIdleTimeMsPerSecond().getCount(), greaterThanOrEqualTo(sleepTimeOutsideMail));
        assertThat(ioMetricGroup.getIdleTimeMsPerSecond().getCount(), Matchers.lessThanOrEqualTo(totalDuration - sleepTimeInsideMail));
        assertThat(ioMetricGroup.getSoftBackPressuredTimePerSecond().getCount(), is(0L));
        assertThat(ioMetricGroup.getHardBackPressuredTimePerSecond().getCount(), is(0L));
    } finally {
        if (waitingThread != null) {
            waitingThread.join();
        }
    }
}

Example 5 with RunnableWithException

use of org.apache.flink.util.function.RunnableWithException in project flink by apache.

the class TaskMailboxImplTest method testUnblocksInternal.

private void testUnblocksInternal(RunnableWithException testMethod, Consumer<TaskMailbox> unblockMethod) throws InterruptedException {
    final Thread[] blockedThreads = new Thread[8];
    final Exception[] exceptions = new Exception[blockedThreads.length];
    CountDownLatch countDownLatch = new CountDownLatch(blockedThreads.length);
    for (int i = 0; i < blockedThreads.length; ++i) {
        final int id = i;
        Thread blocked = new Thread(() -> {
            try {
                countDownLatch.countDown();
                while (true) {
                    testMethod.run();
                }
            } catch (Exception ex) {
                exceptions[id] = ex;
            }
        });
        blockedThreads[i] = blocked;
        blocked.start();
    }
    countDownLatch.await();
    unblockMethod.accept(taskMailbox);
    for (Thread blockedThread : blockedThreads) {
        blockedThread.join();
    }
    for (Exception exception : exceptions) {
        assertEquals(MailboxClosedException.class, exception.getClass());
    }
}