Examples with OneShotLatch org.apache.flink.core.testutils.OneShotLatch used on opensource projects

Example 26 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class InitOutputPathTest method runTest.

private void runTest(final boolean useAwaits) throws Exception {
    final File tempFile = tempDir.newFile();
    final Path path1 = new Path(tempFile.getAbsolutePath(), "1");
    final Path path2 = new Path(tempFile.getAbsolutePath(), "2");
    final OneShotLatch deleteAwaitLatch1 = new OneShotLatch();
    final OneShotLatch deleteAwaitLatch2 = new OneShotLatch();
    final OneShotLatch mkdirsAwaitLatch1 = new OneShotLatch();
    final OneShotLatch mkdirsAwaitLatch2 = new OneShotLatch();
    final OneShotLatch deleteTriggerLatch1 = new OneShotLatch();
    final OneShotLatch deletetriggerLatch2 = new OneShotLatch();
    final OneShotLatch mkdirsTriggerLatch1 = new OneShotLatch();
    final OneShotLatch mkdirsTriggerLatch2 = new OneShotLatch();
    final OneShotLatch createAwaitLatch = new OneShotLatch();
    final OneShotLatch createTriggerLatch = new OneShotLatch();
    // this "new LocalDataOutputStream()" is in the end called by the async threads
    whenNew(LocalDataOutputStream.class).withAnyArguments().thenAnswer(new Answer<LocalDataOutputStream>() {

        @Override
        public LocalDataOutputStream answer(InvocationOnMock invocation) throws Throwable {
            createAwaitLatch.trigger();
            createTriggerLatch.await();
            final File file = (File) invocation.getArguments()[0];
            return new LocalDataOutputStream(file);
        }
    });
    final LocalFileSystem fs1 = new SyncedFileSystem(deleteAwaitLatch1, mkdirsAwaitLatch1, deleteTriggerLatch1, mkdirsTriggerLatch1);
    final LocalFileSystem fs2 = new SyncedFileSystem(deleteAwaitLatch2, mkdirsAwaitLatch2, deletetriggerLatch2, mkdirsTriggerLatch2);
    // start the concurrent file creators
    FileCreator thread1 = new FileCreator(fs1, path1);
    FileCreator thread2 = new FileCreator(fs2, path2);
    thread1.start();
    thread2.start();
    // wait until they both decide to delete the directory
    if (useAwaits) {
        deleteAwaitLatch1.await();
        deleteAwaitLatch2.await();
    } else {
        Thread.sleep(5);
    }
    // now send off #1 to delete the directory (it will pass the 'mkdirs' fast) and wait to create the file
    mkdirsTriggerLatch1.trigger();
    deleteTriggerLatch1.trigger();
    if (useAwaits) {
        createAwaitLatch.await();
    } else {
        // this needs a bit more sleep time, because here mockito is working
        Thread.sleep(100);
    }
    // now send off #2 to delete the directory - it waits at 'mkdirs'
    deletetriggerLatch2.trigger();
    if (useAwaits) {
        mkdirsAwaitLatch2.await();
    } else {
        Thread.sleep(5);
    }
    // let #1 try to create the file and see if it succeeded
    createTriggerLatch.trigger();
    if (useAwaits) {
        thread1.sync();
    } else {
        Thread.sleep(5);
    }
    // now let #1 finish up
    mkdirsTriggerLatch2.trigger();
    thread1.sync();
    thread2.sync();
}

Example 27 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class ContinuousFileProcessingTest method testReaderSnapshotRestore.

@Test
public void testReaderSnapshotRestore() throws Exception {
    String testBasePath = hdfsURI + "/" + UUID.randomUUID() + "/";
    TimestampedFileInputSplit split1 = new TimestampedFileInputSplit(0, 3, new Path("test/test1"), 0, 100, null);
    TimestampedFileInputSplit split2 = new TimestampedFileInputSplit(10, 2, new Path("test/test2"), 101, 200, null);
    TimestampedFileInputSplit split3 = new TimestampedFileInputSplit(10, 1, new Path("test/test2"), 0, 100, null);
    TimestampedFileInputSplit split4 = new TimestampedFileInputSplit(11, 0, new Path("test/test3"), 0, 100, null);
    final OneShotLatch latch = new OneShotLatch();
    BlockingFileInputFormat format = new BlockingFileInputFormat(latch, new Path(testBasePath));
    TypeInformation<FileInputSplit> typeInfo = TypeExtractor.getInputFormatTypes(format);
    ContinuousFileReaderOperator<FileInputSplit> initReader = new ContinuousFileReaderOperator<>(format);
    initReader.setOutputType(typeInfo, new ExecutionConfig());
    OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, FileInputSplit> initTestInstance = new OneInputStreamOperatorTestHarness<>(initReader);
    initTestInstance.setTimeCharacteristic(TimeCharacteristic.EventTime);
    initTestInstance.open();
    // create some state in the reader
    initTestInstance.processElement(new StreamRecord<>(split1));
    initTestInstance.processElement(new StreamRecord<>(split2));
    initTestInstance.processElement(new StreamRecord<>(split3));
    initTestInstance.processElement(new StreamRecord<>(split4));
    // take a snapshot of the operator's state. This will be used
    // to initialize another reader and compare the results of the
    // two operators.
    final OperatorStateHandles snapshot;
    synchronized (initTestInstance.getCheckpointLock()) {
        snapshot = initTestInstance.snapshot(0L, 0L);
    }
    ContinuousFileReaderOperator<FileInputSplit> restoredReader = new ContinuousFileReaderOperator<>(new BlockingFileInputFormat(latch, new Path(testBasePath)));
    restoredReader.setOutputType(typeInfo, new ExecutionConfig());
    OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, FileInputSplit> restoredTestInstance = new OneInputStreamOperatorTestHarness<>(restoredReader);
    restoredTestInstance.setTimeCharacteristic(TimeCharacteristic.EventTime);
    restoredTestInstance.initializeState(snapshot);
    restoredTestInstance.open();
    // now let computation start
    latch.trigger();
    synchronized (initTestInstance.getCheckpointLock()) {
        initTestInstance.close();
    }
    synchronized (restoredTestInstance.getCheckpointLock()) {
        restoredTestInstance.close();
    }
    FileInputSplit fsSplit1 = createSplitFromTimestampedSplit(split1);
    FileInputSplit fsSplit2 = createSplitFromTimestampedSplit(split2);
    FileInputSplit fsSplit3 = createSplitFromTimestampedSplit(split3);
    FileInputSplit fsSplit4 = createSplitFromTimestampedSplit(split4);
    // compare if the results contain what they should contain and also if
    // they are the same, as they should.
    Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit1)));
    Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit2)));
    Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit3)));
    Assert.assertTrue(initTestInstance.getOutput().contains(new StreamRecord<>(fsSplit4)));
    Assert.assertArrayEquals(initTestInstance.getOutput().toArray(), restoredTestInstance.getOutput().toArray());
}

Example 28 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class ContinuousFileProcessingTest method testProcessContinuously.

@Test
public void testProcessContinuously() throws Exception {
    String testBasePath = hdfsURI + "/" + UUID.randomUUID() + "/";
    final OneShotLatch latch = new OneShotLatch();
    // create a single file in the directory
    Tuple2<org.apache.hadoop.fs.Path, String> bootstrap = createFileAndFillWithData(testBasePath, "file", NO_OF_FILES + 1, "This is test line.");
    Assert.assertTrue(hdfs.exists(bootstrap.f0));
    final Set<String> filesToBeRead = new TreeSet<>();
    filesToBeRead.add(bootstrap.f0.getName());
    TextInputFormat format = new TextInputFormat(new Path(testBasePath));
    format.setFilesFilter(FilePathFilter.createDefaultFilter());
    final ContinuousFileMonitoringFunction<String> monitoringFunction = new ContinuousFileMonitoringFunction<>(format, FileProcessingMode.PROCESS_CONTINUOUSLY, 1, INTERVAL);
    // 1 for the bootstrap + NO_OF_FILES
    final int totalNoOfFilesToBeRead = NO_OF_FILES + 1;
    final FileVerifyingSourceContext context = new FileVerifyingSourceContext(latch, monitoringFunction, 1, totalNoOfFilesToBeRead);
    final Thread t = new Thread() {

        @Override
        public void run() {
            try {
                monitoringFunction.open(new Configuration());
                monitoringFunction.run(context);
            } catch (Exception e) {
                Assert.fail(e.getMessage());
            }
        }
    };
    t.start();
    if (!latch.isTriggered()) {
        latch.await();
    }
    // create some additional files that will be processed in the case of PROCESS_CONTINUOUSLY
    final org.apache.hadoop.fs.Path[] filesCreated = new org.apache.hadoop.fs.Path[NO_OF_FILES];
    for (int i = 0; i < NO_OF_FILES; i++) {
        Tuple2<org.apache.hadoop.fs.Path, String> file = createFileAndFillWithData(testBasePath, "file", i, "This is test line.");
        filesCreated[i] = file.f0;
        filesToBeRead.add(file.f0.getName());
    }
    // wait until the monitoring thread exits
    t.join();
    Assert.assertArrayEquals(filesToBeRead.toArray(), context.getSeenFiles().toArray());
    // finally delete the files created for the test.
    hdfs.delete(bootstrap.f0, false);
    for (org.apache.hadoop.fs.Path path : filesCreated) {
        hdfs.delete(path, false);
    }
}

Example 29 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class ContinuousFileProcessingRescalingTest method testReaderScalingUp.

@Test
public void testReaderScalingUp() throws Exception {
    // simulates the scenario of scaling up from 1 to 2 instances
    final OneShotLatch waitingLatch1 = new OneShotLatch();
    final OneShotLatch triggerLatch1 = new OneShotLatch();
    BlockingFileInputFormat format1 = new BlockingFileInputFormat(triggerLatch1, waitingLatch1, new Path("test"), 20, 5);
    FileInputSplit[] splits = format1.createInputSplits(2);
    OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness1 = getTestHarness(format1, 1, 0);
    testHarness1.open();
    testHarness1.processElement(new StreamRecord<>(getTimestampedSplit(0, splits[0])));
    testHarness1.processElement(new StreamRecord<>(getTimestampedSplit(1, splits[1])));
    // wait until its arrives to element 5
    if (!triggerLatch1.isTriggered()) {
        triggerLatch1.await();
    }
    // this will be the state shared by the 2 new instances.
    OperatorStateHandles snapshot = testHarness1.snapshot(0, 0);
    // 1) clear the output of instance so that we can compare it with one created by the new instances, and
    // 2) let the operator process the rest of its state
    testHarness1.getOutput().clear();
    waitingLatch1.trigger();
    // create the second instance and let it process the second split till element 15
    final OneShotLatch triggerLatch2 = new OneShotLatch();
    final OneShotLatch waitingLatch2 = new OneShotLatch();
    BlockingFileInputFormat format2 = new BlockingFileInputFormat(triggerLatch2, waitingLatch2, new Path("test"), 20, 15);
    OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness2 = getTestHarness(format2, 2, 0);
    testHarness2.setup();
    testHarness2.initializeState(snapshot);
    testHarness2.open();
    BlockingFileInputFormat format3 = new BlockingFileInputFormat(triggerLatch2, waitingLatch2, new Path("test"), 20, 15);
    OneInputStreamOperatorTestHarness<TimestampedFileInputSplit, String> testHarness3 = getTestHarness(format3, 2, 1);
    testHarness3.setup();
    testHarness3.initializeState(snapshot);
    testHarness3.open();
    triggerLatch2.trigger();
    waitingLatch2.trigger();
    // and wait for the processing to finish
    synchronized (testHarness1.getCheckpointLock()) {
        testHarness1.close();
    }
    synchronized (testHarness2.getCheckpointLock()) {
        testHarness2.close();
    }
    synchronized (testHarness3.getCheckpointLock()) {
        testHarness3.close();
    }
    Queue<Object> expectedResult = new ArrayDeque<>();
    putElementsInQ(expectedResult, testHarness1.getOutput());
    Queue<Object> actualResult = new ArrayDeque<>();
    putElementsInQ(actualResult, testHarness2.getOutput());
    putElementsInQ(actualResult, testHarness3.getOutput());
    Assert.assertEquals(35, actualResult.size());
    Assert.assertArrayEquals(expectedResult.toArray(), actualResult.toArray());
}

Example 30 with OneShotLatch

use of org.apache.flink.core.testutils.OneShotLatch in project flink by apache.

the class SavepointITCase method testSavepointForJobWithIteration.

@Test
public void testSavepointForJobWithIteration() throws Exception {
    for (int i = 0; i < ITER_TEST_PARALLELISM; ++i) {
        ITER_TEST_SNAPSHOT_WAIT[i] = new OneShotLatch();
        ITER_TEST_RESTORE_WAIT[i] = new OneShotLatch();
        ITER_TEST_CHECKPOINT_VERIFY[i] = 0;
    }
    TemporaryFolder folder = new TemporaryFolder();
    folder.create();
    // Temporary directory for file state backend
    final File tmpDir = folder.newFolder();
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    final IntegerStreamSource source = new IntegerStreamSource();
    IterativeStream<Integer> iteration = env.addSource(source).flatMap(new RichFlatMapFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public void flatMap(Integer in, Collector<Integer> clctr) throws Exception {
            clctr.collect(in);
        }
    }).setParallelism(ITER_TEST_PARALLELISM).keyBy(new KeySelector<Integer, Object>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Object getKey(Integer value) throws Exception {
            return value;
        }
    }).flatMap(new DuplicateFilter()).setParallelism(ITER_TEST_PARALLELISM).iterate();
    DataStream<Integer> iterationBody = iteration.map(new MapFunction<Integer, Integer>() {

        private static final long serialVersionUID = 1L;

        @Override
        public Integer map(Integer value) throws Exception {
            return value;
        }
    }).setParallelism(ITER_TEST_PARALLELISM);
    iteration.closeWith(iterationBody);
    StreamGraph streamGraph = env.getStreamGraph();
    streamGraph.setJobName("Test");
    JobGraph jobGraph = streamGraph.getJobGraph();
    Configuration config = new Configuration();
    config.addAll(jobGraph.getJobConfiguration());
    config.setLong(ConfigConstants.TASK_MANAGER_MEMORY_SIZE_KEY, -1L);
    config.setInteger(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS, 2 * jobGraph.getMaximumParallelism());
    final File checkpointDir = new File(tmpDir, "checkpoints");
    final File savepointDir = new File(tmpDir, "savepoints");
    if (!checkpointDir.mkdir() || !savepointDir.mkdirs()) {
        fail("Test setup failed: failed to create temporary directories.");
    }
    config.setString(CoreOptions.STATE_BACKEND, "filesystem");
    config.setString(FsStateBackendFactory.CHECKPOINT_DIRECTORY_URI_CONF_KEY, checkpointDir.toURI().toString());
    config.setString(FsStateBackendFactory.MEMORY_THRESHOLD_CONF_KEY, "0");
    config.setString(ConfigConstants.SAVEPOINT_DIRECTORY_KEY, savepointDir.toURI().toString());
    TestingCluster cluster = new TestingCluster(config, false);
    String savepointPath = null;
    try {
        cluster.start();
        cluster.submitJobDetached(jobGraph);
        for (OneShotLatch latch : ITER_TEST_SNAPSHOT_WAIT) {
            latch.await();
        }
        savepointPath = cluster.triggerSavepoint(jobGraph.getJobID());
        source.cancel();
        jobGraph = streamGraph.getJobGraph();
        jobGraph.setSavepointRestoreSettings(SavepointRestoreSettings.forPath(savepointPath));
        cluster.submitJobDetached(jobGraph);
        for (OneShotLatch latch : ITER_TEST_RESTORE_WAIT) {
            latch.await();
        }
        source.cancel();
    } finally {
        if (null != savepointPath) {
            cluster.disposeSavepoint(savepointPath);
        }
        cluster.stop();
        cluster.awaitTermination();
    }
}