Search in sources :

Example 1 with Assert.assertThat

use of org.junit.Assert.assertThat in project knime-core by knime.

the class DefaultNodeProgressMonitorTest method internalTestManyMessageEvents.

/**
 * A lot of incremental numeric progress updates + many message events
 * Previously, this took significantly longer due to expensive string construction.
 */
private void internalTestManyMessageEvents(final NodeProgressMonitor toMonitor, final NodeProgressMonitor toControl) throws Exception {
    final int parts = 1000000;
    final MutableLong stringComposeCounter = new MutableLong();
    Function<Integer, String> msgFct = (index) -> {
        stringComposeCounter.increment();
        return "Row " + index + " (Row \"" + RowKey.createRowKey((long) index) + "\")";
    };
    final Pointer<NodeProgress> progressPointer = new Pointer<>();
    String lastExpectedMsg = msgFct.apply(parts);
    final Function<NodeProgress, Boolean> isLastEventFunction = p -> p.getMessage().equals(lastExpectedMsg);
    NodeProgressListener l = createListener(progressPointer, isLastEventFunction);
    toMonitor.addProgressListener(l);
    try {
        for (int i = 1; i < parts + 1; i++) {
            final int index = i;
            // if this line is replaced by a direct string composition this takes an order of magnitude longer
            toControl.setProgress(i / (double) parts, () -> msgFct.apply(index));
        }
        synchronized (isLastEventFunction) {
            isLastEventFunction.wait(500);
        }
        assertThat(progressPointer.get().getProgress(), is(closeTo(1.0, PROG_EPSILON)));
        assertThat(progressPointer.get().getMessage(), is(equalTo(lastExpectedMsg)));
        // the lazy string creation should only be called 4 times a second at most,
        // it must be at least two - one for the reference string creation and one during an event
        // 2020-01-08, BW: increment from max=5 to max=8 -- encountered a longer running test case on Win Server
        Assert.assertThat(stringComposeCounter.getValue(), is(allOf(greaterThanOrEqualTo(2L), lessThanOrEqualTo(8L))));
    } finally {
        toMonitor.removeProgressListener(l);
    }
}
Also used : Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) RowKey(org.knime.core.data.RowKey) Matchers.allOf(org.hamcrest.Matchers.allOf) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) Test(org.junit.Test) Function(java.util.function.Function) NodeProgressEvent(org.knime.core.node.workflow.NodeProgressEvent) Assert.assertThat(org.junit.Assert.assertThat) Matchers.closeTo(org.hamcrest.Matchers.closeTo) NodeProgressListener(org.knime.core.node.workflow.NodeProgressListener) MutableLong(org.apache.commons.lang3.mutable.MutableLong) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Matchers.is(org.hamcrest.Matchers.is) Assert(org.junit.Assert) SubNodeProgressMonitor(org.knime.core.node.DefaultNodeProgressMonitor.SubNodeProgressMonitor) Pointer(org.knime.core.util.Pointer) NodeProgress(org.knime.core.node.workflow.NodeProgress) MutableLong(org.apache.commons.lang3.mutable.MutableLong) NodeProgressListener(org.knime.core.node.workflow.NodeProgressListener) NodeProgress(org.knime.core.node.workflow.NodeProgress) Pointer(org.knime.core.util.Pointer)

Example 2 with Assert.assertThat

use of org.junit.Assert.assertThat in project flink by apache.

the class JobGraphGeneratorTest method testGeneratingJobGraphWithUnconsumedResultPartition.

@Test
public void testGeneratingJobGraphWithUnconsumedResultPartition() {
    ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple2<Long, Long>> input = env.fromElements(new Tuple2<>(1L, 2L)).setParallelism(1);
    DataSet<Tuple2<Long, Long>> ds = input.map(new IdentityMapper<>()).setParallelism(3);
    AbstractID intermediateDataSetID = new AbstractID();
    // this output branch will be excluded.
    ds.output(BlockingShuffleOutputFormat.createOutputFormat(intermediateDataSetID)).setParallelism(1);
    // this is the normal output branch.
    ds.output(new DiscardingOutputFormat<>()).setParallelism(1);
    JobGraph jobGraph = compileJob(env);
    Assert.assertEquals(3, jobGraph.getVerticesSortedTopologicallyFromSources().size());
    JobVertex mapVertex = jobGraph.getVerticesSortedTopologicallyFromSources().get(1);
    Assert.assertThat(mapVertex, Matchers.instanceOf(JobVertex.class));
    // there are 2 output result with one of them is ResultPartitionType.BLOCKING_PERSISTENT
    Assert.assertEquals(2, mapVertex.getProducedDataSets().size());
    Assert.assertTrue(mapVertex.getProducedDataSets().stream().anyMatch(dataSet -> dataSet.getId().equals(new IntermediateDataSetID(intermediateDataSetID)) && dataSet.getResultType() == ResultPartitionType.BLOCKING_PERSISTENT));
}
Also used : CoreMatchers.is(org.hamcrest.CoreMatchers.is) LongSumAggregator(org.apache.flink.api.common.aggregators.LongSumAggregator) JobVertex(org.apache.flink.runtime.jobgraph.JobVertex) Tuple2(org.apache.flink.api.java.tuple.Tuple2) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) ResultPartitionType(org.apache.flink.runtime.io.network.partition.ResultPartitionType) HashMap(java.util.HashMap) JobType(org.apache.flink.runtime.jobgraph.JobType) MapFunction(org.apache.flink.api.common.functions.MapFunction) DataSink(org.apache.flink.api.java.operators.DataSink) Assert.assertThat(org.junit.Assert.assertThat) DataSet(org.apache.flink.api.java.DataSet) JobGraphUtils(org.apache.flink.runtime.jobgraph.JobGraphUtils) DeltaIteration(org.apache.flink.api.java.operators.DeltaIteration) ResourceSpec(org.apache.flink.api.common.operators.ResourceSpec) Map(java.util.Map) Plan(org.apache.flink.api.common.Plan) Optimizer(org.apache.flink.optimizer.Optimizer) BlockingShuffleOutputFormat(org.apache.flink.api.java.io.BlockingShuffleOutputFormat) IdentityMapper(org.apache.flink.optimizer.testfunctions.IdentityMapper) Method(java.lang.reflect.Method) Path(java.nio.file.Path) OptimizedPlan(org.apache.flink.optimizer.plan.OptimizedPlan) DiscardingOutputFormat(org.apache.flink.api.java.io.DiscardingOutputFormat) Files(java.nio.file.Files) AbstractID(org.apache.flink.util.AbstractID) Assert.assertNotNull(org.junit.Assert.assertNotNull) IterativeDataSet(org.apache.flink.api.java.operators.IterativeDataSet) Configuration(org.apache.flink.configuration.Configuration) Matchers(org.hamcrest.Matchers) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) IntermediateDataSetID(org.apache.flink.runtime.jobgraph.IntermediateDataSetID) DistributedCache(org.apache.flink.api.common.cache.DistributedCache) Operator(org.apache.flink.api.java.operators.Operator) FilterFunction(org.apache.flink.api.common.functions.FilterFunction) JobID(org.apache.flink.api.common.JobID) Rule(org.junit.Rule) ExecutionEnvironment(org.apache.flink.api.java.ExecutionEnvironment) Assert.assertFalse(org.junit.Assert.assertFalse) Assert(org.junit.Assert) TemporaryFolder(org.junit.rules.TemporaryFolder) Assert.assertEquals(org.junit.Assert.assertEquals) ExecutionEnvironment(org.apache.flink.api.java.ExecutionEnvironment) JobGraph(org.apache.flink.runtime.jobgraph.JobGraph) JobVertex(org.apache.flink.runtime.jobgraph.JobVertex) IdentityMapper(org.apache.flink.optimizer.testfunctions.IdentityMapper) Tuple2(org.apache.flink.api.java.tuple.Tuple2) IntermediateDataSetID(org.apache.flink.runtime.jobgraph.IntermediateDataSetID) AbstractID(org.apache.flink.util.AbstractID) DiscardingOutputFormat(org.apache.flink.api.java.io.DiscardingOutputFormat) Test(org.junit.Test)

Aggregations

Assert (org.junit.Assert)2 Assert.assertThat (org.junit.Assert.assertThat)2 IOException (java.io.IOException)1 Method (java.lang.reflect.Method)1 Files (java.nio.file.Files)1 Path (java.nio.file.Path)1 HashMap (java.util.HashMap)1 Map (java.util.Map)1 Function (java.util.function.Function)1 MutableLong (org.apache.commons.lang3.mutable.MutableLong)1 JobID (org.apache.flink.api.common.JobID)1 Plan (org.apache.flink.api.common.Plan)1 LongSumAggregator (org.apache.flink.api.common.aggregators.LongSumAggregator)1 DistributedCache (org.apache.flink.api.common.cache.DistributedCache)1 FilterFunction (org.apache.flink.api.common.functions.FilterFunction)1 MapFunction (org.apache.flink.api.common.functions.MapFunction)1 ResourceSpec (org.apache.flink.api.common.operators.ResourceSpec)1 DataSet (org.apache.flink.api.java.DataSet)1 ExecutionEnvironment (org.apache.flink.api.java.ExecutionEnvironment)1 BlockingShuffleOutputFormat (org.apache.flink.api.java.io.BlockingShuffleOutputFormat)1