Search in sources :

Example 1 with TestWithCountDownSink

use of edu.snu.mist.core.task.utils.TestWithCountDownSink in project mist by snuspl.

the class QueryManagerTest method testSubmitComplexQueryHelper.

@SuppressWarnings("unchecked")
private void testSubmitComplexQueryHelper(final Configuration conf) throws Exception {
    final String queryId = "testQuery";
    final List<String> inputs = Arrays.asList("mist is a cloud of tiny water droplets suspended in the atmosphere", "a mist rose out of the river", "the peaks were shrouded in mist");
    // Expected results
    final List<Map<String, Integer>> intermediateResult = getIntermediateResult(inputs);
    final List<String> expectedSink1Output = intermediateResult.stream().map(input -> input.toString()).collect(Collectors.toList());
    final List<Integer> expectedSink2Output = intermediateResult.stream().map(totalCountMapFunc).collect(Collectors.toList());
    // Number of expected outputs
    final CountDownLatch countDownAllOutputs = new CountDownLatch(intermediateResult.size() * 2);
    // Create the execution DAG of the query
    final ExecutionDag executionDag = new ExecutionDag(new AdjacentListDAG<>());
    // Create source
    final TestDataGenerator dataGenerator = new TestDataGenerator(inputs);
    final EventGenerator eventGenerator = new PunctuatedEventGenerator(null, input -> false, null, 0, null, null);
    final PhysicalSource src = new PhysicalSourceImpl("testSource", new HashMap<>(), dataGenerator, eventGenerator);
    final Injector injector = Tang.Factory.getTang().newInjector(conf);
    // Create sinks
    final List<String> sink1Result = new LinkedList<>();
    final List<Integer> sink2Result = new LinkedList<>();
    final PhysicalSink sink1 = new PhysicalSinkImpl<>("sink1", null, new TestWithCountDownSink<String>(sink1Result, countDownAllOutputs));
    final PhysicalSink sink2 = new PhysicalSinkImpl<>("sink2", null, new TestWithCountDownSink<Integer>(sink2Result, countDownAllOutputs));
    // Fake operator chain dag of QueryManager
    final AvroDag fakeAvroDag = new AvroDag();
    fakeAvroDag.setQueryId(queryId);
    // fakeAvroDag.setSuperGroupId("testGroup");
    // Construct execution dag
    constructExecutionDag(executionDag, src, sink1, sink2);
    // Create mock DagGenerator. It returns the above  execution dag
    final ConfigDagGenerator configDagGenerator = mock(ConfigDagGenerator.class);
    final DAG<ConfigVertex, MISTEdge> configDag = mock(DAG.class);
    when(configDagGenerator.generate(fakeAvroDag)).thenReturn(configDag);
    final DagGenerator dagGenerator = mock(DagGenerator.class);
    // when(dagGenerator.generate(configDag, tuple.getValue().getJarFilePaths())).thenReturn(executionDag);
    // Build QueryManager
    final QueryManager queryManager = queryManagerBuild(fakeAvroDag, configDagGenerator, dagGenerator, injector);
    queryManager.create(fakeAvroDag);
    // Wait until all of the outputs are generated
    countDownAllOutputs.await();
    // Check the outputs
    Assert.assertEquals(expectedSink1Output, sink1Result);
    Assert.assertEquals(expectedSink2Output, sink2Result);
    src.close();
    queryManager.close();
    // Delete plan directory and plans
    deletePlans(injector);
}
Also used : DefaultNumEventProcessors(edu.snu.mist.core.task.groupaware.eventprocessor.parameters.DefaultNumEventProcessors) Injector(org.apache.reef.tang.Injector) java.util(java.util) EventGenerator(edu.snu.mist.core.sources.EventGenerator) Tuple2(edu.snu.mist.common.types.Tuple2) MISTEdge(edu.snu.mist.common.graph.MISTEdge) AvroDag(edu.snu.mist.formats.avro.AvroDag) ReduceByKeyOperator(edu.snu.mist.core.operators.ReduceByKeyOperator) PunctuatedEventGenerator(edu.snu.mist.core.sources.PunctuatedEventGenerator) DAG(edu.snu.mist.common.graph.DAG) Configuration(org.apache.reef.tang.Configuration) MISTBiFunction(edu.snu.mist.common.functions.MISTBiFunction) MapOperator(edu.snu.mist.core.operators.MapOperator) TestWithCountDownSink(edu.snu.mist.core.task.utils.TestWithCountDownSink) Tang(org.apache.reef.tang.Tang) MISTFunction(edu.snu.mist.common.functions.MISTFunction) MISTPredicate(edu.snu.mist.common.functions.MISTPredicate) PlanStorePath(edu.snu.mist.core.parameters.PlanStorePath) Mockito.when(org.mockito.Mockito.when) Logger(java.util.logging.Logger) QueryInfoStore(edu.snu.mist.core.task.stores.QueryInfoStore) Collectors(java.util.stream.Collectors) File(java.io.File) FilterOperator(edu.snu.mist.core.operators.FilterOperator) CountDownLatch(java.util.concurrent.CountDownLatch) FlatMapOperator(edu.snu.mist.core.operators.FlatMapOperator) Assert(junit.framework.Assert) Direction(edu.snu.mist.formats.avro.Direction) JavaConfigurationBuilder(org.apache.reef.tang.JavaConfigurationBuilder) TestDataGenerator(edu.snu.mist.core.task.utils.TestDataGenerator) MistTaskConfigs(edu.snu.mist.core.configs.MistTaskConfigs) ClientToTaskPort(edu.snu.mist.core.parameters.ClientToTaskPort) AdjacentListDAG(edu.snu.mist.common.graph.AdjacentListDAG) Mockito.mock(org.mockito.Mockito.mock) TestDataGenerator(edu.snu.mist.core.task.utils.TestDataGenerator) AvroDag(edu.snu.mist.formats.avro.AvroDag) Injector(org.apache.reef.tang.Injector) MISTEdge(edu.snu.mist.common.graph.MISTEdge) EventGenerator(edu.snu.mist.core.sources.EventGenerator) PunctuatedEventGenerator(edu.snu.mist.core.sources.PunctuatedEventGenerator) CountDownLatch(java.util.concurrent.CountDownLatch) PunctuatedEventGenerator(edu.snu.mist.core.sources.PunctuatedEventGenerator)

Aggregations

MISTBiFunction (edu.snu.mist.common.functions.MISTBiFunction)1 MISTFunction (edu.snu.mist.common.functions.MISTFunction)1 MISTPredicate (edu.snu.mist.common.functions.MISTPredicate)1 AdjacentListDAG (edu.snu.mist.common.graph.AdjacentListDAG)1 DAG (edu.snu.mist.common.graph.DAG)1 MISTEdge (edu.snu.mist.common.graph.MISTEdge)1 Tuple2 (edu.snu.mist.common.types.Tuple2)1 MistTaskConfigs (edu.snu.mist.core.configs.MistTaskConfigs)1 FilterOperator (edu.snu.mist.core.operators.FilterOperator)1 FlatMapOperator (edu.snu.mist.core.operators.FlatMapOperator)1 MapOperator (edu.snu.mist.core.operators.MapOperator)1 ReduceByKeyOperator (edu.snu.mist.core.operators.ReduceByKeyOperator)1 ClientToTaskPort (edu.snu.mist.core.parameters.ClientToTaskPort)1 PlanStorePath (edu.snu.mist.core.parameters.PlanStorePath)1 EventGenerator (edu.snu.mist.core.sources.EventGenerator)1 PunctuatedEventGenerator (edu.snu.mist.core.sources.PunctuatedEventGenerator)1 DefaultNumEventProcessors (edu.snu.mist.core.task.groupaware.eventprocessor.parameters.DefaultNumEventProcessors)1 QueryInfoStore (edu.snu.mist.core.task.stores.QueryInfoStore)1 TestDataGenerator (edu.snu.mist.core.task.utils.TestDataGenerator)1 TestWithCountDownSink (edu.snu.mist.core.task.utils.TestWithCountDownSink)1