Examples with MISTPredicate edu.snu.mist.common.functions.MISTPredicate used on opensource projects

Example 1 with MISTPredicate

use of edu.snu.mist.common.functions.MISTPredicate in project mist by snuspl.

the class PhysicalObjectGenerator method newOperator.

/**
 * Get a new operator.
 * @param conf configuration
 * @param classLoader external class loader
 * @return new operator
 */
@SuppressWarnings("unchecked")
public Operator newOperator(final Map<String, String> conf, final ClassLoader classLoader) throws IOException, ClassNotFoundException {
    final String type = conf.get(ConfKeys.OperatorConf.OP_TYPE.name());
    if (type.equals(ConfValues.OperatorType.MAP.name())) {
        return new MapOperator(getObject(conf, ConfKeys.OperatorConf.UDF_STRING.name(), classLoader));
    } else if (type.equals(ConfValues.OperatorType.FILTER.name())) {
        return new FilterOperator(getObject(conf, ConfKeys.OperatorConf.UDF_STRING.name(), classLoader));
    } else if (type.equals(ConfValues.OperatorType.FLAT_MAP.name())) {
        return new FlatMapOperator(getObject(conf, ConfKeys.OperatorConf.UDF_STRING.name(), classLoader));
    } else if (type.equals(ConfValues.OperatorType.APPLY_STATEFUL.name())) {
        return new ApplyStatefulOperator(getObject(conf, ConfKeys.OperatorConf.UDF_STRING.name(), classLoader));
    } else if (type.equals(ConfValues.OperatorType.STATE_TRANSITION.name())) {
        final String initialState = conf.get(ConfKeys.StateTransitionOperator.INITIAL_STATE.name());
        final Map<String, Collection<Tuple2<MISTPredicate, String>>> stateTable = getObject(conf, ConfKeys.StateTransitionOperator.STATE_TABLE.name(), classLoader);
        final Set<String> finalState = getObject(conf, ConfKeys.StateTransitionOperator.FINAL_STATE.name(), classLoader);
        return new StateTransitionOperator(initialState, finalState, stateTable);
    } else if (type.equals(ConfValues.OperatorType.CEP.name())) {
        final List<CepEventPattern> cepEventPatterns = getObject(conf, ConfKeys.CepOperator.CEP_EVENT.name(), classLoader);
        final long windowTime = Long.valueOf(conf.get(ConfKeys.CepOperator.WINDOW_TIME.name()));
        return new CepOperator(cepEventPatterns, windowTime);
    } else if (type.equals(ConfValues.OperatorType.REDUCE_BY_KEY.name())) {
        final int keyFieldNum = Integer.valueOf(conf.get(ConfKeys.ReduceByKeyOperator.KEY_INDEX.name()));
        final MISTBiFunction reduceFunc = getObject(conf, ConfKeys.ReduceByKeyOperator.MIST_BI_FUNC.name(), classLoader);
        return new ReduceByKeyOperator(keyFieldNum, reduceFunc);
    } else if (type.equals(ConfValues.OperatorType.UNION.name())) {
        return new UnionOperator();
    } else if (type.equals(ConfValues.OperatorType.TIME_WINDOW.name())) {
        final int windowSize = Integer.valueOf(conf.get(ConfKeys.WindowOperator.WINDOW_SIZE.name()));
        final int windowInterval = Integer.valueOf(conf.get(ConfKeys.WindowOperator.WINDOW_INTERVAL.name()));
        return new TimeWindowOperator(windowSize, windowInterval);
    } else if (type.equals(ConfValues.OperatorType.COUNT_WINDOW.name())) {
        final int windowSize = Integer.valueOf(conf.get(ConfKeys.WindowOperator.WINDOW_SIZE.name()));
        final int windowInterval = Integer.valueOf(conf.get(ConfKeys.WindowOperator.WINDOW_INTERVAL.name()));
        return new CountWindowOperator(windowSize, windowInterval);
    } else if (type.equals(ConfValues.OperatorType.SESSION_WINDOW.name())) {
        final int windowInterval = Integer.valueOf(conf.get(ConfKeys.WindowOperator.WINDOW_INTERVAL.name()));
        return new SessionWindowOperator(windowInterval);
    } else if (type.equals(ConfValues.OperatorType.JOIN.name())) {
        return new JoinOperator(getObject(conf, ConfKeys.OperatorConf.UDF_STRING.name(), classLoader));
    } else if (type.equals(ConfValues.OperatorType.AGGREGATE_WINDOW.name())) {
        return new AggregateWindowOperator(getObject(conf, ConfKeys.OperatorConf.UDF_STRING.name(), classLoader));
    } else if (type.equals(ConfValues.OperatorType.APPLY_STATEFUL_WINDOW.name())) {
        return new ApplyStatefulWindowOperator(getObject(conf, ConfKeys.OperatorConf.UDF_STRING.name(), classLoader));
    } else {
        throw new RuntimeException("Invalid operator: " + type);
    }
}

Example 2 with MISTPredicate

use of edu.snu.mist.common.functions.MISTPredicate in project mist by snuspl.

the class StateTransitionOperatorTest method testStateTransitionOperatorGetState.

/**
 * Test getting state of StateTransitionOperator.
 */
@Test
public void testStateTransitionOperatorGetState() throws InterruptedException {
    // generate input data event
    final Map<String, Integer> value1 = new HashMap<>();
    value1.put("number", 1);
    final MistDataEvent data1 = new MistDataEvent(value1, 0L);
    final Map<String, Integer> value2 = new HashMap<>();
    value2.put("number", 2);
    final MistDataEvent data2 = new MistDataEvent(value2, 1L);
    // generate a set of final states
    final Set<String> finalSet = new HashSet<>();
    finalSet.add("1");
    finalSet.add("2");
    // generate a state table
    final Map<String, Collection<Tuple2<MISTPredicate, String>>> stateTable = new HashMap<>();
    final Collection<Tuple2<MISTPredicate, String>> list0 = new ArrayList<>();
    list0.add(new Tuple2<>(new RuleBasedEQPredicate("number", 1), "1"));
    final Collection<Tuple2<MISTPredicate, String>> list1 = new ArrayList<>();
    list1.add(new Tuple2<>(new RuleBasedEQPredicate("number", 2), "2"));
    stateTable.put("0", list0);
    stateTable.put("1", list1);
    final StateTransitionOperator stateTransitionOperator = new StateTransitionOperator("0", finalSet, stateTable);
    final List<MistEvent> result = new ArrayList<>();
    stateTransitionOperator.setOutputEmitter(new OutputBufferEmitter(result));
    stateTransitionOperator.processLeftData(data1);
    stateTransitionOperator.processLeftData(data2);
    // Generate the expected state
    final String expectedOperatorState = "2";
    // Get the
    final Map<String, Object> operatorState = stateTransitionOperator.getStateSnapshot();
    final String stateTransitionOperatorState = (String) operatorState.get("stateTransitionOperatorState");
    Assert.assertEquals(expectedOperatorState, stateTransitionOperatorState);
}

Example 3 with MISTPredicate

use of edu.snu.mist.common.functions.MISTPredicate in project mist by snuspl.

the class CepHRMonitoring method submitQuery.

public static APIQueryControlResult submitQuery(final Configuration configuration) throws IOException, InjectionException, URISyntaxException {
    final String sourceSocket = Tang.Factory.getTang().newInjector(configuration).getNamedInstance(NettySourceAddress.class);
    final String[] source = sourceSocket.split(":");
    final String sourceHostname = source[0];
    final int sourcePort = Integer.parseInt(source[1]);
    final CepInput<CepHRClass> input = new CepInput.TextSocketBuilder<CepHRClass>().setSocketAddress(sourceHostname).setSocketPort(sourcePort).setClassGenFunc(new CepHRClassGenFunc()).build();
    final CepSink sink = new CepSink.TextSocketBuilder().setSocketAddress(MISTExampleUtils.SINK_HOSTNAME).setSocketPort(MISTExampleUtils.SINK_PORT).build();
    final MISTPredicate<CepHRClass> conditionD = s -> s.getName().equals("D");
    final MISTPredicate<CepHRClass> conditionP = s -> s.getName().equals("P");
    final CepEventPattern<CepHRClass> eventD = new CepEventPattern.Builder<CepHRClass>().setName("doctor").setCondition(conditionD).setContiguity(CepEventContiguity.NON_DETERMINISTIC_RELAXED).setClass(CepHRClass.class).build();
    final CepEventPattern<CepHRClass> eventP = new CepEventPattern.Builder<CepHRClass>().setName("patient").setCondition(conditionP).setContiguity(CepEventContiguity.NON_DETERMINISTIC_RELAXED).setNOrMore(2).setInnerContiguity(CepEventContiguity.STRICT).setClass(CepHRClass.class).build();
    final MISTCepQuery<CepHRClass> cepQuery = new MISTCepQuery.Builder<CepHRClass>("demo-group", "user1").input(input).setEventPatternSequence(eventD, eventP).setQualifier(new CepHRQualifier()).within(3600000).setAction(new CepAction.Builder().setActionType(CepActionType.TEXT_WRITE).setCepSink(sink).setParams("Alert!").build()).build();
    final MISTQueryBuilder queryBuilder = CepUtils.translate(cepQuery);
    return MISTExampleUtils.submit(queryBuilder, configuration);
}

Example 4 with MISTPredicate

use of edu.snu.mist.common.functions.MISTPredicate in project mist by snuspl.

the class PhysicalObjectGenerator method newEventGenerator.

/**
 * Get a new event generator.
 * @param conf configuration
 * @param classLoader external class loader
 * @param <T> event type
 * @return event generator
 */
@SuppressWarnings("unchecked")
public <T> EventGenerator<T> newEventGenerator(final Map<String, String> conf, final ClassLoader classLoader) throws IOException, ClassNotFoundException {
    final String type = conf.get(ConfKeys.Watermark.EVENT_GENERATOR.name());
    final String tefString = conf.get(ConfKeys.SourceConf.TIMESTAMP_EXTRACT_FUNC.name());
    final MISTFunction timestampExtractFunc;
    if (tefString == null) {
        timestampExtractFunc = null;
    } else {
        timestampExtractFunc = SerializeUtils.deserializeFromString(conf.get(ConfKeys.SourceConf.TIMESTAMP_EXTRACT_FUNC.name()), classLoader);
    }
    if (type.equals(ConfValues.EventGeneratorType.PERIODIC_EVENT_GEN.name())) {
        // periodic event generator
        final long period = Long.valueOf(conf.get(ConfKeys.Watermark.PERIODIC_WATERMARK_PERIOD.name()));
        final long delay = Long.valueOf(conf.get(ConfKeys.Watermark.PERIODIC_WATERMARK_DELAY.name()));
        return new PeriodicEventGenerator(timestampExtractFunc, period, checkpointPeriod, delay, watermarkTimeUnit, scheduler);
    } else if (type.equals(ConfValues.EventGeneratorType.PUNCTUATED_EVENT_GEN.name())) {
        // punctuated event generator
        final MISTPredicate watermarkPredicate = SerializeUtils.deserializeFromString(conf.get(ConfKeys.Watermark.WATERMARK_PREDICATE.name()), classLoader);
        final WatermarkTimestampFunction tf = SerializeUtils.deserializeFromString(conf.get(ConfKeys.Watermark.TIMESTAMP_PARSE_OBJECT.name()), classLoader);
        return new PunctuatedEventGenerator(timestampExtractFunc, watermarkPredicate, tf, checkpointPeriod, watermarkTimeUnit, scheduler);
    } else {
        throw new RuntimeException("Invalid event generator: " + type);
    }
}

Example 5 with MISTPredicate

use of edu.snu.mist.common.functions.MISTPredicate in project mist by snuspl.

the class NettySourceTest method testPunctuatedNettyTextSource.

/**
 * Test whether the created source using DataGenerator by NettyTextDataGeneratorFactory receive event-time data
 * correctly from netty server, and generate proper punctuated watermark and outputs.
 * It creates 4 sources each having data generator using Netty server.
 * @throws Exception
 */
@Test(timeout = 4000L)
public void testPunctuatedNettyTextSource() throws Exception {
    final int numSources = 4;
    final int numData = 3;
    final int numWatermark = 2;
    final List<String> inputStreamWithTimestamp = Arrays.asList("Lorem ipsum dolor sit amet, consectetur adipiscing elit.:100", "In in leo nec erat fringilla mattis eu non massa.:800", "Watermark:1000", "Cras quis diam suscipit, commodo enim id, pulvinar nunc.:1200", "Watermark:1500");
    final List<String> expectedDataWithoutTimestamp = Arrays.asList("Lorem ipsum dolor sit amet, consectetur adipiscing elit.", "In in leo nec erat fringilla mattis eu non massa.", "Cras quis diam suscipit, commodo enim id, pulvinar nunc.");
    final List<Long> expectedPunctuatedWatermark = Arrays.asList(1000L, 1500L);
    final CountDownLatch dataCountDownLatch = new CountDownLatch(numSources * numData);
    final CountDownLatch watermarkCountDownLatch = new CountDownLatch(numSources * numWatermark);
    final CountDownLatch channelCountDown = new CountDownLatch(numSources);
    LOG.log(Level.FINE, "Count down data: {0}", dataCountDownLatch);
    LOG.log(Level.FINE, "Count down watermark: {0}", watermarkCountDownLatch);
    // create netty server
    try (final NettyTextMessageStreamGenerator textMessageStreamGenerator = new NettyTextMessageStreamGenerator(SERVER_ADDR, SERVER_PORT, new TestChannelHandler(channelCountDown))) {
        final Injector injector = Tang.Factory.getTang().newInjector();
        // source list
        final List<Tuple<DataGenerator, EventGenerator>> sources = new LinkedList<>();
        // result data list
        final List<List<String>> punctuatedDataResults = new LinkedList<>();
        // result watermark list
        final List<List<Long>> punctuatedWatermarkResults = new LinkedList<>();
        // Create sources having punctuated watermark
        for (int i = 0; i < numSources; i++) {
            final DataGenerator<String> dataGenerator = new NettyTextDataGenerator(SERVER_ADDR, SERVER_PORT, nettySharedResource);
            final MISTFunction<String, Tuple<String, Long>> extractFunc = (input) -> new Tuple<>(input.toString().split(":")[0], Long.parseLong(input.toString().split(":")[1]));
            final MISTPredicate<String> isWatermark = (input) -> input.toString().split(":")[0].equals("Watermark");
            final WatermarkTimestampFunction<String> parseTsFunc = (input) -> Long.parseLong(input.toString().split(":")[1]);
            final EventGenerator<String> eventGenerator = new PunctuatedEventGenerator<>(extractFunc, isWatermark, parseTsFunc, 0, null, null);
            sources.add(new Tuple<>(dataGenerator, eventGenerator));
            dataGenerator.setEventGenerator(eventGenerator);
            final List<String> receivedData = new LinkedList<>();
            final List<Long> receivedWatermark = new LinkedList<>();
            punctuatedDataResults.add(receivedData);
            punctuatedWatermarkResults.add(receivedWatermark);
            eventGenerator.setOutputEmitter(new SourceTestOutputEmitter<>(receivedData, receivedWatermark, dataCountDownLatch, watermarkCountDownLatch));
        }
        // Start to receive data stream from stream generator
        for (final Tuple<DataGenerator, EventGenerator> source : sources) {
            // start event generator
            source.getValue().start();
            // start data generator
            source.getKey().start();
        }
        // Wait until all sources connect to stream generator
        channelCountDown.await();
        inputStreamWithTimestamp.forEach(textMessageStreamGenerator::write);
        // Wait until all data are sent to source
        dataCountDownLatch.await();
        watermarkCountDownLatch.await();
        for (final List<String> received : punctuatedDataResults) {
            Assert.assertEquals(expectedDataWithoutTimestamp, received);
        }
        for (final List<Long> received : punctuatedWatermarkResults) {
            Assert.assertEquals(expectedPunctuatedWatermark, received);
        }
        // Closes
        for (final Tuple<DataGenerator, EventGenerator> source : sources) {
            // stop data generator
            source.getKey().close();
            // stop event generator
            source.getValue().close();
        }
    }
}