Examples with Event org.apache.flink.streaming.examples.statemachine.event.Event used on opensource projects

Example 1 with Event

use of org.apache.flink.streaming.examples.statemachine.event.Event in project flink by apache.

the class EventsGenerator method nextInvalid.

/**
 * Creates an event for an illegal state transition of one of the internal state machines. If
 * the generator has not yet started any state machines (for example, because no call to {@link
 * #next(int, int)} was made, yet), this will return null.
 *
 * @return An event for a illegal state transition, or null, if not possible.
 */
@Nullable
public Event nextInvalid() {
    final Iterator<Entry<Integer, State>> iter = states.entrySet().iterator();
    if (iter.hasNext()) {
        final Entry<Integer, State> entry = iter.next();
        State currentState = entry.getValue();
        int address = entry.getKey();
        iter.remove();
        EventType event = currentState.randomInvalidTransition(rnd);
        return new Event(event, address);
    } else {
        return null;
    }
}

Example 2 with Event

use of org.apache.flink.streaming.examples.statemachine.event.Event in project flink by apache.

the class StateMachineExample method main.

/**
 * Main entry point for the program.
 *
 * @param args The command line arguments.
 */
public static void main(String[] args) throws Exception {
    // ---- print some usage help ----
    System.out.println("Usage with built-in data generator: StateMachineExample [--error-rate <probability-of-invalid-transition>] [--sleep <sleep-per-record-in-ms>]");
    System.out.println("Usage with Kafka: StateMachineExample --kafka-topic <topic> [--brokers <brokers>]");
    System.out.println("Options for both the above setups: ");
    System.out.println("\t[--backend <hashmap|rocks>]");
    System.out.println("\t[--checkpoint-dir <filepath>]");
    System.out.println("\t[--incremental-checkpoints <true|false>]");
    System.out.println("\t[--output <filepath> OR null for stdout]");
    System.out.println();
    // ---- determine whether to use the built-in source, or read from Kafka ----
    final DataStream<Event> events;
    final ParameterTool params = ParameterTool.fromArgs(args);
    // create the environment to create streams and configure execution
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.enableCheckpointing(2000L);
    final String stateBackend = params.get("backend", "memory");
    if ("hashmap".equals(stateBackend)) {
        final String checkpointDir = params.get("checkpoint-dir");
        env.setStateBackend(new HashMapStateBackend());
        env.getCheckpointConfig().setCheckpointStorage(checkpointDir);
    } else if ("rocks".equals(stateBackend)) {
        final String checkpointDir = params.get("checkpoint-dir");
        boolean incrementalCheckpoints = params.getBoolean("incremental-checkpoints", false);
        env.setStateBackend(new EmbeddedRocksDBStateBackend(incrementalCheckpoints));
        env.getCheckpointConfig().setCheckpointStorage(checkpointDir);
    }
    if (params.has("kafka-topic")) {
        // set up the Kafka reader
        String kafkaTopic = params.get("kafka-topic");
        String brokers = params.get("brokers", "localhost:9092");
        System.out.printf("Reading from kafka topic %s @ %s\n", kafkaTopic, brokers);
        System.out.println();
        KafkaSource<Event> source = KafkaSource.<Event>builder().setBootstrapServers(brokers).setGroupId("stateMachineExample").setTopics(kafkaTopic).setDeserializer(KafkaRecordDeserializationSchema.valueOnly(new EventDeSerializationSchema())).setStartingOffsets(OffsetsInitializer.latest()).build();
        events = env.fromSource(source, WatermarkStrategy.noWatermarks(), "StateMachineExampleSource");
    } else {
        double errorRate = params.getDouble("error-rate", 0.0);
        int sleep = params.getInt("sleep", 1);
        System.out.printf("Using standalone source with error rate %f and sleep delay %s millis\n", errorRate, sleep);
        System.out.println();
        events = env.addSource(new EventsGeneratorSource(errorRate, sleep));
    }
    // ---- main program ----
    final String outputFile = params.get("output");
    // make parameters available in the web interface
    env.getConfig().setGlobalJobParameters(params);
    DataStream<Alert> alerts = events.keyBy(Event::sourceAddress).flatMap(new StateMachineMapper());
    // output the alerts to std-out
    if (outputFile == null) {
        alerts.print();
    } else {
        alerts.sinkTo(FileSink.<Alert>forRowFormat(new Path(outputFile), new SimpleStringEncoder<>()).withRollingPolicy(DefaultRollingPolicy.builder().withMaxPartSize(MemorySize.ofMebiBytes(1)).withRolloverInterval(Duration.ofSeconds(10)).build()).build()).setParallelism(1).name("output");
    }
    // trigger program execution
    env.execute("State machine job");
}

Example 3 with Event

use of org.apache.flink.streaming.examples.statemachine.event.Event in project flink by apache.

the class KafkaEventsGeneratorJob method main.

public static void main(String[] args) throws Exception {
    final ParameterTool params = ParameterTool.fromArgs(args);
    double errorRate = params.getDouble("error-rate", 0.0);
    int sleep = params.getInt("sleep", 1);
    String kafkaTopic = params.get("kafka-topic");
    String brokers = params.get("brokers", "localhost:9092");
    System.out.printf("Generating events to Kafka with standalone source with error rate %f and sleep delay %s millis\n", errorRate, sleep);
    System.out.println();
    final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.addSource(new EventsGeneratorSource(errorRate, sleep)).sinkTo(KafkaSink.<Event>builder().setBootstrapServers(brokers).setRecordSerializer(KafkaRecordSerializationSchema.builder().setValueSerializationSchema(new EventDeSerializationSchema()).setTopic(kafkaTopic).build()).build());
    // trigger program execution
    env.execute("State machine example Kafka events generator job");
}

Example 4 with Event

use of org.apache.flink.streaming.examples.statemachine.event.Event in project flink by apache.

the class EventsGenerator method next.

// ------------------------------------------------------------------------
/**
 * Creates a new random event. This method randomly pick either one of its currently running
 * state machines, or start a new state machine for a random IP address.
 *
 * <p>With {@link #errorProb} probability, the generated event will be from an illegal state
 * transition of one of the currently running state machines.
 *
 * @param minIp The lower bound for the range from which a new IP address may be picked.
 * @param maxIp The upper bound for the range from which a new IP address may be picked.
 * @return A next random event.
 */
public Event next(int minIp, int maxIp) {
    final double p = rnd.nextDouble();
    if (p * 1000 >= states.size()) {
        // create a new state machine
        final int nextIP = rnd.nextInt(maxIp - minIp) + minIp;
        if (!states.containsKey(nextIP)) {
            EventTypeAndState eventAndState = State.Initial.randomTransition(rnd);
            states.put(nextIP, eventAndState.state);
            return new Event(eventAndState.eventType, nextIP);
        } else {
            // collision on IP address, try again
            return next(minIp, maxIp);
        }
    } else {
        // pick an existing state machine
        // skip over some elements in the linked map, then take the next
        // update it, and insert it at the end
        int numToSkip = Math.min(20, rnd.nextInt(states.size()));
        Iterator<Entry<Integer, State>> iter = states.entrySet().iterator();
        for (int i = numToSkip; i > 0; --i) {
            iter.next();
        }
        Entry<Integer, State> entry = iter.next();
        State currentState = entry.getValue();
        int address = entry.getKey();
        iter.remove();
        if (p < errorProb) {
            EventType event = currentState.randomInvalidTransition(rnd);
            return new Event(event, address);
        } else {
            EventTypeAndState eventAndState = currentState.randomTransition(rnd);
            if (!eventAndState.state.isTerminal()) {
                // reinsert
                states.put(address, eventAndState.state);
            }
            return new Event(eventAndState.eventType, address);
        }
    }
}

Example 5 with Event

use of org.apache.flink.streaming.examples.statemachine.event.Event in project flink by apache.

the class EventDeSerializationSchema method deserialize.

@Override
public Event deserialize(byte[] message) throws IOException {
    ByteBuffer buffer = ByteBuffer.wrap(message).order(ByteOrder.LITTLE_ENDIAN);
    int address = buffer.getInt(0);
    int typeOrdinal = buffer.getInt(4);
    return new Event(EventType.values()[typeOrdinal], address);
}