Examples with CommLayerEmitter io.s4.emitter.CommLayerEmitter used on opensource projects

Example 1 with CommLayerEmitter

use of io.s4.emitter.CommLayerEmitter in project core by s4.

the class CommLayerEmitter method run.

@Override
public void run() {
    if (listener == null) {
        if (listenerAppName == null) {
            listenerAppName = appName;
        }
        sender = new SenderProcess(clusterManagerAddress, appName, listenerAppName);
        Map<String, String> map = new HashMap<String, String>();
        map.put("SenderId", "" + senderId);
        sender.setSerializer(new PassThroughSerializer());
        sender.acquireTaskAndCreateSender(map);
    } else {
        Object listenerConfig = null;
        try {
            listenerConfig = listener.getListenerConfig();
            if (listenerConfig == null) {
                logger.info("CommLayerEmitter going to wait for listener to acquire task");
                synchronized (listener) {
                    listenerConfig = listener.getListenerConfig();
                    if (listenerConfig == null) {
                        listener.wait();
                        listenerConfig = listener.getListenerConfig();
                    }
                }
            }
        } catch (Exception e) {
            logger.info("Exception in CommLayerEmitter.run()", e);
        }
        logger.info("Creating sender process with " + listenerConfig);
        String destinationAppName = (listenerAppName != null ? listenerAppName : listener.getAppName());
        sender = new SenderProcess(listener.getClusterManagerAddress(), listener.getAppName(), destinationAppName);
        sender.setSerializer(new PassThroughSerializer());
        sender.createSenderFromConfig(listenerConfig);
        nodeCount = sender.getNumOfPartitions();
    }
    boolean isSent = false;
    while (!Thread.interrupted()) {
        isSent = false;
        try {
            MessageHolder mh = messageQueue.take();
            byte[] rawMessage = mh.getRawMessage();
            if (listener == null) {
                isSent = sender.send(rawMessage);
            } else {
                isSent = sender.sendToPartition(mh.getPartitionId(), rawMessage);
            }
            if (isSent) {
                if (monitor != null) {
                    monitor.increment(low_level_emitter_msg_out_ct.toString(), 1, S4_CORE_METRICS.toString());
                }
            } else {
                if (monitor != null) {
                    monitor.increment(low_level_emitter_out_err_ct.toString(), 1, S4_CORE_METRICS.toString());
                }
                logger.warn("commlayer emit failed ...");
            }
        } catch (InterruptedException ie) {
            if (monitor != null) {
                monitor.increment(low_level_emitter_out_err_ct.toString(), 1, S4_CORE_METRICS.toString());
            }
            Thread.currentThread().interrupt();
        } catch (Exception e) {
            Logger.getLogger("s4").error("Error emitting message", e);
        }
    }
}

Example 2 with CommLayerEmitter

use of io.s4.emitter.CommLayerEmitter in project core by s4.

the class LoadGenerator method main.

public static void main(String[] args) {
    Options options = new Options();
    boolean warmUp = false;
    options.addOption(OptionBuilder.withArgName("rate").hasArg().withDescription("Rate (events per second)").create("r"));
    options.addOption(OptionBuilder.withArgName("display_rate").hasArg().withDescription("Display Rate at specified second boundary").create("d"));
    options.addOption(OptionBuilder.withArgName("start_boundary").hasArg().withDescription("Start boundary in seconds").create("b"));
    options.addOption(OptionBuilder.withArgName("run_for").hasArg().withDescription("Run for a specified number of seconds").create("x"));
    options.addOption(OptionBuilder.withArgName("cluster_manager").hasArg().withDescription("Cluster manager").create("z"));
    options.addOption(OptionBuilder.withArgName("sender_application_name").hasArg().withDescription("Sender application name").create("a"));
    options.addOption(OptionBuilder.withArgName("listener_application_name").hasArg().withDescription("Listener application name").create("g"));
    options.addOption(OptionBuilder.withArgName("sleep_overhead").hasArg().withDescription("Sleep overhead").create("o"));
    options.addOption(new Option("w", "Warm-up"));
    CommandLineParser parser = new GnuParser();
    CommandLine line = null;
    try {
        // parse the command line arguments
        line = parser.parse(options, args);
    } catch (ParseException exp) {
        // oops, something went wrong
        System.err.println("Parsing failed.  Reason: " + exp.getMessage());
        System.exit(1);
    }
    int expectedRate = 250;
    if (line.hasOption("r")) {
        try {
            expectedRate = Integer.parseInt(line.getOptionValue("r"));
        } catch (Exception e) {
            System.err.println("Bad expected rate specified " + line.getOptionValue("r"));
            System.exit(1);
        }
    }
    int displayRateIntervalSeconds = 20;
    if (line.hasOption("d")) {
        try {
            displayRateIntervalSeconds = Integer.parseInt(line.getOptionValue("d"));
        } catch (Exception e) {
            System.err.println("Bad display rate value specified " + line.getOptionValue("d"));
            System.exit(1);
        }
    }
    int startBoundary = 2;
    if (line.hasOption("b")) {
        try {
            startBoundary = Integer.parseInt(line.getOptionValue("b"));
        } catch (Exception e) {
            System.err.println("Bad start boundary value specified " + line.getOptionValue("b"));
            System.exit(1);
        }
    }
    int updateFrequency = 0;
    if (line.hasOption("f")) {
        try {
            updateFrequency = Integer.parseInt(line.getOptionValue("f"));
        } catch (Exception e) {
            System.err.println("Bad query udpdate frequency specified " + line.getOptionValue("f"));
            System.exit(1);
        }
        System.out.printf("Update frequency is %d\n", updateFrequency);
    }
    int runForTime = 0;
    if (line.hasOption("x")) {
        try {
            runForTime = Integer.parseInt(line.getOptionValue("x"));
        } catch (Exception e) {
            System.err.println("Bad run for time specified " + line.getOptionValue("x"));
            System.exit(1);
        }
        System.out.printf("Run for time is %d\n", runForTime);
    }
    String clusterManagerAddress = null;
    if (line.hasOption("z")) {
        clusterManagerAddress = line.getOptionValue("z");
    }
    String senderApplicationName = null;
    if (line.hasOption("a")) {
        senderApplicationName = line.getOptionValue("a");
    }
    String listenerApplicationName = null;
    if (line.hasOption("a")) {
        listenerApplicationName = line.getOptionValue("g");
    }
    if (listenerApplicationName == null) {
        listenerApplicationName = senderApplicationName;
    }
    long sleepOverheadMicros = -1;
    if (line.hasOption("o")) {
        try {
            sleepOverheadMicros = Long.parseLong(line.getOptionValue("o"));
        } catch (NumberFormatException e) {
            System.err.println("Bad sleep overhead specified " + line.getOptionValue("o"));
            System.exit(1);
        }
        System.out.printf("Specified sleep overhead is %d\n", sleepOverheadMicros);
    }
    if (line.hasOption("w")) {
        warmUp = true;
    }
    List loArgs = line.getArgList();
    if (loArgs.size() < 1) {
        System.err.println("No input file specified");
        System.exit(1);
    }
    String inputFilename = (String) loArgs.get(0);
    EventEmitter emitter = null;
    SerializerDeserializer serDeser = new KryoSerDeser();
    CommLayerEmitter clEmitter = new CommLayerEmitter();
    clEmitter.setAppName(senderApplicationName);
    clEmitter.setListenerAppName(listenerApplicationName);
    clEmitter.setClusterManagerAddress(clusterManagerAddress);
    clEmitter.setSenderId(String.valueOf(System.currentTimeMillis() / 1000));
    clEmitter.setSerDeser(serDeser);
    clEmitter.init();
    emitter = clEmitter;
    long endTime = 0;
    if (runForTime > 0) {
        endTime = System.currentTimeMillis() + (runForTime * 1000);
    }
    LoadGenerator loadGenerator = new LoadGenerator();
    loadGenerator.setInputFilename(inputFilename);
    loadGenerator.setEventEmitter(clEmitter);
    loadGenerator.setDisplayRateInterval(displayRateIntervalSeconds);
    loadGenerator.setExpectedRate(expectedRate);
    loadGenerator.run();
    System.exit(0);
}