Examples with ShellProcess backtype.storm.utils.ShellProcess used on opensource projects

Example 1 with ShellProcess

use of backtype.storm.utils.ShellProcess in project storm by nathanmarz.

the class ShellSpout method open.

public void open(Map stormConf, TopologyContext context, SpoutOutputCollector collector) {
    _process = new ShellProcess(_command);
    _collector = collector;
    try {
        Number subpid = _process.launch(stormConf, context);
        LOG.info("Launched subprocess with pid " + subpid);
    } catch (IOException e) {
        throw new RuntimeException("Error when launching multilang subprocess\n" + _process.getErrorsString(), e);
    }
}

Example 2 with ShellProcess

use of backtype.storm.utils.ShellProcess in project storm by nathanmarz.

the class ShellBolt method prepare.

public void prepare(Map stormConf, TopologyContext context, final OutputCollector collector) {
    Object maxPending = stormConf.get(Config.TOPOLOGY_SHELLBOLT_MAX_PENDING);
    if (maxPending != null) {
        this._pendingWrites = new LinkedBlockingQueue(((Number) maxPending).intValue());
    }
    _rand = new Random();
    _process = new ShellProcess(_command);
    _collector = collector;
    try {
        //subprocesses must send their pid first thing
        Number subpid = _process.launch(stormConf, context);
        LOG.info("Launched subprocess with pid " + subpid);
    } catch (IOException e) {
        throw new RuntimeException("Error when launching multilang subprocess\n" + _process.getErrorsString(), e);
    }
    // reader
    _readerThread = new Thread(new Runnable() {

        public void run() {
            while (_running) {
                try {
                    JSONObject action = _process.readMessage();
                    if (action == null) {
                    // ignore sync
                    }
                    String command = (String) action.get("command");
                    if (command.equals("ack")) {
                        handleAck(action);
                    } else if (command.equals("fail")) {
                        handleFail(action);
                    } else if (command.equals("error")) {
                        handleError(action);
                    } else if (command.equals("log")) {
                        String msg = (String) action.get("msg");
                        LOG.info("Shell msg: " + msg);
                    } else if (command.equals("emit")) {
                        handleEmit(action);
                    }
                } catch (InterruptedException e) {
                } catch (Throwable t) {
                    die(t);
                }
            }
        }
    });
    _readerThread.start();
    _writerThread = new Thread(new Runnable() {

        public void run() {
            while (_running) {
                try {
                    Object write = _pendingWrites.poll(1, SECONDS);
                    if (write != null) {
                        _process.writeMessage(write);
                    }
                    // drain the error stream to avoid dead lock because of full error stream buffer
                    _process.drainErrorStream();
                } catch (InterruptedException e) {
                } catch (Throwable t) {
                    die(t);
                }
            }
        }
    });
    _writerThread.start();
}

Example 3 with ShellProcess

use of backtype.storm.utils.ShellProcess in project jstorm by alibaba.

the class ShellSpout method open.

public void open(Map stormConf, TopologyContext context, SpoutOutputCollector collector) {
    _collector = collector;
    _context = context;
    workerTimeoutMills = 1000 * RT.intCast(stormConf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS));
    _process = new ShellProcess(_command);
    Number subpid = _process.launch(stormConf, context);
    LOG.info("Launched subprocess with pid " + subpid);
    heartBeatExecutorService = MoreExecutors.getExitingScheduledExecutorService(new ScheduledThreadPoolExecutor(1));
}

Example 4 with ShellProcess

use of backtype.storm.utils.ShellProcess in project jstorm by alibaba.

the class ShellBolt method prepare.

public void prepare(Map stormConf, TopologyContext context, final OutputCollector collector) {
    Object maxPending = stormConf.get(Config.TOPOLOGY_SHELLBOLT_MAX_PENDING);
    if (maxPending != null) {
        this._pendingWrites = new LinkedBlockingQueue(((Number) maxPending).intValue());
    }
    _rand = new Random();
    _collector = collector;
    _context = context;
    workerTimeoutMills = 1000 * RT.intCast(stormConf.get(Config.SUPERVISOR_WORKER_TIMEOUT_SECS));
    _process = new ShellProcess(_command);
    // subprocesses must send their pid first thing
    Number subpid = _process.launch(stormConf, context);
    LOG.info("Launched subprocess with pid " + subpid);
    // reader
    _readerThread = new Thread(new BoltReaderRunnable());
    _readerThread.start();
    _writerThread = new Thread(new BoltWriterRunnable());
    _writerThread.start();
    heartBeatExecutorService = MoreExecutors.getExitingScheduledExecutorService(new ScheduledThreadPoolExecutor(1));
    heartBeatExecutorService.scheduleAtFixedRate(new BoltHeartbeatTimerTask(this), 1, 1, TimeUnit.SECONDS);
    LOG.info("Start checking heartbeat...");
    setHeartbeat();
}