Examples with PEContainer io.s4.processor.PEContainer used on opensource projects

Example 1 with PEContainer

use of io.s4.processor.PEContainer in project core by s4.

the class MainApp method main.

public static void main(String[] args) throws Exception {
    Options options = new Options();
    options.addOption(OptionBuilder.withArgName("corehome").hasArg().withDescription("core home").create("c"));
    options.addOption(OptionBuilder.withArgName("appshome").hasArg().withDescription("applications home").create("a"));
    options.addOption(OptionBuilder.withArgName("s4clock").hasArg().withDescription("s4 clock").create("d"));
    options.addOption(OptionBuilder.withArgName("seedtime").hasArg().withDescription("event clock initialization time").create("s"));
    options.addOption(OptionBuilder.withArgName("extshome").hasArg().withDescription("extensions home").create("e"));
    options.addOption(OptionBuilder.withArgName("instanceid").hasArg().withDescription("instance id").create("i"));
    options.addOption(OptionBuilder.withArgName("configtype").hasArg().withDescription("configuration type").create("t"));
    CommandLineParser parser = new GnuParser();
    CommandLine commandLine = null;
    String clockType = "wall";
    try {
        commandLine = parser.parse(options, args);
    } catch (ParseException pe) {
        System.err.println(pe.getLocalizedMessage());
        System.exit(1);
    }
    int instanceId = -1;
    if (commandLine.hasOption("i")) {
        String instanceIdStr = commandLine.getOptionValue("i");
        try {
            instanceId = Integer.parseInt(instanceIdStr);
        } catch (NumberFormatException nfe) {
            System.err.println("Bad instance id: %s" + instanceIdStr);
            System.exit(1);
        }
    }
    if (commandLine.hasOption("c")) {
        coreHome = commandLine.getOptionValue("c");
    }
    if (commandLine.hasOption("a")) {
        appsHome = commandLine.getOptionValue("a");
    }
    if (commandLine.hasOption("d")) {
        clockType = commandLine.getOptionValue("d");
    }
    if (commandLine.hasOption("e")) {
        extsHome = commandLine.getOptionValue("e");
    }
    String configType = "typical";
    if (commandLine.hasOption("t")) {
        configType = commandLine.getOptionValue("t");
    }
    long seedTime = 0;
    if (commandLine.hasOption("s")) {
        seedTime = Long.parseLong(commandLine.getOptionValue("s"));
    }
    File coreHomeFile = new File(coreHome);
    if (!coreHomeFile.isDirectory()) {
        System.err.println("Bad core home: " + coreHome);
        System.exit(1);
    }
    File appsHomeFile = new File(appsHome);
    if (!appsHomeFile.isDirectory()) {
        System.err.println("Bad applications home: " + appsHome);
        System.exit(1);
    }
    if (instanceId > -1) {
        System.setProperty("instanceId", "" + instanceId);
    } else {
        System.setProperty("instanceId", "" + S4Util.getPID());
    }
    List loArgs = commandLine.getArgList();
    if (loArgs.size() < 1) {
    // System.err.println("No bean configuration file specified");
    // System.exit(1);
    }
    // String s4ConfigXml = (String) loArgs.get(0);
    // System.out.println("s4ConfigXml is " + s4ConfigXml);
    ClassPathResource propResource = new ClassPathResource("s4_core.properties");
    Properties prop = new Properties();
    if (propResource.exists()) {
        prop.load(propResource.getInputStream());
    } else {
        System.err.println("Unable to find s4_core.properties. It must be available in classpath");
        System.exit(1);
    }
    ApplicationContext coreContext = null;
    String configBase = coreHome + File.separatorChar + "conf" + File.separatorChar + configType;
    String configPath = "";
    List<String> coreConfigUrls = new ArrayList<String>();
    File configFile = null;
    // load clock configuration
    configPath = configBase + File.separatorChar + clockType + "_clock.xml";
    coreConfigUrls.add(configPath);
    // load core config xml
    configPath = configBase + File.separatorChar + "s4_core_conf.xml";
    configFile = new File(configPath);
    if (!configFile.exists()) {
        System.err.printf("S4 core config file %s does not exist\n", configPath);
        System.exit(1);
    }
    coreConfigUrls.add(configPath);
    String[] coreConfigFiles = new String[coreConfigUrls.size()];
    coreConfigUrls.toArray(coreConfigFiles);
    String[] coreConfigFileUrls = new String[coreConfigFiles.length];
    for (int i = 0; i < coreConfigFiles.length; i++) {
        coreConfigFileUrls[i] = "file:" + coreConfigFiles[i];
    }
    coreContext = new FileSystemXmlApplicationContext(coreConfigFileUrls, coreContext);
    ApplicationContext context = coreContext;
    Clock s4Clock = (Clock) context.getBean("clock");
    if (s4Clock instanceof EventClock && seedTime > 0) {
        EventClock s4EventClock = (EventClock) s4Clock;
        s4EventClock.updateTime(seedTime);
        System.out.println("Intializing event clock time with seed time " + s4EventClock.getCurrentTime());
    }
    PEContainer peContainer = (PEContainer) context.getBean("peContainer");
    Watcher w = (Watcher) context.getBean("watcher");
    w.setConfigFilename(configPath);
    // load extension modules
    String[] configFileNames = getModuleConfigFiles(extsHome, prop);
    if (configFileNames.length > 0) {
        String[] configFileUrls = new String[configFileNames.length];
        for (int i = 0; i < configFileNames.length; i++) {
            configFileUrls[i] = "file:" + configFileNames[i];
        }
        context = new FileSystemXmlApplicationContext(configFileUrls, context);
    }
    // load application modules
    configFileNames = getModuleConfigFiles(appsHome, prop);
    if (configFileNames.length > 0) {
        String[] configFileUrls = new String[configFileNames.length];
        for (int i = 0; i < configFileNames.length; i++) {
            configFileUrls[i] = "file:" + configFileNames[i];
        }
        context = new FileSystemXmlApplicationContext(configFileUrls, context);
        // attach any beans that implement ProcessingElement to the PE
        // Container
        String[] processingElementBeanNames = context.getBeanNamesForType(ProcessingElement.class);
        for (String processingElementBeanName : processingElementBeanNames) {
            Object bean = context.getBean(processingElementBeanName);
            try {
                Method getS4ClockMethod = bean.getClass().getMethod("getS4Clock");
                if (getS4ClockMethod.getReturnType().equals(Clock.class)) {
                    if (getS4ClockMethod.invoke(bean) == null) {
                        Method setS4ClockMethod = bean.getClass().getMethod("setS4Clock", Clock.class);
                        setS4ClockMethod.invoke(bean, coreContext.getBean("clock"));
                    }
                }
            } catch (NoSuchMethodException mnfe) {
            // acceptable
            }
            System.out.println("Adding processing element with bean name " + processingElementBeanName + ", id " + ((ProcessingElement) bean).getId());
            peContainer.addProcessor((ProcessingElement) bean);
        }
    }
}

Example 2 with PEContainer

use of io.s4.processor.PEContainer in project core by s4.

the class PEContainer method run.

public void run() {
    long startTime, endTime;
    while (true) {
        EventWrapper eventWrapper = null;
        try {
            eventWrapper = workQueue.take();
            if (s4Clock instanceof EventClock) {
                EventClock eventClock = (EventClock) s4Clock;
                eventClock.update(eventWrapper);
            // To what time to update the clock
            }
            if (trackByKey) {
                boolean foundOne = false;
                for (CompoundKeyInfo compoundKeyInfo : eventWrapper.getCompoundKeys()) {
                    foundOne = true;
                    updateCount(eventWrapper.getStreamName() + " " + compoundKeyInfo.getCompoundKey());
                }
                if (!foundOne) {
                    updateCount(eventWrapper.getStreamName() + " *");
                }
            }
            startTime = System.currentTimeMillis();
            if (logger.isDebugEnabled()) {
                logger.debug("STEP 5 (PEContainer): workQueue.take - " + eventWrapper.toString());
            }
            // "Incoming: " + event.getEventName());
            if (monitor != null) {
                monitor.increment(pecontainer_ev_dq_ct.toString(), 1, S4_CORE_METRICS.toString());
            }
            // printPlainPartitionInfoList(event.getCompoundKeyList());
            boolean ctrlEvent = testControlEvent(eventWrapper);
            // execute the PEs interested in this event
            for (int i = 0; i < prototypeWrappers.size(); i++) {
                if (logger.isDebugEnabled()) {
                    logger.debug("STEP 6 (PEContainer): prototypeWrappers(" + i + ") - " + prototypeWrappers.get(i).toString() + " - " + eventWrapper.getStreamName());
                }
                // so.
                if (ctrlEvent) {
                    if (controlEventProcessor != null) {
                        controlEventProcessor.process(eventWrapper, prototypeWrappers.get(i));
                    }
                    continue;
                }
                // otherwise, continue processing event.
                List<EventAdvice> adviceList = adviceLists.get(i);
                for (EventAdvice eventAdvice : adviceList) {
                    if (eventAdvice.getEventName().equals("*") || eventAdvice.getEventName().equals(eventWrapper.getStreamName())) {
                    // event name matches
                    } else {
                        continue;
                    }
                    if (eventAdvice.getKey().equals("*")) {
                        invokePE(prototypeWrappers.get(i).getPE("*"), eventWrapper, null);
                        continue;
                    }
                    for (CompoundKeyInfo compoundKeyInfo : eventWrapper.getCompoundKeys()) {
                        if (eventAdvice.getKey().equals(compoundKeyInfo.getCompoundKey())) {
                            invokePE(prototypeWrappers.get(i).getPE(compoundKeyInfo.getCompoundValue()), eventWrapper, compoundKeyInfo);
                        }
                    }
                }
            }
            endTime = System.currentTimeMillis();
            if (monitor != null) {
                // TODO: need to be changed for more accurate calc
                monitor.increment(pecontainer_exec_elapse_time.toString(), (int) (endTime - startTime), S4_CORE_METRICS.toString());
            }
        } catch (InterruptedException ie) {
            Logger.getLogger("s4").warn("PEContainer is interrupted", ie);
            return;
        } catch (Exception e) {
            Logger.getLogger("s4").error("Exception choosing processing element to run", e);
        }
    }
}