Examples with EventWrapper io.s4.collector.EventWrapper used on opensource projects

Example 1 with EventWrapper

use of io.s4.collector.EventWrapper in project core by s4.

the class Dispatcher method dispatchEvent.

private void dispatchEvent(String streamName, Object event, boolean variableKey, List<List<String>> compoundKeyNames) {
    synchronized (this) {
        rawEventCount++;
    }
    if (eventEmitter.getNodeCount() <= 0) {
        return;
    } else {
        if (counts == null) {
            counts = new int[eventEmitter.getNodeCount()];
        }
    }
    try {
        synchronized (this) {
            eventCount++;
        }
        List<CompoundKeyInfo> partionInfoList = new ArrayList<CompoundKeyInfo>();
        for (Partitioner partitioner : partitioners) {
            List<CompoundKeyInfo> pInfoList = null;
            if (!variableKey) {
                pInfoList = partitioner.partition(streamName, event, eventEmitter.getNodeCount());
            } else {
                if (partitioner instanceof VariableKeyPartitioner) {
                    VariableKeyPartitioner vp = (VariableKeyPartitioner) partitioner;
                    pInfoList = vp.partition(streamName, compoundKeyNames, event, eventEmitter.getNodeCount());
                }
            }
            if (pInfoList != null) {
                partionInfoList.addAll(pInfoList);
            }
        }
        Map<Integer, List<CompoundKeyInfo>> pInfoMap = new HashMap<Integer, List<CompoundKeyInfo>>();
        for (CompoundKeyInfo partitionInfo : partionInfoList) {
            int partitionId = partitionInfo.getPartitionId();
            List<CompoundKeyInfo> listByPartitionNumber = pInfoMap.get(partitionId);
            if (listByPartitionNumber == null) {
                listByPartitionNumber = new ArrayList<CompoundKeyInfo>();
                pInfoMap.put(partitionId, listByPartitionNumber);
            }
            listByPartitionNumber.add(partitionInfo);
        }
        for (int partitionId : pInfoMap.keySet()) {
            EventWrapper eventWrapper = new EventWrapper(streamName, event, pInfoMap.get(partitionId));
            counts[partitionId]++;
            eventEmitter.emit(partitionId, eventWrapper);
        }
    } catch (Exception e) {
        Logger.getLogger(loggerName).error("Exception in processEvent on thread " + Thread.currentThread().getId() + " at time " + System.currentTimeMillis(), e);
    }
}

Example 2 with EventWrapper

use of io.s4.collector.EventWrapper in project core by s4.

the class ControlEventProcessor method execute.

protected void execute(EventWrapper e, PrototypeWrapper p) {
    List<CompoundKeyInfo> keyInfoList = e.getCompoundKeys();
    Object event = e.getEvent();
    if (event instanceof SinglePERequest) {
        // Handle Requests to individual PEs
        if (keyInfoList.isEmpty())
            return;
        CompoundKeyInfo keyInfo = keyInfoList.get(0);
        String keyVal = keyInfo.getCompoundValue();
        ProcessingElement pe = p.lookupPE(keyVal);
        Response response = ((SinglePERequest) event).evaluate(pe);
        String stream = response.getRInfo().getStream();
        dispatcher.dispatchEvent(stream, response);
    } else if (event instanceof PrototypeRequest) {
        // Or handle aggregate requests to Prototypes.
        Response response = ((PrototypeRequest) event).evaluate(p);
        String stream = response.getRInfo().getStream();
        dispatcher.dispatchEvent(stream, response);
    }
}

Example 3 with EventWrapper

use of io.s4.collector.EventWrapper in project core by s4.

the class PEContainer method run.

public void run() {
    long startTime, endTime;
    while (true) {
        EventWrapper eventWrapper = null;
        try {
            eventWrapper = workQueue.take();
            if (s4Clock instanceof EventClock) {
                EventClock eventClock = (EventClock) s4Clock;
                eventClock.update(eventWrapper);
            // To what time to update the clock
            }
            if (trackByKey) {
                boolean foundOne = false;
                for (CompoundKeyInfo compoundKeyInfo : eventWrapper.getCompoundKeys()) {
                    foundOne = true;
                    updateCount(eventWrapper.getStreamName() + " " + compoundKeyInfo.getCompoundKey());
                }
                if (!foundOne) {
                    updateCount(eventWrapper.getStreamName() + " *");
                }
            }
            startTime = System.currentTimeMillis();
            if (logger.isDebugEnabled()) {
                logger.debug("STEP 5 (PEContainer): workQueue.take - " + eventWrapper.toString());
            }
            // "Incoming: " + event.getEventName());
            if (monitor != null) {
                monitor.increment(pecontainer_ev_dq_ct.toString(), 1, S4_CORE_METRICS.toString());
            }
            // printPlainPartitionInfoList(event.getCompoundKeyList());
            boolean ctrlEvent = testControlEvent(eventWrapper);
            // execute the PEs interested in this event
            for (int i = 0; i < prototypeWrappers.size(); i++) {
                if (logger.isDebugEnabled()) {
                    logger.debug("STEP 6 (PEContainer): prototypeWrappers(" + i + ") - " + prototypeWrappers.get(i).toString() + " - " + eventWrapper.getStreamName());
                }
                // so.
                if (ctrlEvent) {
                    if (controlEventProcessor != null) {
                        controlEventProcessor.process(eventWrapper, prototypeWrappers.get(i));
                    }
                    continue;
                }
                // otherwise, continue processing event.
                List<EventAdvice> adviceList = adviceLists.get(i);
                for (EventAdvice eventAdvice : adviceList) {
                    if (eventAdvice.getEventName().equals("*") || eventAdvice.getEventName().equals(eventWrapper.getStreamName())) {
                    // event name matches
                    } else {
                        continue;
                    }
                    if (eventAdvice.getKey().equals("*")) {
                        invokePE(prototypeWrappers.get(i).getPE("*"), eventWrapper, null);
                        continue;
                    }
                    for (CompoundKeyInfo compoundKeyInfo : eventWrapper.getCompoundKeys()) {
                        if (eventAdvice.getKey().equals(compoundKeyInfo.getCompoundKey())) {
                            invokePE(prototypeWrappers.get(i).getPE(compoundKeyInfo.getCompoundValue()), eventWrapper, compoundKeyInfo);
                        }
                    }
                }
            }
            endTime = System.currentTimeMillis();
            if (monitor != null) {
                // TODO: need to be changed for more accurate calc
                monitor.increment(pecontainer_exec_elapse_time.toString(), (int) (endTime - startTime), S4_CORE_METRICS.toString());
            }
        } catch (InterruptedException ie) {
            Logger.getLogger("s4").warn("PEContainer is interrupted", ie);
            return;
        } catch (Exception e) {
            Logger.getLogger("s4").error("Exception choosing processing element to run", e);
        }
    }
}

Example 4 with EventWrapper

use of io.s4.collector.EventWrapper in project core by s4.

the class LoadGenerator method run.

public void run() {
    // for now, no warm-up mechanism
    adjustedExpectedRate = expectedRate;
    long startTime = 0;
    long intervalStart = 0;
    int emitCountStart = 0;
    long[] rateInfo = new long[2];
    // start with a sleep time of 100
    rateInfo[0] = 100;
    BufferedReader br = null;
    Reader inputReader = null;
    try {
        if (inputFilename.equals("-")) {
            inputReader = new InputStreamReader(System.in);
        } else {
            inputReader = new FileReader(inputFilename);
        }
        br = new BufferedReader(inputReader);
        String inputLine = null;
        boolean firstLine = true;
        EventWrapper eventWrapper = null;
        for (startTime = System.nanoTime(); (inputLine = br.readLine()) != null; startTime = System.nanoTime()) {
            if (firstLine) {
                JSONObject jsonRecord = new JSONObject(inputLine);
                createEventTypeInfo(jsonRecord);
                System.out.println(eventTypeInfoMap);
                if (eventTypeInfoMap.size() == 0) {
                    return;
                }
                firstLine = false;
                continue;
            }
            try {
                JSONObject jsonRecord = new JSONObject(inputLine);
                int classIndex = jsonRecord.getInt("_index");
                EventTypeInfo eventTypeInfo = eventTypeInfoMap.get(classIndex);
                if (eventTypeInfo == null) {
                    System.err.printf("Invalid _index value %d\n", classIndex);
                    return;
                }
                Object event = makeRecord(jsonRecord, eventTypeInfo.getSchema());
                eventWrapper = new EventWrapper(eventTypeInfo.getStreamName(), event, new ArrayList<CompoundKeyInfo>());
            // System.out.println(eventWrapper.getStreamName() + ": " +
            // eventWrapper.getEvent());
            } catch (Exception e) {
                e.printStackTrace();
                System.err.printf("Bad input data %s\n", inputLine);
                continue;
            }
            int partition = Math.abs(rand.nextInt()) % eventEmitter.getNodeCount();
            eventEmitter.emit(partition, eventWrapper);
            emitCount++;
            // the rest of the stuff in this block is just to maintain the
            // rate
            processTimes[processTimePointer] = System.nanoTime() - startTime;
            processTimePointer = (processTimePointer == PROCESS_TIME_LIST_MAX_SIZE - 1) ? 0 : processTimePointer + 1;
            if (emitCount == 1 || emitCount % 20 == 0) {
                rateInfo = getRateInfo(rateInfo);
            }
            // if it's time, display the actual emit rate
            if (intervalStart == 0) {
                intervalStart = System.currentTimeMillis();
            } else {
                long interval = System.currentTimeMillis() - intervalStart;
                if (interval >= (displayRateInterval * 1000)) {
                    double rate = (emitCount - emitCountStart) / (interval / 1000.0);
                    System.out.println("Rate is " + rate);
                    intervalStart = System.currentTimeMillis();
                    emitCountStart = emitCount;
                }
            }
            if (rateInfo[1] == 0 || emitCount % rateInfo[1] == 0) {
                try {
                    Thread.sleep(rateInfo[0]);
                } catch (InterruptedException ie) {
                }
            }
        }
        System.out.printf("Emitted %d events\n", emitCount);
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        try {
            br.close();
        } catch (Exception e) {
        }
        try {
            inputReader.close();
        } catch (Exception e) {
        }
    }
}

Example 5 with EventWrapper

use of io.s4.collector.EventWrapper in project core by s4.

the class Adapter method clientTest.

public static void clientTest() throws IOException, InterruptedException {
    BasicConfigurator.configure();
    TestDispatcher disp = new TestDispatcher();
    Adapter adapter = new Adapter();
    adapter.setDispatcher(disp);
    GenericJsonClientStub stub = new GenericJsonClientStub();
    stub.setConnectionPort(2334);
    InputStub[] in = { stub };
    OutputStub[] out = { stub };
    adapter.setInputStubs(Arrays.asList(in));
    adapter.setOutputStubs(Arrays.asList(out));
    adapter.init();
    stub.init();
    while (true) {
        Thread.sleep(10000);
        TestReturnType r = new TestReturnType(100, 200);
        adapter.eventReader.queueWork(new EventWrapper("TESTSTREAM", r, null));
    }
}