Examples with CompoundKeyInfo io.s4.dispatcher.partitioner.CompoundKeyInfo used on opensource projects

Example 6 with CompoundKeyInfo

use of io.s4.dispatcher.partitioner.CompoundKeyInfo in project core by s4.

the class ControlEventProcessor method execute.

protected void execute(EventWrapper e, PrototypeWrapper p) {
    List<CompoundKeyInfo> keyInfoList = e.getCompoundKeys();
    Object event = e.getEvent();
    if (event instanceof SinglePERequest) {
        // Handle Requests to individual PEs
        if (keyInfoList.isEmpty())
            return;
        CompoundKeyInfo keyInfo = keyInfoList.get(0);
        String keyVal = keyInfo.getCompoundValue();
        ProcessingElement pe = p.lookupPE(keyVal);
        Response response = ((SinglePERequest) event).evaluate(pe);
        String stream = response.getRInfo().getStream();
        dispatcher.dispatchEvent(stream, response);
    } else if (event instanceof PrototypeRequest) {
        // Or handle aggregate requests to Prototypes.
        Response response = ((PrototypeRequest) event).evaluate(p);
        String stream = response.getRInfo().getStream();
        dispatcher.dispatchEvent(stream, response);
    }
}

Example 7 with CompoundKeyInfo

use of io.s4.dispatcher.partitioner.CompoundKeyInfo in project core by s4.

the class PEContainer method run.

public void run() {
    long startTime, endTime;
    while (true) {
        EventWrapper eventWrapper = null;
        try {
            eventWrapper = workQueue.take();
            if (s4Clock instanceof EventClock) {
                EventClock eventClock = (EventClock) s4Clock;
                eventClock.update(eventWrapper);
            // To what time to update the clock
            }
            if (trackByKey) {
                boolean foundOne = false;
                for (CompoundKeyInfo compoundKeyInfo : eventWrapper.getCompoundKeys()) {
                    foundOne = true;
                    updateCount(eventWrapper.getStreamName() + " " + compoundKeyInfo.getCompoundKey());
                }
                if (!foundOne) {
                    updateCount(eventWrapper.getStreamName() + " *");
                }
            }
            startTime = System.currentTimeMillis();
            if (logger.isDebugEnabled()) {
                logger.debug("STEP 5 (PEContainer): workQueue.take - " + eventWrapper.toString());
            }
            // "Incoming: " + event.getEventName());
            if (monitor != null) {
                monitor.increment(pecontainer_ev_dq_ct.toString(), 1, S4_CORE_METRICS.toString());
            }
            // printPlainPartitionInfoList(event.getCompoundKeyList());
            boolean ctrlEvent = testControlEvent(eventWrapper);
            // execute the PEs interested in this event
            for (int i = 0; i < prototypeWrappers.size(); i++) {
                if (logger.isDebugEnabled()) {
                    logger.debug("STEP 6 (PEContainer): prototypeWrappers(" + i + ") - " + prototypeWrappers.get(i).toString() + " - " + eventWrapper.getStreamName());
                }
                // so.
                if (ctrlEvent) {
                    if (controlEventProcessor != null) {
                        controlEventProcessor.process(eventWrapper, prototypeWrappers.get(i));
                    }
                    continue;
                }
                // otherwise, continue processing event.
                List<EventAdvice> adviceList = adviceLists.get(i);
                for (EventAdvice eventAdvice : adviceList) {
                    if (eventAdvice.getEventName().equals("*") || eventAdvice.getEventName().equals(eventWrapper.getStreamName())) {
                    // event name matches
                    } else {
                        continue;
                    }
                    if (eventAdvice.getKey().equals("*")) {
                        invokePE(prototypeWrappers.get(i).getPE("*"), eventWrapper, null);
                        continue;
                    }
                    for (CompoundKeyInfo compoundKeyInfo : eventWrapper.getCompoundKeys()) {
                        if (eventAdvice.getKey().equals(compoundKeyInfo.getCompoundKey())) {
                            invokePE(prototypeWrappers.get(i).getPE(compoundKeyInfo.getCompoundValue()), eventWrapper, compoundKeyInfo);
                        }
                    }
                }
            }
            endTime = System.currentTimeMillis();
            if (monitor != null) {
                // TODO: need to be changed for more accurate calc
                monitor.increment(pecontainer_exec_elapse_time.toString(), (int) (endTime - startTime), S4_CORE_METRICS.toString());
            }
        } catch (InterruptedException ie) {
            Logger.getLogger("s4").warn("PEContainer is interrupted", ie);
            return;
        } catch (Exception e) {
            Logger.getLogger("s4").error("Exception choosing processing element to run", e);
        }
    }
}

Example 8 with CompoundKeyInfo

use of io.s4.dispatcher.partitioner.CompoundKeyInfo in project core by s4.

the class LoadGenerator method run.

public void run() {
    // for now, no warm-up mechanism
    adjustedExpectedRate = expectedRate;
    long startTime = 0;
    long intervalStart = 0;
    int emitCountStart = 0;
    long[] rateInfo = new long[2];
    // start with a sleep time of 100
    rateInfo[0] = 100;
    BufferedReader br = null;
    Reader inputReader = null;
    try {
        if (inputFilename.equals("-")) {
            inputReader = new InputStreamReader(System.in);
        } else {
            inputReader = new FileReader(inputFilename);
        }
        br = new BufferedReader(inputReader);
        String inputLine = null;
        boolean firstLine = true;
        EventWrapper eventWrapper = null;
        for (startTime = System.nanoTime(); (inputLine = br.readLine()) != null; startTime = System.nanoTime()) {
            if (firstLine) {
                JSONObject jsonRecord = new JSONObject(inputLine);
                createEventTypeInfo(jsonRecord);
                System.out.println(eventTypeInfoMap);
                if (eventTypeInfoMap.size() == 0) {
                    return;
                }
                firstLine = false;
                continue;
            }
            try {
                JSONObject jsonRecord = new JSONObject(inputLine);
                int classIndex = jsonRecord.getInt("_index");
                EventTypeInfo eventTypeInfo = eventTypeInfoMap.get(classIndex);
                if (eventTypeInfo == null) {
                    System.err.printf("Invalid _index value %d\n", classIndex);
                    return;
                }
                Object event = makeRecord(jsonRecord, eventTypeInfo.getSchema());
                eventWrapper = new EventWrapper(eventTypeInfo.getStreamName(), event, new ArrayList<CompoundKeyInfo>());
            // System.out.println(eventWrapper.getStreamName() + ": " +
            // eventWrapper.getEvent());
            } catch (Exception e) {
                e.printStackTrace();
                System.err.printf("Bad input data %s\n", inputLine);
                continue;
            }
            int partition = Math.abs(rand.nextInt()) % eventEmitter.getNodeCount();
            eventEmitter.emit(partition, eventWrapper);
            emitCount++;
            // the rest of the stuff in this block is just to maintain the
            // rate
            processTimes[processTimePointer] = System.nanoTime() - startTime;
            processTimePointer = (processTimePointer == PROCESS_TIME_LIST_MAX_SIZE - 1) ? 0 : processTimePointer + 1;
            if (emitCount == 1 || emitCount % 20 == 0) {
                rateInfo = getRateInfo(rateInfo);
            }
            // if it's time, display the actual emit rate
            if (intervalStart == 0) {
                intervalStart = System.currentTimeMillis();
            } else {
                long interval = System.currentTimeMillis() - intervalStart;
                if (interval >= (displayRateInterval * 1000)) {
                    double rate = (emitCount - emitCountStart) / (interval / 1000.0);
                    System.out.println("Rate is " + rate);
                    intervalStart = System.currentTimeMillis();
                    emitCountStart = emitCount;
                }
            }
            if (rateInfo[1] == 0 || emitCount % rateInfo[1] == 0) {
                try {
                    Thread.sleep(rateInfo[0]);
                } catch (InterruptedException ie) {
                }
            }
        }
        System.out.printf("Emitted %d events\n", emitCount);
    } catch (Exception e) {
        throw new RuntimeException(e);
    } finally {
        try {
            br.close();
        } catch (Exception e) {
        }
        try {
            inputReader.close();
        } catch (Exception e) {
        }
    }
}

Example 9 with CompoundKeyInfo

use of io.s4.dispatcher.partitioner.CompoundKeyInfo in project core by s4.

the class SinglePERequest method partition.

public List<CompoundKeyInfo> partition(Hasher h, String delim, int partCount) {
    List<String> valueList = this.getTarget();
    if (valueList == null)
        return null;
    // First, build the key
    KeyInfo keyInfo = new KeyInfo();
    // special kay name to denote request
    keyInfo.addElementToPath("#req");
    // for value, concatenate list of values from Request's target field.
    String stringValue = StringUtils.collectionToDelimitedString(valueList, delim);
    keyInfo.setValue(stringValue);
    // partition id is derived form string value, as usual
    int partitionId = (int) (h.hash(stringValue) % partCount);
    CompoundKeyInfo partitionInfo = new CompoundKeyInfo();
    partitionInfo.addKeyInfo(keyInfo);
    partitionInfo.setCompoundValue(stringValue);
    partitionInfo.setPartitionId(partitionId);
    List<CompoundKeyInfo> partitionInfoList = new ArrayList<CompoundKeyInfo>();
    partitionInfoList.add(partitionInfo);
    return partitionInfoList;
}