Examples with CompoundKeyInfo io.s4.dispatcher.partitioner.CompoundKeyInfo used on opensource projects

Example 1 with CompoundKeyInfo

use of io.s4.dispatcher.partitioner.CompoundKeyInfo in project core by s4.

the class Dispatcher method dispatchEvent.

private void dispatchEvent(String streamName, Object event, boolean variableKey, List<List<String>> compoundKeyNames) {
    synchronized (this) {
        rawEventCount++;
    }
    if (eventEmitter.getNodeCount() <= 0) {
        return;
    } else {
        if (counts == null) {
            counts = new int[eventEmitter.getNodeCount()];
        }
    }
    try {
        synchronized (this) {
            eventCount++;
        }
        List<CompoundKeyInfo> partionInfoList = new ArrayList<CompoundKeyInfo>();
        for (Partitioner partitioner : partitioners) {
            List<CompoundKeyInfo> pInfoList = null;
            if (!variableKey) {
                pInfoList = partitioner.partition(streamName, event, eventEmitter.getNodeCount());
            } else {
                if (partitioner instanceof VariableKeyPartitioner) {
                    VariableKeyPartitioner vp = (VariableKeyPartitioner) partitioner;
                    pInfoList = vp.partition(streamName, compoundKeyNames, event, eventEmitter.getNodeCount());
                }
            }
            if (pInfoList != null) {
                partionInfoList.addAll(pInfoList);
            }
        }
        Map<Integer, List<CompoundKeyInfo>> pInfoMap = new HashMap<Integer, List<CompoundKeyInfo>>();
        for (CompoundKeyInfo partitionInfo : partionInfoList) {
            int partitionId = partitionInfo.getPartitionId();
            List<CompoundKeyInfo> listByPartitionNumber = pInfoMap.get(partitionId);
            if (listByPartitionNumber == null) {
                listByPartitionNumber = new ArrayList<CompoundKeyInfo>();
                pInfoMap.put(partitionId, listByPartitionNumber);
            }
            listByPartitionNumber.add(partitionInfo);
        }
        for (int partitionId : pInfoMap.keySet()) {
            EventWrapper eventWrapper = new EventWrapper(streamName, event, pInfoMap.get(partitionId));
            counts[partitionId]++;
            eventEmitter.emit(partitionId, eventWrapper);
        }
    } catch (Exception e) {
        Logger.getLogger(loggerName).error("Exception in processEvent on thread " + Thread.currentThread().getId() + " at time " + System.currentTimeMillis(), e);
    }
}

Example 2 with CompoundKeyInfo

use of io.s4.dispatcher.partitioner.CompoundKeyInfo in project core by s4.

the class DefaultPartitioner method partition.

public List<CompoundKeyInfo> partition(String streamName, List<List<String>> compoundKeyNames, Object event, int partitionCount) {
    if (streamName != null && streamNameSet != null && !streamNameSet.contains(streamName)) {
        return null;
    }
    // Some event types that need special handling
    if (event instanceof io.s4.message.Request) {
        // construct key from request's target
        io.s4.message.Request r = (io.s4.message.Request) event;
        return r.partition(hasher, delimiter, partitionCount);
    } else if (event instanceof io.s4.message.Response) {
        // partition id is encoded in Response, so use it directly.
        io.s4.message.Response r = (io.s4.message.Response) event;
        return r.partition(partitionCount);
    } else if (compoundKeyNames == null) {
        // if compoundKeyNames is null, then assign to a random partition.
        return partitionRandom(partitionCount);
    }
    // have to compute key value and
    // partition based on hash of that value
    Schema schema = schemaContainer.getSchema(event.getClass());
    if (debug) {
        System.out.println(schema);
    }
    List<CompoundKeyInfo> partitionInfoList = new ArrayList<CompoundKeyInfo>();
    // fast path for single top-level key
    if (fastPath || (compoundKeyNames.size() == 1 && compoundKeyNames.get(0).size() == 1)) {
        String simpleKeyName = compoundKeyNames.get(0).get(0);
        if (debug) {
            System.out.println("Using fast path!");
        }
        fastPath = true;
        KeyInfo keyInfo = new KeyInfo();
        Property property = schema.getProperties().get(simpleKeyName);
        if (property == null) {
            return null;
        }
        Object value = null;
        try {
            value = property.getGetterMethod().invoke(event);
        } catch (Exception e) {
            if (debug) {
                e.printStackTrace();
            }
        }
        if (value == null) {
            if (debug) {
                System.out.println("Fast path: Null value encountered");
            }
            return null;
        }
        keyInfo.addElementToPath(simpleKeyName);
        String stringValue = String.valueOf(value);
        keyInfo.setValue(stringValue);
        CompoundKeyInfo partitionInfo = new CompoundKeyInfo();
        partitionInfo.addKeyInfo(keyInfo);
        int partitionId = (int) (hasher.hash(stringValue) % partitionCount);
        partitionInfo.setPartitionId(partitionId);
        partitionInfo.setCompoundValue(stringValue);
        partitionInfoList.add(partitionInfo);
        if (debug) {
            System.out.printf("Value %s, partition id %d\n", stringValue, partitionInfo.getPartitionId());
        }
        return partitionInfoList;
    }
    List<List<KeyInfo>> valueLists = new ArrayList<List<KeyInfo>>();
    int maxSize = 0;
    for (List<String> simpleKeyPath : compoundKeyNames) {
        List<KeyInfo> keyInfoList = new ArrayList<KeyInfo>();
        KeyInfo keyInfo = new KeyInfo();
        keyInfoList = getKeyValues(event, schema, simpleKeyPath, 0, keyInfoList, keyInfo);
        if (keyInfoList == null || keyInfoList.size() == 0) {
            if (debug) {
                System.out.println("Null value encountered");
            }
            // do no partitioning if any simple key's value
            return null;
        // resolves to null
        }
        valueLists.add(keyInfoList);
        maxSize = Math.max(maxSize, keyInfoList.size());
        if (debug) {
            printKeyInfoList(keyInfoList);
        }
    }
    for (int i = 0; i < maxSize; i++) {
        String compoundValue = "";
        CompoundKeyInfo partitionInfo = new CompoundKeyInfo();
        for (List<KeyInfo> keyInfoList : valueLists) {
            if (i < keyInfoList.size()) {
                compoundValue += (compoundValue.length() > 0 ? delimiter : "") + keyInfoList.get(i).getValue();
                partitionInfo.addKeyInfo(keyInfoList.get(i));
            } else {
                compoundValue += (compoundValue.length() > 0 ? delimiter : "") + keyInfoList.get(keyInfoList.size() - 1).getValue();
                partitionInfo.addKeyInfo(keyInfoList.get(keyInfoList.size() - 1));
            }
        }
        // get the partition id
        int partitionId = (int) (hasher.hash(compoundValue) % partitionCount);
        partitionInfo.setPartitionId(partitionId);
        partitionInfo.setCompoundValue(compoundValue);
        partitionInfoList.add(partitionInfo);
        if (debug) {
            System.out.printf("Value %s, partition id %d\n", compoundValue, partitionInfo.getPartitionId());
        }
    }
    return partitionInfoList;
}

Example 3 with CompoundKeyInfo

use of io.s4.dispatcher.partitioner.CompoundKeyInfo in project core by s4.

the class PrototypeRequest method partition.

public List<CompoundKeyInfo> partition(Hasher h, String delim, int partCount) {
    // send to all partitions
    List<CompoundKeyInfo> partitionInfoList = new ArrayList<CompoundKeyInfo>();
    for (int i = 0; i < partCount; ++i) {
        CompoundKeyInfo partitionInfo = new CompoundKeyInfo();
        partitionInfo.setPartitionId(i);
        partitionInfoList.add(partitionInfo);
    }
    return partitionInfoList;
}

Example 4 with CompoundKeyInfo

use of io.s4.dispatcher.partitioner.CompoundKeyInfo in project core by s4.

the class Response method partition.

public List<CompoundKeyInfo> partition(int partCount) {
    // partition id is available from the request info object
    int p = this.getRInfo().getPartition();
    List<CompoundKeyInfo> partitionInfoList = null;
    if (p >= 0 && p < partCount) {
        CompoundKeyInfo partitionInfo = new CompoundKeyInfo();
        partitionInfo.setPartitionId(p);
        partitionInfoList = new ArrayList<CompoundKeyInfo>();
        partitionInfoList.add(partitionInfo);
    }
    return partitionInfoList;
}

Example 5 with CompoundKeyInfo

use of io.s4.dispatcher.partitioner.CompoundKeyInfo in project core by s4.

the class AbstractPE method setKeyValue.

private void setKeyValue(Object event, CompoundKeyInfo compoundKeyInfo) {
    if (compoundKeyInfo == null) {
        return;
    }
    keyValue = new ArrayList<Object>();
    Schema schema = schemaContainer.getSchema(event.getClass());
    // get the value for each keyInfo
    for (KeyInfo keyInfo : compoundKeyInfo.getKeyInfoList()) {
        Object value = null;
        Object record = event;
        List<?> list = null;
        Property property = null;
        for (KeyPathElement keyPathElement : keyInfo.getKeyPath()) {
            if (keyPathElement instanceof KeyPathElementIndex) {
                record = list.get(((KeyPathElementIndex) keyPathElement).getIndex());
                schema = property.getComponentProperty().getSchema();
            } else {
                String keyPathElementName = ((KeyPathElementName) keyPathElement).getKeyName();
                property = schema.getProperties().get(keyPathElementName);
                value = null;
                try {
                    value = property.getGetterMethod().invoke(record);
                } catch (Exception e) {
                    Logger.getLogger("s4").error(e);
                    return;
                }
                if (value == null) {
                    Logger.getLogger("s4").error("Value for " + keyPathElementName + " is null!");
                    return;
                }
                if (property.getType().isPrimitive() || property.isNumber() || property.getType().equals(String.class)) {
                    keyValue.add(value);
                    if (saveKeyRecord) {
                        if (keyRecord == null) {
                            keyRecord = new ArrayList<Object>();
                        }
                        keyRecord.add(record);
                    }
                    continue;
                } else if (property.isList()) {
                    try {
                        list = (List) property.getGetterMethod().invoke(record);
                    } catch (Exception e) {
                        Logger.getLogger("s4").error(e);
                        return;
                    }
                } else {
                    try {
                        record = property.getGetterMethod().invoke(record);
                    } catch (Exception e) {
                        Logger.getLogger("s4").error(e);
                        return;
                    }
                    schema = property.getSchema();
                }
            }
        }
    }
}