Examples with KeyInfo io.s4.dispatcher.partitioner.KeyInfo used on opensource projects

Example 1 with KeyInfo

use of io.s4.dispatcher.partitioner.KeyInfo in project core by s4.

the class DefaultPartitioner method partition.

public List<CompoundKeyInfo> partition(String streamName, List<List<String>> compoundKeyNames, Object event, int partitionCount) {
    if (streamName != null && streamNameSet != null && !streamNameSet.contains(streamName)) {
        return null;
    }
    // Some event types that need special handling
    if (event instanceof io.s4.message.Request) {
        // construct key from request's target
        io.s4.message.Request r = (io.s4.message.Request) event;
        return r.partition(hasher, delimiter, partitionCount);
    } else if (event instanceof io.s4.message.Response) {
        // partition id is encoded in Response, so use it directly.
        io.s4.message.Response r = (io.s4.message.Response) event;
        return r.partition(partitionCount);
    } else if (compoundKeyNames == null) {
        // if compoundKeyNames is null, then assign to a random partition.
        return partitionRandom(partitionCount);
    }
    // have to compute key value and
    // partition based on hash of that value
    Schema schema = schemaContainer.getSchema(event.getClass());
    if (debug) {
        System.out.println(schema);
    }
    List<CompoundKeyInfo> partitionInfoList = new ArrayList<CompoundKeyInfo>();
    // fast path for single top-level key
    if (fastPath || (compoundKeyNames.size() == 1 && compoundKeyNames.get(0).size() == 1)) {
        String simpleKeyName = compoundKeyNames.get(0).get(0);
        if (debug) {
            System.out.println("Using fast path!");
        }
        fastPath = true;
        KeyInfo keyInfo = new KeyInfo();
        Property property = schema.getProperties().get(simpleKeyName);
        if (property == null) {
            return null;
        }
        Object value = null;
        try {
            value = property.getGetterMethod().invoke(event);
        } catch (Exception e) {
            if (debug) {
                e.printStackTrace();
            }
        }
        if (value == null) {
            if (debug) {
                System.out.println("Fast path: Null value encountered");
            }
            return null;
        }
        keyInfo.addElementToPath(simpleKeyName);
        String stringValue = String.valueOf(value);
        keyInfo.setValue(stringValue);
        CompoundKeyInfo partitionInfo = new CompoundKeyInfo();
        partitionInfo.addKeyInfo(keyInfo);
        int partitionId = (int) (hasher.hash(stringValue) % partitionCount);
        partitionInfo.setPartitionId(partitionId);
        partitionInfo.setCompoundValue(stringValue);
        partitionInfoList.add(partitionInfo);
        if (debug) {
            System.out.printf("Value %s, partition id %d\n", stringValue, partitionInfo.getPartitionId());
        }
        return partitionInfoList;
    }
    List<List<KeyInfo>> valueLists = new ArrayList<List<KeyInfo>>();
    int maxSize = 0;
    for (List<String> simpleKeyPath : compoundKeyNames) {
        List<KeyInfo> keyInfoList = new ArrayList<KeyInfo>();
        KeyInfo keyInfo = new KeyInfo();
        keyInfoList = getKeyValues(event, schema, simpleKeyPath, 0, keyInfoList, keyInfo);
        if (keyInfoList == null || keyInfoList.size() == 0) {
            if (debug) {
                System.out.println("Null value encountered");
            }
            // do no partitioning if any simple key's value
            return null;
        // resolves to null
        }
        valueLists.add(keyInfoList);
        maxSize = Math.max(maxSize, keyInfoList.size());
        if (debug) {
            printKeyInfoList(keyInfoList);
        }
    }
    for (int i = 0; i < maxSize; i++) {
        String compoundValue = "";
        CompoundKeyInfo partitionInfo = new CompoundKeyInfo();
        for (List<KeyInfo> keyInfoList : valueLists) {
            if (i < keyInfoList.size()) {
                compoundValue += (compoundValue.length() > 0 ? delimiter : "") + keyInfoList.get(i).getValue();
                partitionInfo.addKeyInfo(keyInfoList.get(i));
            } else {
                compoundValue += (compoundValue.length() > 0 ? delimiter : "") + keyInfoList.get(keyInfoList.size() - 1).getValue();
                partitionInfo.addKeyInfo(keyInfoList.get(keyInfoList.size() - 1));
            }
        }
        // get the partition id
        int partitionId = (int) (hasher.hash(compoundValue) % partitionCount);
        partitionInfo.setPartitionId(partitionId);
        partitionInfo.setCompoundValue(compoundValue);
        partitionInfoList.add(partitionInfo);
        if (debug) {
            System.out.printf("Value %s, partition id %d\n", compoundValue, partitionInfo.getPartitionId());
        }
    }
    return partitionInfoList;
}

Example 2 with KeyInfo

use of io.s4.dispatcher.partitioner.KeyInfo in project core by s4.

the class AbstractPE method setKeyValue.

private void setKeyValue(Object event, CompoundKeyInfo compoundKeyInfo) {
    if (compoundKeyInfo == null) {
        return;
    }
    keyValue = new ArrayList<Object>();
    Schema schema = schemaContainer.getSchema(event.getClass());
    // get the value for each keyInfo
    for (KeyInfo keyInfo : compoundKeyInfo.getKeyInfoList()) {
        Object value = null;
        Object record = event;
        List<?> list = null;
        Property property = null;
        for (KeyPathElement keyPathElement : keyInfo.getKeyPath()) {
            if (keyPathElement instanceof KeyPathElementIndex) {
                record = list.get(((KeyPathElementIndex) keyPathElement).getIndex());
                schema = property.getComponentProperty().getSchema();
            } else {
                String keyPathElementName = ((KeyPathElementName) keyPathElement).getKeyName();
                property = schema.getProperties().get(keyPathElementName);
                value = null;
                try {
                    value = property.getGetterMethod().invoke(record);
                } catch (Exception e) {
                    Logger.getLogger("s4").error(e);
                    return;
                }
                if (value == null) {
                    Logger.getLogger("s4").error("Value for " + keyPathElementName + " is null!");
                    return;
                }
                if (property.getType().isPrimitive() || property.isNumber() || property.getType().equals(String.class)) {
                    keyValue.add(value);
                    if (saveKeyRecord) {
                        if (keyRecord == null) {
                            keyRecord = new ArrayList<Object>();
                        }
                        keyRecord.add(record);
                    }
                    continue;
                } else if (property.isList()) {
                    try {
                        list = (List) property.getGetterMethod().invoke(record);
                    } catch (Exception e) {
                        Logger.getLogger("s4").error(e);
                        return;
                    }
                } else {
                    try {
                        record = property.getGetterMethod().invoke(record);
                    } catch (Exception e) {
                        Logger.getLogger("s4").error(e);
                        return;
                    }
                    schema = property.getSchema();
                }
            }
        }
    }
}

Example 3 with KeyInfo

use of io.s4.dispatcher.partitioner.KeyInfo in project core by s4.

the class ControlEventProcessor method execute.

protected void execute(EventWrapper e, PrototypeWrapper p) {
    List<CompoundKeyInfo> keyInfoList = e.getCompoundKeys();
    Object event = e.getEvent();
    if (event instanceof SinglePERequest) {
        // Handle Requests to individual PEs
        if (keyInfoList.isEmpty())
            return;
        CompoundKeyInfo keyInfo = keyInfoList.get(0);
        String keyVal = keyInfo.getCompoundValue();
        ProcessingElement pe = p.lookupPE(keyVal);
        Response response = ((SinglePERequest) event).evaluate(pe);
        String stream = response.getRInfo().getStream();
        dispatcher.dispatchEvent(stream, response);
    } else if (event instanceof PrototypeRequest) {
        // Or handle aggregate requests to Prototypes.
        Response response = ((PrototypeRequest) event).evaluate(p);
        String stream = response.getRInfo().getStream();
        dispatcher.dispatchEvent(stream, response);
    }
}

Example 4 with KeyInfo

use of io.s4.dispatcher.partitioner.KeyInfo in project core by s4.

the class DefaultPartitioner method getKeyValues.

private List<KeyInfo> getKeyValues(Object record, Schema schema, List<String> keyNameElements, int elementIndex, List<KeyInfo> keyInfoList, KeyInfo keyInfo) {
    String keyElement = keyNameElements.get(elementIndex);
    Property property = schema.getProperties().get(keyElement);
    if (property == null) {
        return null;
    }
    keyInfo.addElementToPath(keyElement);
    Object value = null;
    try {
        value = property.getGetterMethod().invoke(record);
    } catch (Exception e) {
        if (debug) {
            System.out.println("key element is " + keyElement);
            e.printStackTrace();
        }
    }
    if (value == null) {
        // return a null KeyInfo list if we hit a null value
        return null;
    }
    if (property.isList()) {
        List list = (List) value;
        // TODO: handle case where key does not include property of
        // component type
        Schema componentSchema = property.getComponentProperty().getSchema();
        int listLength = list.size();
        for (int i = 0; i < listLength; i++) {
            Object listEntry = list.get(i);
            KeyInfo keyInfoForListEntry = keyInfo.copy();
            keyInfoForListEntry.addElementToPath(i);
            Object partialList = getKeyValues(listEntry, componentSchema, keyNameElements, elementIndex + 1, keyInfoList, keyInfoForListEntry);
            if (partialList == null) {
                return null;
            }
        }
    } else if (property.getSchema() != null) {
        return getKeyValues(value, property.getSchema(), keyNameElements, elementIndex + 1, keyInfoList, keyInfo);
    } else {
        keyInfo.setValue(String.valueOf(value));
        keyInfoList.add(keyInfo);
    }
    return keyInfoList;
}

Example 5 with KeyInfo

use of io.s4.dispatcher.partitioner.KeyInfo in project core by s4.

the class SinglePERequest method partition.

public List<CompoundKeyInfo> partition(Hasher h, String delim, int partCount) {
    List<String> valueList = this.getTarget();
    if (valueList == null)
        return null;
    // First, build the key
    KeyInfo keyInfo = new KeyInfo();
    // special kay name to denote request
    keyInfo.addElementToPath("#req");
    // for value, concatenate list of values from Request's target field.
    String stringValue = StringUtils.collectionToDelimitedString(valueList, delim);
    keyInfo.setValue(stringValue);
    // partition id is derived form string value, as usual
    int partitionId = (int) (h.hash(stringValue) % partCount);
    CompoundKeyInfo partitionInfo = new CompoundKeyInfo();
    partitionInfo.addKeyInfo(keyInfo);
    partitionInfo.setCompoundValue(stringValue);
    partitionInfo.setPartitionId(partitionId);
    List<CompoundKeyInfo> partitionInfoList = new ArrayList<CompoundKeyInfo>();
    partitionInfoList.add(partitionInfo);
    return partitionInfoList;
}