Examples with Property io.s4.schema.Schema.Property used on opensource projects

Example 1 with Property

use of io.s4.schema.Schema.Property in project core by s4.

the class DefaultPartitioner method partition.

public List<CompoundKeyInfo> partition(String streamName, List<List<String>> compoundKeyNames, Object event, int partitionCount) {
    if (streamName != null && streamNameSet != null && !streamNameSet.contains(streamName)) {
        return null;
    }
    // Some event types that need special handling
    if (event instanceof io.s4.message.Request) {
        // construct key from request's target
        io.s4.message.Request r = (io.s4.message.Request) event;
        return r.partition(hasher, delimiter, partitionCount);
    } else if (event instanceof io.s4.message.Response) {
        // partition id is encoded in Response, so use it directly.
        io.s4.message.Response r = (io.s4.message.Response) event;
        return r.partition(partitionCount);
    } else if (compoundKeyNames == null) {
        // if compoundKeyNames is null, then assign to a random partition.
        return partitionRandom(partitionCount);
    }
    // have to compute key value and
    // partition based on hash of that value
    Schema schema = schemaContainer.getSchema(event.getClass());
    if (debug) {
        System.out.println(schema);
    }
    List<CompoundKeyInfo> partitionInfoList = new ArrayList<CompoundKeyInfo>();
    // fast path for single top-level key
    if (fastPath || (compoundKeyNames.size() == 1 && compoundKeyNames.get(0).size() == 1)) {
        String simpleKeyName = compoundKeyNames.get(0).get(0);
        if (debug) {
            System.out.println("Using fast path!");
        }
        fastPath = true;
        KeyInfo keyInfo = new KeyInfo();
        Property property = schema.getProperties().get(simpleKeyName);
        if (property == null) {
            return null;
        }
        Object value = null;
        try {
            value = property.getGetterMethod().invoke(event);
        } catch (Exception e) {
            if (debug) {
                e.printStackTrace();
            }
        }
        if (value == null) {
            if (debug) {
                System.out.println("Fast path: Null value encountered");
            }
            return null;
        }
        keyInfo.addElementToPath(simpleKeyName);
        String stringValue = String.valueOf(value);
        keyInfo.setValue(stringValue);
        CompoundKeyInfo partitionInfo = new CompoundKeyInfo();
        partitionInfo.addKeyInfo(keyInfo);
        int partitionId = (int) (hasher.hash(stringValue) % partitionCount);
        partitionInfo.setPartitionId(partitionId);
        partitionInfo.setCompoundValue(stringValue);
        partitionInfoList.add(partitionInfo);
        if (debug) {
            System.out.printf("Value %s, partition id %d\n", stringValue, partitionInfo.getPartitionId());
        }
        return partitionInfoList;
    }
    List<List<KeyInfo>> valueLists = new ArrayList<List<KeyInfo>>();
    int maxSize = 0;
    for (List<String> simpleKeyPath : compoundKeyNames) {
        List<KeyInfo> keyInfoList = new ArrayList<KeyInfo>();
        KeyInfo keyInfo = new KeyInfo();
        keyInfoList = getKeyValues(event, schema, simpleKeyPath, 0, keyInfoList, keyInfo);
        if (keyInfoList == null || keyInfoList.size() == 0) {
            if (debug) {
                System.out.println("Null value encountered");
            }
            // do no partitioning if any simple key's value
            return null;
        // resolves to null
        }
        valueLists.add(keyInfoList);
        maxSize = Math.max(maxSize, keyInfoList.size());
        if (debug) {
            printKeyInfoList(keyInfoList);
        }
    }
    for (int i = 0; i < maxSize; i++) {
        String compoundValue = "";
        CompoundKeyInfo partitionInfo = new CompoundKeyInfo();
        for (List<KeyInfo> keyInfoList : valueLists) {
            if (i < keyInfoList.size()) {
                compoundValue += (compoundValue.length() > 0 ? delimiter : "") + keyInfoList.get(i).getValue();
                partitionInfo.addKeyInfo(keyInfoList.get(i));
            } else {
                compoundValue += (compoundValue.length() > 0 ? delimiter : "") + keyInfoList.get(keyInfoList.size() - 1).getValue();
                partitionInfo.addKeyInfo(keyInfoList.get(keyInfoList.size() - 1));
            }
        }
        // get the partition id
        int partitionId = (int) (hasher.hash(compoundValue) % partitionCount);
        partitionInfo.setPartitionId(partitionId);
        partitionInfo.setCompoundValue(compoundValue);
        partitionInfoList.add(partitionInfo);
        if (debug) {
            System.out.printf("Value %s, partition id %d\n", compoundValue, partitionInfo.getPartitionId());
        }
    }
    return partitionInfoList;
}

Example 2 with Property

use of io.s4.schema.Schema.Property in project core by s4.

the class AbstractPE method setKeyValue.

private void setKeyValue(Object event, CompoundKeyInfo compoundKeyInfo) {
    if (compoundKeyInfo == null) {
        return;
    }
    keyValue = new ArrayList<Object>();
    Schema schema = schemaContainer.getSchema(event.getClass());
    // get the value for each keyInfo
    for (KeyInfo keyInfo : compoundKeyInfo.getKeyInfoList()) {
        Object value = null;
        Object record = event;
        List<?> list = null;
        Property property = null;
        for (KeyPathElement keyPathElement : keyInfo.getKeyPath()) {
            if (keyPathElement instanceof KeyPathElementIndex) {
                record = list.get(((KeyPathElementIndex) keyPathElement).getIndex());
                schema = property.getComponentProperty().getSchema();
            } else {
                String keyPathElementName = ((KeyPathElementName) keyPathElement).getKeyName();
                property = schema.getProperties().get(keyPathElementName);
                value = null;
                try {
                    value = property.getGetterMethod().invoke(record);
                } catch (Exception e) {
                    Logger.getLogger("s4").error(e);
                    return;
                }
                if (value == null) {
                    Logger.getLogger("s4").error("Value for " + keyPathElementName + " is null!");
                    return;
                }
                if (property.getType().isPrimitive() || property.isNumber() || property.getType().equals(String.class)) {
                    keyValue.add(value);
                    if (saveKeyRecord) {
                        if (keyRecord == null) {
                            keyRecord = new ArrayList<Object>();
                        }
                        keyRecord.add(record);
                    }
                    continue;
                } else if (property.isList()) {
                    try {
                        list = (List) property.getGetterMethod().invoke(record);
                    } catch (Exception e) {
                        Logger.getLogger("s4").error(e);
                        return;
                    }
                } else {
                    try {
                        record = property.getGetterMethod().invoke(record);
                    } catch (Exception e) {
                        Logger.getLogger("s4").error(e);
                        return;
                    }
                    schema = property.getSchema();
                }
            }
        }
    }
}

Example 3 with Property

use of io.s4.schema.Schema.Property in project core by s4.

the class JoinPE method copyField.

private void copyField(String propertyName, Schema sourceSchema, Schema targetSchema, Object source, Object target) {
    Property sourceProperty = sourceSchema.getProperties().get(propertyName);
    Property targetProperty = targetSchema.getProperties().get(propertyName);
    if (sourceProperty == null || targetProperty == null || !sourceProperty.getType().equals(targetProperty.getType())) {
        throw new RuntimeException("Specified property " + propertyName + " doesn't exist or is not consistent");
    }
    try {
        Object sourceValue = sourceProperty.getGetterMethod().invoke(source);
        if (sourceValue == null) {
            return;
        }
        if (sourceProperty.getType().isPrimitive()) {
            if (sourceValue instanceof Number) {
                if (((Number) sourceValue).doubleValue() == 0.0) {
                    return;
                }
            }
            if (sourceValue instanceof Boolean) {
                if (((Boolean) sourceValue).equals(Boolean.FALSE)) {
                    return;
                }
            }
        }
        targetProperty.getSetterMethod().invoke(target, sourceValue);
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 4 with Property

use of io.s4.schema.Schema.Property in project core by s4.

the class LoadGenerator method makeList.

public Object makeList(Property property, JSONArray jsonArray) {
    Property componentProperty = property.getComponentProperty();
    int size = jsonArray.length();
    List<Object> list = new ArrayList<Object>(size);
    try {
        for (int i = 0; i < size; i++) {
            Object value = jsonArray.get(i);
            list.add(makeSettableValue(componentProperty, value));
        }
    } catch (JSONException je) {
        throw new RuntimeException(je);
    }
    return list;
}

Example 5 with Property

use of io.s4.schema.Schema.Property in project core by s4.

the class AbstractWindowingPE method processEvent.

public void processEvent(Object event) {
    long currentTime = getCurrentTime();
    long maybeCurrentTime = -1;
    if (timestampFields != null) {
        Schema schema = schemaContainer.getSchema(event.getClass());
        String fieldName = timestampFields.get(getStreamName());
        if (fieldName != null) {
            Property property = schema.getProperties().get(fieldName);
            if (property != null && (property.getType().equals(Long.TYPE) || property.getType().equals(Long.class))) {
                try {
                    maybeCurrentTime = (Long) property.getGetterMethod().invoke(event);
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        }
    }
    if (maybeCurrentTime > -1) {
        currentTime = maybeCurrentTime;
        lastTimestamp = currentTime;
    }
    // convert
    long slotTime = slotUtils.getSlotAtTime(currentTime / 1000);
    if (slots == null) {
        slots = Collections.synchronizedMap(new HashMap<Long, Slot>());
    }
    Slot slot = slots.get(slotTime);
    if (slot == null) {
        try {
            slot = (Slot) slotClass.newInstance();
        } catch (IllegalAccessException iae) {
            throw new RuntimeException(iae);
        } catch (InstantiationException ie) {
            throw new RuntimeException(ie);
        }
        slots.put(slotTime, slot);
    }
    overloadDispatcher.dispatch(slot, event, slotTime, this);
}