Examples with Utf8 org.apache.avro.util.Utf8 used on opensource projects

Example 71 with Utf8

use of org.apache.avro.util.Utf8 in project databus by linkedin.

the class MapFieldGenerate method generateMap.

public Map<Utf8, Object> generateMap() throws UnknownTypeException {
    Map<Utf8, Object> map = new HashMap<Utf8, Object>(randGenerator.getNextInt() % maxNumberOfMapFields);
    Field fakeField = new Field(field.name(), field.schema().getValueType(), null, null);
    int count = randGenerator.getNextInt() % getMaxNumberOfMapFields();
    for (int i = 0; i < count; i++) {
        // create a new filler each time to emulate null-able fields
        SchemaFiller filler = SchemaFiller.createRandomField(fakeField);
        map.put(new Utf8(field.name() + i), filler.generateRandomObject());
    }
    return map;
}

Example 72 with Utf8

use of org.apache.avro.util.Utf8 in project databus by linkedin.

the class PersonConsumer method processEvent.

private ConsumerCallbackResult processEvent(DbusEvent event, DbusEventDecoder eventDecoder) {
    GenericRecord decodedEvent = eventDecoder.getGenericRecord(event, null);
    try {
        Utf8 firstName = (Utf8) decodedEvent.get("firstName");
        Utf8 lastName = (Utf8) decodedEvent.get("lastName");
        Long birthDate = (Long) decodedEvent.get("birthDate");
        Utf8 deleted = (Utf8) decodedEvent.get("deleted");
        LOG.info("firstName: " + firstName.toString() + ", lastName: " + lastName.toString() + ", birthDate: " + birthDate + ", deleted: " + deleted.toString());
    } catch (Exception e) {
        LOG.error("error decoding event ", e);
        return ConsumerCallbackResult.ERROR;
    }
    return ConsumerCallbackResult.SUCCESS;
}

Example 73 with Utf8

use of org.apache.avro.util.Utf8 in project pinot by linkedin.

the class AvroQueryGenerator method transformAvroArrayToObjectArray.

private static Object[] transformAvroArrayToObjectArray(Array arr) {
    if (arr == null) {
        return new Object[0];
    }
    final Object[] ret = new Object[arr.size()];
    final Iterator iterator = arr.iterator();
    int i = 0;
    while (iterator.hasNext()) {
        Object value = iterator.next();
        if (value instanceof Record) {
            value = ((Record) value).get(0);
        }
        if (value instanceof Utf8) {
            value = ((Utf8) value).toString();
        }
        ret[i++] = value;
    }
    return ret;
}

Example 74 with Utf8

use of org.apache.avro.util.Utf8 in project pinot by linkedin.

the class AvroQueryGenerator method generateSimpleAggregationOnSingleColumnFilters.

public void generateSimpleAggregationOnSingleColumnFilters() throws IOException {
    final Map<String, Map<Object, Integer>> cardinalityCountsMap = new HashMap<String, Map<Object, Integer>>();
    final Map<String, Map<Object, Map<String, Double>>> sumMap = new HashMap<String, Map<Object, Map<String, Double>>>();
    // here string key is columnName:columnValue:MetricName:GroupColumnName:groupKey:metricValue
    final Map<String, Map<Object, Double>> sumGroupBy = new HashMap<String, Map<Object, Double>>();
    aggregationQueries = new ArrayList<AvroQueryGenerator.TestSimpleAggreationQuery>();
    groupByQueries = new ArrayList<AvroQueryGenerator.TestGroupByAggreationQuery>();
    for (final Field f : schema.getFields()) {
        final String fieldName = f.name();
        if (dimensions.contains(fieldName) || metrics.contains(fieldName) || time.equals(fieldName)) {
            isSingleValueMap.put(fieldName, isSingleValueField(f));
            dataTypeMap.put(fieldName, getColumnType(f));
            if (!metrics.contains(fieldName)) {
                cardinalityCountsMap.put(fieldName, new HashMap<Object, Integer>());
            }
        }
    }
    for (final String column : cardinalityCountsMap.keySet()) {
        sumMap.put(column, new HashMap<Object, Map<String, Double>>());
    }
    while (dataStream.hasNext()) {
        final GenericRecord record = dataStream.next();
        for (final String column : cardinalityCountsMap.keySet()) {
            Object value = record.get(column);
            if (value == null) {
                switch(schema.getField(column).schema().getType()) {
                    case INT:
                        value = 0;
                        break;
                    case FLOAT:
                        value = 0F;
                        break;
                    case LONG:
                        value = 0L;
                        break;
                    case DOUBLE:
                        value = 0D;
                        break;
                    case STRING:
                    case BOOLEAN:
                        value = "null";
                        break;
                }
            }
            if (value instanceof Utf8) {
                value = ((Utf8) value).toString();
            }
            if (value instanceof Array) {
                continue;
            }
            for (final String metricName : metrics) {
                final String groupbyKeyBase = column + ":" + record.get(column) + ":" + metricName;
                int dimCounter = 1;
                for (final String dim : cardinalityCountsMap.keySet()) {
                    if (!dim.equals(column)) {
                        dimCounter++;
                        final String groupbyKey = groupbyKeyBase + ":" + dim;
                        if (sumGroupBy.containsKey(groupbyKey)) {
                            if (sumGroupBy.get(groupbyKey).containsKey(record.get(dim))) {
                                sumGroupBy.get(groupbyKey).put(record.get(dim), getAppropriateNumberType(metricName, record.get(metricName), sumGroupBy.get(groupbyKey).get(record.get(dim))));
                            } else {
                                sumGroupBy.get(groupbyKey).put(record.get(dim), Double.parseDouble(record.get(metricName).toString()));
                            }
                        } else {
                            sumGroupBy.put(groupbyKey, new HashMap<Object, Double>());
                            sumGroupBy.get(groupbyKey).put(record.get(dim), Double.parseDouble(record.get(metricName).toString()));
                        }
                    }
                    if (dimCounter == 4) {
                        break;
                    }
                }
            }
            if (cardinalityCountsMap.get(column).containsKey(value)) {
                cardinalityCountsMap.get(column).put(value, cardinalityCountsMap.get(column).get(value) + 1);
            } else {
                cardinalityCountsMap.get(column).put(value, 1);
            }
            if (!sumMap.get(column).containsKey(value)) {
                sumMap.get(column).put(value, new HashMap<String, Double>());
            }
            for (final String metric : metrics) {
                if (!sumMap.get(column).get(value).containsKey(metric)) {
                    sumMap.get(column).get(value).put(metric, getAppropriateNumberType(metric, record.get(metric), 0D));
                } else {
                    sumMap.get(column).get(value).put(metric, getAppropriateNumberType(metric, record.get(metric), sumMap.get(column).get(value).get(metric)));
                }
            }
        // here string key is columnName:columnValue:MetricName:GroupColumnName:groupKey:metricValue
        }
    }
    dataStream.close();
    if (!isRealtimeSegment) {
        for (final String column : cardinalityCountsMap.keySet()) {
            for (final Object entry : cardinalityCountsMap.get(column).keySet()) {
                final StringBuilder bld = new StringBuilder();
                bld.append("select count(*) from ");
                bld.append(resourceName);
                bld.append(" where ");
                bld.append(column);
                bld.append("=");
                bld.append("'");
                bld.append(entry);
                bld.append("'");
                bld.append(" ");
                bld.append("limit 0");
                String queryString = bld.toString();
                if (!queryString.contains("null")) {
                    aggregationQueries.add(new TestSimpleAggreationQuery(queryString, new Double(cardinalityCountsMap.get(column).get(entry))));
                }
            }
        }
    }
    for (final String column : sumMap.keySet()) {
        for (final Object value : sumMap.get(column).keySet()) {
            for (final String metric : sumMap.get(column).get(value).keySet()) {
                final StringBuilder bld = new StringBuilder();
                bld.append("select sum('" + metric + "') from ");
                bld.append(resourceName);
                bld.append(" where ");
                bld.append(column);
                bld.append("=");
                bld.append("'");
                bld.append(value);
                bld.append("'");
                bld.append(" ");
                bld.append("limit 0");
                String queryString = bld.toString();
                if (!queryString.contains("null")) {
                    aggregationQueries.add(new TestSimpleAggreationQuery(bld.toString(), sumMap.get(column).get(value).get(metric)));
                }
            }
        }
    }
    for (final String groupKey : sumGroupBy.keySet()) {
        final String columnName = groupKey.split(":")[0];
        final String columnValue = groupKey.split(":")[1];
        final String metricColumn = groupKey.split(":")[2];
        final String groupByColumnName = groupKey.split(":")[3];
        final StringBuilder bld = new StringBuilder();
        bld.append("select sum('" + metricColumn + "') from ");
        bld.append(resourceName);
        bld.append(" where ");
        bld.append(columnName);
        bld.append("=");
        bld.append("'");
        bld.append(columnValue);
        bld.append("'");
        bld.append(" ");
        bld.append(" group by ");
        bld.append(groupByColumnName);
        bld.append(" top 10 ");
        bld.append("limit 0");
        String queryString = bld.toString();
        if (!queryString.contains("null")) {
            groupByQueries.add(new TestGroupByAggreationQuery(bld.toString(), sumGroupBy.get(groupKey)));
        }
    }
}

Example 75 with Utf8

use of org.apache.avro.util.Utf8 in project rest.li by linkedin.

the class AnyRecordTranslator method dataToAvroGeneric.

@Override
public Object dataToAvroGeneric(DataTranslatorContext context, Object data, DataSchema schema, Schema avroSchema) {
    Object result = null;
    DataMap dataMap;
    try {
        dataMap = (DataMap) data;
    } catch (ClassCastException e) {
        context.appendMessage("Error translating %1$s, it is not a DataMap", data);
        dataMap = null;
    }
    if (dataMap != null) {
        if (dataMap.size() != 1) {
            context.appendMessage("Error translating %1$s, DataMap has more than one element", data);
        } else {
            try {
                Map.Entry<String, Object> entry = dataMap.entrySet().iterator().next();
                String key = entry.getKey();
                Object value = entry.getValue();
                GenericRecord record = new GenericData.Record(avroSchema);
                record.put(TYPE, new Utf8(key));
                record.put(VALUE, new Utf8(_codec.mapToBytes((DataMap) value)));
                result = record;
            } catch (IOException e) {
                context.appendMessage("Error translating %1$s, %2$s", data, e);
            }
        }
    }
    return result;
}