Examples with BasicBSONObject org.bson.BasicBSONObject used on opensource projects

Example 66 with BasicBSONObject

use of org.bson.BasicBSONObject in project mongo-hadoop by mongodb.

the class BSONSerDe method serializeStruct.

/**
     * Turn a Hive struct into a BasicBSONObject.
     * @param obj the Hive struct
     * @param structOI an {@code ObjectInspector} for the struct
     * @param ext the field name
     * @return a BasicBSONObject representing the Hive struct
     */
private Object serializeStruct(final Object obj, final StructObjectInspector structOI, final String ext) {
    if (ext.length() > 0 && isObjectIdStruct(obj, structOI)) {
        String objectIdString = "";
        for (StructField s : structOI.getAllStructFieldRefs()) {
            if (s.getFieldName().equals(OID)) {
                objectIdString = structOI.getStructFieldData(obj, s).toString();
                break;
            }
        }
        return new ObjectId(objectIdString);
    } else {
        BasicBSONObject bsonObject = new BasicBSONObject();
        // fields is the list of all variable names and information within the struct obj
        List<? extends StructField> fields = structOI.getAllStructFieldRefs();
        for (int i = 0; i < fields.size(); i++) {
            StructField field = fields.get(i);
            String fieldName, hiveMapping;
            // get corresponding mongoDB field  
            if (ext.length() == 0) {
                fieldName = columnNames.get(i);
                hiveMapping = fieldName;
            } else {
                fieldName = field.getFieldName();
                hiveMapping = ext + "." + fieldName;
            }
            ObjectInspector fieldOI = field.getFieldObjectInspector();
            Object fieldObj = structOI.getStructFieldData(obj, field);
            if (hiveToMongo != null && hiveToMongo.containsKey(hiveMapping)) {
                String mongoMapping = hiveToMongo.get(hiveMapping);
                int lastDotPos = mongoMapping.lastIndexOf(".");
                String lastMapping = lastDotPos == -1 ? mongoMapping : mongoMapping.substring(lastDotPos + 1);
                bsonObject.put(lastMapping, serializeObject(fieldObj, fieldOI, hiveMapping));
            } else {
                bsonObject.put(fieldName, serializeObject(fieldObj, fieldOI, hiveMapping));
            }
        }
        return bsonObject;
    }
}

Example 67 with BasicBSONObject

use of org.bson.BasicBSONObject in project mongo-hadoop by mongodb.

the class TreasuryYieldUpdateReducer method reduce.

@Override
public void reduce(final IntWritable pKey, final Iterable<DoubleWritable> pValues, final Context pContext) throws IOException, InterruptedException {
    int count = 0;
    double sum = 0;
    for (final DoubleWritable value : pValues) {
        sum += value.get();
        count++;
    }
    final double avg = sum / count;
    if (LOG.isDebugEnabled()) {
        LOG.debug("Average 10 Year Treasury for " + pKey.get() + " was " + avg);
    }
    BasicBSONObject query = new BasicBSONObject("_id", pKey.get());
    BasicBSONObject modifiers = new BasicBSONObject();
    modifiers.put("$set", BasicDBObjectBuilder.start().add("count", count).add("avg", avg).add("sum", sum).get());
    modifiers.put("$push", new BasicBSONObject("calculatedAt", new Date()));
    modifiers.put("$inc", new BasicBSONObject("numCalculations", 1));
    reduceResult.setQuery(query);
    reduceResult.setModifiers(modifiers);
    pContext.write(null, reduceResult);
}

Example 68 with BasicBSONObject

use of org.bson.BasicBSONObject in project mongo-hadoop by mongodb.

the class LogReducer method reduce.

@Override
public void reduce(final Text key, final Iterator<IntWritable> values, final OutputCollector<NullWritable, MongoUpdateWritable> output, final Reporter reporter) throws IOException {
    int count = 0;
    while (values.hasNext()) {
        count += values.next().get();
    }
    BasicBSONObject query = new BasicBSONObject("devices", new ObjectId(key.toString()));
    BasicBSONObject update = new BasicBSONObject("$inc", new BasicBSONObject("logs_count", count));
    if (LOG.isDebugEnabled()) {
        LOG.debug("query: " + query);
        LOG.debug("update: " + update);
    }
    reduceResult.setQuery(query);
    reduceResult.setModifiers(update);
    output.collect(null, reduceResult);
}

Example 69 with BasicBSONObject

use of org.bson.BasicBSONObject in project mongo-spark by plaa.

the class JavaWordCount method main.

public static void main(String[] args) {
    JavaSparkContext sc = new JavaSparkContext("local", "Java Word Count");
    Configuration config = new Configuration();
    config.set("mongo.input.uri", "mongodb://127.0.0.1:27017/beowulf.input");
    config.set("mongo.output.uri", "mongodb://127.0.0.1:27017/beowulf.output");
    JavaPairRDD<Object, BSONObject> mongoRDD = sc.newAPIHadoopRDD(config, com.mongodb.hadoop.MongoInputFormat.class, Object.class, BSONObject.class);
    // Input contains tuples of (ObjectId, BSONObject)
    JavaRDD<String> words = mongoRDD.flatMap(new FlatMapFunction<Tuple2<Object, BSONObject>, String>() {

        @Override
        public Iterable<String> call(Tuple2<Object, BSONObject> arg) {
            Object o = arg._2.get("text");
            if (o instanceof String) {
                String str = (String) o;
                str = str.toLowerCase().replaceAll("[.,!?\n]", " ");
                return Arrays.asList(str.split(" "));
            } else {
                return Collections.emptyList();
            }
        }
    });
    JavaPairRDD<String, Integer> ones = words.map(new PairFunction<String, String, Integer>() {

        public Tuple2<String, Integer> call(String s) {
            return new Tuple2<>(s, 1);
        }
    });
    JavaPairRDD<String, Integer> counts = ones.reduceByKey(new Function2<Integer, Integer, Integer>() {

        public Integer call(Integer i1, Integer i2) {
            return i1 + i2;
        }
    });
    // Output contains tuples of (null, BSONObject) - ObjectId will be generated by Mongo driver if null
    JavaPairRDD<Object, BSONObject> save = counts.map(new PairFunction<Tuple2<String, Integer>, Object, BSONObject>() {

        @Override
        public Tuple2<Object, BSONObject> call(Tuple2<String, Integer> tuple) {
            BSONObject bson = new BasicBSONObject();
            bson.put("word", tuple._1);
            bson.put("count", tuple._2);
            return new Tuple2<>(null, bson);
        }
    });
    // Only MongoOutputFormat and config are relevant
    save.saveAsNewAPIHadoopFile("file:///bogus", Object.class, Object.class, MongoOutputFormat.class, config);
}

Example 70 with BasicBSONObject

use of org.bson.BasicBSONObject in project jackrabbit-oak by apache.

the class ReplicaSetInfo method getHiddenMembers.

List<String> getHiddenMembers() {
    BasicDBObject result;
    try {
        result = getReplicaConfig();
    } catch (MongoException e) {
        LOG.error("Can't get replica configuration", e);
        return null;
    }
    @SuppressWarnings("unchecked") Iterable<BasicBSONObject> members = (Iterable<BasicBSONObject>) result.get("members");
    if (members == null) {
        members = Collections.emptyList();
    }
    List<String> hiddenMembers = new ArrayList<String>();
    for (BasicBSONObject member : members) {
        if (member.getBoolean("hidden")) {
            hiddenMembers.add(member.getString("host"));
        }
    }
    return hiddenMembers;
}