Examples with DBObject com.mongodb.DBObject used on opensource projects

Example 41 with DBObject

use of com.mongodb.DBObject in project mongo-hadoop by mongodb.

the class MongoRecordWriter method write.

@Override
public void write(final K key, final V value) throws IOException {
    if (value instanceof MongoUpdateWritable) {
        outputStream.writeInt(MongoWritableTypes.MONGO_UPDATE_WRITABLE);
        ((MongoUpdateWritable) value).write(outputStream);
    } else {
        DBObject o = new BasicDBObject();
        if (key instanceof BSONWritable) {
            o.put("_id", ((BSONWritable) key).getDoc());
        } else if (key instanceof BSONObject) {
            o.put("_id", key);
        } else {
            o.put("_id", BSONWritable.toBSON(key));
        }
        if (value instanceof BSONWritable) {
            o.putAll(((BSONWritable) value).getDoc());
        } else if (value instanceof MongoOutput) {
            ((MongoOutput) value).appendAsValue(o);
        } else if (value instanceof BSONObject) {
            o.putAll((BSONObject) value);
        } else if (value instanceof Map) {
            o.putAll((Map) value);
        } else {
            o.put("value", BSONWritable.toBSON(value));
        }
        outputStream.writeInt(MongoWritableTypes.BSON_WRITABLE);
        bsonWritable.setDoc(o);
        bsonWritable.write(outputStream);
    }
}

Example 42 with DBObject

use of com.mongodb.DBObject in project mongo-hadoop by mongodb.

the class BookstoreTest method tagsIndex.

@Test
public void tagsIndex() throws URISyntaxException, UnknownHostException, IllegalAccessException {
    MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", "bookstore_tags")).build();
    MongoClient mongoClient = new MongoClient(uri);
    DBCollection collection = mongoClient.getDB(uri.getDatabase()).getCollection(uri.getCollection());
    MapReduceJob job = new MapReduceJob(BookstoreConfig.class.getName()).jar(JAR_PATH).inputUris(INVENTORY_BSON).outputUri(uri).param("mapred.input.dir", INVENTORY_BSON.toString());
    if (!HADOOP_VERSION.startsWith("1.")) {
        job.inputFormat(BSONFileInputFormat.class);
    } else {
        job.mapredInputFormat(com.mongodb.hadoop.mapred.BSONFileInputFormat.class);
        job.mapredOutputFormat(MongoOutputFormat.class);
        job.outputCommitter(MongoOutputCommitter.class);
    }
    job.execute(false);
    DBObject object = collection.findOne(new BasicDBObject("_id", "history"));
    assertNotNull(object);
    List books = (List) object.get("books");
    Assert.assertEquals("Should find only 8 books", books.size(), 8);
}

Example 43 with DBObject

use of com.mongodb.DBObject in project mongo-hadoop by mongodb.

the class MongoPaginatingSplitterTest method testQuery.

@Test
public void testQuery() throws SplitFailedException {
    Configuration conf = new Configuration();
    MongoConfigUtil.setInputURI(conf, uri);
    MongoConfigUtil.setRangeQueryEnabled(conf, true);
    MongoConfigUtil.setInputSplitMinDocs(conf, 5000);
    DBObject query = new BasicDBObject("$or", new BasicDBObject[] { new BasicDBObject("value", new BasicDBObject("$lt", 25000)), new BasicDBObject("value", new BasicDBObject("$gte", 31000)) });
    MongoConfigUtil.setQuery(conf, query);
    MongoPaginatingSplitter splitter = new MongoPaginatingSplitter(conf);
    List<InputSplit> splits = splitter.calculateSplits();
    assertEquals(7, splits.size());
    assertSplitRange((MongoInputSplit) splits.get(0), null, 5000);
    assertSplitRange((MongoInputSplit) splits.get(1), 5000, 10000);
    assertSplitRange((MongoInputSplit) splits.get(2), 10000, 15000);
    assertSplitRange((MongoInputSplit) splits.get(3), 15000, 20000);
    assertSplitRange((MongoInputSplit) splits.get(4), 20000, 31000);
    assertSplitRange((MongoInputSplit) splits.get(5), 31000, 36000);
    assertSplitRange((MongoInputSplit) splits.get(6), 36000, null);
    // 6000 documents excluded by query.
    assertSplitsCount(collection.count() - 6000, splits);
}

Example 44 with DBObject

use of com.mongodb.DBObject in project mongo-hadoop by mongodb.

the class StandaloneMongoSplitterTest method unshardedCollectionMinMax.

@Test
public void unshardedCollectionMinMax() throws UnknownHostException, SplitFailedException {
    Configuration config = new Configuration();
    StandaloneMongoSplitter splitter = new StandaloneMongoSplitter(config);
    MongoConfigUtil.setInputURI(config, uri);
    DBObject inputSplitKey = BasicDBObjectBuilder.start("value", 1).get();
    MongoConfigUtil.setInputSplitKey(config, inputSplitKey);
    MongoConfigUtil.setSplitSize(config, 1);
    List<InputSplit> regularSplits = splitter.calculateSplits();
    MongoConfigUtil.setMinSplitKey(config, "{value:100}");
    MongoConfigUtil.setMaxSplitKey(config, "{value:39900}");
    List<InputSplit> inputSplits = splitter.calculateSplits();
    assertTrue("should be fewer splits with min/max set", regularSplits.size() >= inputSplits.size());
}

Example 45 with DBObject

use of com.mongodb.DBObject in project mongo-hadoop by mongodb.

the class StandaloneMongoSplitterTest method testFilterEmptySplits.

@Test
public void testFilterEmptySplits() throws SplitFailedException {
    Configuration config = new Configuration();
    DBObject query = new BasicDBObject("$or", new BasicDBObject[] { new BasicDBObject("value", new BasicDBObject("$lt", 20000)), new BasicDBObject("value", new BasicDBObject("$gt", 35000)) });
    MongoConfigUtil.setInputURI(config, uri);
    MongoConfigUtil.setEnableFilterEmptySplits(config, true);
    MongoConfigUtil.setQuery(config, query);
    // 1 MB per document results in 4 splits; the 3rd one is empty per
    // the above query.
    MongoConfigUtil.setSplitSize(config, 1);
    StandaloneMongoSplitter splitter = new StandaloneMongoSplitter(config);
    List<InputSplit> splits = splitter.calculateSplits();
    // No splits are empty.
    for (InputSplit split : splits) {
        // Cursor is closed on the split, so copy it to create a new one.
        MongoInputSplit mis = new MongoInputSplit((MongoInputSplit) split);
        assertNotEquals(0, mis.getCursor().itcount());
    }
    assertSplitsCount(collection.count(query), splits);
}