Examples with MongoClientURIBuilder com.mongodb.hadoop.util.MongoClientURIBuilder used on opensource projects

Example 1 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class TestSharded method testDirectAccess.

public void testDirectAccess() {
    DBCollection collection = getMongos().getDB("mongo_hadoop").getCollection("yield_historical.out");
    collection.drop();
    // HADOOP61 - simulate a failed migration by having some docs from one chunk
    // also exist on another shard who does not own that chunk(duplicates)
    DB config = getMongos().getDB("config");
    DBObject chunk = config.getCollection("chunks").findOne(new BasicDBObject("shard", "sh01"));
    DBObject query = new BasicDBObject("_id", new BasicDBObject("$gte", ((DBObject) chunk.get("min")).get("_id")).append("$lt", ((DBObject) chunk.get("max")).get("_id")));
    List<DBObject> data = toList(getMongos().getDB("mongo_hadoop").getCollection("yield_historical.in").find(query));
    DBCollection destination = getShard().getDB("mongo_hadoop").getCollection("yield_historical.in");
    for (DBObject doc : data) {
        destination.insert(doc, WriteConcern.UNACKNOWLEDGED);
    }
    MapReduceJob job = new MapReduceJob(TreasuryYieldXMLConfig.class.getName()).jar(JOBJAR_PATH).param(SPLITS_SLAVE_OK, "true").param(SPLITS_USE_SHARDS, "true").param(SPLITS_USE_CHUNKS, "false").inputUris(new MongoClientURIBuilder(getInputUri()).readPreference(ReadPreference.secondary()).build());
    if (isHadoopV1()) {
        job.outputCommitter(MongoOutputCommitter.class);
    }
    job.execute(isRunTestInVm());
    compareResults(collection, getReference());
    collection.drop();
    MapReduceJob jobWithChunks = new MapReduceJob(TreasuryYieldXMLConfig.class.getName()).jar(JOBJAR_PATH).inputUris(new MongoClientURIBuilder(getInputUri()).readPreference(ReadPreference.secondary()).build()).param(SPLITS_SLAVE_OK, "true").param(SPLITS_USE_SHARDS, "true").param(SPLITS_USE_CHUNKS, "true");
    if (isHadoopV1()) {
        jobWithChunks.outputCommitter(MongoOutputCommitter.class);
    }
    jobWithChunks.execute(isRunTestInVm());
    compareResults(collection, getReference());
}

Example 2 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class HiveMappingTest method queryBasedHiveTable.

@Test
public void queryBasedHiveTable() throws SQLException {
    String tableName = "filtered";
    DBCollection collection = getCollection(tableName);
    collection.drop();
    dropTable(tableName);
    int size = 1000;
    for (int i = 0; i < size; i++) {
        collection.insert(new BasicDBObject("_id", i).append("intField", i % 10).append("booleanField", i % 2 == 0).append("stringField", "" + (i % 2 == 0)));
    }
    MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", collection.getName())).build();
    ColumnMapping map = new ColumnMapping().map("id", "_id", "INT").map("ints", "intField", "INT").map("booleans", "booleanField", "BOOLEAN").map("strings", "stringField", "STRING");
    HiveTableBuilder builder = new HiveTableBuilder().mapping(map).name(tableName).uri(uri).tableProperty(MongoConfigUtil.INPUT_QUERY, "{_id : {\"$gte\" : 900 }}");
    execute(builder.toString());
    assertEquals(format("Should find %d items", size), collection.count(), size);
    Results execute = query(format("SELECT * from %s where id=1", tableName));
    assertTrue(execute.size() == 0);
    int expected = size - 900;
    assertEquals(format("Should find only %d items", expected), query("SELECT count(*) as count from " + tableName).iterator().next().get(0), "" + expected);
}

Example 3 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class HiveMappingTest method nestedObjects.

@Test
public void nestedObjects() throws SQLException {
    DBCollection collection = getCollection("hive_addresses");
    collection.drop();
    dropTable("hive_addresses");
    collection.insert(user(1, "Jim", "Beam", "Clermont", "KY"));
    collection.insert(user(2, "Don", "Draper", "New York", "NY"));
    collection.insert(user(3, "John", "Elway", "Denver", "CO"));
    MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", collection.getName())).build();
    ColumnMapping map = new ColumnMapping().map("id", "_id", "INT").map("firstName", "firstName", "STRING").map("lastName", "lastName", "STRING").map("city", "address.city", "STRING").map("state", "address.state", "STRING");
    // , lastName STRING
    execute(format("CREATE TABLE hive_addresses (id INT, firstName STRING, lastName STRING, city STRING, state STRING)\n" + "STORED BY '%s'\n" + "WITH SERDEPROPERTIES('mongo.columns.mapping'='%s')\n" + "TBLPROPERTIES ('mongo.uri'='%s')", MongoStorageHandler.class.getName(), map.toSerDePropertiesString(), uri));
    Results execute = query("SELECT * from hive_addresses");
    assertEquals("KY", execute.getRow(0).get("state"));
    assertEquals("Don", execute.getRow(1).get("firstname"));
    assertEquals("Denver", execute.getRow(2).get("city"));
}

Example 4 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class StandaloneMongoSplitterTest method setUp.

@BeforeClass
public static void setUp() {
    MongoClient client = new MongoClient("localhost", 27017);
    uri = new MongoClientURIBuilder().collection("mongo_hadoop", "splitter_test").build();
    collection = client.getDB(uri.getDatabase()).getCollection(uri.getCollection());
    collection.drop();
    collection.createIndex("value");
    for (int i = 0; i < 40000; i++) {
        collection.insert(new BasicDBObject("_id", i).append("value", i));
    }
}

Example 5 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class TestStandalone method collectionSettings.

private JsonNode collectionSettings() {
    ArrayNode settings = new ArrayNode(JsonNodeFactory.instance);
    ObjectNode node = new ObjectNode(JsonNodeFactory.instance);
    node.put(INPUT_URI, getInputUri().toString());
    ObjectNode dow = new ObjectNode(JsonNodeFactory.instance);
    dow.put("dayOfWeek", "FRIDAY");
    node.put("query", dow);
    node.put(MONGO_SPLITTER_CLASS, SingleMongoSplitter.class.getName());
    node.put(SPLITS_USE_RANGEQUERY, true);
    node.put(INPUT_NOTIMEOUT, true);
    settings.add(node);
    MongoClientURI inputUri3 = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", "yield_historical.in3")).build();
    node = new ObjectNode(JsonNodeFactory.instance);
    node.put(INPUT_URI, inputUri3.toString());
    node.put(SPLITS_USE_RANGEQUERY, true);
    node.put(INPUT_NOTIMEOUT, true);
    settings.add(node);
    return settings;
}