Examples with MongoClientURIBuilder com.mongodb.hadoop.util.MongoClientURIBuilder used on opensource projects

Example 6 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class HiveMappingTest method queryBasedHiveTable.

@Test
public void queryBasedHiveTable() throws SQLException {
    String tableName = "filtered";
    DBCollection collection = getCollection(tableName);
    collection.drop();
    dropTable(tableName);
    int size = 1000;
    for (int i = 0; i < size; i++) {
        collection.insert(new BasicDBObject("_id", i).append("intField", i % 10).append("booleanField", i % 2 == 0).append("stringField", "" + (i % 2 == 0)));
    }
    MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", collection.getName())).build();
    ColumnMapping map = new ColumnMapping().map("id", "_id", "INT").map("ints", "intField", "INT").map("booleans", "booleanField", "BOOLEAN").map("strings", "stringField", "STRING");
    HiveTableBuilder builder = new HiveTableBuilder().mapping(map).name(tableName).uri(uri).tableProperty(MongoConfigUtil.INPUT_QUERY, "{_id : {\"$gte\" : 900 }}");
    execute(builder.toString());
    assertEquals(format("Should find %d items", size), collection.count(), size);
    Results execute = query(format("SELECT * from %s where id=1", tableName));
    assertTrue(execute.size() == 0);
    int expected = size - 900;
    assertEquals(format("Should find only %d items", expected), query("SELECT count(*) as count from " + tableName).iterator().next().get(0), "" + expected);
}

Example 7 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class HiveMappingTest method nestedObjects.

@Test
public void nestedObjects() throws SQLException {
    DBCollection collection = getCollection("hive_addresses");
    collection.drop();
    dropTable("hive_addresses");
    collection.insert(user(1, "Jim", "Beam", "Clermont", "KY"));
    collection.insert(user(2, "Don", "Draper", "New York", "NY"));
    collection.insert(user(3, "John", "Elway", "Denver", "CO"));
    MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", collection.getName())).build();
    ColumnMapping map = new ColumnMapping().map("id", "_id", "INT").map("firstName", "firstName", "STRING").map("lastName", "lastName", "STRING").map("city", "address.city", "STRING").map("state", "address.state", "STRING");
    //, lastName STRING
    execute(format("CREATE TABLE hive_addresses (id INT, firstName STRING, lastName STRING, city STRING, state STRING)\n" + "STORED BY '%s'\n" + "WITH SERDEPROPERTIES('mongo.columns.mapping'='%s')\n" + "TBLPROPERTIES ('mongo.uri'='%s')", MongoStorageHandler.class.getName(), map.toSerDePropertiesString(), uri));
    Results execute = query("SELECT * from hive_addresses");
    assertEquals("KY", execute.getRow(0).get("state"));
    assertEquals("Don", execute.getRow(1).get("firstname"));
    assertEquals("Denver", execute.getRow(2).get("city"));
}

Example 8 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class MongoRecordReaderTest method testGetCurrentKey.

@Test
public void testGetCurrentKey() throws Exception {
    MongoClient client = new MongoClient("localhost", 27017);
    MongoClientURI uri = new MongoClientURIBuilder().collection("mongo_hadoop", "mongo_record_reader_test").build();
    DBCollection collection = client.getDB(uri.getDatabase()).getCollection(uri.getCollection());
    collection.drop();
    BasicDBList colors = new BasicDBList() {

        {
            add(new BasicBSONObject("red", 255));
            add(new BasicBSONObject("blue", 255));
            add(new BasicBSONObject("green", 0));
        }
    };
    collection.insert(new BasicDBObject("_id", 0).append("address", new BasicDBObject("street", "foo street")).append("colors", colors));
    // Default case: "_id" is used as inputKey.
    MongoInputSplit split = new MongoInputSplit();
    split.setInputURI(uri);
    MongoRecordReader reader = new MongoRecordReader(split);
    assertTrue(reader.nextKeyValue());
    assertEquals(reader.getCurrentKey(), 0);
    // Use a nested field as inputKey.
    split = new MongoInputSplit();
    split.setInputURI(uri);
    split.setKeyField("address.street");
    reader = new MongoRecordReader(split);
    assertTrue(reader.nextKeyValue());
    assertEquals(reader.getCurrentKey(), "foo street");
    // Use a key within an array as the inputKey.
    split = new MongoInputSplit();
    split.setInputURI(uri);
    split.setKeyField("colors.1");
    reader = new MongoRecordReader(split);
    assertTrue(reader.nextKeyValue());
    assertEquals(reader.getCurrentKey(), new BasicBSONObject("blue", 255));
}

Example 9 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class HiveMappingTest method nestedColumns.

@Test
public void nestedColumns() throws SQLException {
    DBCollection collection = getCollection("hive_addresses");
    collection.drop();
    dropTable("hive_addresses");
    collection.insert(user(1, "Jim", "Beam", "Clermont", "KY"));
    collection.insert(user(2, "Don", "Draper", "New York", "NY"));
    collection.insert(user(3, "John", "Elway", "Denver", "CO"));
    MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", collection.getName())).build();
    Map<String, String> map = new HashMap<String, String>() {

        {
            put("id", "_id");
            put("firstName", "firstName");
            put("lastName", "lastName");
            put("place.municipality", "address.city");
            put("place.region", "address.state");
        }
    };
    execute(format("CREATE TABLE hive_addresses " + "(id INT, firstName STRING, lastName STRING, " + "place STRUCT<municipality:STRING, region:STRING>)\n" + "STORED BY '%s'\n" + "WITH SERDEPROPERTIES('mongo.columns.mapping'='%s')\n" + "TBLPROPERTIES ('mongo.uri'='%s')", MongoStorageHandler.class.getName(), JSON.serialize(map), uri));
    // Alias inner fields to avoid retrieving entire struct as a String.
    Results execute = query("SELECT place.municipality AS city, place.region AS state, firstname from hive_addresses");
    assertEquals("KY", execute.getRow(0).get("state"));
    assertEquals("Don", execute.getRow(1).get("firstname"));
    assertEquals("Denver", execute.getRow(2).get("city"));
}

Example 10 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class HiveTest method createMongoBackedTable.

public void createMongoBackedTable(final boolean withSerDeProps) throws SQLException {
    dropTable(MONGO_BACKED_TABLE);
    final MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", MONGO_COLLECTION)).build();
    execute(format("CREATE TABLE %s %s\n" + "STORED BY '%s'\n" + (withSerDeProps ? format("WITH SERDEPROPERTIES(%s)\n", SERDE_PROPERTIES) : "") + "TBLPROPERTIES ('mongo.uri'='%s')", MONGO_BACKED_TABLE, TEST_SCHEMA, MongoStorageHandler.class.getName(), uri));
}