Search in sources :

Example 6 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class HiveMappingTest method queryBasedHiveTable.

@Test
public void queryBasedHiveTable() throws SQLException {
    String tableName = "filtered";
    DBCollection collection = getCollection(tableName);
    collection.drop();
    dropTable(tableName);
    int size = 1000;
    for (int i = 0; i < size; i++) {
        collection.insert(new BasicDBObject("_id", i).append("intField", i % 10).append("booleanField", i % 2 == 0).append("stringField", "" + (i % 2 == 0)));
    }
    MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", collection.getName())).build();
    ColumnMapping map = new ColumnMapping().map("id", "_id", "INT").map("ints", "intField", "INT").map("booleans", "booleanField", "BOOLEAN").map("strings", "stringField", "STRING");
    HiveTableBuilder builder = new HiveTableBuilder().mapping(map).name(tableName).uri(uri).tableProperty(MongoConfigUtil.INPUT_QUERY, "{_id : {\"$gte\" : 900 }}");
    execute(builder.toString());
    assertEquals(format("Should find %d items", size), collection.count(), size);
    Results execute = query(format("SELECT * from %s where id=1", tableName));
    assertTrue(execute.size() == 0);
    int expected = size - 900;
    assertEquals(format("Should find only %d items", expected), query("SELECT count(*) as count from " + tableName).iterator().next().get(0), "" + expected);
}
Also used : DBCollection(com.mongodb.DBCollection) BasicDBObject(com.mongodb.BasicDBObject) MongoClientURIBuilder(com.mongodb.hadoop.util.MongoClientURIBuilder) MongoClientURI(com.mongodb.MongoClientURI) Test(org.junit.Test)

Example 7 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class HiveMappingTest method nestedObjects.

@Test
public void nestedObjects() throws SQLException {
    DBCollection collection = getCollection("hive_addresses");
    collection.drop();
    dropTable("hive_addresses");
    collection.insert(user(1, "Jim", "Beam", "Clermont", "KY"));
    collection.insert(user(2, "Don", "Draper", "New York", "NY"));
    collection.insert(user(3, "John", "Elway", "Denver", "CO"));
    MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", collection.getName())).build();
    ColumnMapping map = new ColumnMapping().map("id", "_id", "INT").map("firstName", "firstName", "STRING").map("lastName", "lastName", "STRING").map("city", "address.city", "STRING").map("state", "address.state", "STRING");
    //, lastName STRING
    execute(format("CREATE TABLE hive_addresses (id INT, firstName STRING, lastName STRING, city STRING, state STRING)\n" + "STORED BY '%s'\n" + "WITH SERDEPROPERTIES('mongo.columns.mapping'='%s')\n" + "TBLPROPERTIES ('mongo.uri'='%s')", MongoStorageHandler.class.getName(), map.toSerDePropertiesString(), uri));
    Results execute = query("SELECT * from hive_addresses");
    assertEquals("KY", execute.getRow(0).get("state"));
    assertEquals("Don", execute.getRow(1).get("firstname"));
    assertEquals("Denver", execute.getRow(2).get("city"));
}
Also used : DBCollection(com.mongodb.DBCollection) MongoClientURIBuilder(com.mongodb.hadoop.util.MongoClientURIBuilder) MongoClientURI(com.mongodb.MongoClientURI) Test(org.junit.Test)

Example 8 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class MongoRecordReaderTest method testGetCurrentKey.

@Test
public void testGetCurrentKey() throws Exception {
    MongoClient client = new MongoClient("localhost", 27017);
    MongoClientURI uri = new MongoClientURIBuilder().collection("mongo_hadoop", "mongo_record_reader_test").build();
    DBCollection collection = client.getDB(uri.getDatabase()).getCollection(uri.getCollection());
    collection.drop();
    BasicDBList colors = new BasicDBList() {

        {
            add(new BasicBSONObject("red", 255));
            add(new BasicBSONObject("blue", 255));
            add(new BasicBSONObject("green", 0));
        }
    };
    collection.insert(new BasicDBObject("_id", 0).append("address", new BasicDBObject("street", "foo street")).append("colors", colors));
    // Default case: "_id" is used as inputKey.
    MongoInputSplit split = new MongoInputSplit();
    split.setInputURI(uri);
    MongoRecordReader reader = new MongoRecordReader(split);
    assertTrue(reader.nextKeyValue());
    assertEquals(reader.getCurrentKey(), 0);
    // Use a nested field as inputKey.
    split = new MongoInputSplit();
    split.setInputURI(uri);
    split.setKeyField("address.street");
    reader = new MongoRecordReader(split);
    assertTrue(reader.nextKeyValue());
    assertEquals(reader.getCurrentKey(), "foo street");
    // Use a key within an array as the inputKey.
    split = new MongoInputSplit();
    split.setInputURI(uri);
    split.setKeyField("colors.1");
    reader = new MongoRecordReader(split);
    assertTrue(reader.nextKeyValue());
    assertEquals(reader.getCurrentKey(), new BasicBSONObject("blue", 255));
}
Also used : MongoClient(com.mongodb.MongoClient) DBCollection(com.mongodb.DBCollection) BasicDBList(com.mongodb.BasicDBList) BasicBSONObject(org.bson.BasicBSONObject) BasicDBObject(com.mongodb.BasicDBObject) MongoInputSplit(com.mongodb.hadoop.input.MongoInputSplit) MongoClientURIBuilder(com.mongodb.hadoop.util.MongoClientURIBuilder) MongoRecordReader(com.mongodb.hadoop.input.MongoRecordReader) MongoClientURI(com.mongodb.MongoClientURI) Test(org.junit.Test)

Example 9 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class HiveMappingTest method nestedColumns.

@Test
public void nestedColumns() throws SQLException {
    DBCollection collection = getCollection("hive_addresses");
    collection.drop();
    dropTable("hive_addresses");
    collection.insert(user(1, "Jim", "Beam", "Clermont", "KY"));
    collection.insert(user(2, "Don", "Draper", "New York", "NY"));
    collection.insert(user(3, "John", "Elway", "Denver", "CO"));
    MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", collection.getName())).build();
    Map<String, String> map = new HashMap<String, String>() {

        {
            put("id", "_id");
            put("firstName", "firstName");
            put("lastName", "lastName");
            put("place.municipality", "address.city");
            put("place.region", "address.state");
        }
    };
    execute(format("CREATE TABLE hive_addresses " + "(id INT, firstName STRING, lastName STRING, " + "place STRUCT<municipality:STRING, region:STRING>)\n" + "STORED BY '%s'\n" + "WITH SERDEPROPERTIES('mongo.columns.mapping'='%s')\n" + "TBLPROPERTIES ('mongo.uri'='%s')", MongoStorageHandler.class.getName(), JSON.serialize(map), uri));
    // Alias inner fields to avoid retrieving entire struct as a String.
    Results execute = query("SELECT place.municipality AS city, place.region AS state, firstname from hive_addresses");
    assertEquals("KY", execute.getRow(0).get("state"));
    assertEquals("Don", execute.getRow(1).get("firstname"));
    assertEquals("Denver", execute.getRow(2).get("city"));
}
Also used : DBCollection(com.mongodb.DBCollection) MongoClientURIBuilder(com.mongodb.hadoop.util.MongoClientURIBuilder) HashMap(java.util.HashMap) MongoClientURI(com.mongodb.MongoClientURI) Test(org.junit.Test)

Example 10 with MongoClientURIBuilder

use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.

the class HiveTest method createMongoBackedTable.

public void createMongoBackedTable(final boolean withSerDeProps) throws SQLException {
    dropTable(MONGO_BACKED_TABLE);
    final MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", MONGO_COLLECTION)).build();
    execute(format("CREATE TABLE %s %s\n" + "STORED BY '%s'\n" + (withSerDeProps ? format("WITH SERDEPROPERTIES(%s)\n", SERDE_PROPERTIES) : "") + "TBLPROPERTIES ('mongo.uri'='%s')", MONGO_BACKED_TABLE, TEST_SCHEMA, MongoStorageHandler.class.getName(), uri));
}
Also used : MongoClientURIBuilder(com.mongodb.hadoop.util.MongoClientURIBuilder) MongoClientURI(com.mongodb.MongoClientURI)

Aggregations

MongoClientURIBuilder (com.mongodb.hadoop.util.MongoClientURIBuilder)10 MongoClientURI (com.mongodb.MongoClientURI)8 DBCollection (com.mongodb.DBCollection)7 BasicDBObject (com.mongodb.BasicDBObject)6 Test (org.junit.Test)5 MongoClient (com.mongodb.MongoClient)4 DBObject (com.mongodb.DBObject)3 BasicDBList (com.mongodb.BasicDBList)2 MongoInputSplit (com.mongodb.hadoop.input.MongoInputSplit)2 MapReduceJob (com.mongodb.hadoop.testutils.MapReduceJob)2 CommandResult (com.mongodb.CommandResult)1 DB (com.mongodb.DB)1 DBCursor (com.mongodb.DBCursor)1 MongoException (com.mongodb.MongoException)1 MongoRecordReader (com.mongodb.hadoop.input.MongoRecordReader)1 SingleMongoSplitter (com.mongodb.hadoop.splitter.SingleMongoSplitter)1 BaseHadoopTest (com.mongodb.hadoop.testutils.BaseHadoopTest)1 HashMap (java.util.HashMap)1 List (java.util.List)1 InputSplit (org.apache.hadoop.mapreduce.InputSplit)1