use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.
the class HiveMappingTest method queryBasedHiveTable.
@Test
public void queryBasedHiveTable() throws SQLException {
String tableName = "filtered";
DBCollection collection = getCollection(tableName);
collection.drop();
dropTable(tableName);
int size = 1000;
for (int i = 0; i < size; i++) {
collection.insert(new BasicDBObject("_id", i).append("intField", i % 10).append("booleanField", i % 2 == 0).append("stringField", "" + (i % 2 == 0)));
}
MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", collection.getName())).build();
ColumnMapping map = new ColumnMapping().map("id", "_id", "INT").map("ints", "intField", "INT").map("booleans", "booleanField", "BOOLEAN").map("strings", "stringField", "STRING");
HiveTableBuilder builder = new HiveTableBuilder().mapping(map).name(tableName).uri(uri).tableProperty(MongoConfigUtil.INPUT_QUERY, "{_id : {\"$gte\" : 900 }}");
execute(builder.toString());
assertEquals(format("Should find %d items", size), collection.count(), size);
Results execute = query(format("SELECT * from %s where id=1", tableName));
assertTrue(execute.size() == 0);
int expected = size - 900;
assertEquals(format("Should find only %d items", expected), query("SELECT count(*) as count from " + tableName).iterator().next().get(0), "" + expected);
}
use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.
the class HiveMappingTest method nestedObjects.
@Test
public void nestedObjects() throws SQLException {
DBCollection collection = getCollection("hive_addresses");
collection.drop();
dropTable("hive_addresses");
collection.insert(user(1, "Jim", "Beam", "Clermont", "KY"));
collection.insert(user(2, "Don", "Draper", "New York", "NY"));
collection.insert(user(3, "John", "Elway", "Denver", "CO"));
MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", collection.getName())).build();
ColumnMapping map = new ColumnMapping().map("id", "_id", "INT").map("firstName", "firstName", "STRING").map("lastName", "lastName", "STRING").map("city", "address.city", "STRING").map("state", "address.state", "STRING");
//, lastName STRING
execute(format("CREATE TABLE hive_addresses (id INT, firstName STRING, lastName STRING, city STRING, state STRING)\n" + "STORED BY '%s'\n" + "WITH SERDEPROPERTIES('mongo.columns.mapping'='%s')\n" + "TBLPROPERTIES ('mongo.uri'='%s')", MongoStorageHandler.class.getName(), map.toSerDePropertiesString(), uri));
Results execute = query("SELECT * from hive_addresses");
assertEquals("KY", execute.getRow(0).get("state"));
assertEquals("Don", execute.getRow(1).get("firstname"));
assertEquals("Denver", execute.getRow(2).get("city"));
}
use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.
the class MongoRecordReaderTest method testGetCurrentKey.
@Test
public void testGetCurrentKey() throws Exception {
MongoClient client = new MongoClient("localhost", 27017);
MongoClientURI uri = new MongoClientURIBuilder().collection("mongo_hadoop", "mongo_record_reader_test").build();
DBCollection collection = client.getDB(uri.getDatabase()).getCollection(uri.getCollection());
collection.drop();
BasicDBList colors = new BasicDBList() {
{
add(new BasicBSONObject("red", 255));
add(new BasicBSONObject("blue", 255));
add(new BasicBSONObject("green", 0));
}
};
collection.insert(new BasicDBObject("_id", 0).append("address", new BasicDBObject("street", "foo street")).append("colors", colors));
// Default case: "_id" is used as inputKey.
MongoInputSplit split = new MongoInputSplit();
split.setInputURI(uri);
MongoRecordReader reader = new MongoRecordReader(split);
assertTrue(reader.nextKeyValue());
assertEquals(reader.getCurrentKey(), 0);
// Use a nested field as inputKey.
split = new MongoInputSplit();
split.setInputURI(uri);
split.setKeyField("address.street");
reader = new MongoRecordReader(split);
assertTrue(reader.nextKeyValue());
assertEquals(reader.getCurrentKey(), "foo street");
// Use a key within an array as the inputKey.
split = new MongoInputSplit();
split.setInputURI(uri);
split.setKeyField("colors.1");
reader = new MongoRecordReader(split);
assertTrue(reader.nextKeyValue());
assertEquals(reader.getCurrentKey(), new BasicBSONObject("blue", 255));
}
use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.
the class HiveMappingTest method nestedColumns.
@Test
public void nestedColumns() throws SQLException {
DBCollection collection = getCollection("hive_addresses");
collection.drop();
dropTable("hive_addresses");
collection.insert(user(1, "Jim", "Beam", "Clermont", "KY"));
collection.insert(user(2, "Don", "Draper", "New York", "NY"));
collection.insert(user(3, "John", "Elway", "Denver", "CO"));
MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", collection.getName())).build();
Map<String, String> map = new HashMap<String, String>() {
{
put("id", "_id");
put("firstName", "firstName");
put("lastName", "lastName");
put("place.municipality", "address.city");
put("place.region", "address.state");
}
};
execute(format("CREATE TABLE hive_addresses " + "(id INT, firstName STRING, lastName STRING, " + "place STRUCT<municipality:STRING, region:STRING>)\n" + "STORED BY '%s'\n" + "WITH SERDEPROPERTIES('mongo.columns.mapping'='%s')\n" + "TBLPROPERTIES ('mongo.uri'='%s')", MongoStorageHandler.class.getName(), JSON.serialize(map), uri));
// Alias inner fields to avoid retrieving entire struct as a String.
Results execute = query("SELECT place.municipality AS city, place.region AS state, firstname from hive_addresses");
assertEquals("KY", execute.getRow(0).get("state"));
assertEquals("Don", execute.getRow(1).get("firstname"));
assertEquals("Denver", execute.getRow(2).get("city"));
}
use of com.mongodb.hadoop.util.MongoClientURIBuilder in project mongo-hadoop by mongodb.
the class HiveTest method createMongoBackedTable.
public void createMongoBackedTable(final boolean withSerDeProps) throws SQLException {
dropTable(MONGO_BACKED_TABLE);
final MongoClientURI uri = authCheck(new MongoClientURIBuilder().collection("mongo_hadoop", MONGO_COLLECTION)).build();
execute(format("CREATE TABLE %s %s\n" + "STORED BY '%s'\n" + (withSerDeProps ? format("WITH SERDEPROPERTIES(%s)\n", SERDE_PROPERTIES) : "") + "TBLPROPERTIES ('mongo.uri'='%s')", MONGO_BACKED_TABLE, TEST_SCHEMA, MongoStorageHandler.class.getName(), uri));
}
Aggregations