Search in sources :

Example 6 with BSONWritable

use of com.mongodb.hadoop.io.BSONWritable in project mongo-hadoop by mongodb.

the class BSONSerDeTest method testString.

@Test
public void testString() throws SerDeException {
    String columnNames = "s";
    String columnTypes = "string";
    String value = "value";
    BSONSerDe serde = new BSONSerDe();
    Object result = helpDeserialize(serde, columnNames, columnTypes, value);
    assertThat(value, equalTo(result));
    ObjectInspector innerInspector = PrimitiveObjectInspectorFactory.getPrimitiveObjectInspectorFromClass(String.class);
    BasicBSONObject bObject = new BasicBSONObject();
    Object serialized = helpSerialize(columnNames, innerInspector, bObject, value, serde);
    assertThat(new BSONWritable(bObject), equalTo(serialized));
}
Also used : BSONWritable(com.mongodb.hadoop.io.BSONWritable) BasicBSONObject(org.bson.BasicBSONObject) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) BasicBSONObject(org.bson.BasicBSONObject) Test(org.junit.Test)

Example 7 with BSONWritable

use of com.mongodb.hadoop.io.BSONWritable in project mongo-hadoop by mongodb.

the class BSONSerDeTest method testMap.

@Test
public void testMap() throws SerDeException {
    String columnNames = "m";
    String columnTypes = "map<string,int>";
    BasicBSONObject value = new BasicBSONObject();
    String oneKey = "one";
    int oneValue = 10;
    value.put(oneKey, oneValue);
    String twoKey = "two";
    int twoValue = 20;
    value.put(twoKey, twoValue);
    BSONSerDe serde = new BSONSerDe();
    Object result = helpDeserialize(serde, columnNames, columnTypes, value);
    assertThat(value.toMap(), equalTo(result));
    // Since objectid is currently taken to be a string
    ObjectInspector keyInspector = PrimitiveObjectInspectorFactory.getPrimitiveObjectInspectorFromClass(String.class);
    ObjectInspector valueInspector = PrimitiveObjectInspectorFactory.getPrimitiveObjectInspectorFromClass(Integer.class);
    MapObjectInspector mapInspector = ObjectInspectorFactory.getStandardMapObjectInspector(keyInspector, valueInspector);
    BasicBSONObject bObject = new BasicBSONObject();
    Object serialized = helpSerialize(columnNames, mapInspector, bObject, value, serde);
    assertThat(new BSONWritable(bObject), equalTo(serialized));
}
Also used : BSONWritable(com.mongodb.hadoop.io.BSONWritable) BasicBSONObject(org.bson.BasicBSONObject) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) BasicBSONObject(org.bson.BasicBSONObject) Test(org.junit.Test)

Example 8 with BSONWritable

use of com.mongodb.hadoop.io.BSONWritable in project mongo-hadoop by mongodb.

the class BSONSerDeTest method testInt.

@Test
public void testInt() throws SerDeException {
    String columnNames = "i";
    String columnTypes = "int";
    Integer value = 1234;
    BSONSerDe serde = new BSONSerDe();
    Object result = helpDeserialize(serde, columnNames, columnTypes, value);
    assertThat(value, equalTo(result));
    ObjectInspector innerInspector = PrimitiveObjectInspectorFactory.getPrimitiveObjectInspectorFromClass(Integer.class);
    BasicBSONObject bObject = new BasicBSONObject();
    Object serialized = helpSerialize(columnNames, innerInspector, bObject, value, serde);
    assertThat(new BSONWritable(bObject), equalTo(serialized));
}
Also used : BSONWritable(com.mongodb.hadoop.io.BSONWritable) BasicBSONObject(org.bson.BasicBSONObject) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) BasicBSONObject(org.bson.BasicBSONObject) Test(org.junit.Test)

Example 9 with BSONWritable

use of com.mongodb.hadoop.io.BSONWritable in project mongo-hadoop by mongodb.

the class BSONSerDeTest method testBoolean.

@Test
public void testBoolean() throws SerDeException {
    String columnNames = "bool";
    String columnTypes = "boolean";
    Boolean value = false;
    BSONSerDe serde = new BSONSerDe();
    Object result = helpDeserialize(serde, columnNames, columnTypes, value);
    assertThat(value, equalTo(result));
    ObjectInspector innerInspector = PrimitiveObjectInspectorFactory.getPrimitiveObjectInspectorFromClass(Boolean.class);
    BasicBSONObject bObject = new BasicBSONObject();
    Object serialized = helpSerialize(columnNames, innerInspector, bObject, value, serde);
    assertThat(new BSONWritable(bObject), equalTo(serialized));
}
Also used : BSONWritable(com.mongodb.hadoop.io.BSONWritable) BasicBSONObject(org.bson.BasicBSONObject) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) BasicBSONObject(org.bson.BasicBSONObject) Test(org.junit.Test)

Example 10 with BSONWritable

use of com.mongodb.hadoop.io.BSONWritable in project mongo-hadoop by mongodb.

the class BSONSerDeTest method testDates.

@Test
public void testDates() throws SerDeException {
    String columnNames = "d";
    String columnTypes = "timestamp";
    Date d = new Date();
    Timestamp value = new Timestamp(d.getTime());
    BSONSerDe serde = new BSONSerDe();
    Object result = helpDeserialize(serde, columnNames, columnTypes, value);
    assertThat(value, equalTo(result));
    result = serde.deserializeField(d, serde.columnTypes.get(0), "");
    assertThat(value, equalTo(result));
    BSONTimestamp bts = new BSONTimestamp(((Long) (d.getTime() / 1000L)).intValue(), 1);
    result = serde.deserializeField(bts, serde.columnTypes.get(0), "");
    // BSONTimestamp only takes an int, so the long returned in the Timestamp won't be the same
    assertThat((long) bts.getTime(), equalTo(((Timestamp) result).getTime() / 1000L));
    // Utilizes a timestampWritable because there's no native timestamp type in java for
    // object inspector class to relate to
    ObjectInspector innerInspector = PrimitiveObjectInspectorFactory.getPrimitiveObjectInspectorFromClass(TimestampWritable.class);
    BasicBSONObject bObject = new BasicBSONObject();
    BSONWritable serialized = (BSONWritable) helpSerialize(columnNames, innerInspector, bObject, new TimestampWritable(value), serde);
    // The time going in to serialize is Timestamp but it comes out as BSONTimestamp
    BasicBSONObject bsonWithTimestamp = new BasicBSONObject();
    bsonWithTimestamp.put(columnNames, bts);
    assertThat(value.getTime(), equalTo(((Date) serialized.getDoc().get(columnNames)).getTime()));
}
Also used : BSONWritable(com.mongodb.hadoop.io.BSONWritable) BasicBSONObject(org.bson.BasicBSONObject) ListObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector) MapObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector) StructObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector) ObjectInspector(org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector) BSONTimestamp(org.bson.types.BSONTimestamp) TimestampWritable(org.apache.hadoop.hive.serde2.io.TimestampWritable) BasicBSONObject(org.bson.BasicBSONObject) Timestamp(java.sql.Timestamp) BSONTimestamp(org.bson.types.BSONTimestamp) Date(java.util.Date) Test(org.junit.Test)

Aggregations

BSONWritable (com.mongodb.hadoop.io.BSONWritable)21 BasicBSONObject (org.bson.BasicBSONObject)14 Test (org.junit.Test)13 ListObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector)11 MapObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector)11 ObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector)11 StructObjectInspector (org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector)11 BasicDBObject (com.mongodb.BasicDBObject)4 MongoUpdateWritable (com.mongodb.hadoop.io.MongoUpdateWritable)4 ArrayList (java.util.ArrayList)4 BSONObject (org.bson.BSONObject)4 DBObject (com.mongodb.DBObject)3 File (java.io.File)2 IOException (java.io.IOException)2 SerDeException (org.apache.hadoop.hive.serde2.SerDeException)2 NullWritable (org.apache.hadoop.io.NullWritable)2 FileSplit (org.apache.hadoop.mapred.FileSplit)2 JobConf (org.apache.hadoop.mapred.JobConf)2 BulkUpdateRequestBuilder (com.mongodb.BulkUpdateRequestBuilder)1 BulkWriteOperation (com.mongodb.BulkWriteOperation)1