Search in sources :

Example 61 with BooleanWritable

use of org.apache.hadoop.io.BooleanWritable in project presto by prestodb.

the class RcFileTester method decodeRecordReaderValue.

private static Object decodeRecordReaderValue(Type type, Object actualValue) {
    if (actualValue instanceof LazyPrimitive) {
        actualValue = ((LazyPrimitive<?, ?>) actualValue).getWritableObject();
    }
    if (actualValue instanceof BooleanWritable) {
        actualValue = ((BooleanWritable) actualValue).get();
    } else if (actualValue instanceof ByteWritable) {
        actualValue = ((ByteWritable) actualValue).get();
    } else if (actualValue instanceof BytesWritable) {
        actualValue = new SqlVarbinary(((BytesWritable) actualValue).copyBytes());
    } else if (actualValue instanceof DateWritable) {
        actualValue = new SqlDate(((DateWritable) actualValue).getDays());
    } else if (actualValue instanceof DoubleWritable) {
        actualValue = ((DoubleWritable) actualValue).get();
    } else if (actualValue instanceof FloatWritable) {
        actualValue = ((FloatWritable) actualValue).get();
    } else if (actualValue instanceof IntWritable) {
        actualValue = ((IntWritable) actualValue).get();
    } else if (actualValue instanceof LongWritable) {
        actualValue = ((LongWritable) actualValue).get();
    } else if (actualValue instanceof ShortWritable) {
        actualValue = ((ShortWritable) actualValue).get();
    } else if (actualValue instanceof HiveDecimalWritable) {
        DecimalType decimalType = (DecimalType) type;
        HiveDecimalWritable writable = (HiveDecimalWritable) actualValue;
        // writable messes with the scale so rescale the values to the Presto type
        BigInteger rescaledValue = rescale(writable.getHiveDecimal().unscaledValue(), writable.getScale(), decimalType.getScale());
        actualValue = new SqlDecimal(rescaledValue, decimalType.getPrecision(), decimalType.getScale());
    } else if (actualValue instanceof Text) {
        actualValue = actualValue.toString();
    } else if (actualValue instanceof TimestampWritable) {
        TimestampWritable timestamp = (TimestampWritable) actualValue;
        actualValue = new SqlTimestamp((timestamp.getSeconds() * 1000) + (timestamp.getNanos() / 1000000L), UTC_KEY);
    } else if (actualValue instanceof StructObject) {
        StructObject structObject = (StructObject) actualValue;
        actualValue = decodeRecordReaderStruct(type, structObject.getFieldsAsList());
    } else if (actualValue instanceof LazyBinaryArray) {
        actualValue = decodeRecordReaderList(type, ((LazyBinaryArray) actualValue).getList());
    } else if (actualValue instanceof LazyBinaryMap) {
        actualValue = decodeRecordReaderMap(type, ((LazyBinaryMap) actualValue).getMap());
    } else if (actualValue instanceof LazyArray) {
        actualValue = decodeRecordReaderList(type, ((LazyArray) actualValue).getList());
    } else if (actualValue instanceof LazyMap) {
        actualValue = decodeRecordReaderMap(type, ((LazyMap) actualValue).getMap());
    } else if (actualValue instanceof List) {
        actualValue = decodeRecordReaderList(type, ((List<?>) actualValue));
    }
    return actualValue;
}
Also used : SqlVarbinary(com.facebook.presto.spi.type.SqlVarbinary) TimestampWritable(org.apache.hadoop.hive.serde2.io.TimestampWritable) DoubleWritable(org.apache.hadoop.io.DoubleWritable) SqlTimestamp(com.facebook.presto.spi.type.SqlTimestamp) LazyBinaryArray(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryArray) ShortWritable(org.apache.hadoop.hive.serde2.io.ShortWritable) LazyPrimitive(org.apache.hadoop.hive.serde2.lazy.LazyPrimitive) StructObject(org.apache.hadoop.hive.serde2.StructObject) ArrayList(java.util.ArrayList) List(java.util.List) ImmutableList(com.google.common.collect.ImmutableList) Collectors.toList(java.util.stream.Collectors.toList) LongWritable(org.apache.hadoop.io.LongWritable) ByteWritable(org.apache.hadoop.io.ByteWritable) IntWritable(org.apache.hadoop.io.IntWritable) DateWritable(org.apache.hadoop.hive.serde2.io.DateWritable) HiveDecimalWritable(org.apache.hadoop.hive.serde2.io.HiveDecimalWritable) LazyMap(org.apache.hadoop.hive.serde2.lazy.LazyMap) BytesWritable(org.apache.hadoop.io.BytesWritable) SqlDecimal(com.facebook.presto.spi.type.SqlDecimal) Text(org.apache.hadoop.io.Text) FloatWritable(org.apache.hadoop.io.FloatWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) SqlDate(com.facebook.presto.spi.type.SqlDate) DecimalType(com.facebook.presto.spi.type.DecimalType) BigInteger(java.math.BigInteger) LazyArray(org.apache.hadoop.hive.serde2.lazy.LazyArray) LazyBinaryMap(org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryMap)

Example 62 with BooleanWritable

use of org.apache.hadoop.io.BooleanWritable in project mongo-hadoop by mongodb.

the class BSONWritable method toBSON.

/**
     * Unwrap a (usually Writable) Object, getting back a value suitable for
     * putting into a BSONObject. If the given object is not Writable, then
     * simply return the Object back.
     *
     * @param x the Object to turn into BSON.
     * @return the BSON representation of the Object.
     */
@SuppressWarnings("unchecked")
public static Object toBSON(final Object x) {
    if (x == null) {
        return null;
    }
    if (x instanceof Text) {
        return x.toString();
    }
    if (x instanceof BSONWritable) {
        return ((BSONWritable) x).getDoc();
    }
    if (x instanceof Writable) {
        if (x instanceof AbstractMapWritable) {
            if (!(x instanceof Map)) {
                throw new IllegalArgumentException(String.format("Cannot turn %s into BSON, since it does " + "not implement java.util.Map.", x.getClass().getName()));
            }
            Map<Writable, Writable> map = (Map<Writable, Writable>) x;
            BasicBSONObject bson = new BasicBSONObject();
            for (Map.Entry<Writable, Writable> entry : map.entrySet()) {
                bson.put(entry.getKey().toString(), toBSON(entry.getValue()));
            }
            return bson;
        }
        if (x instanceof ArrayWritable) {
            Writable[] o = ((ArrayWritable) x).get();
            Object[] a = new Object[o.length];
            for (int i = 0; i < o.length; i++) {
                a[i] = toBSON(o[i]);
            }
            return a;
        }
        if (x instanceof NullWritable) {
            return null;
        }
        if (x instanceof BooleanWritable) {
            return ((BooleanWritable) x).get();
        }
        if (x instanceof BytesWritable) {
            return ((BytesWritable) x).getBytes();
        }
        if (x instanceof ByteWritable) {
            return ((ByteWritable) x).get();
        }
        if (x instanceof DoubleWritable) {
            return ((DoubleWritable) x).get();
        }
        if (x instanceof FloatWritable) {
            return ((FloatWritable) x).get();
        }
        if (x instanceof LongWritable) {
            return ((LongWritable) x).get();
        }
        if (x instanceof IntWritable) {
            return ((IntWritable) x).get();
        }
    // TODO - Support counters
    }
    return x;
}
Also used : NullWritable(org.apache.hadoop.io.NullWritable) Writable(org.apache.hadoop.io.Writable) DoubleWritable(org.apache.hadoop.io.DoubleWritable) LongWritable(org.apache.hadoop.io.LongWritable) ByteWritable(org.apache.hadoop.io.ByteWritable) BytesWritable(org.apache.hadoop.io.BytesWritable) ArrayWritable(org.apache.hadoop.io.ArrayWritable) IntWritable(org.apache.hadoop.io.IntWritable) AbstractMapWritable(org.apache.hadoop.io.AbstractMapWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) FloatWritable(org.apache.hadoop.io.FloatWritable) Text(org.apache.hadoop.io.Text) BytesWritable(org.apache.hadoop.io.BytesWritable) DoubleWritable(org.apache.hadoop.io.DoubleWritable) NullWritable(org.apache.hadoop.io.NullWritable) BasicBSONObject(org.bson.BasicBSONObject) FloatWritable(org.apache.hadoop.io.FloatWritable) ArrayWritable(org.apache.hadoop.io.ArrayWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) AbstractMapWritable(org.apache.hadoop.io.AbstractMapWritable) BasicBSONObject(org.bson.BasicBSONObject) BasicDBObject(com.mongodb.BasicDBObject) BSONObject(org.bson.BSONObject) LongWritable(org.apache.hadoop.io.LongWritable) Map(java.util.Map) ByteWritable(org.apache.hadoop.io.ByteWritable) IntWritable(org.apache.hadoop.io.IntWritable)

Example 63 with BooleanWritable

use of org.apache.hadoop.io.BooleanWritable in project mongo-hadoop by mongodb.

the class BSONWritableTest method testToBSON.

@Test
public void testToBSON() {
    assertEquals(null, toBSON(null));
    assertEquals(null, toBSON(NullWritable.get()));
    assertEquals("hello", toBSON(new Text("hello")));
    DBObject obj = new BasicDBObject("hello", "world");
    assertEquals(obj, toBSON(new BSONWritable(obj)));
    final BasicBSONObject bsonResult = new BasicBSONObject("one", 1);
    SortedMapWritable smw = new SortedMapWritable();
    smw.put(new Text("one"), new IntWritable(1));
    assertEquals(bsonResult, toBSON(smw));
    MapWritable mw = new MapWritable();
    mw.put(new Text("one"), new IntWritable(1));
    assertEquals(bsonResult, toBSON(mw));
    String[] expectedObjects = new String[] { "one", "two" };
    Writable[] writableObjects = new Writable[] { new Text("one"), new Text("two") };
    ArrayWritable aw = new ArrayWritable(Text.class, writableObjects);
    Object[] actual = (Object[]) toBSON(aw);
    assertTrue(Arrays.equals(expectedObjects, actual));
    assertEquals(false, toBSON(new BooleanWritable(false)));
    byte[] bytes = new byte[] { '0', '1', '2' };
    assertEquals(bytes, toBSON(new BytesWritable(bytes)));
    byte b = (byte) 'c';
    assertEquals(b, toBSON(new ByteWritable(b)));
    assertEquals(3.14159, toBSON(new DoubleWritable(3.14159)));
    assertEquals(3.14159f, toBSON(new FloatWritable(3.14159f)));
    assertEquals(42L, toBSON(new LongWritable(42L)));
    assertEquals(42, toBSON(new IntWritable(42)));
    // Catchall
    assertEquals("hi", toBSON("hi"));
}
Also used : NullWritable(org.apache.hadoop.io.NullWritable) SortedMapWritable(org.apache.hadoop.io.SortedMapWritable) Writable(org.apache.hadoop.io.Writable) MapWritable(org.apache.hadoop.io.MapWritable) DoubleWritable(org.apache.hadoop.io.DoubleWritable) LongWritable(org.apache.hadoop.io.LongWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) ByteWritable(org.apache.hadoop.io.ByteWritable) BytesWritable(org.apache.hadoop.io.BytesWritable) ArrayWritable(org.apache.hadoop.io.ArrayWritable) FloatWritable(org.apache.hadoop.io.FloatWritable) IntWritable(org.apache.hadoop.io.IntWritable) Text(org.apache.hadoop.io.Text) BytesWritable(org.apache.hadoop.io.BytesWritable) DoubleWritable(org.apache.hadoop.io.DoubleWritable) SortedMapWritable(org.apache.hadoop.io.SortedMapWritable) SortedMapWritable(org.apache.hadoop.io.SortedMapWritable) MapWritable(org.apache.hadoop.io.MapWritable) BasicDBObject(com.mongodb.BasicDBObject) DBObject(com.mongodb.DBObject) BasicDBObject(com.mongodb.BasicDBObject) BasicBSONObject(org.bson.BasicBSONObject) FloatWritable(org.apache.hadoop.io.FloatWritable) ArrayWritable(org.apache.hadoop.io.ArrayWritable) BooleanWritable(org.apache.hadoop.io.BooleanWritable) BasicBSONObject(org.bson.BasicBSONObject) BasicDBObject(com.mongodb.BasicDBObject) DBObject(com.mongodb.DBObject) LongWritable(org.apache.hadoop.io.LongWritable) ByteWritable(org.apache.hadoop.io.ByteWritable) IntWritable(org.apache.hadoop.io.IntWritable) Test(org.junit.Test)

Aggregations

BooleanWritable (org.apache.hadoop.io.BooleanWritable)63 IntWritable (org.apache.hadoop.io.IntWritable)41 LongWritable (org.apache.hadoop.io.LongWritable)40 FloatWritable (org.apache.hadoop.io.FloatWritable)37 Text (org.apache.hadoop.io.Text)31 DoubleWritable (org.apache.hadoop.hive.serde2.io.DoubleWritable)27 ShortWritable (org.apache.hadoop.hive.serde2.io.ShortWritable)26 BytesWritable (org.apache.hadoop.io.BytesWritable)26 ByteWritable (org.apache.hadoop.hive.serde2.io.ByteWritable)25 Writable (org.apache.hadoop.io.Writable)17 Test (org.junit.Test)17 ArrayList (java.util.ArrayList)15 Configuration (org.apache.hadoop.conf.Configuration)12 TimestampWritable (org.apache.hadoop.hive.serde2.io.TimestampWritable)12 Random (java.util.Random)11 HiveDecimalWritable (org.apache.hadoop.hive.serde2.io.HiveDecimalWritable)10 DateWritable (org.apache.hadoop.hive.serde2.io.DateWritable)9 KeyValue (org.apache.hadoop.hbase.KeyValue)7 Result (org.apache.hadoop.hbase.client.Result)7 HiveChar (org.apache.hadoop.hive.common.type.HiveChar)7