use of org.apache.hadoop.hive.serde2.thrift.test.IntString in project hive by apache.
the class TestObjectInspectorUtils method testObjectInspectorUtils.
public void testObjectInspectorUtils() throws Throwable {
try {
ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector(Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
// metadata
assertEquals(Category.STRUCT, oi1.getCategory());
// standard ObjectInspector
StructObjectInspector soi = (StructObjectInspector) ObjectInspectorUtils.getStandardObjectInspector(oi1);
List<? extends StructField> fields = soi.getAllStructFieldRefs();
assertEquals(10, fields.size());
assertEquals(fields.get(0), soi.getStructFieldRef("aint"));
// null
for (int i = 0; i < fields.size(); i++) {
assertNull(soi.getStructFieldData(null, fields.get(i)));
}
// real object
Complex cc = new Complex();
cc.setAint(1);
cc.setAString("test");
List<Integer> c2 = Arrays.asList(new Integer[] { 1, 2, 3 });
cc.setLint(c2);
List<String> c3 = Arrays.asList(new String[] { "one", "two" });
cc.setLString(c3);
List<IntString> c4 = new ArrayList<IntString>();
cc.setLintString(c4);
cc.setMStringString(null);
// standard object
Object c = ObjectInspectorUtils.copyToStandardObject(cc, oi1);
assertEquals(1, soi.getStructFieldData(c, fields.get(0)));
assertEquals("test", soi.getStructFieldData(c, fields.get(1)));
assertEquals(c2, soi.getStructFieldData(c, fields.get(2)));
assertEquals(c3, soi.getStructFieldData(c, fields.get(3)));
assertEquals(c4, soi.getStructFieldData(c, fields.get(4)));
assertNull(soi.getStructFieldData(c, fields.get(5)));
ArrayList<Object> cfields = new ArrayList<Object>();
for (int i = 0; i < 10; i++) {
cfields.add(soi.getStructFieldData(c, fields.get(i)));
}
assertEquals(cfields, soi.getStructFieldsDataAsList(c));
// sub fields
assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, fields.get(0).getFieldObjectInspector());
assertEquals(PrimitiveObjectInspectorFactory.javaStringObjectInspector, fields.get(1).getFieldObjectInspector());
assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector), fields.get(2).getFieldObjectInspector());
assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(3).getFieldObjectInspector());
assertEquals(ObjectInspectorUtils.getStandardObjectInspector(ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT))), fields.get(4).getFieldObjectInspector());
assertEquals(ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(5).getFieldObjectInspector());
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}
use of org.apache.hadoop.hive.serde2.thrift.test.IntString in project hive by apache.
the class TestProtocolBuffersObjectInspectors method testProtocolBuffersObjectInspectors.
public void testProtocolBuffersObjectInspectors() throws Throwable {
try {
ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector(Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.PROTOCOL_BUFFERS);
ObjectInspector oi2 = ObjectInspectorFactory.getReflectionObjectInspector(Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.PROTOCOL_BUFFERS);
assertEquals(oi1, oi2);
// metadata
assertEquals(Category.STRUCT, oi1.getCategory());
StructObjectInspector soi = (StructObjectInspector) oi1;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
assertEquals(5, fields.size());
assertEquals(fields.get(0), soi.getStructFieldRef("aint_"));
// null
for (int i = 0; i < fields.size(); i++) {
assertNull(soi.getStructFieldData(null, fields.get(i)));
}
// real object
List<Integer> c2 = Arrays.asList(new Integer[] { 1, 2, 3 });
List<String> c3 = Arrays.asList(new String[] { "one", "two" });
List<IntString> c4 = new ArrayList<IntString>();
Complex c = new Complex(1, "test", c2, c3, c4);
assertEquals(1, soi.getStructFieldData(c, fields.get(0)));
assertEquals("test", soi.getStructFieldData(c, fields.get(1)));
assertEquals(c2, soi.getStructFieldData(c, fields.get(2)));
assertEquals(c3, soi.getStructFieldData(c, fields.get(3)));
assertEquals(c4, soi.getStructFieldData(c, fields.get(4)));
ArrayList<Object> cfields = new ArrayList<Object>();
for (int i = 0; i < 5; i++) {
cfields.add(soi.getStructFieldData(c, fields.get(i)));
}
assertEquals(cfields, soi.getStructFieldsDataAsList(c));
// sub fields
assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, fields.get(0).getFieldObjectInspector());
assertEquals(PrimitiveObjectInspectorFactory.javaStringObjectInspector, fields.get(1).getFieldObjectInspector());
assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector), fields.get(2).getFieldObjectInspector());
assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(3).getFieldObjectInspector());
assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, ObjectInspectorFactory.ObjectInspectorOptions.PROTOCOL_BUFFERS)), fields.get(4).getFieldObjectInspector());
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}
use of org.apache.hadoop.hive.serde2.thrift.test.IntString in project hive by apache.
the class TestThriftObjectInspectors method testThriftObjectInspectors.
public void testThriftObjectInspectors() throws Throwable {
try {
ObjectInspector oi1 = ObjectInspectorFactory.getReflectionObjectInspector(Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
ObjectInspector oi2 = ObjectInspectorFactory.getReflectionObjectInspector(Complex.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
assertEquals(oi1, oi2);
// metadata
assertEquals(Category.STRUCT, oi1.getCategory());
StructObjectInspector soi = (StructObjectInspector) oi1;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
assertEquals(10, fields.size());
assertEquals(fields.get(0), soi.getStructFieldRef("aint"));
// null
for (int i = 0; i < fields.size(); i++) {
assertNull(soi.getStructFieldData(null, fields.get(i)));
}
ObjectInspector oi = ObjectInspectorFactory.getReflectionObjectInspector(PropValueUnion.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT);
assertNotNull(oi.toString());
// real object
Complex c = new Complex();
c.setAint(1);
c.setAString("test");
List<Integer> c2 = Arrays.asList(new Integer[] { 1, 2, 3 });
c.setLint(c2);
List<String> c3 = Arrays.asList(new String[] { "one", "two" });
c.setLString(c3);
List<IntString> c4 = new ArrayList<IntString>();
c.setLintString(c4);
c.setMStringString(null);
c.setAttributes(null);
c.setUnionField1(null);
c.setUnionField2(null);
c.setUnionField3(null);
assertEquals(1, soi.getStructFieldData(c, fields.get(0)));
assertEquals("test", soi.getStructFieldData(c, fields.get(1)));
assertEquals(c2, soi.getStructFieldData(c, fields.get(2)));
assertEquals(c3, soi.getStructFieldData(c, fields.get(3)));
assertEquals(c4, soi.getStructFieldData(c, fields.get(4)));
assertNull(soi.getStructFieldData(c, fields.get(5)));
assertNull(soi.getStructFieldData(c, fields.get(6)));
assertNull(soi.getStructFieldData(c, fields.get(7)));
assertNull(soi.getStructFieldData(c, fields.get(8)));
assertNull(soi.getStructFieldData(c, fields.get(9)));
ArrayList<Object> cfields = new ArrayList<Object>();
for (int i = 0; i < 10; i++) {
cfields.add(soi.getStructFieldData(c, fields.get(i)));
}
assertEquals(cfields, soi.getStructFieldsDataAsList(c));
// sub fields
assertEquals(PrimitiveObjectInspectorFactory.javaIntObjectInspector, fields.get(0).getFieldObjectInspector());
assertEquals(PrimitiveObjectInspectorFactory.javaStringObjectInspector, fields.get(1).getFieldObjectInspector());
assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaIntObjectInspector), fields.get(2).getFieldObjectInspector());
assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(3).getFieldObjectInspector());
assertEquals(ObjectInspectorFactory.getStandardListObjectInspector(ObjectInspectorFactory.getReflectionObjectInspector(IntString.class, ObjectInspectorFactory.ObjectInspectorOptions.THRIFT)), fields.get(4).getFieldObjectInspector());
assertEquals(ObjectInspectorFactory.getStandardMapObjectInspector(PrimitiveObjectInspectorFactory.javaStringObjectInspector, PrimitiveObjectInspectorFactory.javaStringObjectInspector), fields.get(5).getFieldObjectInspector());
} catch (Throwable e) {
e.printStackTrace();
throw e;
}
}
use of org.apache.hadoop.hive.serde2.thrift.test.IntString in project hive by apache.
the class CreateSequenceFile method main.
public static void main(String[] args) throws Exception {
// Read parameters
int lines = 10;
List<String> extraArgs = new ArrayList<String>();
for (int ai = 0; ai < args.length; ai++) {
if (args[ai].equals("-line") && ai + 1 < args.length) {
lines = Integer.parseInt(args[ai + 1]);
ai++;
} else {
extraArgs.add(args[ai]);
}
}
if (extraArgs.size() != 1) {
usage();
}
JobConf conf = new JobConf(CreateSequenceFile.class);
ThriftSerializer serializer = new ThriftSerializer();
// Open files
SequenceFile.Writer writer = new SequenceFile.Writer(FileSystem.get(conf), conf, new Path(extraArgs.get(0)), BytesWritable.class, BytesWritable.class);
// write to file
BytesWritable key = new BytesWritable();
Random rand = new Random(20081215);
for (int i = 0; i < lines; i++) {
ArrayList<Integer> alist = new ArrayList<Integer>();
alist.add(i);
alist.add(i * 2);
alist.add(i * 3);
ArrayList<String> slist = new ArrayList<String>();
slist.add("" + i * 10);
slist.add("" + i * 100);
slist.add("" + i * 1000);
ArrayList<IntString> islist = new ArrayList<IntString>();
islist.add(new IntString(i * i, "" + i * i * i, i));
HashMap<String, String> hash = new HashMap<String, String>();
hash.put("key_" + i, "value_" + i);
Map<String, Map<String, Map<String, PropValueUnion>>> unionMap = new HashMap<String, Map<String, Map<String, PropValueUnion>>>();
Map<String, Map<String, PropValueUnion>> erMap = new HashMap<String, Map<String, PropValueUnion>>();
Map<String, PropValueUnion> attrMap = new HashMap<String, PropValueUnion>();
erMap.put("erVal" + i, attrMap);
attrMap.put("value_" + i, PropValueUnion.doubleValue(1.0));
unionMap.put("key_" + i, erMap);
Complex complex = new Complex(rand.nextInt(), "record_" + (new Integer(i)).toString(), alist, slist, islist, hash, unionMap, PropValueUnion.stringValue("test" + i), PropValueUnion.unionMStringString(hash), PropValueUnion.lString(slist));
Writable value = serializer.serialize(complex);
writer.append(key, value);
}
// Add an all-null record
Complex complex = new Complex(0, null, null, null, null, null, null, null, null, null);
Writable value = serializer.serialize(complex);
writer.append(key, value);
// Close files
writer.close();
}
use of org.apache.hadoop.hive.serde2.thrift.test.IntString in project hive by apache.
the class TestHCatHiveThriftCompatibility method setUp.
@Before
@Override
public void setUp() throws Exception {
super.setUp();
if (setUpComplete) {
return;
}
ByteArrayOutputStream out = new ByteArrayOutputStream();
TIOStreamTransport transport = new TIOStreamTransport(out);
TBinaryProtocol protocol = new TBinaryProtocol(transport);
IntString intString = new IntString(1, "one", 1);
intString.write(protocol);
BytesWritable bytesWritable = new BytesWritable(out.toByteArray());
intStringSeq = new Path(TEST_DATA_DIR + "/data/intString.seq");
LOG.info("Creating data file: " + intStringSeq);
SequenceFile.Writer seqFileWriter = SequenceFile.createWriter(intStringSeq.getFileSystem(hiveConf), hiveConf, intStringSeq, NullWritable.class, BytesWritable.class);
seqFileWriter.append(NullWritable.get(), bytesWritable);
seqFileWriter.close();
setUpComplete = true;
}
Aggregations