use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.
the class ObjectInspectorUtils method getConstantObjectInspector.
public static ConstantObjectInspector getConstantObjectInspector(ObjectInspector oi, Object value) {
if (oi instanceof ConstantObjectInspector) {
return (ConstantObjectInspector) oi;
}
ObjectInspector writableOI = getStandardObjectInspector(oi, ObjectInspectorCopyOption.WRITABLE);
Object writableValue = value == null ? value : ObjectInspectorConverters.getConverter(oi, writableOI).convert(value);
switch(writableOI.getCategory()) {
case PRIMITIVE:
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
return PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(poi.getTypeInfo(), writableValue);
case LIST:
ListObjectInspector loi = (ListObjectInspector) oi;
return ObjectInspectorFactory.getStandardConstantListObjectInspector(getStandardObjectInspector(loi.getListElementObjectInspector(), ObjectInspectorCopyOption.WRITABLE), (List<?>) writableValue);
case MAP:
MapObjectInspector moi = (MapObjectInspector) oi;
return ObjectInspectorFactory.getStandardConstantMapObjectInspector(getStandardObjectInspector(moi.getMapKeyObjectInspector(), ObjectInspectorCopyOption.WRITABLE), getStandardObjectInspector(moi.getMapValueObjectInspector(), ObjectInspectorCopyOption.WRITABLE), (Map<?, ?>) writableValue);
case STRUCT:
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> fields = soi.getAllStructFieldRefs();
List<String> fieldNames = new ArrayList<String>(fields.size());
List<ObjectInspector> fieldObjectInspectors = new ArrayList<ObjectInspector>(fields.size());
for (StructField f : fields) {
fieldNames.add(f.getFieldName());
fieldObjectInspectors.add(getStandardObjectInspector(f.getFieldObjectInspector(), ObjectInspectorCopyOption.WRITABLE));
}
if (value != null && (writableValue.getClass().isArray())) {
writableValue = java.util.Arrays.asList((Object[]) writableValue);
}
return ObjectInspectorFactory.getStandardConstantStructObjectInspector(fieldNames, fieldObjectInspectors, (List<?>) writableValue);
default:
throw new IllegalArgumentException(writableOI.getCategory() + " not yet supported for constant OI");
}
}
use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.
the class ObjectInspectorUtils method hasAllFieldsSettable.
/**
* @param oi - Input object inspector
* @param oiSettableProperties - Lookup map to cache the result.(If no caching, pass null)
* @return - true if : (1) oi is an instance of settable<DataType>OI.
* (2) All the embedded object inspectors are instances of settable<DataType>OI.
* If (1) or (2) is false, return false.
*/
public static boolean hasAllFieldsSettable(ObjectInspector oi, Map<ObjectInspector, Boolean> oiSettableProperties) {
// If the result is already present in the cache, return it.
if (!(oiSettableProperties == null) && oiSettableProperties.containsKey(oi)) {
return oiSettableProperties.get(oi).booleanValue();
}
// If the top-level object inspector is non-settable return false
if (!(isInstanceOfSettableOI(oi))) {
return setOISettablePropertiesMap(oi, oiSettableProperties, false);
}
Boolean returnValue = true;
switch(oi.getCategory()) {
case PRIMITIVE:
break;
case STRUCT:
StructObjectInspector structOutputOI = (StructObjectInspector) oi;
List<? extends StructField> listFields = structOutputOI.getAllStructFieldRefs();
for (StructField listField : listFields) {
if (!hasAllFieldsSettable(listField.getFieldObjectInspector(), oiSettableProperties)) {
returnValue = false;
break;
}
}
break;
case LIST:
ListObjectInspector listOutputOI = (ListObjectInspector) oi;
returnValue = hasAllFieldsSettable(listOutputOI.getListElementObjectInspector(), oiSettableProperties);
break;
case MAP:
MapObjectInspector mapOutputOI = (MapObjectInspector) oi;
returnValue = hasAllFieldsSettable(mapOutputOI.getMapKeyObjectInspector(), oiSettableProperties) && hasAllFieldsSettable(mapOutputOI.getMapValueObjectInspector(), oiSettableProperties);
break;
case UNION:
UnionObjectInspector unionOutputOI = (UnionObjectInspector) oi;
List<ObjectInspector> unionListFields = unionOutputOI.getObjectInspectors();
for (ObjectInspector listField : unionListFields) {
if (!hasAllFieldsSettable(listField, oiSettableProperties)) {
returnValue = false;
break;
}
}
break;
default:
throw new RuntimeException("Hive internal error inside hasAllFieldsSettable : " + oi.getTypeName() + " not supported yet.");
}
return setOISettablePropertiesMap(oi, oiSettableProperties, returnValue);
}
use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.
the class ObjectInspectorUtils method compareTypes.
/**
* Compares two types identified by the given object inspectors. This method
* compares the types as follows:
* <ol>
* <li>If the given inspectors do not belong to same category, the result is
* negative.</li>
* <li>If the given inspectors are for <code>PRIMITIVE</code> type, the result
* is the comparison of their type names.</li>
* <li>If the given inspectors are for <code>LIST</code> type, then the result
* is recursive call to compare the type of list elements.</li>
* <li>If the given inspectors are <code>MAP</code> type, then the result is a
* recursive call to compare the map key and value types.</li>
* <li>If the given inspectors are <code>STRUCT</code> type, then the result
* is negative if they do not have the same number of fields. If they do have
* the same number of fields, the result is a recursive call to compare each
* of the field types.</li>
* <li>If none of the above, the result is negative.</li>
* </ol>
* @param o1
* @param o2
* @return true if the given object inspectors represent the same types.
*/
public static boolean compareTypes(ObjectInspector o1, ObjectInspector o2) {
Category c1 = o1.getCategory();
Category c2 = o2.getCategory();
// Return false if categories are not equal
if (!c1.equals(c2)) {
return false;
}
// If both categories are primitive return the comparison of type names.
if (c1.equals(Category.PRIMITIVE)) {
return o1.getTypeName().equals(o2.getTypeName());
}
// If lists, recursively compare the list element types
if (c1.equals(Category.LIST)) {
ObjectInspector child1 = ((ListObjectInspector) o1).getListElementObjectInspector();
ObjectInspector child2 = ((ListObjectInspector) o2).getListElementObjectInspector();
return compareTypes(child1, child2);
}
// If maps, recursively compare the key and value types
if (c1.equals(Category.MAP)) {
MapObjectInspector mapOI1 = (MapObjectInspector) o1;
MapObjectInspector mapOI2 = (MapObjectInspector) o2;
ObjectInspector childKey1 = mapOI1.getMapKeyObjectInspector();
ObjectInspector childKey2 = mapOI2.getMapKeyObjectInspector();
if (compareTypes(childKey1, childKey2)) {
ObjectInspector childVal1 = mapOI1.getMapValueObjectInspector();
ObjectInspector childVal2 = mapOI2.getMapValueObjectInspector();
if (compareTypes(childVal1, childVal2)) {
return true;
}
}
return false;
}
// If structs, recursively compare the fields
if (c1.equals(Category.STRUCT)) {
StructObjectInspector structOI1 = (StructObjectInspector) o1;
StructObjectInspector structOI2 = (StructObjectInspector) o2;
List<? extends StructField> childFieldsList1 = structOI1.getAllStructFieldRefs();
List<? extends StructField> childFieldsList2 = structOI2.getAllStructFieldRefs();
if (childFieldsList1 == null && childFieldsList2 == null) {
return true;
} else if (childFieldsList1 == null || childFieldsList2 == null) {
return false;
} else if (childFieldsList1.size() != childFieldsList2.size()) {
return false;
}
Iterator<? extends StructField> it1 = childFieldsList1.iterator();
Iterator<? extends StructField> it2 = childFieldsList2.iterator();
while (it1.hasNext()) {
StructField field1 = it1.next();
StructField field2 = it2.next();
if (!compareTypes(field1.getFieldObjectInspector(), field2.getFieldObjectInspector())) {
return false;
}
}
return true;
}
if (c1.equals(Category.UNION)) {
UnionObjectInspector uoi1 = (UnionObjectInspector) o1;
UnionObjectInspector uoi2 = (UnionObjectInspector) o2;
List<ObjectInspector> ois1 = uoi1.getObjectInspectors();
List<ObjectInspector> ois2 = uoi2.getObjectInspectors();
if (ois1 == null && ois2 == null) {
return true;
} else if (ois1 == null || ois2 == null) {
return false;
} else if (ois1.size() != ois2.size()) {
return false;
}
Iterator<? extends ObjectInspector> it1 = ois1.iterator();
Iterator<? extends ObjectInspector> it2 = ois2.iterator();
while (it1.hasNext()) {
if (!compareTypes(it1.next(), it2.next())) {
return false;
}
}
return true;
}
// Unknown category
throw new RuntimeException("Unknown category encountered: " + c1);
}
use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.
the class ObjectInspectorUtils method compare.
/**
* Compare two objects with their respective ObjectInspectors.
* if nullValueOpt is MAXVALUE, treat null as maximum value.
* if nullValueOpt is MINVALUE, treat null as minimum value.
*/
public static int compare(Object o1, ObjectInspector oi1, Object o2, ObjectInspector oi2, MapEqualComparer mapEqualComparer, NullValueOption nullValueOpt) {
if (oi1.getCategory() != oi2.getCategory()) {
return oi1.getCategory().compareTo(oi2.getCategory());
}
int nullCmpRtn = -1;
switch(nullValueOpt) {
case MAXVALUE:
nullCmpRtn = 1;
break;
case MINVALUE:
nullCmpRtn = -1;
break;
}
if (o1 == null) {
return o2 == null ? 0 : nullCmpRtn;
} else if (o2 == null) {
return -nullCmpRtn;
}
switch(oi1.getCategory()) {
case PRIMITIVE:
{
PrimitiveObjectInspector poi1 = ((PrimitiveObjectInspector) oi1);
PrimitiveObjectInspector poi2 = ((PrimitiveObjectInspector) oi2);
if (poi1.getPrimitiveCategory() != poi2.getPrimitiveCategory()) {
return poi1.getPrimitiveCategory().compareTo(poi2.getPrimitiveCategory());
}
switch(poi1.getPrimitiveCategory()) {
case VOID:
return 0;
case BOOLEAN:
{
int v1 = ((BooleanObjectInspector) poi1).get(o1) ? 1 : 0;
int v2 = ((BooleanObjectInspector) poi2).get(o2) ? 1 : 0;
return v1 - v2;
}
case BYTE:
{
int v1 = ((ByteObjectInspector) poi1).get(o1);
int v2 = ((ByteObjectInspector) poi2).get(o2);
return v1 - v2;
}
case SHORT:
{
int v1 = ((ShortObjectInspector) poi1).get(o1);
int v2 = ((ShortObjectInspector) poi2).get(o2);
return v1 - v2;
}
case INT:
{
int v1 = ((IntObjectInspector) poi1).get(o1);
int v2 = ((IntObjectInspector) poi2).get(o2);
return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0);
}
case LONG:
{
long v1 = ((LongObjectInspector) poi1).get(o1);
long v2 = ((LongObjectInspector) poi2).get(o2);
return v1 > v2 ? 1 : (v1 < v2 ? -1 : 0);
}
case FLOAT:
{
float v1 = ((FloatObjectInspector) poi1).get(o1);
float v2 = ((FloatObjectInspector) poi2).get(o2);
// The IEEE 754 floating point spec specifies that signed -0.0 and 0.0 should be treated as equal.
if (v1 == 0.0f && v2 == 0.0f) {
return 0;
} else {
// Float.compare() treats -0.0 and 0.0 as different
return Float.compare(v1, v2);
}
}
case DOUBLE:
{
double v1 = ((DoubleObjectInspector) poi1).get(o1);
double v2 = ((DoubleObjectInspector) poi2).get(o2);
// The IEEE 754 floating point spec specifies that signed -0.0 and 0.0 should be treated as equal.
if (v1 == 0.0d && v2 == 0.0d) {
return 0;
} else {
// Double.compare() treats -0.0 and 0.0 as different
return Double.compare(v1, v2);
}
}
case STRING:
{
if (poi1.preferWritable() || poi2.preferWritable()) {
Text t1 = (Text) poi1.getPrimitiveWritableObject(o1);
Text t2 = (Text) poi2.getPrimitiveWritableObject(o2);
return t1 == null ? (t2 == null ? 0 : -1) : (t2 == null ? 1 : t1.compareTo(t2));
} else {
String s1 = (String) poi1.getPrimitiveJavaObject(o1);
String s2 = (String) poi2.getPrimitiveJavaObject(o2);
return s1 == null ? (s2 == null ? 0 : -1) : (s2 == null ? 1 : s1.compareTo(s2));
}
}
case CHAR:
{
HiveCharWritable t1 = ((HiveCharObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveCharWritable t2 = ((HiveCharObjectInspector) poi2).getPrimitiveWritableObject(o2);
return t1.compareTo(t2);
}
case VARCHAR:
{
HiveVarcharWritable t1 = ((HiveVarcharObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveVarcharWritable t2 = ((HiveVarcharObjectInspector) poi2).getPrimitiveWritableObject(o2);
return t1.compareTo(t2);
}
case BINARY:
{
BytesWritable bw1 = ((BinaryObjectInspector) poi1).getPrimitiveWritableObject(o1);
BytesWritable bw2 = ((BinaryObjectInspector) poi2).getPrimitiveWritableObject(o2);
return bw1.compareTo(bw2);
}
case DATE:
{
DateWritable d1 = ((DateObjectInspector) poi1).getPrimitiveWritableObject(o1);
DateWritable d2 = ((DateObjectInspector) poi2).getPrimitiveWritableObject(o2);
return d1.compareTo(d2);
}
case TIMESTAMP:
{
TimestampWritable t1 = ((TimestampObjectInspector) poi1).getPrimitiveWritableObject(o1);
TimestampWritable t2 = ((TimestampObjectInspector) poi2).getPrimitiveWritableObject(o2);
return t1.compareTo(t2);
}
case TIMESTAMPLOCALTZ:
{
TimestampLocalTZWritable tstz1 = ((TimestampLocalTZObjectInspector) poi1).getPrimitiveWritableObject(o1);
TimestampLocalTZWritable tstz2 = ((TimestampLocalTZObjectInspector) poi2).getPrimitiveWritableObject(o2);
return tstz1.compareTo(tstz2);
}
case INTERVAL_YEAR_MONTH:
{
HiveIntervalYearMonthWritable i1 = ((HiveIntervalYearMonthObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveIntervalYearMonthWritable i2 = ((HiveIntervalYearMonthObjectInspector) poi2).getPrimitiveWritableObject(o2);
return i1.compareTo(i2);
}
case INTERVAL_DAY_TIME:
{
HiveIntervalDayTimeWritable i1 = ((HiveIntervalDayTimeObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveIntervalDayTimeWritable i2 = ((HiveIntervalDayTimeObjectInspector) poi2).getPrimitiveWritableObject(o2);
return i1.compareTo(i2);
}
case DECIMAL:
{
HiveDecimalWritable t1 = ((HiveDecimalObjectInspector) poi1).getPrimitiveWritableObject(o1);
HiveDecimalWritable t2 = ((HiveDecimalObjectInspector) poi2).getPrimitiveWritableObject(o2);
return t1.compareTo(t2);
}
default:
{
throw new RuntimeException("Unknown type: " + poi1.getPrimitiveCategory());
}
}
}
case STRUCT:
{
StructObjectInspector soi1 = (StructObjectInspector) oi1;
StructObjectInspector soi2 = (StructObjectInspector) oi2;
List<? extends StructField> fields1 = soi1.getAllStructFieldRefs();
List<? extends StructField> fields2 = soi2.getAllStructFieldRefs();
int minimum = Math.min(fields1.size(), fields2.size());
for (int i = 0; i < minimum; i++) {
int r = compare(soi1.getStructFieldData(o1, fields1.get(i)), fields1.get(i).getFieldObjectInspector(), soi2.getStructFieldData(o2, fields2.get(i)), fields2.get(i).getFieldObjectInspector(), mapEqualComparer, nullValueOpt);
if (r != 0) {
return r;
}
}
return fields1.size() - fields2.size();
}
case LIST:
{
ListObjectInspector loi1 = (ListObjectInspector) oi1;
ListObjectInspector loi2 = (ListObjectInspector) oi2;
int minimum = Math.min(loi1.getListLength(o1), loi2.getListLength(o2));
for (int i = 0; i < minimum; i++) {
int r = compare(loi1.getListElement(o1, i), loi1.getListElementObjectInspector(), loi2.getListElement(o2, i), loi2.getListElementObjectInspector(), mapEqualComparer, nullValueOpt);
if (r != 0) {
return r;
}
}
return loi1.getListLength(o1) - loi2.getListLength(o2);
}
case MAP:
{
if (mapEqualComparer == null) {
throw new RuntimeException("Compare on map type not supported!");
} else {
return mapEqualComparer.compare(o1, (MapObjectInspector) oi1, o2, (MapObjectInspector) oi2);
}
}
case UNION:
{
UnionObjectInspector uoi1 = (UnionObjectInspector) oi1;
UnionObjectInspector uoi2 = (UnionObjectInspector) oi2;
byte tag1 = uoi1.getTag(o1);
byte tag2 = uoi2.getTag(o2);
if (tag1 != tag2) {
return tag1 - tag2;
}
return compare(uoi1.getField(o1), uoi1.getObjectInspectors().get(tag1), uoi2.getField(o2), uoi2.getObjectInspectors().get(tag2), mapEqualComparer, nullValueOpt);
}
default:
throw new RuntimeException("Compare on unknown type: " + oi1.getCategory());
}
}
use of org.apache.hadoop.hive.serde2.objectinspector.StructField in project hive by apache.
the class SerDeUtils method buildJSONString.
static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi, String nullStr) {
switch(oi.getCategory()) {
case PRIMITIVE:
{
PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
if (o == null) {
sb.append(nullStr);
} else {
switch(poi.getPrimitiveCategory()) {
case BOOLEAN:
{
boolean b = ((BooleanObjectInspector) poi).get(o);
sb.append(b ? "true" : "false");
break;
}
case BYTE:
{
sb.append(((ByteObjectInspector) poi).get(o));
break;
}
case SHORT:
{
sb.append(((ShortObjectInspector) poi).get(o));
break;
}
case INT:
{
sb.append(((IntObjectInspector) poi).get(o));
break;
}
case LONG:
{
sb.append(((LongObjectInspector) poi).get(o));
break;
}
case FLOAT:
{
sb.append(((FloatObjectInspector) poi).get(o));
break;
}
case DOUBLE:
{
sb.append(((DoubleObjectInspector) poi).get(o));
break;
}
case STRING:
{
sb.append('"');
sb.append(escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o)));
sb.append('"');
break;
}
case CHAR:
{
sb.append('"');
sb.append(escapeString(((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
sb.append('"');
break;
}
case VARCHAR:
{
sb.append('"');
sb.append(escapeString(((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString()));
sb.append('"');
break;
}
case DATE:
{
sb.append('"');
sb.append(((DateObjectInspector) poi).getPrimitiveWritableObject(o));
sb.append('"');
break;
}
case TIMESTAMP:
{
sb.append('"');
sb.append(((TimestampObjectInspector) poi).getPrimitiveWritableObject(o));
sb.append('"');
break;
}
case TIMESTAMPLOCALTZ:
{
sb.append('"');
sb.append(((TimestampLocalTZObjectInspector) poi).getPrimitiveWritableObject(o));
sb.append('"');
break;
}
case BINARY:
{
BytesWritable bw = ((BinaryObjectInspector) oi).getPrimitiveWritableObject(o);
Text txt = new Text();
txt.set(bw.getBytes(), 0, bw.getLength());
sb.append(txt.toString());
break;
}
case DECIMAL:
{
sb.append(((HiveDecimalObjectInspector) oi).getPrimitiveJavaObject(o));
break;
}
case INTERVAL_YEAR_MONTH:
{
sb.append(((HiveIntervalYearMonthObjectInspector) oi).getPrimitiveJavaObject(o));
break;
}
case INTERVAL_DAY_TIME:
{
sb.append(((HiveIntervalDayTimeObjectInspector) oi).getPrimitiveJavaObject(o));
break;
}
default:
throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
}
}
break;
}
case LIST:
{
ListObjectInspector loi = (ListObjectInspector) oi;
ObjectInspector listElementObjectInspector = loi.getListElementObjectInspector();
List<?> olist = loi.getList(o);
if (olist == null) {
sb.append(nullStr);
} else {
sb.append(LBRACKET);
for (int i = 0; i < olist.size(); i++) {
if (i > 0) {
sb.append(COMMA);
}
buildJSONString(sb, olist.get(i), listElementObjectInspector, JSON_NULL);
}
sb.append(RBRACKET);
}
break;
}
case MAP:
{
MapObjectInspector moi = (MapObjectInspector) oi;
ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
ObjectInspector mapValueObjectInspector = moi.getMapValueObjectInspector();
Map<?, ?> omap = moi.getMap(o);
if (omap == null) {
sb.append(nullStr);
} else {
sb.append(LBRACE);
boolean first = true;
for (Object entry : omap.entrySet()) {
if (first) {
first = false;
} else {
sb.append(COMMA);
}
Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
buildJSONString(sb, e.getKey(), mapKeyObjectInspector, JSON_NULL);
sb.append(COLON);
buildJSONString(sb, e.getValue(), mapValueObjectInspector, JSON_NULL);
}
sb.append(RBRACE);
}
break;
}
case STRUCT:
{
StructObjectInspector soi = (StructObjectInspector) oi;
List<? extends StructField> structFields = soi.getAllStructFieldRefs();
if (o == null) {
sb.append(nullStr);
} else {
sb.append(LBRACE);
for (int i = 0; i < structFields.size(); i++) {
if (i > 0) {
sb.append(COMMA);
}
sb.append(QUOTE);
sb.append(structFields.get(i).getFieldName());
sb.append(QUOTE);
sb.append(COLON);
buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)), structFields.get(i).getFieldObjectInspector(), JSON_NULL);
}
sb.append(RBRACE);
}
break;
}
case UNION:
{
UnionObjectInspector uoi = (UnionObjectInspector) oi;
if (o == null) {
sb.append(nullStr);
} else {
sb.append(LBRACE);
sb.append(uoi.getTag(o));
sb.append(COLON);
buildJSONString(sb, uoi.getField(o), uoi.getObjectInspectors().get(uoi.getTag(o)), JSON_NULL);
sb.append(RBRACE);
}
break;
}
default:
throw new RuntimeException("Unknown type in ObjectInspector!");
}
}
Aggregations