use of org.apache.carbondata.processing.loading.complexobjects.StructObject in project carbondata by apache.
the class AvroCarbonWriter method avroFieldToObjectForUnionType.
/**
* fill fieldvalue for union type
*
* @param avroField
* @param fieldValue
* @param avroFields
* @return
*/
private Object avroFieldToObjectForUnionType(Schema avroField, Object fieldValue, Schema.Field avroFields) {
Object out = null;
Schema.Type type = avroField.getType();
LogicalType logicalType = avroField.getLogicalType();
switch(type) {
case RECORD:
if (fieldValue instanceof GenericData.Record) {
List<Schema.Field> fields = avroField.getFields();
Object[] structChildObjects = new Object[fields.size()];
for (int i = 0; i < fields.size(); i++) {
Object childObject = avroFieldToObject(fields.get(i), ((GenericData.Record) fieldValue).get(i));
if (childObject != null) {
structChildObjects[i] = childObject;
}
}
out = new StructObject(structChildObjects);
} else {
out = null;
}
break;
case ARRAY:
if (fieldValue instanceof GenericData.Array || fieldValue instanceof ArrayList) {
Object[] arrayChildObjects;
if (fieldValue instanceof GenericData.Array) {
int size = ((GenericData.Array) fieldValue).size();
arrayChildObjects = new Object[size];
for (int i = 0; i < size; i++) {
Object childObject = avroFieldToObject(new Schema.Field(avroFields.name(), avroField.getElementType(), avroFields.doc(), avroFields.defaultVal()), ((GenericData.Array) fieldValue).get(i));
if (childObject != null) {
arrayChildObjects[i] = childObject;
}
}
} else {
int size = ((ArrayList) fieldValue).size();
arrayChildObjects = new Object[size];
for (int i = 0; i < size; i++) {
Object childObject = avroFieldToObject(new Schema.Field(avroFields.name(), avroField.getElementType(), avroFields.doc(), avroFields.defaultVal()), ((ArrayList) fieldValue).get(i));
if (childObject != null) {
arrayChildObjects[i] = childObject;
}
}
}
out = new ArrayObject(arrayChildObjects);
} else {
out = null;
}
break;
case MAP:
// Map will be internally stored as Array<Struct<Key,Value>>
if (fieldValue instanceof HashMap) {
Map mapEntries = (HashMap) fieldValue;
Object[] arrayMapChildObjects = new Object[mapEntries.size()];
if (!mapEntries.isEmpty()) {
Iterator iterator = mapEntries.entrySet().iterator();
int counter = 0;
while (iterator.hasNext()) {
// size is 2 because map will have key and value
Object[] mapChildObjects = new Object[2];
Map.Entry mapEntry = (Map.Entry) iterator.next();
// evaluate key
Object keyObject = avroFieldToObject(new Schema.Field(avroFields.name(), Schema.create(Schema.Type.STRING), avroFields.doc(), avroFields.defaultVal()), mapEntry.getKey());
// evaluate value
Object valueObject = avroFieldToObject(new Schema.Field(avroFields.name(), avroField.getValueType(), avroFields.doc(), avroFields.defaultVal()), mapEntry.getValue());
if (keyObject != null) {
mapChildObjects[0] = keyObject;
}
if (valueObject != null) {
mapChildObjects[1] = valueObject;
}
StructObject keyValueObject = new StructObject(mapChildObjects);
arrayMapChildObjects[counter++] = keyValueObject;
}
}
out = new ArrayObject(arrayMapChildObjects);
} else {
out = null;
}
break;
case BYTES:
// set to "decimal" and a specified precision and scale
if (logicalType instanceof LogicalTypes.Decimal) {
out = extractDecimalValue(fieldValue, ((LogicalTypes.Decimal) avroField.getLogicalType()).getScale(), ((LogicalTypes.Decimal) avroField.getLogicalType()).getPrecision());
}
break;
default:
out = avroPrimitiveFieldToObject(type, logicalType, fieldValue);
}
return out;
}
use of org.apache.carbondata.processing.loading.complexobjects.StructObject in project carbondata by apache.
the class AvroCarbonWriter method avroFieldToObject.
private Object avroFieldToObject(Schema.Field avroField, Object fieldValue) {
Object out = null;
Schema.Type type = avroField.schema().getType();
LogicalType logicalType = avroField.schema().getLogicalType();
switch(type) {
case MAP:
// Note: Avro object takes care of removing the duplicates so we should not handle it again
// Map will be internally stored as Array<Struct<Key,Value>>
Map mapEntries = (HashMap) fieldValue;
Object[] arrayMapChildObjects = new Object[mapEntries.size()];
if (!mapEntries.isEmpty()) {
Iterator iterator = mapEntries.entrySet().iterator();
int counter = 0;
while (iterator.hasNext()) {
// size is 2 because map will have key and value
Object[] mapChildObjects = new Object[2];
Map.Entry mapEntry = (Map.Entry) iterator.next();
// evaluate key
Object keyObject = avroFieldToObject(new Schema.Field(avroField.name(), Schema.create(Schema.Type.STRING), avroField.doc(), avroField.defaultVal()), mapEntry.getKey());
// evaluate value
Object valueObject = avroFieldToObject(new Schema.Field(avroField.name(), avroField.schema().getValueType(), avroField.doc(), avroField.defaultVal()), mapEntry.getValue());
if (keyObject != null) {
mapChildObjects[0] = keyObject;
}
if (valueObject != null) {
mapChildObjects[1] = valueObject;
}
StructObject keyValueObject = new StructObject(mapChildObjects);
arrayMapChildObjects[counter++] = keyValueObject;
}
}
out = new ArrayObject(arrayMapChildObjects);
break;
case RECORD:
List<Schema.Field> fields = avroField.schema().getFields();
Object[] structChildObjects = new Object[fields.size()];
for (int i = 0; i < fields.size(); i++) {
Object childObject = avroFieldToObject(fields.get(i), ((GenericData.Record) fieldValue).get(i));
if (childObject != null) {
structChildObjects[i] = childObject;
}
}
StructObject structObject = new StructObject(structChildObjects);
out = structObject;
break;
case ARRAY:
Object[] arrayChildObjects;
if (fieldValue instanceof GenericData.Array) {
int size = ((GenericData.Array) fieldValue).size();
arrayChildObjects = new Object[size];
for (int i = 0; i < size; i++) {
Object childObject = avroFieldToObject(new Schema.Field(avroField.name(), avroField.schema().getElementType(), avroField.doc(), avroField.defaultVal()), ((GenericData.Array) fieldValue).get(i));
if (childObject != null) {
arrayChildObjects[i] = childObject;
}
}
} else {
int size = ((ArrayList) fieldValue).size();
arrayChildObjects = new Object[size];
for (int i = 0; i < size; i++) {
Object childObject = avroFieldToObject(new Schema.Field(avroField.name(), avroField.schema().getElementType(), avroField.doc(), avroField.defaultVal()), ((ArrayList) fieldValue).get(i));
if (childObject != null) {
arrayChildObjects[i] = childObject;
}
}
}
out = new ArrayObject(arrayChildObjects);
break;
case UNION:
// Union type will be internally stored as Struct<col:type>
// Fill data object only if fieldvalue is instance of datatype
// For other field datatypes, fill value as Null
List<Schema> unionFields = avroField.schema().getTypes();
int notNullUnionFieldsCount = 0;
for (Schema unionField : unionFields) {
if (!unionField.getType().equals(Schema.Type.NULL)) {
notNullUnionFieldsCount++;
}
}
Object[] values = new Object[notNullUnionFieldsCount];
int j = 0;
for (Schema unionField : unionFields) {
if (unionField.getType().equals(Schema.Type.NULL)) {
continue;
}
// hence check for schema also in case of union of multiple record or enum or fixed type
if (validateUnionFieldValue(unionField.getType(), fieldValue, unionField)) {
values[j] = avroFieldToObjectForUnionType(unionField, fieldValue, avroField);
break;
}
j++;
}
out = new StructObject(values);
break;
case BYTES:
// set to "decimal" and a specified precision and scale
if (logicalType instanceof LogicalTypes.Decimal) {
out = extractDecimalValue(fieldValue, ((LogicalTypes.Decimal) avroField.schema().getLogicalType()).getScale(), ((LogicalTypes.Decimal) avroField.schema().getLogicalType()).getPrecision());
}
break;
default:
out = avroPrimitiveFieldToObject(type, logicalType, fieldValue);
}
return out;
}
use of org.apache.carbondata.processing.loading.complexobjects.StructObject in project carbondata by apache.
the class JsonRowParser method jsonToCarbonObject.
private Object jsonToCarbonObject(Map<String, Object> jsonNodeMap, CarbonColumn column) {
DataType type = column.getDataType();
if (DataTypes.isArrayType(type)) {
CarbonDimension carbonDimension = (CarbonDimension) column;
ArrayList array = (ArrayList) jsonNodeMap.get(extractChildColumnName(column));
if ((array == null) || (array.size() == 0)) {
return null;
}
// stored as array in carbonObject
Object[] arrayChildObjects = new Object[array.size()];
for (int i = 0; i < array.size(); i++) {
// array column will have only one child, hence get(0).
// But data can have n elements, hence the loop.
CarbonDimension childCol = carbonDimension.getListOfChildDimensions().get(0);
arrayChildObjects[i] = jsonChildElementToCarbonChildElement(array.get(i), childCol);
}
return new ArrayObject(arrayChildObjects);
} else if (DataTypes.isStructType(type)) {
CarbonDimension carbonDimension = (CarbonDimension) column;
int size = carbonDimension.getNumberOfChild();
Map<String, Object> jsonMap = (Map<String, Object>) jsonNodeMap.get(extractChildColumnName(column));
if (jsonMap == null) {
return null;
}
Object[] structChildObjects = new Object[size];
for (int i = 0; i < size; i++) {
CarbonDimension childCol = carbonDimension.getListOfChildDimensions().get(i);
Object childObject = jsonChildElementToCarbonChildElement(jsonMap.get(extractChildColumnName(childCol)), childCol);
structChildObjects[i] = childObject;
}
return new StructObject(structChildObjects);
} else {
// primitive type
if (jsonNodeMap.get(extractChildColumnName(column)) == null) {
return null;
}
return jsonNodeMap.get(extractChildColumnName(column)).toString();
}
}
use of org.apache.carbondata.processing.loading.complexobjects.StructObject in project carbondata by apache.
the class JsonRowParser method jsonChildElementToCarbonChildElement.
private Object jsonChildElementToCarbonChildElement(Object childObject, CarbonDimension column) {
if (childObject == null) {
return null;
}
DataType type = column.getDataType();
if (DataTypes.isArrayType(type)) {
ArrayList array = (ArrayList) childObject;
if (array.size() == 0) {
// handling empty array
return null;
}
// stored as array in carbonObject
Object[] arrayChildObjects = new Object[array.size()];
for (int i = 0; i < array.size(); i++) {
// array column will have only one child, hence get(0).
// But data can have n elements, hence the loop.
CarbonDimension childCol = column.getListOfChildDimensions().get(0);
arrayChildObjects[i] = jsonChildElementToCarbonChildElement(array.get(i), childCol);
}
return new ArrayObject(arrayChildObjects);
} else if (DataTypes.isStructType(type)) {
Map<String, Object> childFieldsMap = (Map<String, Object>) childObject;
int size = column.getNumberOfChild();
Object[] structChildObjects = new Object[size];
for (int i = 0; i < size; i++) {
CarbonDimension childCol = column.getListOfChildDimensions().get(i);
Object child = jsonChildElementToCarbonChildElement(childFieldsMap.get(extractChildColumnName(childCol)), childCol);
structChildObjects[i] = child;
}
return new StructObject(structChildObjects);
} else {
// primitive type
return childObject.toString();
}
}
Aggregations