Examples with Writable org.apache.hadoop.io.Writable used on opensource projects

Example 96 with Writable

use of org.apache.hadoop.io.Writable in project asterixdb by apache.

the class HiveRecordParser method parse.

@Override
public void parse(IRawRecord<? extends Writable> record, DataOutput out) throws HyracksDataException {
    try {
        Writable hiveRawRecord = record.get();
        Object hiveObject = hiveSerde.deserialize(hiveRawRecord);
        int n = recordType.getFieldNames().length;
        List<Object> attributesValues = oi.getStructFieldsDataAsList(hiveObject);
        recBuilder.reset(recordType);
        recBuilder.init();
        for (int i = 0; i < n; i++) {
            final Object value = attributesValues.get(i);
            final ObjectInspector foi = fieldRefs.get(i).getFieldObjectInspector();
            fieldValueBuffer.reset();
            final DataOutput dataOutput = fieldValueBuffer.getDataOutput();
            dataOutput.writeByte(fieldTypeTags[i]);
            //get field type
            parseItem(fieldTypes[i], value, foi, dataOutput, false);
            recBuilder.addField(i, fieldValueBuffer);
        }
        recBuilder.write(out, true);
    } catch (Exception e) {
        throw new HyracksDataException(e);
    }
}

Example 97 with Writable

use of org.apache.hadoop.io.Writable in project carbondata by apache.

the class CarbonArrayInspector method getListElement.

@Override
public Object getListElement(final Object data, final int index) {
    if (data == null) {
        return null;
    }
    if (data instanceof ArrayWritable) {
        final Writable[] listContainer = ((ArrayWritable) data).get();
        if (listContainer == null || listContainer.length == 0) {
            return null;
        }
        final Writable subObj = listContainer[0];
        if (subObj == null) {
            return null;
        }
        if (index >= 0 && index < ((ArrayWritable) subObj).get().length) {
            return ((ArrayWritable) subObj).get()[index];
        } else {
            return null;
        }
    }
    throw new UnsupportedOperationException("Cannot inspect " + data.getClass().getCanonicalName());
}

Example 98 with Writable

use of org.apache.hadoop.io.Writable in project carbondata by apache.

the class CarbonArrayInspector method getList.

@Override
public List<?> getList(final Object data) {
    if (data == null) {
        return null;
    }
    if (data instanceof ArrayWritable) {
        final Writable[] listContainer = ((ArrayWritable) data).get();
        if (listContainer == null || listContainer.length == 0) {
            return null;
        }
        final Writable subObj = listContainer[0];
        if (subObj == null) {
            return null;
        }
        final Writable[] array = ((ArrayWritable) subObj).get();
        final List<Writable> list = Arrays.asList(array);
        for (final Writable obj : array) {
            list.add(obj);
        }
        return list;
    }
    throw new UnsupportedOperationException("Cannot inspect " + data.getClass().getCanonicalName());
}

Example 99 with Writable

use of org.apache.hadoop.io.Writable in project carbondata by apache.

the class CarbonHiveRecordReader method createArray.

private ArrayWritable createArray(Object obj, ListObjectInspector inspector) throws SerDeException {
    List sourceArray = inspector.getList(obj);
    ObjectInspector subInspector = inspector.getListElementObjectInspector();
    List array = new ArrayList();
    Iterator iterator;
    if (sourceArray != null) {
        for (iterator = sourceArray.iterator(); iterator.hasNext(); ) {
            Object curObj = iterator.next();
            Writable newObj = createObject(curObj, subInspector);
            if (newObj != null) {
                array.add(newObj);
            }
        }
    }
    if (array.size() > 0) {
        ArrayWritable subArray = new ArrayWritable(((Writable) array.get(0)).getClass(), (Writable[]) array.toArray(new Writable[array.size()]));
        return new ArrayWritable(Writable.class, new Writable[] { subArray });
    }
    return null;
}

Example 100 with Writable

use of org.apache.hadoop.io.Writable in project carbondata by apache.

the class CarbonHiveRecordReader method initialize.

public void initialize(InputSplit inputSplit, Configuration conf) throws IOException {
    // The input split can contain single HDFS block or multiple blocks, so firstly get all the
    // blocks and then set them in the query model.
    List<CarbonHiveInputSplit> splitList;
    if (inputSplit instanceof CarbonHiveInputSplit) {
        splitList = new ArrayList<>(1);
        splitList.add((CarbonHiveInputSplit) inputSplit);
    } else {
        throw new RuntimeException("unsupported input split type: " + inputSplit);
    }
    List<TableBlockInfo> tableBlockInfoList = CarbonHiveInputSplit.createBlocks(splitList);
    queryModel.setTableBlockInfos(tableBlockInfoList);
    readSupport.initialize(queryModel.getProjectionColumns(), queryModel.getAbsoluteTableIdentifier());
    try {
        carbonIterator = new ChunkRowIterator(queryExecutor.execute(queryModel));
    } catch (QueryExecutionException e) {
        throw new IOException(e.getMessage(), e.getCause());
    }
    if (valueObj == null) {
        valueObj = new ArrayWritable(Writable.class, new Writable[queryModel.getProjectionColumns().length]);
    }
    final TypeInfo rowTypeInfo;
    final List<String> columnNames;
    List<TypeInfo> columnTypes;
    // Get column names and sort order
    final String colIds = conf.get("hive.io.file.readcolumn.ids");
    final String columnNameProperty = conf.get("hive.io.file.readcolumn.names");
    final String columnTypeProperty = conf.get(serdeConstants.LIST_COLUMN_TYPES);
    if (columnNameProperty.length() == 0) {
        columnNames = new ArrayList<String>();
    } else {
        columnNames = Arrays.asList(columnNameProperty.split(","));
    }
    if (columnTypeProperty.length() == 0) {
        columnTypes = new ArrayList<TypeInfo>();
    } else {
        columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
    }
    String[] arraySelectedColId = colIds.split(",");
    List<TypeInfo> reqColTypes = new ArrayList<TypeInfo>();
    for (String anArrayColId : arraySelectedColId) {
        reqColTypes.add(columnTypes.get(Integer.parseInt(anArrayColId)));
    }
    // Create row related objects
    rowTypeInfo = TypeInfoFactory.getStructTypeInfo(columnNames, reqColTypes);
    this.objInspector = new CarbonObjectInspector((StructTypeInfo) rowTypeInfo);
}