Examples with CarbonColumnVector org.apache.carbondata.core.scan.result.vector.CarbonColumnVector used on opensource projects

Example 1 with CarbonColumnVector

use of org.apache.carbondata.core.scan.result.vector.CarbonColumnVector in project carbondata by apache.

the class FixedLengthDimensionDataChunk method fillConvertedChunkData.

/**
   * Fill the data to vector
   *
   * @param rowMapping
   * @param vectorInfo
   * @param column
   * @param restructuringInfo
   * @return next column index
   */
@Override
public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo, int column, KeyStructureInfo restructuringInfo) {
    ColumnVectorInfo columnVectorInfo = vectorInfo[column];
    int offset = columnVectorInfo.offset;
    int vectorOffset = columnVectorInfo.vectorOffset;
    int len = columnVectorInfo.size + offset;
    CarbonColumnVector vector = columnVectorInfo.vector;
    for (int j = offset; j < len; j++) {
        int dict = dataChunkStore.getSurrogate(rowMapping[j]);
        if (columnVectorInfo.directDictionaryGenerator == null) {
            vector.putInt(vectorOffset++, dict);
        } else {
            Object valueFromSurrogate = columnVectorInfo.directDictionaryGenerator.getValueFromSurrogate(dict);
            if (valueFromSurrogate == null) {
                vector.putNull(vectorOffset++);
            } else {
                switch(columnVectorInfo.directDictionaryGenerator.getReturnType()) {
                    case INT:
                        vector.putInt(vectorOffset++, (int) valueFromSurrogate);
                        break;
                    case LONG:
                        vector.putLong(vectorOffset++, (long) valueFromSurrogate);
                        break;
                }
            }
        }
    }
    return column + 1;
}

Example 2 with CarbonColumnVector

use of org.apache.carbondata.core.scan.result.vector.CarbonColumnVector in project carbondata by apache.

the class VariableLengthDimensionDataChunk method fillConvertedChunkData.

/**
   * Fill the data to vector
   *
   * @param rowMapping
   * @param vectorInfo
   * @param column
   * @param restructuringInfo
   * @return next column index
   */
@Override
public int fillConvertedChunkData(int[] rowMapping, ColumnVectorInfo[] vectorInfo, int column, KeyStructureInfo restructuringInfo) {
    ColumnVectorInfo columnVectorInfo = vectorInfo[column];
    CarbonColumnVector vector = columnVectorInfo.vector;
    int offset = columnVectorInfo.offset;
    int vectorOffset = columnVectorInfo.vectorOffset;
    int len = offset + columnVectorInfo.size;
    for (int i = offset; i < len; i++) {
        // Considering only String case now as we support only
        // string in no dictionary case at present.
        dataChunkStore.fillRow(rowMapping[i], vector, vectorOffset++);
    }
    return column + 1;
}

Example 3 with CarbonColumnVector

use of org.apache.carbondata.core.scan.result.vector.CarbonColumnVector in project carbondata by apache.

the class VectorizedCarbonRecordReader method initBatch.

/**
   * Returns the ColumnarBatch object that will be used for all rows returned by this reader.
   * This object is reused. Calling this enables the vectorized reader. This should be called
   * before any calls to nextKeyValue/nextBatch.
   */
private void initBatch(MemoryMode memMode) {
    List<QueryDimension> queryDimension = queryModel.getQueryDimension();
    List<QueryMeasure> queryMeasures = queryModel.getQueryMeasures();
    StructField[] fields = new StructField[queryDimension.size() + queryMeasures.size()];
    for (int i = 0; i < queryDimension.size(); i++) {
        QueryDimension dim = queryDimension.get(i);
        if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
            DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(dim.getDimension().getDataType());
            fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(generator.getReturnType()), true, null);
        } else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) {
            fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true, null);
        } else if (dim.getDimension().isComplex()) {
            fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true, null);
        } else {
            fields[dim.getQueryOrder()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(DataType.INT), true, null);
        }
    }
    for (int i = 0; i < queryMeasures.size(); i++) {
        QueryMeasure msr = queryMeasures.get(i);
        switch(msr.getMeasure().getDataType()) {
            case SHORT:
            case INT:
            case LONG:
                fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(msr.getMeasure().getDataType()), true, null);
                break;
            case DECIMAL:
                fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), new DecimalType(msr.getMeasure().getPrecision(), msr.getMeasure().getScale()), true, null);
                break;
            default:
                fields[msr.getQueryOrder()] = new StructField(msr.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(DataType.DOUBLE), true, null);
        }
    }
    columnarBatch = ColumnarBatch.allocate(new StructType(fields), memMode);
    CarbonColumnVector[] vectors = new CarbonColumnVector[fields.length];
    boolean[] filteredRows = new boolean[columnarBatch.capacity()];
    for (int i = 0; i < fields.length; i++) {
        vectors[i] = new ColumnarVectorWrapper(columnarBatch.column(i), filteredRows);
    }
    carbonColumnarBatch = new CarbonColumnarBatch(vectors, columnarBatch.capacity(), filteredRows);
}

Example 4 with CarbonColumnVector

use of org.apache.carbondata.core.scan.result.vector.CarbonColumnVector in project carbondata by apache.

the class RestructureBasedVectorResultCollector method fillDataForNonExistingMeasures.

/**
   * This method will fill the default values of non existing measures in the current block
   */
private void fillDataForNonExistingMeasures() {
    for (int i = 0; i < tableBlockExecutionInfos.getActualQueryMeasures().length; i++) {
        if (!measureInfo.getMeasureExists()[i]) {
            int queryOrder = tableBlockExecutionInfos.getActualQueryMeasures()[i].getQueryOrder();
            CarbonMeasure measure = tableBlockExecutionInfos.getActualQueryMeasures()[i].getMeasure();
            ColumnVectorInfo columnVectorInfo = allColumnInfo[queryOrder];
            CarbonColumnVector vector = columnVectorInfo.vector;
            Object defaultValue = measureDefaultValues[i];
            if (null == defaultValue) {
                vector.putNulls(columnVectorInfo.vectorOffset, columnVectorInfo.size);
            } else {
                switch(measureInfo.getMeasureDataTypes()[i]) {
                    case SHORT:
                        vector.putShorts(columnVectorInfo.vectorOffset, columnVectorInfo.size, (short) defaultValue);
                        break;
                    case INT:
                        vector.putInts(columnVectorInfo.vectorOffset, columnVectorInfo.size, (int) defaultValue);
                        break;
                    case LONG:
                        vector.putLongs(columnVectorInfo.vectorOffset, columnVectorInfo.size, (long) defaultValue);
                        break;
                    case DECIMAL:
                        vector.putDecimals(columnVectorInfo.vectorOffset, columnVectorInfo.size, (Decimal) defaultValue, measure.getPrecision());
                        break;
                    default:
                        vector.putDoubles(columnVectorInfo.vectorOffset, columnVectorInfo.size, (double) defaultValue);
                }
            }
        }
    }
}

Example 5 with CarbonColumnVector

use of org.apache.carbondata.core.scan.result.vector.CarbonColumnVector in project carbondata by apache.

the class AbstractScannedResult method fillColumnarImplicitBatch.

/**
   * Fill the column data to vector
   */
public void fillColumnarImplicitBatch(ColumnVectorInfo[] vectorInfo) {
    int column = 0;
    for (int i = 0; i < vectorInfo.length; i++) {
        ColumnVectorInfo columnVectorInfo = vectorInfo[column];
        CarbonColumnVector vector = columnVectorInfo.vector;
        int offset = columnVectorInfo.offset;
        int vectorOffset = columnVectorInfo.vectorOffset;
        int len = offset + columnVectorInfo.size;
        for (int j = offset; j < len; j++) {
            // Considering only String case now as we support only
            String data = getBlockletId();
            if (CarbonCommonConstants.CARBON_IMPLICIT_COLUMN_TUPLEID.equals(columnVectorInfo.dimension.getColumnName())) {
                data = data + CarbonCommonConstants.FILE_SEPARATOR + (rowMapping == null ? j : rowMapping[pageCounter][j]);
            }
            vector.putBytes(vectorOffset++, offset, data.length(), data.getBytes());
        }
    }
}