Examples with ProjectionDimension org.apache.carbondata.core.scan.model.ProjectionDimension used on opensource projects

Example 6 with ProjectionDimension

use of org.apache.carbondata.core.scan.model.ProjectionDimension in project carbondata by apache.

the class VectorizedCarbonRecordReader method initBatch.

/**
 * Returns the ColumnarBatch object that will be used for all rows returned by this reader.
 * This object is reused. Calling this enables the vectorized reader. This should be called
 * before any calls to nextKeyValue/nextBatch.
 */
private void initBatch(MemoryMode memMode) {
    List<ProjectionDimension> queryDimension = queryModel.getProjectionDimensions();
    List<ProjectionMeasure> queryMeasures = queryModel.getProjectionMeasures();
    StructField[] fields = new StructField[queryDimension.size() + queryMeasures.size()];
    for (int i = 0; i < queryDimension.size(); i++) {
        ProjectionDimension dim = queryDimension.get(i);
        if (dim.getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
            DirectDictionaryGenerator generator = DirectDictionaryKeyGeneratorFactory.getDirectDictionaryGenerator(dim.getDimension().getDataType());
            fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(generator.getReturnType()), true, null);
        } else if (!dim.getDimension().hasEncoding(Encoding.DICTIONARY)) {
            fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true, null);
        } else if (dim.getDimension().isComplex()) {
            fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(dim.getDimension().getDataType()), true, null);
        } else {
            fields[dim.getOrdinal()] = new StructField(dim.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(DataTypes.INT), true, null);
        }
    }
    for (int i = 0; i < queryMeasures.size(); i++) {
        ProjectionMeasure msr = queryMeasures.get(i);
        DataType dataType = msr.getMeasure().getDataType();
        if (dataType == DataTypes.BOOLEAN || dataType == DataTypes.SHORT || dataType == DataTypes.INT || dataType == DataTypes.LONG) {
            fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(msr.getMeasure().getDataType()), true, null);
        } else if (DataTypes.isDecimal(dataType)) {
            fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), new DecimalType(msr.getMeasure().getPrecision(), msr.getMeasure().getScale()), true, null);
        } else {
            fields[msr.getOrdinal()] = new StructField(msr.getColumnName(), CarbonScalaUtil.convertCarbonToSparkDataType(DataTypes.DOUBLE), true, null);
        }
    }
    columnarBatch = ColumnarBatch.allocate(new StructType(fields), memMode);
    CarbonColumnVector[] vectors = new CarbonColumnVector[fields.length];
    boolean[] filteredRows = new boolean[columnarBatch.capacity()];
    for (int i = 0; i < fields.length; i++) {
        vectors[i] = new ColumnarVectorWrapper(columnarBatch.column(i), filteredRows);
    }
    carbonColumnarBatch = new CarbonColumnarBatch(vectors, columnarBatch.capacity(), filteredRows);
}

Example 7 with ProjectionDimension

use of org.apache.carbondata.core.scan.model.ProjectionDimension in project carbondata by apache.

the class CarbonUtilTest method testToGetDirectDictionaryEncodingArray.

@Test
public void testToGetDirectDictionaryEncodingArray() {
    ColumnSchema column1Schema = new ColumnSchema();
    ColumnSchema column2Schema = new ColumnSchema();
    column1Schema.setColumnName("Column1");
    List<Encoding> encoding = new ArrayList<>();
    encoding.add(Encoding.DIRECT_DICTIONARY);
    column1Schema.setEncodingList(encoding);
    ProjectionDimension column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
    column2Schema.setColumnName("Column2");
    List<Encoding> encoding2 = new ArrayList<>();
    encoding2.add(Encoding.DELTA);
    column2Schema.setEncodingList(encoding2);
    ProjectionDimension column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
    ProjectionDimension[] queryDimensions = { column1, column2 };
    boolean[] dictionaryEncoding = CarbonUtil.getDirectDictionaryEncodingArray(queryDimensions);
    boolean[] expectedDictionaryEncoding = { true, false };
    for (int i = 0; i < dictionaryEncoding.length; i++) {
        assertEquals(dictionaryEncoding[i], expectedDictionaryEncoding[i]);
    }
}

Example 8 with ProjectionDimension

use of org.apache.carbondata.core.scan.model.ProjectionDimension in project carbondata by apache.

the class QueryUtilTest method testGetMaskedByteRangeGivingProperMaksedByteRangeOnlyForDictionaryKey.

@Test
public void testGetMaskedByteRangeGivingProperMaksedByteRangeOnlyForDictionaryKey() {
    List<ProjectionDimension> dimensions = new ArrayList<ProjectionDimension>();
    for (int i = 0; i < 2; i++) {
        ProjectionDimension dimension = new ProjectionDimension(segmentProperties.getDimensions().get(i));
        dimensions.add(dimension);
    }
    int[] maskedByteRange = QueryUtil.getMaskedByteRange(dimensions, segmentProperties.getDimensionKeyGenerator());
    int[] expectedMaskedByteRange = { 0 };
    for (int i = 0; i < maskedByteRange.length; i++) {
        assertEquals(expectedMaskedByteRange[i], maskedByteRange[i]);
    }
}

Example 9 with ProjectionDimension

use of org.apache.carbondata.core.scan.model.ProjectionDimension in project carbondata by apache.

the class QueryUtilTest method testGetSortDimensionIndexes.

@Test
public void testGetSortDimensionIndexes() {
    List<ProjectionDimension> sortedDimensions = new ArrayList<ProjectionDimension>();
    for (int i = 0; i < 2; i++) {
        ProjectionDimension dimension = new ProjectionDimension(segmentProperties.getDimensions().get(i));
        sortedDimensions.add(dimension);
    }
    List<ProjectionDimension> queryDimensions = new ArrayList<ProjectionDimension>();
    for (int i = 0; i < 2; i++) {
        ProjectionDimension dimension = new ProjectionDimension(segmentProperties.getDimensions().get(i));
        queryDimensions.add(dimension);
    }
    byte[] actualValue = QueryUtil.getSortDimensionIndexes(sortedDimensions, queryDimensions);
    byte[] expectedValue = { 0, 0 };
    assertArrayEquals(expectedValue, actualValue);
}

Example 10 with ProjectionDimension

use of org.apache.carbondata.core.scan.model.ProjectionDimension in project carbondata by apache.

the class QueryUtilTest method testGetMaskedByteRangeGivingProperMaksedByteRange.

@Test
public void testGetMaskedByteRangeGivingProperMaksedByteRange() {
    ProjectionDimension dimension = new ProjectionDimension(segmentProperties.getDimensions().get(0));
    int[] maskedByteRange = QueryUtil.getMaskedByteRange(Arrays.asList(dimension), segmentProperties.getDimensionKeyGenerator());
    int[] expectedMaskedByteRange = { 0 };
    for (int i = 0; i < maskedByteRange.length; i++) {
        assertEquals(expectedMaskedByteRange[i], maskedByteRange[i]);
    }
}