Examples with CarbonMeasure org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure used on opensource projects

Example 36 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class CarbonFactDataHandlerModel method getCarbonFactDataHandlerModel.

/**
 * This method will create a model object for carbon fact data handler
 *
 * @param loadModel
 * @return
 */
public static CarbonFactDataHandlerModel getCarbonFactDataHandlerModel(CarbonLoadModel loadModel, CarbonTable carbonTable, SegmentProperties segmentProperties, String tableName, String[] tempStoreLocation, String carbonDataDirectoryPath) {
    CarbonFactDataHandlerModel carbonFactDataHandlerModel = new CarbonFactDataHandlerModel();
    carbonFactDataHandlerModel.setSchemaUpdatedTimeStamp(carbonTable.getTableLastUpdatedTime());
    carbonFactDataHandlerModel.setDatabaseName(loadModel.getDatabaseName());
    carbonFactDataHandlerModel.setTableName(tableName);
    carbonFactDataHandlerModel.setMeasureCount(segmentProperties.getMeasures().size());
    carbonFactDataHandlerModel.setStoreLocation(tempStoreLocation);
    carbonFactDataHandlerModel.setDimLens(segmentProperties.getDimColumnsCardinality());
    carbonFactDataHandlerModel.setSegmentProperties(segmentProperties);
    carbonFactDataHandlerModel.setNoDictionaryCount(segmentProperties.getNumberOfNoDictionaryDimension());
    carbonFactDataHandlerModel.setDimensionCount(segmentProperties.getDimensions().size() - carbonFactDataHandlerModel.getNoDictionaryCount());
    List<ColumnSchema> wrapperColumnSchema = CarbonUtil.getColumnSchemaList(carbonTable.getDimensionByTableName(tableName), carbonTable.getMeasureByTableName(tableName));
    carbonFactDataHandlerModel.setWrapperColumnSchema(wrapperColumnSchema);
    // get the cardinality for all all the columns including no dictionary columns
    int[] formattedCardinality = CarbonUtil.getFormattedCardinality(segmentProperties.getDimColumnsCardinality(), wrapperColumnSchema);
    carbonFactDataHandlerModel.setColCardinality(formattedCardinality);
    // TO-DO Need to handle complex types here .
    Map<Integer, GenericDataType> complexIndexMap = new HashMap<Integer, GenericDataType>(segmentProperties.getComplexDimensions().size());
    carbonFactDataHandlerModel.setComplexIndexMap(complexIndexMap);
    DataType[] measureDataTypes = new DataType[segmentProperties.getMeasures().size()];
    int i = 0;
    for (CarbonMeasure msr : segmentProperties.getMeasures()) {
        measureDataTypes[i++] = msr.getDataType();
    }
    carbonFactDataHandlerModel.setMeasureDataType(measureDataTypes);
    CarbonUtil.checkAndCreateFolderWithPermission(carbonDataDirectoryPath);
    carbonFactDataHandlerModel.setCarbonDataDirectoryPath(carbonDataDirectoryPath);
    List<CarbonDimension> dimensionByTableName = carbonTable.getDimensionByTableName(tableName);
    boolean[] isUseInvertedIndexes = new boolean[dimensionByTableName.size()];
    int index = 0;
    for (CarbonDimension dimension : dimensionByTableName) {
        isUseInvertedIndexes[index++] = dimension.isUseInvertedIndex();
    }
    carbonFactDataHandlerModel.setIsUseInvertedIndex(isUseInvertedIndexes);
    carbonFactDataHandlerModel.setPrimitiveDimLens(segmentProperties.getDimColumnsCardinality());
    carbonFactDataHandlerModel.setBlockSizeInMB(carbonTable.getBlockSizeInMB());
    carbonFactDataHandlerModel.tableSpec = new TableSpec(loadModel.getCarbonDataLoadSchema().getCarbonTable());
    DataMapWriterListener listener = new DataMapWriterListener();
    listener.registerAllWriter(loadModel.getCarbonDataLoadSchema().getCarbonTable(), loadModel.getSegmentId(), tempStoreLocation[new Random().nextInt(tempStoreLocation.length)]);
    carbonFactDataHandlerModel.dataMapWriterlistener = listener;
    return carbonFactDataHandlerModel;
}

Example 37 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class CarbonCompactionUtil method updateColumnSchemaAndGetCardinality.

/**
 * This method will return the updated cardinality according to the master schema
 *
 * @param columnCardinalityMap
 * @param carbonTable
 * @param updatedColumnSchemaList
 * @return
 */
public static int[] updateColumnSchemaAndGetCardinality(Map<String, Integer> columnCardinalityMap, CarbonTable carbonTable, List<ColumnSchema> updatedColumnSchemaList) {
    List<CarbonDimension> masterDimensions = carbonTable.getDimensionByTableName(carbonTable.getTableName());
    List<Integer> updatedCardinalityList = new ArrayList<>(columnCardinalityMap.size());
    for (CarbonDimension dimension : masterDimensions) {
        Integer value = columnCardinalityMap.get(dimension.getColumnId());
        if (null == value) {
            updatedCardinalityList.add(getDimensionDefaultCardinality(dimension));
        } else {
            updatedCardinalityList.add(value);
        }
        updatedColumnSchemaList.add(dimension.getColumnSchema());
    }
    // add measures to the column schema list
    List<CarbonMeasure> masterSchemaMeasures = carbonTable.getMeasureByTableName(carbonTable.getTableName());
    for (CarbonMeasure measure : masterSchemaMeasures) {
        updatedColumnSchemaList.add(measure.getColumnSchema());
    }
    return ArrayUtils.toPrimitive(updatedCardinalityList.toArray(new Integer[updatedCardinalityList.size()]));
}

Example 38 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class BlockletDataMap method updateMinValues.

/**
 * Fill the measures min values with minimum , this is needed for backward version compatability
 * as older versions don't store min values for measures
 */
private byte[][] updateMinValues(byte[][] minValues, int[] minMaxLen) {
    byte[][] updatedValues = minValues;
    if (minValues.length < minMaxLen.length) {
        updatedValues = new byte[minMaxLen.length][];
        System.arraycopy(minValues, 0, updatedValues, 0, minValues.length);
        List<CarbonMeasure> measures = segmentProperties.getMeasures();
        ByteBuffer buffer = ByteBuffer.allocate(8);
        for (int i = 0; i < measures.size(); i++) {
            buffer.rewind();
            DataType dataType = measures.get(i).getDataType();
            if (dataType == DataTypes.BYTE) {
                buffer.putLong(Byte.MIN_VALUE);
                updatedValues[minValues.length + i] = buffer.array().clone();
            } else if (dataType == DataTypes.SHORT) {
                buffer.putLong(Short.MIN_VALUE);
                updatedValues[minValues.length + i] = buffer.array().clone();
            } else if (dataType == DataTypes.INT) {
                buffer.putLong(Integer.MIN_VALUE);
                updatedValues[minValues.length + i] = buffer.array().clone();
            } else if (dataType == DataTypes.LONG) {
                buffer.putLong(Long.MIN_VALUE);
                updatedValues[minValues.length + i] = buffer.array().clone();
            } else if (DataTypes.isDecimal(dataType)) {
                updatedValues[minValues.length + i] = DataTypeUtil.bigDecimalToByte(BigDecimal.valueOf(Long.MIN_VALUE));
            } else {
                buffer.putDouble(Double.MIN_VALUE);
                updatedValues[minValues.length + i] = buffer.array().clone();
            }
        }
    }
    return updatedValues;
}

Example 39 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class BlockletDataMap method updateMaxValues.

/**
 * Fill the measures max values with maximum , this is needed for backward version compatability
 * as older versions don't store max values for measures
 */
private byte[][] updateMaxValues(byte[][] maxValues, int[] minMaxLen) {
    byte[][] updatedValues = maxValues;
    if (maxValues.length < minMaxLen.length) {
        updatedValues = new byte[minMaxLen.length][];
        System.arraycopy(maxValues, 0, updatedValues, 0, maxValues.length);
        List<CarbonMeasure> measures = segmentProperties.getMeasures();
        ByteBuffer buffer = ByteBuffer.allocate(8);
        for (int i = 0; i < measures.size(); i++) {
            buffer.rewind();
            DataType dataType = measures.get(i).getDataType();
            if (dataType == DataTypes.BYTE) {
                buffer.putLong(Byte.MAX_VALUE);
                updatedValues[maxValues.length + i] = buffer.array().clone();
            } else if (dataType == DataTypes.SHORT) {
                buffer.putLong(Short.MAX_VALUE);
                updatedValues[maxValues.length + i] = buffer.array().clone();
            } else if (dataType == DataTypes.INT) {
                buffer.putLong(Integer.MAX_VALUE);
                updatedValues[maxValues.length + i] = buffer.array().clone();
            } else if (dataType == DataTypes.LONG) {
                buffer.putLong(Long.MAX_VALUE);
                updatedValues[maxValues.length + i] = buffer.array().clone();
            } else if (DataTypes.isDecimal(dataType)) {
                updatedValues[maxValues.length + i] = DataTypeUtil.bigDecimalToByte(BigDecimal.valueOf(Long.MAX_VALUE));
            } else {
                buffer.putDouble(Double.MAX_VALUE);
                updatedValues[maxValues.length + i] = buffer.array().clone();
            }
        }
    }
    return updatedValues;
}

Example 40 with CarbonMeasure

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonMeasure in project carbondata by apache.

the class RestructureUtilTest method testToGetUpdatedQueryDimension.

@Test
public void testToGetUpdatedQueryDimension() {
    BlockExecutionInfo blockExecutionInfo = new BlockExecutionInfo();
    List<Encoding> encodingList = new ArrayList<Encoding>();
    encodingList.add(Encoding.DICTIONARY);
    ColumnSchema columnSchema1 = new ColumnSchema();
    columnSchema1.setColumnName("Id");
    columnSchema1.setDataType(DataTypes.STRING);
    columnSchema1.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema1.setEncodingList(encodingList);
    ColumnSchema columnSchema2 = new ColumnSchema();
    columnSchema2.setColumnName("Name");
    columnSchema2.setDataType(DataTypes.STRING);
    columnSchema2.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema2.setEncodingList(encodingList);
    ColumnSchema columnSchema3 = new ColumnSchema();
    columnSchema3.setColumnName("Age");
    columnSchema3.setDataType(DataTypes.INT);
    columnSchema3.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema3.setEncodingList(encodingList);
    ColumnSchema columnSchema4 = new ColumnSchema();
    columnSchema4.setColumnName("Salary");
    columnSchema4.setDataType(DataTypes.INT);
    columnSchema4.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema4.setEncodingList(encodingList);
    ColumnSchema columnSchema5 = new ColumnSchema();
    columnSchema5.setColumnName("Address");
    columnSchema5.setDataType(DataTypes.STRING);
    columnSchema5.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema5.setEncodingList(encodingList);
    CarbonDimension tableBlockDimension1 = new CarbonDimension(columnSchema1, 1, 1, 1, 1);
    CarbonDimension tableBlockDimension2 = new CarbonDimension(columnSchema2, 5, 5, 5, 5);
    List<CarbonDimension> tableBlockDimensions = Arrays.asList(tableBlockDimension1, tableBlockDimension2);
    CarbonDimension tableComplexDimension1 = new CarbonDimension(columnSchema3, 4, 4, 4, 4);
    CarbonDimension tableComplexDimension2 = new CarbonDimension(columnSchema4, 2, 2, 2, 2);
    List<CarbonDimension> tableComplexDimensions = Arrays.asList(tableComplexDimension1, tableComplexDimension2);
    ProjectionDimension queryDimension1 = new ProjectionDimension(tableBlockDimension1);
    ProjectionDimension queryDimension2 = new ProjectionDimension(tableComplexDimension2);
    ProjectionDimension queryDimension3 = new ProjectionDimension(new CarbonDimension(columnSchema5, 3, 3, 3, 3));
    ProjectionMeasure queryMeasure1 = new ProjectionMeasure(new CarbonMeasure(columnSchema3, 2));
    ProjectionMeasure queryMeasure2 = new ProjectionMeasure(new CarbonMeasure(columnSchema4, 4));
    List<ProjectionMeasure> queryMeasures = Arrays.asList(queryMeasure1, queryMeasure2);
    List<ProjectionDimension> queryDimensions = Arrays.asList(queryDimension1, queryDimension2, queryDimension3);
    List<ProjectionDimension> result = null;
    result = RestructureUtil.createDimensionInfoAndGetCurrentBlockQueryDimension(blockExecutionInfo, queryDimensions, tableBlockDimensions, tableComplexDimensions, queryMeasures.size());
    List<CarbonDimension> resultDimension = new ArrayList<>(result.size());
    for (ProjectionDimension queryDimension : result) {
        resultDimension.add(queryDimension.getDimension());
    }
    assertThat(resultDimension, is(equalTo(Arrays.asList(queryDimension1.getDimension(), queryDimension2.getDimension()))));
}