Examples with CarbonDimension org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension used on opensource projects

Example 76 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class CarbonUtilTest method testToFindDimension.

@Test
public void testToFindDimension() {
    ColumnSchema column1Schema = new ColumnSchema();
    ColumnSchema column2Schema = new ColumnSchema();
    column1Schema.setColumnName("Column1");
    column2Schema.setColumnName("Column2");
    List<CarbonDimension> carbonDimension = new ArrayList<>();
    carbonDimension.add(new CarbonDimension(column1Schema, 1, 1, 1, 1));
    carbonDimension.add(new CarbonDimension(column2Schema, 2, 1, 2, 1));
    assertEquals(CarbonUtil.findDimension(carbonDimension, "Column1"), new CarbonDimension(column1Schema, 1, 1, 1, 1));
}

Example 77 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class CarbonUtilTest method testToIdentifyDimensionType.

@Test
public void testToIdentifyDimensionType() {
    ColumnSchema column1Schema = new ColumnSchema();
    ColumnSchema column2Schema = new ColumnSchema();
    ColumnSchema column3Schema = new ColumnSchema();
    column1Schema.setColumnName("Column1");
    column1Schema.setColumnar(true);
    column1Schema.setEncodingList(Arrays.asList(Encoding.DELTA, Encoding.DICTIONARY));
    column2Schema.setColumnName("Column2");
    column2Schema.setColumnar(false);
    column2Schema.setEncodingList(Arrays.asList(Encoding.DELTA, Encoding.DICTIONARY));
    column3Schema.setColumnName("Column3");
    column3Schema.setColumnar(true);
    column3Schema.setEncodingList(Arrays.asList(Encoding.DELTA, Encoding.INVERTED_INDEX));
    CarbonDimension carbonDimension = new CarbonDimension(column1Schema, 1, 1, 1, 1);
    CarbonDimension carbonDimension2 = new CarbonDimension(column2Schema, 2, 2, 2, 2);
    CarbonDimension carbonDimension3 = new CarbonDimension(column3Schema, 3, 3, 3, 3);
    List<CarbonDimension> carbonDimensions = Arrays.asList(carbonDimension, carbonDimension2, carbonDimension3);
    boolean[] result = CarbonUtil.identifyDimensionType(carbonDimensions);
    assertThat(result, is(equalTo(new boolean[] { true, true, false })));
}

Example 78 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class RestructureUtil method addChildrenForComplexTypeDimension.

/**
   * Below method is to add dimension children for complex type dimension as
   * internally we are creating dimension column for each each complex
   * dimension so when complex query dimension request will come in the query,
   * we need to add its children as it is hidden from the user For example if
   * complex dimension is of Array of String[2] so we are storing 3 dimension
   * and when user will query for complex type i.e. array type we need to add
   * its children and then we will read respective block and create a tuple
   * based on all three dimension
   *
   * @param queryDimensions      current query dimensions
   * @param tableBlockDimensions dimensions which is present in the table block
   * @return updated dimension(after adding complex type children)
   */
public static List<CarbonDimension> addChildrenForComplexTypeDimension(List<CarbonDimension> queryDimensions, List<CarbonDimension> tableBlockDimensions) {
    List<CarbonDimension> updatedQueryDimension = new ArrayList<CarbonDimension>();
    int numberOfChildren = 0;
    for (CarbonDimension queryDimension : queryDimensions) {
        // so directly add it query dimension
        if (queryDimension.numberOfChild() == 0) {
            updatedQueryDimension.add(queryDimension);
        }
        // if number of child is more than 1 then add all its children
        numberOfChildren = queryDimension.getOrdinal() + queryDimension.numberOfChild();
        for (int j = queryDimension.getOrdinal(); j < numberOfChildren; j++) {
            updatedQueryDimension.add(tableBlockDimensions.get(j));
        }
    }
    return updatedQueryDimension;
}

Example 79 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class RestructureUtil method createDimensionInfoAndGetCurrentBlockQueryDimension.

/**
   * Below method will be used to get the updated query dimension updation
   * means, after restructuring some dimension will be not present in older
   * table blocks in that case we need to select only those dimension out of
   * query dimension which is present in the current table block
   *
   * @param blockExecutionInfo
   * @param queryDimensions
   * @param tableBlockDimensions
   * @param tableComplexDimension
   * @return list of query dimension which is present in the table block
   */
public static List<QueryDimension> createDimensionInfoAndGetCurrentBlockQueryDimension(BlockExecutionInfo blockExecutionInfo, List<QueryDimension> queryDimensions, List<CarbonDimension> tableBlockDimensions, List<CarbonDimension> tableComplexDimension) {
    List<QueryDimension> presentDimension = new ArrayList<QueryDimension>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    boolean[] isDimensionExists = new boolean[queryDimensions.size()];
    Object[] defaultValues = new Object[queryDimensions.size()];
    // create dimension information instance
    DimensionInfo dimensionInfo = new DimensionInfo(isDimensionExists, defaultValues);
    int newDictionaryColumnCount = 0;
    int newNoDictionaryColumnCount = 0;
    // selecting only those dimension which is present in the query
    int dimIndex = 0;
    for (QueryDimension queryDimension : queryDimensions) {
        if (queryDimension.getDimension().hasEncoding(Encoding.IMPLICIT)) {
            presentDimension.add(queryDimension);
            isDimensionExists[dimIndex] = true;
        } else {
            for (CarbonDimension tableDimension : tableBlockDimensions) {
                if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) {
                    QueryDimension currentBlockDimension = new QueryDimension(tableDimension.getColName());
                    tableDimension.getColumnSchema().setDataType(queryDimension.getDimension().getDataType());
                    tableDimension.getColumnSchema().setPrecision(queryDimension.getDimension().getColumnSchema().getPrecision());
                    tableDimension.getColumnSchema().setScale(queryDimension.getDimension().getColumnSchema().getScale());
                    tableDimension.getColumnSchema().setDefaultValue(queryDimension.getDimension().getDefaultValue());
                    currentBlockDimension.setDimension(tableDimension);
                    currentBlockDimension.setQueryOrder(queryDimension.getQueryOrder());
                    presentDimension.add(currentBlockDimension);
                    isDimensionExists[dimIndex] = true;
                    break;
                }
            }
            // if dimension is found then no need to search in the complex dimensions list
            if (isDimensionExists[dimIndex]) {
                dimIndex++;
                continue;
            }
            for (CarbonDimension tableDimension : tableComplexDimension) {
                if (tableDimension.getColumnId().equals(queryDimension.getDimension().getColumnId())) {
                    QueryDimension currentBlockDimension = new QueryDimension(tableDimension.getColName());
                    // TODO: for complex dimension set scale and precision by traversing
                    // the child dimensions
                    currentBlockDimension.setDimension(tableDimension);
                    currentBlockDimension.setQueryOrder(queryDimension.getQueryOrder());
                    presentDimension.add(currentBlockDimension);
                    isDimensionExists[dimIndex] = true;
                    break;
                }
            }
            // add default value only in case query dimension is not found in the current block
            if (!isDimensionExists[dimIndex]) {
                defaultValues[dimIndex] = validateAndGetDefaultValue(queryDimension.getDimension());
                blockExecutionInfo.setRestructuredBlock(true);
                // newly added columns data need to be filled
                if (queryDimension.getDimension().hasEncoding(Encoding.DICTIONARY)) {
                    dimensionInfo.setDictionaryColumnAdded(true);
                    newDictionaryColumnCount++;
                } else {
                    dimensionInfo.setNoDictionaryColumnAdded(true);
                    newNoDictionaryColumnCount++;
                }
            }
        }
        dimIndex++;
    }
    dimensionInfo.setNewDictionaryColumnCount(newDictionaryColumnCount);
    dimensionInfo.setNewNoDictionaryColumnCount(newNoDictionaryColumnCount);
    blockExecutionInfo.setDimensionInfo(dimensionInfo);
    return presentDimension;
}

Example 80 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class RestructureBasedRawResultCollector method initRestructuredKeyGenerator.

/**
   * This method will create a new key generator for generating mdKey according to latest schema
   */
private void initRestructuredKeyGenerator() {
    SegmentProperties segmentProperties = tableBlockExecutionInfos.getDataBlock().getSegmentProperties();
    QueryDimension[] queryDimensions = tableBlockExecutionInfos.getActualQueryDimensions();
    List<Integer> updatedColumnCardinality = new ArrayList<>(queryDimensions.length);
    List<Integer> updatedDimensionPartitioner = new ArrayList<>(queryDimensions.length);
    int[] dictionaryColumnBlockIndex = tableBlockExecutionInfos.getDictionaryColumnBlockIndex();
    int dimCounterInCurrentBlock = 0;
    for (int i = 0; i < queryDimensions.length; i++) {
        if (queryDimensions[i].getDimension().hasEncoding(Encoding.DICTIONARY)) {
            if (tableBlockExecutionInfos.getDimensionInfo().getDimensionExists()[i]) {
                // get the dictionary key ordinal as column cardinality in segment properties
                // will only be for dictionary encoded columns
                CarbonDimension currentBlockDimension = segmentProperties.getDimensions().get(dictionaryColumnBlockIndex[dimCounterInCurrentBlock]);
                updatedColumnCardinality.add(segmentProperties.getDimColumnsCardinality()[currentBlockDimension.getKeyOrdinal()]);
                updatedDimensionPartitioner.add(segmentProperties.getDimensionPartitions()[currentBlockDimension.getKeyOrdinal()]);
                dimCounterInCurrentBlock++;
            } else {
                // partitioner index will be 1 every column will be in columnar format
                updatedDimensionPartitioner.add(1);
                // for direct dictionary 4 bytes need to be allocated else 1
                if (queryDimensions[i].getDimension().hasEncoding(Encoding.DIRECT_DICTIONARY)) {
                    updatedColumnCardinality.add(Integer.MAX_VALUE);
                } else {
                    // cardinality will be 2 will user has provided a default value
                    byte[] defaultValue = queryDimensions[i].getDimension().getDefaultValue();
                    if (null != defaultValue) {
                        updatedColumnCardinality.add(CarbonCommonConstants.DICTIONARY_DEFAULT_CARDINALITY + 1);
                    } else {
                        updatedColumnCardinality.add(CarbonCommonConstants.DICTIONARY_DEFAULT_CARDINALITY);
                    }
                }
            }
        }
    }
    if (!updatedColumnCardinality.isEmpty()) {
        int[] latestColumnCardinality = ArrayUtils.toPrimitive(updatedColumnCardinality.toArray(new Integer[updatedColumnCardinality.size()]));
        int[] latestColumnPartitioner = ArrayUtils.toPrimitive(updatedDimensionPartitioner.toArray(new Integer[updatedDimensionPartitioner.size()]));
        int[] dimensionBitLength = CarbonUtil.getDimensionBitLength(latestColumnCardinality, latestColumnPartitioner);
        restructuredKeyGenerator = new MultiDimKeyVarLengthGenerator(dimensionBitLength);
    }
}