Examples with CarbonDimension org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension used on opensource projects

Example 21 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class SegmentProperties method fillDimensionAndMeasureDetails.

/**
   * below method will fill dimension and measure detail of the block.
   *
   * @param columnsInTable
   * @param columnCardinality
   */
private void fillDimensionAndMeasureDetails(List<ColumnSchema> columnsInTable, int[] columnCardinality) {
    ColumnSchema columnSchema = null;
    // ordinal will be required to read the data from file block
    int dimensonOrdinal = 0;
    int measureOrdinal = -1;
    // table ordinal is actually a schema ordinal this is required as
    // cardinality array
    // which is stored in segment info contains -1 if that particular column
    // is n
    int tableOrdinal = -1;
    // creating a list as we do not know how many dimension not participated
    // in the mdkey
    List<Integer> cardinalityIndexForNormalDimensionColumn = new ArrayList<Integer>(columnsInTable.size());
    // creating a list as we do not know how many dimension not participated
    // in the mdkey
    List<Integer> cardinalityIndexForComplexDimensionColumn = new ArrayList<Integer>(columnsInTable.size());
    boolean isComplexDimensionStarted = false;
    CarbonDimension carbonDimension = null;
    // to store the position of dimension in surrogate key array which is
    // participating in mdkey
    int keyOrdinal = 0;
    int previousColumnGroup = -1;
    // to store the ordinal of the column group ordinal
    int columnGroupOrdinal = 0;
    int counter = 0;
    int complexTypeOrdinal = -1;
    while (counter < columnsInTable.size()) {
        columnSchema = columnsInTable.get(counter);
        if (columnSchema.isDimensionColumn()) {
            tableOrdinal++;
            // column as it was not the part of mdkey
            if (CarbonUtil.hasEncoding(columnSchema.getEncodingList(), Encoding.DICTIONARY) && !isComplexDimensionStarted && columnSchema.getNumberOfChild() == 0) {
                cardinalityIndexForNormalDimensionColumn.add(tableOrdinal);
                if (columnSchema.isSortColumn()) {
                    this.numberOfSortColumns++;
                }
                if (columnSchema.isColumnar()) {
                    // if it is a columnar dimension participated in mdkey then added
                    // key ordinal and dimension ordinal
                    carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, keyOrdinal++, -1, -1);
                } else {
                    // in this case ordinal of the column group will be 0
                    if (previousColumnGroup != columnSchema.getColumnGroupId()) {
                        columnGroupOrdinal = 0;
                        carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, keyOrdinal++, columnGroupOrdinal++, -1);
                    } else // if previous dimension  column group id is same as current then
                    // then its belongs to same row group
                    {
                        carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, keyOrdinal++, columnGroupOrdinal++, -1);
                    }
                    previousColumnGroup = columnSchema.getColumnGroupId();
                }
            } else // will be added to complex type
            if (isComplexDimensionStarted || CarbonUtil.hasDataType(columnSchema.getDataType(), new DataType[] { DataType.ARRAY, DataType.STRUCT })) {
                cardinalityIndexForComplexDimensionColumn.add(tableOrdinal);
                carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, ++complexTypeOrdinal);
                carbonDimension.initializeChildDimensionsList(columnSchema.getNumberOfChild());
                complexDimensions.add(carbonDimension);
                isComplexDimensionStarted = true;
                int previouseOrdinal = dimensonOrdinal;
                dimensonOrdinal = readAllComplexTypeChildren(dimensonOrdinal, columnSchema.getNumberOfChild(), columnsInTable, carbonDimension, complexTypeOrdinal);
                int numberOfChildrenDimensionAdded = dimensonOrdinal - previouseOrdinal;
                for (int i = 0; i < numberOfChildrenDimensionAdded; i++) {
                    cardinalityIndexForComplexDimensionColumn.add(++tableOrdinal);
                }
                counter = dimensonOrdinal;
                complexTypeOrdinal = assignComplexOrdinal(carbonDimension, complexTypeOrdinal);
                continue;
            } else {
                // for no dictionary dimension
                carbonDimension = new CarbonDimension(columnSchema, dimensonOrdinal++, -1, -1, -1);
                numberOfNoDictionaryDimension++;
                if (columnSchema.isSortColumn()) {
                    this.numberOfSortColumns++;
                    this.numberOfNoDictSortColumns++;
                }
            }
            dimensions.add(carbonDimension);
        } else {
            measures.add(new CarbonMeasure(columnSchema, ++measureOrdinal));
        }
        counter++;
    }
    dimColumnsCardinality = new int[cardinalityIndexForNormalDimensionColumn.size()];
    complexDimColumnCardinality = new int[cardinalityIndexForComplexDimensionColumn.size()];
    int index = 0;
    // generator
    for (Integer cardinalityArrayIndex : cardinalityIndexForNormalDimensionColumn) {
        dimColumnsCardinality[index++] = columnCardinality[cardinalityArrayIndex];
    }
    index = 0;
    // key generator
    for (Integer cardinalityArrayIndex : cardinalityIndexForComplexDimensionColumn) {
        complexDimColumnCardinality[index++] = columnCardinality[cardinalityArrayIndex];
    }
}

Example 22 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class DictionaryClientTest method setUp.

@Before
public void setUp() throws Exception {
    // enable lru cache by setting cache size
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "10");
    // Create two column schemas and dimensions for the table
    empColumnSchema = new ColumnSchema();
    empColumnSchema.setColumnName("empNameCol");
    empColumnSchema.setColumnUniqueId("empNameCol");
    empColumnSchema.setDimensionColumn(true);
    empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
    empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
    ageColumnSchema = new ColumnSchema();
    ageColumnSchema.setColumnName("empNameCol");
    ageColumnSchema.setColumnUniqueId("empNameCol");
    ageColumnSchema.setDimensionColumn(true);
    ageColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
    ageDimension = new CarbonDimension(ageColumnSchema, 0, 0, 0, 0, 0);
    // Create a Table
    tableSchema = new TableSchema();
    tableSchema.setTableName("TestTable");
    tableSchema.setListOfColumns(Arrays.asList(empColumnSchema, ageColumnSchema));
    CarbonMetadata metadata = CarbonMetadata.getInstance();
    tableInfo = new TableInfo();
    tableInfo.setFactTable(tableSchema);
    tableInfo.setTableUniqueName("TestTable");
    tableInfo.setDatabaseName("test");
    storePath = System.getProperty("java.io.tmpdir") + "/tmp";
    tableInfo.setStorePath(storePath);
    CarbonTable carbonTable = new CarbonTable();
    carbonTable.loadCarbonTable(tableInfo);
    // Add the created table to metadata
    metadata.addCarbonTable(carbonTable);
    // Start the server for testing the client
    server = DictionaryServer.getInstance(5678);
}

Example 23 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class FilterUtilTest method testGetMaskKey.

@Test
public void testGetMaskKey() {
    int surrogate = 1;
    int[] keys = new int[] { 1, 2 };
    MultiDimKeyVarLengthGenerator multiDimKeyVarLengthGenerator = new MultiDimKeyVarLengthGenerator(keys);
    int ordinal = 1;
    int keyOrdinal = 1;
    int columnGroupOrdinal = 1;
    int complexTypeOrdinal = 1;
    ColumnSchema columnSchema = new ColumnSchema();
    columnSchema.setColumnar(true);
    columnSchema.setColumnName("IMEI");
    columnSchema.setColumnUniqueId(UUID.randomUUID().toString());
    columnSchema.setDataType(DataType.STRING);
    columnSchema.setDimensionColumn(true);
    CarbonDimension carbonDimension = new CarbonDimension(columnSchema, ordinal, keyOrdinal, columnGroupOrdinal, complexTypeOrdinal);
    byte[] expectedResult = new byte[] { 1 };
    byte[] actualResult = FilterUtil.getMaskKey(surrogate, carbonDimension, multiDimKeyVarLengthGenerator);
    assertArrayEquals(expectedResult, actualResult);
}

Example 24 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class RangeFilterProcessorTest method createFilterTree_flavor2.

@Test
public void createFilterTree_flavor2() {
    // Build 3rd BTree a >= '11' or a > '12' or a <= '20' or a <= '15'
    Expression inputFilter;
    boolean result = false;
    ColumnSchema empColumnSchema = new ColumnSchema();
    empColumnSchema.setColumnName("a");
    empColumnSchema.setColumnUniqueId("a");
    empColumnSchema.setDimensionColumn(true);
    empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
    empColumnSchema.setDataType(DataType.STRING);
    CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
    ColumnExpression cola1 = new ColumnExpression("a", DataType.STRING);
    cola1.setDimension(true);
    cola1.setDimension(empDimension);
    ColumnExpression cola2 = new ColumnExpression("a", DataType.STRING);
    cola2.setDimension(true);
    cola2.setDimension(empDimension);
    ColumnExpression cola3 = new ColumnExpression("a", DataType.STRING);
    cola3.setDimension(true);
    cola3.setDimension(empDimension);
    ColumnExpression cola4 = new ColumnExpression("a", DataType.STRING);
    cola4.setDimension(true);
    cola4.setDimension(empDimension);
    Expression lessThan1 = new LessThanEqualToExpression(cola1, new LiteralExpression("15", DataType.STRING));
    Expression lessThan2 = new LessThanEqualToExpression(cola2, new LiteralExpression("20", DataType.STRING));
    Expression greaterThan1 = new GreaterThanExpression(cola3, new LiteralExpression("12", DataType.STRING));
    Expression greaterThan2 = new GreaterThanEqualToExpression(cola4, new LiteralExpression("11", DataType.STRING));
    Expression Or1 = new OrExpression(new NotEqualsExpression(null, null), greaterThan2);
    Expression Or2 = new OrExpression(Or1, greaterThan1);
    Expression Or3 = new OrExpression(Or2, lessThan2);
    inputFilter = new OrExpression(Or3, lessThan1);
    // Build The output
    ColumnExpression colb1 = new ColumnExpression("a", DataType.STRING);
    cola1.setDimension(true);
    cola1.setDimension(empDimension);
    ColumnExpression colb2 = new ColumnExpression("a", DataType.STRING);
    cola2.setDimension(true);
    cola2.setDimension(empDimension);
    ColumnExpression colb3 = new ColumnExpression("a", DataType.STRING);
    cola3.setDimension(true);
    cola3.setDimension(empDimension);
    ColumnExpression colb4 = new ColumnExpression("a", DataType.STRING);
    cola4.setDimension(true);
    cola4.setDimension(empDimension);
    Expression lessThanb1 = new LessThanEqualToExpression(colb1, new LiteralExpression("15", DataType.STRING));
    Expression lessThanb2 = new LessThanEqualToExpression(colb2, new LiteralExpression("20", DataType.STRING));
    Expression greaterThanb1 = new GreaterThanExpression(colb3, new LiteralExpression("12", DataType.STRING));
    Expression greaterThanb2 = new GreaterThanEqualToExpression(colb4, new LiteralExpression("11", DataType.STRING));
    Expression Orb1 = new OrExpression(new NotEqualsExpression(null, null), greaterThanb2);
    Expression Orb2 = new OrExpression(Orb1, greaterThanb1);
    Expression Orb3 = new OrExpression(Orb2, lessThanb2);
    FilterOptimizer rangeFilterOptimizer = new RangeFilterOptmizer(new FilterOptimizerBasic(), inputFilter);
    rangeFilterOptimizer.optimizeFilter();
    result = checkBothTrees(inputFilter, new OrExpression(Orb3, lessThanb1));
    // no change
    Assert.assertTrue(result);
}

Example 25 with CarbonDimension

use of org.apache.carbondata.core.metadata.schema.table.column.CarbonDimension in project carbondata by apache.

the class CarbonMetadataTest method testGetCarbonDimensionBasedOnColIdentifierNullCase.

@Test
public void testGetCarbonDimensionBasedOnColIdentifierNullCase() {
    CarbonTable carbonTable = new CarbonTable();
    String columnIdentifier = "3";
    final List<CarbonDimension> carbonDimensions = new ArrayList();
    ColumnSchema colSchema1 = new ColumnSchema();
    colSchema1.setColumnUniqueId("1");
    colSchema1.setNumberOfChild(1);
    CarbonDimension carbonDimension = new CarbonDimension(colSchema1, 1, 1, 2, 1);
    carbonDimensions.add(carbonDimension);
    final List<CarbonDimension> carbonChildDimensions = new ArrayList();
    ColumnSchema colSchema2 = new ColumnSchema();
    colSchema2.setColumnUniqueId("9");
    colSchema2.setNumberOfChild(0);
    carbonChildDimensions.add(new CarbonDimension(colSchema2, 1, 1, 2, 1));
    new MockUp<CarbonTable>() {

        @Mock
        public String getFactTableName() {
            return "carbonTestTable";
        }

        @Mock
        public List<CarbonDimension> getDimensionByTableName(String tableName) {
            return carbonDimensions;
        }
    };
    new MockUp<CarbonDimension>() {

        @Mock
        public List<CarbonDimension> getListOfChildDimensions() {
            return carbonChildDimensions;
        }
    };
    CarbonDimension result = carbonMetadata.getCarbonDimensionBasedOnColIdentifier(carbonTable, columnIdentifier);
    assertNull(result);
}