Examples with ColumnSchema org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema used on opensource projects

Example 26 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class RangeFilterProcessorTest method createFilterTree_noChange.

@Test
public void createFilterTree_noChange() {
    // Build 2nd Tree no change a < 5 and a > 20
    Expression inputFilter;
    boolean result = false;
    ColumnExpression cola = new ColumnExpression("a", DataTypes.STRING);
    cola.setDimension(true);
    ColumnSchema empColumnSchema = new ColumnSchema();
    empColumnSchema.setColumnName("empNameCol");
    empColumnSchema.setColumnUniqueId("empNameCol");
    empColumnSchema.setDimensionColumn(true);
    empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
    empColumnSchema.setDataType(DataTypes.STRING);
    CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
    cola.setDimension(empDimension);
    Expression greaterThan = new GreaterThanEqualToExpression(cola, new LiteralExpression("20", DataTypes.STRING));
    ColumnExpression colb = new ColumnExpression("a", DataTypes.STRING);
    colb.setDimension(true);
    colb.setDimension(empDimension);
    Expression lessThan = new LessThanEqualToExpression(colb, new LiteralExpression("05", DataTypes.STRING));
    inputFilter = new AndExpression(greaterThan, lessThan);
    Expression output = new AndExpression(new GreaterThanEqualToExpression(new ColumnExpression("a", DataTypes.STRING), new LiteralExpression("20", DataTypes.STRING)), new LessThanEqualToExpression(new ColumnExpression("a", DataTypes.STRING), new LiteralExpression("05", DataTypes.STRING)));
    FilterOptimizer rangeFilterOptimizer = new RangeFilterOptmizer(inputFilter);
    rangeFilterOptimizer.optimizeFilter();
    result = checkBothTrees(inputFilter, output);
    // no change
    Assert.assertTrue(result);
}

Example 27 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class RangeFilterProcessorTest method createFilterTree_flavor2.

@Test
public void createFilterTree_flavor2() {
    // Build 3rd BTree a >= '11' or a > '12' or a <= '20' or a <= '15'
    Expression inputFilter;
    boolean result = false;
    ColumnSchema empColumnSchema = new ColumnSchema();
    empColumnSchema.setColumnName("a");
    empColumnSchema.setColumnUniqueId("a");
    empColumnSchema.setDimensionColumn(true);
    empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
    empColumnSchema.setDataType(DataTypes.STRING);
    CarbonDimension empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
    ColumnExpression cola1 = new ColumnExpression("a", DataTypes.STRING);
    cola1.setDimension(true);
    cola1.setDimension(empDimension);
    ColumnExpression cola2 = new ColumnExpression("a", DataTypes.STRING);
    cola2.setDimension(true);
    cola2.setDimension(empDimension);
    ColumnExpression cola3 = new ColumnExpression("a", DataTypes.STRING);
    cola3.setDimension(true);
    cola3.setDimension(empDimension);
    ColumnExpression cola4 = new ColumnExpression("a", DataTypes.STRING);
    cola4.setDimension(true);
    cola4.setDimension(empDimension);
    Expression lessThan1 = new LessThanEqualToExpression(cola1, new LiteralExpression("15", DataTypes.STRING));
    Expression lessThan2 = new LessThanEqualToExpression(cola2, new LiteralExpression("20", DataTypes.STRING));
    Expression greaterThan1 = new GreaterThanExpression(cola3, new LiteralExpression("12", DataTypes.STRING));
    Expression greaterThan2 = new GreaterThanEqualToExpression(cola4, new LiteralExpression("11", DataTypes.STRING));
    Expression Or1 = new OrExpression(new NotEqualsExpression(null, null), greaterThan2);
    Expression Or2 = new OrExpression(Or1, greaterThan1);
    Expression Or3 = new OrExpression(Or2, lessThan2);
    inputFilter = new OrExpression(Or3, lessThan1);
    // Build The output
    ColumnExpression colb1 = new ColumnExpression("a", DataTypes.STRING);
    cola1.setDimension(true);
    cola1.setDimension(empDimension);
    ColumnExpression colb2 = new ColumnExpression("a", DataTypes.STRING);
    cola2.setDimension(true);
    cola2.setDimension(empDimension);
    ColumnExpression colb3 = new ColumnExpression("a", DataTypes.STRING);
    cola3.setDimension(true);
    cola3.setDimension(empDimension);
    ColumnExpression colb4 = new ColumnExpression("a", DataTypes.STRING);
    cola4.setDimension(true);
    cola4.setDimension(empDimension);
    Expression lessThanb1 = new LessThanEqualToExpression(colb1, new LiteralExpression("15", DataTypes.STRING));
    Expression lessThanb2 = new LessThanEqualToExpression(colb2, new LiteralExpression("20", DataTypes.STRING));
    Expression greaterThanb1 = new GreaterThanExpression(colb3, new LiteralExpression("12", DataTypes.STRING));
    Expression greaterThanb2 = new GreaterThanEqualToExpression(colb4, new LiteralExpression("11", DataTypes.STRING));
    Expression Orb1 = new OrExpression(new NotEqualsExpression(null, null), greaterThanb2);
    Expression Orb2 = new OrExpression(Orb1, greaterThanb1);
    Expression Orb3 = new OrExpression(Orb2, lessThanb2);
    FilterOptimizer rangeFilterOptimizer = new RangeFilterOptmizer(inputFilter);
    rangeFilterOptimizer.optimizeFilter();
    result = checkBothTrees(inputFilter, new OrExpression(Orb3, lessThanb1));
    // no change
    Assert.assertTrue(result);
}

Example 28 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class CarbondataMetadata method getColumnHandles.

@Override
public Map<String, ColumnHandle> getColumnHandles(ConnectorSession session, ConnectorTableHandle tableHandle) {
    CarbondataTableHandle handle = checkType(tableHandle, CarbondataTableHandle.class, "tableHandle");
    checkArgument(handle.getConnectorId().equals(connectorId), "tableHandle is not for this connector");
    String schemaName = handle.getSchemaTableName().getSchemaName();
    if (!listSchemaNamesInternal().contains(schemaName)) {
        throw new SchemaNotFoundException(schemaName);
    }
    // CarbonTable(official struct) is stored in CarbonMetadata(official struct)
    CarbonTable cb = carbonTableReader.getTable(handle.getSchemaTableName());
    ImmutableMap.Builder<String, ColumnHandle> columnHandles = ImmutableMap.builder();
    String tableName = handle.getSchemaTableName().getTableName();
    for (CarbonDimension column : cb.getDimensionByTableName(tableName)) {
        ColumnSchema cs = column.getColumnSchema();
        Type spiType = carbonDataType2SpiMapper(cs);
        columnHandles.put(cs.getColumnName(), new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType, column.getSchemaOrdinal(), column.getKeyOrdinal(), column.getColumnGroupOrdinal(), false, cs.getColumnGroupId(), cs.getColumnUniqueId(), cs.isUseInvertedIndex(), cs.getPrecision(), cs.getScale()));
    }
    for (CarbonMeasure measure : cb.getMeasureByTableName(tableName)) {
        ColumnSchema cs = measure.getColumnSchema();
        Type spiType = carbonDataType2SpiMapper(cs);
        columnHandles.put(cs.getColumnName(), new CarbondataColumnHandle(connectorId, cs.getColumnName(), spiType, cs.getSchemaOrdinal(), measure.getOrdinal(), cs.getColumnGroupId(), true, cs.getColumnGroupId(), cs.getColumnUniqueId(), cs.isUseInvertedIndex(), cs.getPrecision(), cs.getScale()));
    }
    // should i cache it?
    columnHandleMap = columnHandles.build();
    return columnHandleMap;
}

Example 29 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class DictionaryClientTest method setUp.

@BeforeClass
public static void setUp() throws Exception {
    // enable lru cache by setting cache size
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "10");
    // Create two column schemas and dimensions for the table
    empColumnSchema = new ColumnSchema();
    empColumnSchema.setColumnName("empNameCol");
    empColumnSchema.setColumnUniqueId("empNameCol");
    empColumnSchema.setDimensionColumn(true);
    empColumnSchema.setDataType(DataTypes.STRING);
    empColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
    empDimension = new CarbonDimension(empColumnSchema, 0, 0, 0, 0, 0);
    ageColumnSchema = new ColumnSchema();
    ageColumnSchema.setColumnName("ageNameCol");
    ageColumnSchema.setDataType(DataTypes.SHORT_INT);
    ageColumnSchema.setColumnUniqueId("ageNameCol");
    ageColumnSchema.setDimensionColumn(true);
    ageColumnSchema.setEncodingList(Arrays.asList(Encoding.DICTIONARY));
    ageDimension = new CarbonDimension(ageColumnSchema, 0, 0, 0, 0, 0);
    // Create a Table
    tableSchema = new TableSchema();
    tableSchema.setTableName("TestTable");
    tableSchema.setTableId("1");
    tableSchema.setListOfColumns(Arrays.asList(empColumnSchema, ageColumnSchema));
    CarbonMetadata metadata = CarbonMetadata.getInstance();
    tableInfo = new TableInfo();
    tableInfo.setFactTable(tableSchema);
    tableInfo.setTableUniqueName("TestTable");
    tableInfo.setDatabaseName("test");
    storePath = System.getProperty("java.io.tmpdir") + "/tmp";
    tableInfo.setTablePath(storePath + "/" + "test" + "/" + "TestTable");
    CarbonTable carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
    // Add the created table to metadata
    metadata.addCarbonTable(carbonTable);
    // Start the server for testing the client
    server = NonSecureDictionaryServer.getInstance(5678, carbonTable);
    host = server.getHost();
}

Example 30 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class IncrementalColumnDictionaryGeneratorTest method writeDictionaryData.

@Test
public void writeDictionaryData() throws Exception {
    // Create required column schema
    ColumnSchema columnSchema = new ColumnSchema();
    columnSchema.setColumnName("empNameCol");
    columnSchema.setDataType(DataTypes.STRING);
    columnSchema.setColumnUniqueId("empNameCol");
    CarbonDimension carbonDimension = new CarbonDimension(columnSchema, 0, 0, 0, 0, 0);
    // Create the generator and add the keys to dictionary
    IncrementalColumnDictionaryGenerator generator = new IncrementalColumnDictionaryGenerator(carbonDimension, 10, carbonTable);
    // Create a table schema for saving the dictionary
    TableSchema tableSchema = new TableSchema();
    tableSchema.setTableName("TestTable");
    tableSchema.setListOfColumns(Arrays.asList(columnSchema));
    CarbonMetadata metadata = CarbonMetadata.getInstance();
    TableInfo tableInfo = new TableInfo();
    tableInfo.setFactTable(tableSchema);
    tableInfo.setTableUniqueName("TestTable");
    tableInfo.setDatabaseName("test");
    String storePath = System.getProperty("java.io.tmpdir") + "/tmp";
    File dictPath = new File(storePath + "/test/TestTable/Metadata/");
    System.out.print(dictPath.mkdirs());
    tableInfo.setTablePath(storePath + "/test" + "/TestTable");
    CarbonTable carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
    // Add the table to metadata
    metadata.addCarbonTable(carbonTable);
    // / Write the dictionary and verify whether its written successfully
    generator.writeDictionaryData();
    File dictionaryFile = new File(dictPath, "empNameCol.dict");
    System.out.println(dictionaryFile.getCanonicalPath());
    assertTrue(dictionaryFile.exists());
    dictionaryFile.delete();
    // cleanup created files
    metadata.removeTable(carbonTable.getTableUniqueName());
    cleanUpDirectory(new File(storePath));
}