Examples with ColumnSchema org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema used on opensource projects

Example 21 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class StoreCreator method createTable.

public static CarbonTable createTable(AbsoluteTableIdentifier identifier) throws IOException {
    TableInfo tableInfo = new TableInfo();
    tableInfo.setDatabaseName(identifier.getCarbonTableIdentifier().getDatabaseName());
    TableSchema tableSchema = new TableSchema();
    tableSchema.setTableName(identifier.getCarbonTableIdentifier().getTableName());
    List<ColumnSchema> columnSchemas = new ArrayList<ColumnSchema>();
    ArrayList<Encoding> encodings = new ArrayList<>();
    encodings.add(Encoding.DICTIONARY);
    ColumnSchema id = new ColumnSchema();
    id.setColumnName("ID");
    id.setColumnar(true);
    id.setDataType(DataTypes.INT);
    id.setEncodingList(encodings);
    id.setColumnUniqueId(UUID.randomUUID().toString());
    id.setColumnReferenceId(id.getColumnUniqueId());
    id.setDimensionColumn(true);
    id.setColumnGroup(1);
    columnSchemas.add(id);
    ColumnSchema date = new ColumnSchema();
    date.setColumnName("date");
    date.setColumnar(true);
    date.setDataType(DataTypes.STRING);
    date.setEncodingList(encodings);
    date.setColumnUniqueId(UUID.randomUUID().toString());
    date.setDimensionColumn(true);
    date.setColumnGroup(2);
    date.setSortColumn(true);
    date.setColumnReferenceId(id.getColumnUniqueId());
    columnSchemas.add(date);
    ColumnSchema country = new ColumnSchema();
    country.setColumnName("country");
    country.setColumnar(true);
    country.setDataType(DataTypes.STRING);
    country.setEncodingList(encodings);
    country.setColumnUniqueId(UUID.randomUUID().toString());
    country.setDimensionColumn(true);
    country.setColumnGroup(3);
    country.setSortColumn(true);
    country.setColumnReferenceId(id.getColumnUniqueId());
    columnSchemas.add(country);
    ColumnSchema name = new ColumnSchema();
    name.setColumnName("name");
    name.setColumnar(true);
    name.setDataType(DataTypes.STRING);
    name.setEncodingList(encodings);
    name.setColumnUniqueId(UUID.randomUUID().toString());
    name.setDimensionColumn(true);
    name.setColumnGroup(4);
    name.setSortColumn(true);
    name.setColumnReferenceId(id.getColumnUniqueId());
    columnSchemas.add(name);
    ColumnSchema phonetype = new ColumnSchema();
    phonetype.setColumnName("phonetype");
    phonetype.setColumnar(true);
    phonetype.setDataType(DataTypes.STRING);
    phonetype.setEncodingList(encodings);
    phonetype.setColumnUniqueId(UUID.randomUUID().toString());
    phonetype.setDimensionColumn(true);
    phonetype.setColumnGroup(5);
    phonetype.setSortColumn(true);
    phonetype.setColumnReferenceId(id.getColumnUniqueId());
    columnSchemas.add(phonetype);
    ColumnSchema serialname = new ColumnSchema();
    serialname.setColumnName("serialname");
    serialname.setColumnar(true);
    serialname.setDataType(DataTypes.STRING);
    serialname.setEncodingList(encodings);
    serialname.setColumnUniqueId(UUID.randomUUID().toString());
    serialname.setDimensionColumn(true);
    serialname.setColumnGroup(6);
    serialname.setSortColumn(true);
    serialname.setColumnReferenceId(id.getColumnUniqueId());
    columnSchemas.add(serialname);
    ColumnSchema salary = new ColumnSchema();
    salary.setColumnName("salary");
    salary.setColumnar(true);
    salary.setDataType(DataTypes.INT);
    salary.setEncodingList(new ArrayList<Encoding>());
    salary.setColumnUniqueId(UUID.randomUUID().toString());
    salary.setDimensionColumn(false);
    salary.setColumnReferenceId(id.getColumnUniqueId());
    salary.setColumnGroup(7);
    columnSchemas.add(salary);
    tableSchema.setListOfColumns(columnSchemas);
    SchemaEvolution schemaEvol = new SchemaEvolution();
    schemaEvol.setSchemaEvolutionEntryList(new ArrayList<SchemaEvolutionEntry>());
    tableSchema.setSchemaEvalution(schemaEvol);
    tableSchema.setTableId(UUID.randomUUID().toString());
    tableInfo.setTableUniqueName(identifier.getCarbonTableIdentifier().getTableUniqueName());
    tableInfo.setLastUpdatedTime(System.currentTimeMillis());
    tableInfo.setFactTable(tableSchema);
    tableInfo.setTablePath(identifier.getTablePath());
    String schemaFilePath = CarbonTablePath.getSchemaFilePath(identifier.getTablePath());
    String schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath);
    CarbonMetadata.getInstance().loadTableMetadata(tableInfo);
    SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
    org.apache.carbondata.format.TableInfo thriftTableInfo = schemaConverter.fromWrapperToExternalTableInfo(tableInfo, tableInfo.getDatabaseName(), tableInfo.getFactTable().getTableName());
    org.apache.carbondata.format.SchemaEvolutionEntry schemaEvolutionEntry = new org.apache.carbondata.format.SchemaEvolutionEntry(tableInfo.getLastUpdatedTime());
    thriftTableInfo.getFact_table().getSchema_evolution().getSchema_evolution_history().add(schemaEvolutionEntry);
    FileFactory.FileType fileType = FileFactory.getFileType(schemaMetadataPath);
    if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
        FileFactory.mkdirs(schemaMetadataPath, fileType);
    }
    ThriftWriter thriftWriter = new ThriftWriter(schemaFilePath, false);
    thriftWriter.open();
    thriftWriter.write(thriftTableInfo);
    thriftWriter.close();
    return CarbonMetadata.getInstance().getCarbonTable(tableInfo.getTableUniqueName());
}

Example 22 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class CarbonUtilTest method testToGetComplexDataTypeArray.

@Test
public void testToGetComplexDataTypeArray() {
    ColumnSchema column1Schema = new ColumnSchema();
    ColumnSchema column2Schema = new ColumnSchema();
    column1Schema.setColumnName("Column1");
    column1Schema.setDataType(DataTypes.DATE);
    ProjectionDimension column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
    column2Schema.setColumnName("Column2");
    column2Schema.setDataType(DataTypes.createDefaultArrayType());
    ProjectionDimension column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
    ProjectionDimension[] queryDimensions = { column1, column2 };
    boolean[] dictionaryEncoding = CarbonUtil.getComplexDataTypeArray(queryDimensions);
    boolean[] expectedDictionaryEncoding = { false, true };
    for (int i = 0; i < dictionaryEncoding.length; i++) {
        assertEquals(dictionaryEncoding[i], expectedDictionaryEncoding[i]);
    }
}

Example 23 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class CarbonUtilTest method testToGetDictionaryEncodingArray.

@Test
public void testToGetDictionaryEncodingArray() {
    ColumnSchema column1Schema = new ColumnSchema();
    ColumnSchema column2Schema = new ColumnSchema();
    column1Schema.setColumnName("Column1");
    List<Encoding> encoding = new ArrayList<>();
    encoding.add(Encoding.DICTIONARY);
    column1Schema.setEncodingList(encoding);
    ProjectionDimension column1 = new ProjectionDimension(new CarbonDimension(column1Schema, 1, 1, 1, 1));
    column2Schema.setColumnName("Column2");
    List<Encoding> encoding2 = new ArrayList<>();
    encoding2.add(Encoding.DELTA);
    column2Schema.setEncodingList(encoding2);
    ProjectionDimension column2 = new ProjectionDimension(new CarbonDimension(column2Schema, 1, 1, 1, 1));
    ProjectionDimension[] queryDimensions = { column1, column2 };
    boolean[] dictionaryEncoding = CarbonUtil.getDictionaryEncodingArray(queryDimensions);
    boolean[] expectedDictionaryEncoding = { true, false };
    for (int i = 0; i < dictionaryEncoding.length; i++) {
        assertEquals(dictionaryEncoding[i], expectedDictionaryEncoding[i]);
    }
}

Example 24 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class CarbonUtilTest method testUpdateMinMaxValues.

@Test
public void testUpdateMinMaxValues() {
    // create dimension and measure column schema
    ColumnSchema dimensionColumnSchema = createColumnSchema(DataTypes.STRING, true);
    ColumnSchema measureColumnSchema = createColumnSchema(DataTypes.DOUBLE, false);
    List<ColumnSchema> columnSchemas = new ArrayList<>(2);
    columnSchemas.add(dimensionColumnSchema);
    columnSchemas.add(measureColumnSchema);
    // create data file footer object
    DataFileFooter fileFooter = new DataFileFooter();
    fileFooter.setColumnInTable(columnSchemas);
    // initialise the expected values
    int expectedMaxValue = 5;
    int expectedMinValue = 2;
    double expectedMeasureMaxValue = 28.74;
    double expectedMeasureMinValue = -21.46;
    // initialise the minValues
    byte[][] minValues = new byte[2][];
    minValues[0] = new byte[] { 2 };
    ByteBuffer buffer = ByteBuffer.allocate(8);
    minValues[1] = (byte[]) buffer.putDouble(28.74).flip().array();
    buffer = ByteBuffer.allocate(8);
    // initialise the maxValues
    byte[][] maxValues = new byte[2][];
    maxValues[0] = new byte[] { 5 };
    maxValues[1] = (byte[]) buffer.putDouble(-21.46).flip().array();
    byte[][] updateMaxValues = CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, false);
    byte[][] updateMinValues = CarbonUtil.updateMinMaxValues(fileFooter, maxValues, minValues, true);
    // compare max values
    assert (expectedMaxValue == ByteBuffer.wrap(updateMaxValues[0]).get());
    assert (expectedMeasureMaxValue == ByteBuffer.wrap(updateMaxValues[1]).getDouble());
    // compare min values
    assert (expectedMinValue == ByteBuffer.wrap(updateMinValues[0]).get());
    assert (expectedMeasureMinValue == ByteBuffer.wrap(updateMinValues[1]).getDouble());
}

Example 25 with ColumnSchema

use of org.apache.carbondata.core.metadata.schema.table.column.ColumnSchema in project carbondata by apache.

the class DataTypeUtilTest method testGetMeasureValueBasedOnDataType.

@Test
public void testGetMeasureValueBasedOnDataType() {
    ColumnSchema columnSchema = new ColumnSchema();
    CarbonMeasure carbonMeasure = new CarbonMeasure(columnSchema, 1);
    Object resultInt = getMeasureValueBasedOnDataType("1", DataTypes.INT, carbonMeasure);
    Object expectedInt = Double.valueOf(1).intValue();
    assertEquals(expectedInt, resultInt);
    Object resultLong = getMeasureValueBasedOnDataType("1", DataTypes.LONG, carbonMeasure);
    Object expectedLong = Long.valueOf(1);
    assertEquals(expectedLong, resultLong);
    Object resultDefault = getMeasureValueBasedOnDataType("1", DataTypes.DOUBLE, carbonMeasure);
    Double expectedDefault = Double.valueOf(1);
    assertEquals(expectedDefault, resultDefault);
}