Examples with Encoding org.apache.carbondata.core.metadata.encoder.Encoding used on opensource projects

Example 1 with Encoding

use of org.apache.carbondata.core.metadata.encoder.Encoding in project carbondata by apache.

the class AbstractDataFileFooterConverter method getDataChunk.

/**
   * Below method will be used to convert the thrift data chunk to wrapper
   * data chunk
   *
   * @param datachunkThrift
   * @return wrapper data chunk
   */
protected DataChunk getDataChunk(org.apache.carbondata.format.DataChunk datachunkThrift, boolean isPresenceMetaPresent) {
    DataChunk dataChunk = new DataChunk();
    dataChunk.setDataPageLength(datachunkThrift.getData_page_length());
    dataChunk.setDataPageOffset(datachunkThrift.getData_page_offset());
    if (isPresenceMetaPresent) {
        dataChunk.setNullValueIndexForColumn(getPresenceMeta(datachunkThrift.getPresence()));
    }
    dataChunk.setRlePageLength(datachunkThrift.getRle_page_length());
    dataChunk.setRlePageOffset(datachunkThrift.getRle_page_offset());
    dataChunk.setRowMajor(datachunkThrift.isRowMajor());
    dataChunk.setRowIdPageLength(datachunkThrift.getRowid_page_length());
    dataChunk.setRowIdPageOffset(datachunkThrift.getRowid_page_offset());
    List<Encoding> encodingList = new ArrayList<Encoding>(datachunkThrift.getEncoders().size());
    for (int i = 0; i < datachunkThrift.getEncoders().size(); i++) {
        encodingList.add(fromExternalToWrapperEncoding(datachunkThrift.getEncoders().get(i)));
    }
    dataChunk.setEncodingList(encodingList);
    if (encodingList.contains(Encoding.DELTA)) {
        List<ByteBuffer> thriftEncoderMeta = datachunkThrift.getEncoder_meta();
        List<ValueEncoderMeta> encodeMetaList = new ArrayList<ValueEncoderMeta>(thriftEncoderMeta.size());
        for (int i = 0; i < thriftEncoderMeta.size(); i++) {
            encodeMetaList.add(CarbonUtil.deserializeEncoderMeta(thriftEncoderMeta.get(i).array()));
        }
        dataChunk.setValueEncoderMeta(encodeMetaList);
    }
    return dataChunk;
}

Example 2 with Encoding

use of org.apache.carbondata.core.metadata.encoder.Encoding in project carbondata by apache.

the class ThriftWrapperSchemaConverterImpl method fromExternalToWrapperColumnSchema.

/* (non-Javadoc)
   * convert from external to wrapper columnschema
   */
@Override
public ColumnSchema fromExternalToWrapperColumnSchema(org.apache.carbondata.format.ColumnSchema externalColumnSchema) {
    ColumnSchema wrapperColumnSchema = new ColumnSchema();
    wrapperColumnSchema.setColumnUniqueId(externalColumnSchema.getColumn_id());
    wrapperColumnSchema.setColumnName(externalColumnSchema.getColumn_name());
    wrapperColumnSchema.setColumnar(externalColumnSchema.isColumnar());
    wrapperColumnSchema.setDataType(fromExternalToWrapperDataType(externalColumnSchema.data_type));
    wrapperColumnSchema.setDimensionColumn(externalColumnSchema.isDimension());
    List<Encoding> encoders = new ArrayList<Encoding>();
    for (org.apache.carbondata.format.Encoding encoder : externalColumnSchema.getEncoders()) {
        encoders.add(fromExternalToWrapperEncoding(encoder));
    }
    wrapperColumnSchema.setEncodingList(encoders);
    wrapperColumnSchema.setNumberOfChild(externalColumnSchema.getNum_child());
    wrapperColumnSchema.setPrecision(externalColumnSchema.getPrecision());
    wrapperColumnSchema.setColumnGroup(externalColumnSchema.getColumn_group_id());
    wrapperColumnSchema.setScale(externalColumnSchema.getScale());
    wrapperColumnSchema.setDefaultValue(externalColumnSchema.getDefault_value());
    wrapperColumnSchema.setInvisible(externalColumnSchema.isInvisible());
    wrapperColumnSchema.setColumnReferenceId(externalColumnSchema.getColumnReferenceId());
    wrapperColumnSchema.setSchemaOrdinal(externalColumnSchema.getSchemaOrdinal());
    wrapperColumnSchema.setSortColumn(false);
    Map<String, String> properties = externalColumnSchema.getColumnProperties();
    if (properties != null) {
        String sortColumns = properties.get(CarbonCommonConstants.SORT_COLUMNS);
        if (sortColumns != null) {
            wrapperColumnSchema.setSortColumn(true);
        }
    }
    return wrapperColumnSchema;
}

Example 3 with Encoding

use of org.apache.carbondata.core.metadata.encoder.Encoding in project carbondata by apache.

the class ThriftWrapperSchemaConverterImplTest method testFromWrapperToExternalTableSchema.

@Test
public void testFromWrapperToExternalTableSchema() {
    TableSchema wrapperTableSchema = new TableSchema();
    new MockUp<ColumnSchema>() {

        @Mock
        public List<Encoding> getEncodingList() {
            return encodings;
        }

        @Mock
        public int getSchemaOrdinal() {
            return 1;
        }

        @Mock
        public DataType getDataType() {
            return DataType.BOOLEAN;
        }

        @Mock
        public String getColumnName() {
            return "columnName";
        }

        @Mock
        public String getColumnUniqueId() {
            return "1";
        }

        @Mock
        public boolean isColumnar() {
            return true;
        }

        @Mock
        public boolean isDimensionColumn() {
            return true;
        }

        @Mock
        public int getColumnGroupId() {
            return 1;
        }

        @Mock
        public int getScale() {
            return 1;
        }

        @Mock
        public int getPrecision() {
            return 1;
        }

        @Mock
        public int getNumberOfChild() {
            return 1;
        }

        @Mock
        public byte[] getDefaultValue() {
            return new byte[] { 1, 2 };
        }

        @Mock
        public Map<String, String> getColumnProperties() {
            return columnPropertyMap;
        }

        @Mock
        public boolean isInvisible() {
            return true;
        }

        @Mock
        public String getColumnReferenceId() {
            return "1";
        }
    };
    new MockUp<SchemaEvolutionEntry>() {

        @Mock
        public List<ColumnSchema> getAdded() {
            return columnSchemas;
        }

        @Mock
        public List<ColumnSchema> getRemoved() {
            return columnSchemas;
        }
    };
    final SchemaEvolution schemaEvolution = new SchemaEvolution();
    final Map mapTableProperties = new HashMap<String, String>();
    new MockUp<TableSchema>() {

        @Mock
        public List<ColumnSchema> getListOfColumns() {
            return columnSchemas;
        }

        @Mock
        public SchemaEvolution getSchemaEvalution() {
            return schemaEvolution;
        }

        @Mock
        public String getTableId() {
            return "tableId";
        }

        @Mock
        public Map<String, String> getTableProperties() {
            return mapTableProperties;
        }
    };
    org.apache.carbondata.format.TableSchema expectedResult = new org.apache.carbondata.format.TableSchema("tableId", thriftColumnSchemas, schemaEvol);
    org.apache.carbondata.format.TableSchema actualResult = thriftWrapperSchemaConverter.fromWrapperToExternalTableSchema(wrapperTableSchema);
    assertEquals(expectedResult, actualResult);
}

Example 4 with Encoding

use of org.apache.carbondata.core.metadata.encoder.Encoding in project carbondata by apache.

the class CarbonTableWithComplexTypesTest method getColumnarDimensionColumn.

private List<ColumnSchema> getColumnarDimensionColumn() {
    List<ColumnSchema> cols = new ArrayList<ColumnSchema>();
    ColumnSchema dimColumn = new ColumnSchema();
    dimColumn.setColumnar(true);
    dimColumn.setColumnName("IMEI");
    dimColumn.setColumnUniqueId(UUID.randomUUID().toString());
    dimColumn.setDataType(DataType.STRING);
    dimColumn.setDimensionColumn(true);
    List<Encoding> encodeList = new ArrayList<Encoding>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    encodeList.add(Encoding.DICTIONARY);
    dimColumn.setEncodingList(encodeList);
    dimColumn.setNumberOfChild(0);
    cols.add(dimColumn);
    ColumnSchema structColumn = new ColumnSchema();
    structColumn.setColumnar(true);
    structColumn.setColumnName("mobile");
    structColumn.setColumnUniqueId(UUID.randomUUID().toString());
    structColumn.setDataType(DataType.STRUCT);
    structColumn.setDimensionColumn(true);
    structColumn.setEncodingList(encodeList);
    structColumn.setNumberOfChild(2);
    cols.add(structColumn);
    ColumnSchema primitiveColumn = new ColumnSchema();
    primitiveColumn.setColumnar(true);
    primitiveColumn.setColumnName("mobile.stdcode");
    primitiveColumn.setColumnUniqueId(UUID.randomUUID().toString());
    primitiveColumn.setDataType(DataType.STRING);
    primitiveColumn.setDimensionColumn(true);
    primitiveColumn.setEncodingList(encodeList);
    primitiveColumn.setNumberOfChild(0);
    cols.add(primitiveColumn);
    ColumnSchema arrayColumn = new ColumnSchema();
    arrayColumn.setColumnar(true);
    arrayColumn.setColumnName("mobile.val");
    arrayColumn.setColumnUniqueId(UUID.randomUUID().toString());
    arrayColumn.setDataType(DataType.ARRAY);
    arrayColumn.setDimensionColumn(true);
    arrayColumn.setEncodingList(encodeList);
    arrayColumn.setNumberOfChild(1);
    cols.add(arrayColumn);
    ColumnSchema primitiveColumn1 = new ColumnSchema();
    primitiveColumn1.setColumnar(true);
    primitiveColumn1.setColumnName("mobile.val.phoneno");
    primitiveColumn1.setColumnUniqueId(UUID.randomUUID().toString());
    primitiveColumn1.setDataType(DataType.STRING);
    primitiveColumn1.setDimensionColumn(true);
    primitiveColumn1.setEncodingList(encodeList);
    primitiveColumn1.setNumberOfChild(0);
    cols.add(primitiveColumn1);
    return cols;
}

Example 5 with Encoding

use of org.apache.carbondata.core.metadata.encoder.Encoding in project carbondata by apache.

the class ThriftWrapperSchemaConverterImplTest method testFromWrapperToExternalTableInfo.

@Test
public void testFromWrapperToExternalTableInfo() {
    TableInfo wrapperTableInfo = new TableInfo();
    String dbName = "dbName";
    String tableName = "TableName";
    final TableSchema wrapperTableSchema = new TableSchema();
    final List<TableSchema> tableSchemas = new ArrayList<>();
    tableSchemas.add(wrapperTableSchema);
    new MockUp<SchemaEvolutionEntry>() {

        @Mock
        public List<ColumnSchema> getAdded() {
            return columnSchemas;
        }

        @Mock
        public List<ColumnSchema> getRemoved() {
            return columnSchemas;
        }
    };
    new MockUp<ColumnSchema>() {

        @Mock
        public List<Encoding> getEncodingList() {
            return encodings;
        }

        @Mock
        public int getSchemaOrdinal() {
            return 1;
        }

        @Mock
        public DataType getDataType() {
            return DataType.BOOLEAN;
        }

        @Mock
        public String getColumnName() {
            return "columnName";
        }

        @Mock
        public String getColumnUniqueId() {
            return "1";
        }

        @Mock
        public boolean isColumnar() {
            return true;
        }

        @Mock
        public boolean isDimensionColumn() {
            return true;
        }

        @Mock
        public int getColumnGroupId() {
            return 1;
        }

        @Mock
        public int getScale() {
            return 1;
        }

        @Mock
        public int getPrecision() {
            return 1;
        }

        @Mock
        public int getNumberOfChild() {
            return 1;
        }

        @Mock
        public byte[] getDefaultValue() {
            return new byte[] { 1, 2 };
        }

        @Mock
        public Map<String, String> getColumnProperties() {
            return columnPropertyMap;
        }

        @Mock
        public boolean isInvisible() {
            return true;
        }

        @Mock
        public String getColumnReferenceId() {
            return "1";
        }
    };
    new MockUp<TableInfo>() {

        @Mock
        public TableSchema getFactTable() {
            return wrapperTableSchema;
        }

        @Mock
        public List<TableSchema> getAggregateTableList() {
            return tableSchemas;
        }
    };
    new MockUp<TableSchema>() {

        @Mock
        public List<ColumnSchema> getListOfColumns() {
            return columnSchemas;
        }

        final SchemaEvolution schemaEvolution = new SchemaEvolution();

        final Map mapTableProperties = new HashMap<String, String>();

        @Mock
        public SchemaEvolution getSchemaEvalution() {
            return schemaEvolution;
        }

        @Mock
        public String getTableId() {
            return "tableId";
        }

        @Mock
        public Map<String, String> getTableProperties() {
            return mapTableProperties;
        }
    };
    org.apache.carbondata.format.TableSchema thriftFactTable = new org.apache.carbondata.format.TableSchema("tableId", thriftColumnSchemas, schemaEvol);
    List<org.apache.carbondata.format.TableSchema> thriftAggTables = new ArrayList<>();
    thriftAggTables.add(thriftFactTable);
    org.apache.carbondata.format.TableInfo actualResult = thriftWrapperSchemaConverter.fromWrapperToExternalTableInfo(wrapperTableInfo, dbName, tableName);
    org.apache.carbondata.format.TableInfo expectedResult = new org.apache.carbondata.format.TableInfo(thriftFactTable, thriftAggTables);
    assertEquals(expectedResult, actualResult);
}