Examples with SchemaEvolutionEntry org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry used on opensource projects

Example 6 with SchemaEvolutionEntry

use of org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry in project carbondata by apache.

the class ThriftWrapperSchemaConverterImplTest method testFromExternalToWrapperSchemaEvolutionEntry.

@Test
public void testFromExternalToWrapperSchemaEvolutionEntry() {
    long time = 1112745600000L;
    ColumnSchema wrapperColumnSchema = new ColumnSchema();
    wrapperColumnSchema.setColumnUniqueId("1");
    wrapperColumnSchema.setColumnName("columnName");
    wrapperColumnSchema.setColumnar(true);
    wrapperColumnSchema.setDataType(DataTypes.STRING);
    wrapperColumnSchema.setDimensionColumn(true);
    wrapperColumnSchema.setEncodingList(encodings);
    wrapperColumnSchema.setNumberOfChild(1);
    wrapperColumnSchema.setColumnGroup(1);
    wrapperColumnSchema.setDefaultValue(new byte[] { 1, 2 });
    wrapperColumnSchema.setColumnProperties(columnPropertyMap);
    wrapperColumnSchema.setInvisible(true);
    wrapperColumnSchema.setColumnReferenceId("1");
    List<ColumnSchema> wrapperAddedColumns = new ArrayList<ColumnSchema>();
    wrapperAddedColumns.add(wrapperColumnSchema);
    SchemaEvolutionEntry wrapperSchemaEvolutionEntry = new SchemaEvolutionEntry();
    List<ColumnSchema> wrapperRemovedColumns = new ArrayList<ColumnSchema>();
    wrapperRemovedColumns.add(wrapperColumnSchema);
    wrapperSchemaEvolutionEntry.setTimeStamp(time);
    wrapperSchemaEvolutionEntry.setAdded(wrapperAddedColumns);
    wrapperSchemaEvolutionEntry.setRemoved(wrapperRemovedColumns);
    SchemaEvolutionEntry actualResult = thriftWrapperSchemaConverter.fromExternalToWrapperSchemaEvolutionEntry(schemaEvolEntry);
    assertEquals(wrapperSchemaEvolutionEntry.getAdded().get(0), actualResult.getAdded().get(0));
}

Example 7 with SchemaEvolutionEntry

use of org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry in project carbondata by apache.

the class ThriftWrapperSchemaConverterImplTest method testFromWrapperToExternalSchemaEvolution.

@Test
public void testFromWrapperToExternalSchemaEvolution() {
    SchemaEvolution schemaEvolution = new SchemaEvolution();
    new MockUp<SchemaEvolutionEntry>() {

        @Mock
        public List<ColumnSchema> getAdded() {
            return columnSchemas;
        }

        @Mock
        public List<ColumnSchema> getRemoved() {
            return columnSchemas;
        }
    };
    new MockUp<ColumnSchema>() {

        @Mock
        public List<Encoding> getEncodingList() {
            return encodings;
        }

        @Mock
        public int getSchemaOrdinal() {
            return 1;
        }

        @Mock
        public DataType getDataType() {
            return DataTypes.BOOLEAN;
        }

        @Mock
        public String getColumnName() {
            return "columnName";
        }

        @Mock
        public String getColumnUniqueId() {
            return "1";
        }

        @Mock
        public boolean isColumnar() {
            return true;
        }

        @Mock
        public boolean isDimensionColumn() {
            return true;
        }

        @Mock
        public int getColumnGroupId() {
            return 1;
        }

        @Mock
        public int getScale() {
            return 1;
        }

        @Mock
        public int getPrecision() {
            return 1;
        }

        @Mock
        public int getNumberOfChild() {
            return 1;
        }

        @Mock
        public byte[] getDefaultValue() {
            return new byte[] { 1, 2 };
        }

        @Mock
        public Map<String, String> getColumnProperties() {
            return columnPropertyMap;
        }

        @Mock
        public boolean isInvisible() {
            return true;
        }

        @Mock
        public String getColumnReferenceId() {
            return "1";
        }
    };
    List<org.apache.carbondata.format.SchemaEvolutionEntry> thriftSchemaEvolutionEntries = new ArrayList<>();
    thriftSchemaEvolutionEntries.add(schemaEvolEntry);
    org.apache.carbondata.format.SchemaEvolution actualResult = thriftWrapperSchemaConverter.fromWrapperToExternalSchemaEvolution(schemaEvolution);
    org.apache.carbondata.format.SchemaEvolution expectedResult = new org.apache.carbondata.format.SchemaEvolution(thriftSchemaEvolutionEntries);
    assertEquals(expectedResult, actualResult);
}

Example 8 with SchemaEvolutionEntry

use of org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry in project carbondata by apache.

the class ThriftWrapperSchemaConverterImplTest method setUp.

@BeforeClass
public static void setUp() {
    thriftWrapperSchemaConverter = new ThriftWrapperSchemaConverterImpl();
    schemaEvolEntry = new org.apache.carbondata.format.SchemaEvolutionEntry();
    schemaEvolutionEntries = new ArrayList();
    schemaEvolutionEntries.add(new SchemaEvolutionEntry());
    columnSchemas = new ArrayList();
    columnSchemas.add(new ColumnSchema());
    encodings = new ArrayList<>();
    encodings.add(Encoding.INVERTED_INDEX);
    encodings.add(Encoding.DELTA);
    encodings.add(Encoding.BIT_PACKED);
    encodings.add(Encoding.DICTIONARY);
    encodings.add(Encoding.RLE);
    encodings.add(Encoding.DIRECT_DICTIONARY);
    encoders = new ArrayList<org.apache.carbondata.format.Encoding>();
    encoders.add(org.apache.carbondata.format.Encoding.INVERTED_INDEX);
    encoders.add(org.apache.carbondata.format.Encoding.DELTA);
    encoders.add(org.apache.carbondata.format.Encoding.BIT_PACKED);
    encoders.add(org.apache.carbondata.format.Encoding.DICTIONARY);
    encoders.add(org.apache.carbondata.format.Encoding.RLE);
    encoders.add(org.apache.carbondata.format.Encoding.DIRECT_DICTIONARY);
    columnPropertyMap = new HashMap<String, String>();
    columnPropertyMap.put("property", "value");
    thriftColumnSchema = new org.apache.carbondata.format.ColumnSchema(org.apache.carbondata.format.DataType.STRING, "columnName", "1", true, encoders, true);
    thriftColumnSchema.setSchemaOrdinal(1);
    thriftColumnSchema.setAggregate_function("");
    thriftColumnSchemas = new ArrayList<org.apache.carbondata.format.ColumnSchema>();
    thriftColumnSchemas.add(thriftColumnSchema);
    thriftSchemaEvolutionEntries = new ArrayList<>();
    thriftSchemaEvolutionEntries.add(schemaEvolEntry);
    schemaEvol = new org.apache.carbondata.format.SchemaEvolution(thriftSchemaEvolutionEntries);
    new MockUp<SchemaEvolution>() {

        @Mock
        public List<SchemaEvolutionEntry> getSchemaEvolutionEntryList() {
            return schemaEvolutionEntries;
        }
    };
    new MockUp<org.apache.carbondata.format.SchemaEvolutionEntry>() {

        @Mock
        public org.apache.carbondata.format.SchemaEvolutionEntry setAdded(List<org.apache.carbondata.format.ColumnSchema> added) {
            return schemaEvolEntry;
        }

        @Mock
        public org.apache.carbondata.format.SchemaEvolutionEntry setRemoved(List<org.apache.carbondata.format.ColumnSchema> removed) {
            return schemaEvolEntry;
        }

        @Mock
        public long getTime_stamp() {
            return 1112745600000L;
        }

        @Mock
        public List<org.apache.carbondata.format.ColumnSchema> getAdded() {
            return thriftColumnSchemas;
        }

        @Mock
        public List<org.apache.carbondata.format.ColumnSchema> getRemoved() {
            return thriftColumnSchemas;
        }
    };
    new MockUp<org.apache.carbondata.format.ColumnSchema>() {

        @Mock
        public org.apache.carbondata.format.ColumnSchema setColumn_group_id(int column_group_id) {
            return thriftColumnSchema;
        }

        @Mock
        public org.apache.carbondata.format.ColumnSchema setScale(int scale) {
            return thriftColumnSchema;
        }

        @Mock
        public org.apache.carbondata.format.ColumnSchema setPrecision(int precision) {
            return thriftColumnSchema;
        }

        @Mock
        public org.apache.carbondata.format.ColumnSchema setNum_child(int num_child) {
            return thriftColumnSchema;
        }

        @Mock
        public org.apache.carbondata.format.ColumnSchema setDefault_value(byte[] default_value) {
            return thriftColumnSchema;
        }

        @Mock
        public org.apache.carbondata.format.ColumnSchema setColumnProperties(Map<String, String> columnProperties) {
            return thriftColumnSchema;
        }

        @Mock
        public org.apache.carbondata.format.ColumnSchema setInvisible(boolean invisible) {
            return thriftColumnSchema;
        }

        @Mock
        public org.apache.carbondata.format.ColumnSchema setColumnReferenceId(String columnReferenceId) {
            return thriftColumnSchema;
        }

        @Mock
        public String getColumn_id() {
            return "1";
        }

        @Mock
        public String getColumn_name() {
            return "columnName";
        }

        @Mock
        public boolean isColumnar() {
            return true;
        }

        @Mock
        public boolean isDimension() {
            return true;
        }

        @Mock
        public List<org.apache.carbondata.format.Encoding> getEncoders() {
            return encoders;
        }

        @Mock
        public int getNum_child() {
            return 1;
        }

        @Mock
        public int getPrecision() {
            return 1;
        }

        @Mock
        public int getColumn_group_id() {
            return 1;
        }

        @Mock
        public int getScale() {
            return 1;
        }

        @Mock
        public byte[] getDefault_value() {
            return new byte[] { 1, 2 };
        }

        @Mock
        public String getAggregate_function() {
            return "";
        }

        @Mock
        public Map<String, String> getColumnProperties() {
            return columnPropertyMap;
        }

        @Mock
        public boolean isInvisible() {
            return true;
        }

        @Mock
        public String getColumnReferenceId() {
            return "1";
        }
    };
    final Map mapTableProperties = new HashMap<String, String>();
    tabSchema = new org.apache.carbondata.format.TableSchema();
    new MockUp<org.apache.carbondata.format.TableSchema>() {

        @Mock
        public org.apache.carbondata.format.TableSchema setTableProperties(Map<String, String> tableProperties) {
            return tabSchema;
        }

        @Mock
        public String getTable_id() {
            return "1";
        }

        @Mock
        public Map<String, String> getTableProperties() {
            return mapTableProperties;
        }

        @Mock
        public List<org.apache.carbondata.format.ColumnSchema> getTable_columns() {
            return thriftColumnSchemas;
        }

        @Mock
        public org.apache.carbondata.format.SchemaEvolution getSchema_evolution() {
            return schemaEvol;
        }
    };
    new MockUp<org.apache.carbondata.format.SchemaEvolution>() {

        @Mock
        public List<org.apache.carbondata.format.SchemaEvolutionEntry> getSchema_evolution_history() {
            return thriftSchemaEvolutionEntries;
        }
    };
}

Example 9 with SchemaEvolutionEntry

use of org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry in project carbondata by apache.

the class CarbonUtil method inferSchema.

/**
 * This method will read the schema file from a given path
 *
 * @param schemaFilePath
 * @return
 */
public static org.apache.carbondata.format.TableInfo inferSchema(String carbonDataFilePath, AbsoluteTableIdentifier absoluteTableIdentifier, boolean schemaExists) throws IOException {
    TBaseCreator createTBase = new ThriftReader.TBaseCreator() {

        public org.apache.thrift.TBase<org.apache.carbondata.format.TableInfo, org.apache.carbondata.format.TableInfo._Fields> create() {
            return new org.apache.carbondata.format.TableInfo();
        }
    };
    if (schemaExists == false) {
        List<String> filePaths = getFilePathExternalFilePath(carbonDataFilePath + "/Fact/Part0/Segment_null");
        String fistFilePath = null;
        try {
            fistFilePath = filePaths.get(0);
        } catch (Exception e) {
            LOGGER.error("CarbonData file is not present in the table location");
        }
        CarbonHeaderReader carbonHeaderReader = new CarbonHeaderReader(fistFilePath);
        FileHeader fileHeader = carbonHeaderReader.readHeader();
        List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
        List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema();
        for (int i = 0; i < table_columns.size(); i++) {
            ColumnSchema col = thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i));
            col.setColumnReferenceId(col.getColumnUniqueId());
            columnSchemaList.add(col);
        }
        TableSchema tableSchema = new TableSchema();
        tableSchema.setTableName(absoluteTableIdentifier.getTableName());
        tableSchema.setBucketingInfo(null);
        tableSchema.setSchemaEvalution(null);
        tableSchema.setTableId(UUID.randomUUID().toString());
        tableSchema.setListOfColumns(columnSchemaList);
        ThriftWrapperSchemaConverterImpl thriftWrapperSchemaConverter = new ThriftWrapperSchemaConverterImpl();
        SchemaEvolutionEntry schemaEvolutionEntry = new SchemaEvolutionEntry();
        schemaEvolutionEntry.setTimeStamp(System.currentTimeMillis());
        SchemaEvolution schemaEvol = new SchemaEvolution();
        List<SchemaEvolutionEntry> schEntryList = new ArrayList<>();
        schEntryList.add(schemaEvolutionEntry);
        schemaEvol.setSchemaEvolutionEntryList(schEntryList);
        tableSchema.setSchemaEvalution(schemaEvol);
        org.apache.carbondata.format.TableSchema thriftFactTable = thriftWrapperSchemaConverter.fromWrapperToExternalTableSchema(tableSchema);
        org.apache.carbondata.format.TableInfo tableInfo = new org.apache.carbondata.format.TableInfo(thriftFactTable, new ArrayList<org.apache.carbondata.format.TableSchema>());
        tableInfo.setDataMapSchemas(null);
        return tableInfo;
    } else {
        ThriftReader thriftReader = new ThriftReader(carbonDataFilePath, createTBase);
        thriftReader.open();
        org.apache.carbondata.format.TableInfo tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
        thriftReader.close();
        return tableInfo;
    }
}

Example 10 with SchemaEvolutionEntry

use of org.apache.carbondata.core.metadata.schema.SchemaEvolutionEntry in project carbondata by apache.

the class ThriftWrapperSchemaConverterImpl method fromExternalToWrapperSchemaEvolutionEntry.

/* (non-Javadoc)
   * convert from external to wrapper schema evolution entry
   */
@Override
public SchemaEvolutionEntry fromExternalToWrapperSchemaEvolutionEntry(org.apache.carbondata.format.SchemaEvolutionEntry externalSchemaEvolutionEntry) {
    SchemaEvolutionEntry wrapperSchemaEvolutionEntry = new SchemaEvolutionEntry();
    wrapperSchemaEvolutionEntry.setTimeStamp(externalSchemaEvolutionEntry.getTime_stamp());
    List<ColumnSchema> wrapperAddedColumns = new ArrayList<ColumnSchema>();
    if (null != externalSchemaEvolutionEntry.getAdded()) {
        for (org.apache.carbondata.format.ColumnSchema externalColumnSchema : externalSchemaEvolutionEntry.getAdded()) {
            wrapperAddedColumns.add(fromExternalToWrapperColumnSchema(externalColumnSchema));
        }
    }
    List<ColumnSchema> wrapperRemovedColumns = new ArrayList<ColumnSchema>();
    if (null != externalSchemaEvolutionEntry.getRemoved()) {
        for (org.apache.carbondata.format.ColumnSchema externalColumnSchema : externalSchemaEvolutionEntry.getRemoved()) {
            wrapperRemovedColumns.add(fromExternalToWrapperColumnSchema(externalColumnSchema));
        }
    }
    wrapperSchemaEvolutionEntry.setAdded(wrapperAddedColumns);
    wrapperSchemaEvolutionEntry.setRemoved(wrapperRemovedColumns);
    return wrapperSchemaEvolutionEntry;
}