Examples with ThriftWrapperSchemaConverterImpl org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl used on opensource projects

Example 1 with ThriftWrapperSchemaConverterImpl

use of org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl in project carbondata by apache.

the class SchemaReader method readCarbonTableFromStore.

public static CarbonTable readCarbonTableFromStore(AbsoluteTableIdentifier identifier) throws IOException {
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(identifier);
    String schemaFilePath = carbonTablePath.getSchemaFilePath();
    if (FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.LOCAL) || FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.HDFS) || FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.VIEWFS)) {
        String tableName = identifier.getCarbonTableIdentifier().getTableName();
        ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() {

            public TBase create() {
                return new org.apache.carbondata.format.TableInfo();
            }
        };
        ThriftReader thriftReader = new ThriftReader(carbonTablePath.getSchemaFilePath(), createTBase);
        thriftReader.open();
        org.apache.carbondata.format.TableInfo tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
        thriftReader.close();
        SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
        TableInfo wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(tableInfo, identifier.getCarbonTableIdentifier().getDatabaseName(), tableName, identifier.getStorePath());
        wrapperTableInfo.setMetaDataFilepath(CarbonTablePath.getFolderContainingFile(schemaFilePath));
        CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo);
        return CarbonMetadata.getInstance().getCarbonTable(identifier.getCarbonTableIdentifier().getTableUniqueName());
    } else {
        throw new IOException("File does not exist: " + schemaFilePath);
    }
}

Example 2 with ThriftWrapperSchemaConverterImpl

use of org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl in project carbondata by apache.

the class StoreCreator method createTable.

private static CarbonTable createTable() throws IOException {
    TableInfo tableInfo = new TableInfo();
    tableInfo.setStorePath(absoluteTableIdentifier.getStorePath());
    tableInfo.setDatabaseName(absoluteTableIdentifier.getCarbonTableIdentifier().getDatabaseName());
    TableSchema tableSchema = new TableSchema();
    tableSchema.setTableName(absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
    List<ColumnSchema> columnSchemas = new ArrayList<ColumnSchema>();
    ArrayList<Encoding> encodings = new ArrayList<>();
    encodings.add(Encoding.DICTIONARY);
    ColumnSchema id = new ColumnSchema();
    id.setColumnName("ID");
    id.setColumnar(true);
    id.setDataType(DataType.INT);
    id.setEncodingList(encodings);
    id.setColumnUniqueId(UUID.randomUUID().toString());
    id.setDimensionColumn(true);
    id.setColumnGroup(1);
    columnSchemas.add(id);
    ColumnSchema date = new ColumnSchema();
    date.setColumnName("date");
    date.setColumnar(true);
    date.setDataType(DataType.STRING);
    date.setEncodingList(encodings);
    date.setColumnUniqueId(UUID.randomUUID().toString());
    date.setDimensionColumn(true);
    date.setColumnGroup(2);
    date.setSortColumn(true);
    columnSchemas.add(date);
    ColumnSchema country = new ColumnSchema();
    country.setColumnName("country");
    country.setColumnar(true);
    country.setDataType(DataType.STRING);
    country.setEncodingList(encodings);
    country.setColumnUniqueId(UUID.randomUUID().toString());
    country.setDimensionColumn(true);
    country.setColumnGroup(3);
    country.setSortColumn(true);
    columnSchemas.add(country);
    ColumnSchema name = new ColumnSchema();
    name.setColumnName("name");
    name.setColumnar(true);
    name.setDataType(DataType.STRING);
    name.setEncodingList(encodings);
    name.setColumnUniqueId(UUID.randomUUID().toString());
    name.setDimensionColumn(true);
    name.setColumnGroup(4);
    name.setSortColumn(true);
    columnSchemas.add(name);
    ColumnSchema phonetype = new ColumnSchema();
    phonetype.setColumnName("phonetype");
    phonetype.setColumnar(true);
    phonetype.setDataType(DataType.STRING);
    phonetype.setEncodingList(encodings);
    phonetype.setColumnUniqueId(UUID.randomUUID().toString());
    phonetype.setDimensionColumn(true);
    phonetype.setColumnGroup(5);
    phonetype.setSortColumn(true);
    columnSchemas.add(phonetype);
    ColumnSchema serialname = new ColumnSchema();
    serialname.setColumnName("serialname");
    serialname.setColumnar(true);
    serialname.setDataType(DataType.STRING);
    serialname.setEncodingList(encodings);
    serialname.setColumnUniqueId(UUID.randomUUID().toString());
    serialname.setDimensionColumn(true);
    serialname.setColumnGroup(6);
    serialname.setSortColumn(true);
    columnSchemas.add(serialname);
    ColumnSchema salary = new ColumnSchema();
    salary.setColumnName("salary");
    salary.setColumnar(true);
    salary.setDataType(DataType.INT);
    salary.setEncodingList(new ArrayList<Encoding>());
    salary.setColumnUniqueId(UUID.randomUUID().toString());
    salary.setDimensionColumn(false);
    salary.setColumnGroup(7);
    columnSchemas.add(salary);
    tableSchema.setListOfColumns(columnSchemas);
    SchemaEvolution schemaEvol = new SchemaEvolution();
    schemaEvol.setSchemaEvolutionEntryList(new ArrayList<SchemaEvolutionEntry>());
    tableSchema.setSchemaEvalution(schemaEvol);
    tableSchema.setTableId(UUID.randomUUID().toString());
    tableInfo.setTableUniqueName(absoluteTableIdentifier.getCarbonTableIdentifier().getDatabaseName() + "_" + absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
    tableInfo.setLastUpdatedTime(System.currentTimeMillis());
    tableInfo.setFactTable(tableSchema);
    tableInfo.setAggregateTableList(new ArrayList<TableSchema>());
    CarbonTablePath carbonTablePath = CarbonStorePath.getCarbonTablePath(absoluteTableIdentifier.getStorePath(), absoluteTableIdentifier.getCarbonTableIdentifier());
    String schemaFilePath = carbonTablePath.getSchemaFilePath();
    String schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath);
    tableInfo.setMetaDataFilepath(schemaMetadataPath);
    CarbonMetadata.getInstance().loadTableMetadata(tableInfo);
    SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
    org.apache.carbondata.format.TableInfo thriftTableInfo = schemaConverter.fromWrapperToExternalTableInfo(tableInfo, tableInfo.getDatabaseName(), tableInfo.getFactTable().getTableName());
    org.apache.carbondata.format.SchemaEvolutionEntry schemaEvolutionEntry = new org.apache.carbondata.format.SchemaEvolutionEntry(tableInfo.getLastUpdatedTime());
    thriftTableInfo.getFact_table().getSchema_evolution().getSchema_evolution_history().add(schemaEvolutionEntry);
    FileFactory.FileType fileType = FileFactory.getFileType(schemaMetadataPath);
    if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
        FileFactory.mkdirs(schemaMetadataPath, fileType);
    }
    ThriftWriter thriftWriter = new ThriftWriter(schemaFilePath, false);
    thriftWriter.open();
    thriftWriter.write(thriftTableInfo);
    thriftWriter.close();
    return CarbonMetadata.getInstance().getCarbonTable(tableInfo.getTableUniqueName());
}

Example 3 with ThriftWrapperSchemaConverterImpl

use of org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl in project carbondata by apache.

the class StoreCreator method createTable.

public static CarbonTable createTable(AbsoluteTableIdentifier identifier) throws IOException {
    TableInfo tableInfo = new TableInfo();
    tableInfo.setDatabaseName(identifier.getCarbonTableIdentifier().getDatabaseName());
    TableSchema tableSchema = new TableSchema();
    tableSchema.setTableName(identifier.getCarbonTableIdentifier().getTableName());
    List<ColumnSchema> columnSchemas = new ArrayList<ColumnSchema>();
    ArrayList<Encoding> encodings = new ArrayList<>();
    encodings.add(Encoding.DICTIONARY);
    ColumnSchema id = new ColumnSchema();
    id.setColumnName("ID");
    id.setColumnar(true);
    id.setDataType(DataTypes.INT);
    id.setEncodingList(encodings);
    id.setColumnUniqueId(UUID.randomUUID().toString());
    id.setColumnReferenceId(id.getColumnUniqueId());
    id.setDimensionColumn(true);
    id.setColumnGroup(1);
    columnSchemas.add(id);
    ColumnSchema date = new ColumnSchema();
    date.setColumnName("date");
    date.setColumnar(true);
    date.setDataType(DataTypes.STRING);
    date.setEncodingList(encodings);
    date.setColumnUniqueId(UUID.randomUUID().toString());
    date.setDimensionColumn(true);
    date.setColumnGroup(2);
    date.setSortColumn(true);
    date.setColumnReferenceId(id.getColumnUniqueId());
    columnSchemas.add(date);
    ColumnSchema country = new ColumnSchema();
    country.setColumnName("country");
    country.setColumnar(true);
    country.setDataType(DataTypes.STRING);
    country.setEncodingList(encodings);
    country.setColumnUniqueId(UUID.randomUUID().toString());
    country.setDimensionColumn(true);
    country.setColumnGroup(3);
    country.setSortColumn(true);
    country.setColumnReferenceId(id.getColumnUniqueId());
    columnSchemas.add(country);
    ColumnSchema name = new ColumnSchema();
    name.setColumnName("name");
    name.setColumnar(true);
    name.setDataType(DataTypes.STRING);
    name.setEncodingList(encodings);
    name.setColumnUniqueId(UUID.randomUUID().toString());
    name.setDimensionColumn(true);
    name.setColumnGroup(4);
    name.setSortColumn(true);
    name.setColumnReferenceId(id.getColumnUniqueId());
    columnSchemas.add(name);
    ColumnSchema phonetype = new ColumnSchema();
    phonetype.setColumnName("phonetype");
    phonetype.setColumnar(true);
    phonetype.setDataType(DataTypes.STRING);
    phonetype.setEncodingList(encodings);
    phonetype.setColumnUniqueId(UUID.randomUUID().toString());
    phonetype.setDimensionColumn(true);
    phonetype.setColumnGroup(5);
    phonetype.setSortColumn(true);
    phonetype.setColumnReferenceId(id.getColumnUniqueId());
    columnSchemas.add(phonetype);
    ColumnSchema serialname = new ColumnSchema();
    serialname.setColumnName("serialname");
    serialname.setColumnar(true);
    serialname.setDataType(DataTypes.STRING);
    serialname.setEncodingList(encodings);
    serialname.setColumnUniqueId(UUID.randomUUID().toString());
    serialname.setDimensionColumn(true);
    serialname.setColumnGroup(6);
    serialname.setSortColumn(true);
    serialname.setColumnReferenceId(id.getColumnUniqueId());
    columnSchemas.add(serialname);
    ColumnSchema salary = new ColumnSchema();
    salary.setColumnName("salary");
    salary.setColumnar(true);
    salary.setDataType(DataTypes.INT);
    salary.setEncodingList(new ArrayList<Encoding>());
    salary.setColumnUniqueId(UUID.randomUUID().toString());
    salary.setDimensionColumn(false);
    salary.setColumnReferenceId(id.getColumnUniqueId());
    salary.setColumnGroup(7);
    columnSchemas.add(salary);
    tableSchema.setListOfColumns(columnSchemas);
    SchemaEvolution schemaEvol = new SchemaEvolution();
    schemaEvol.setSchemaEvolutionEntryList(new ArrayList<SchemaEvolutionEntry>());
    tableSchema.setSchemaEvalution(schemaEvol);
    tableSchema.setTableId(UUID.randomUUID().toString());
    tableInfo.setTableUniqueName(identifier.getCarbonTableIdentifier().getTableUniqueName());
    tableInfo.setLastUpdatedTime(System.currentTimeMillis());
    tableInfo.setFactTable(tableSchema);
    tableInfo.setTablePath(identifier.getTablePath());
    String schemaFilePath = CarbonTablePath.getSchemaFilePath(identifier.getTablePath());
    String schemaMetadataPath = CarbonTablePath.getFolderContainingFile(schemaFilePath);
    CarbonMetadata.getInstance().loadTableMetadata(tableInfo);
    SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
    org.apache.carbondata.format.TableInfo thriftTableInfo = schemaConverter.fromWrapperToExternalTableInfo(tableInfo, tableInfo.getDatabaseName(), tableInfo.getFactTable().getTableName());
    org.apache.carbondata.format.SchemaEvolutionEntry schemaEvolutionEntry = new org.apache.carbondata.format.SchemaEvolutionEntry(tableInfo.getLastUpdatedTime());
    thriftTableInfo.getFact_table().getSchema_evolution().getSchema_evolution_history().add(schemaEvolutionEntry);
    FileFactory.FileType fileType = FileFactory.getFileType(schemaMetadataPath);
    if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
        FileFactory.mkdirs(schemaMetadataPath, fileType);
    }
    ThriftWriter thriftWriter = new ThriftWriter(schemaFilePath, false);
    thriftWriter.open();
    thriftWriter.write(thriftTableInfo);
    thriftWriter.close();
    return CarbonMetadata.getInstance().getCarbonTable(tableInfo.getTableUniqueName());
}

Example 4 with ThriftWrapperSchemaConverterImpl

use of org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl in project carbondata by apache.

the class CarbonUtil method inferSchema.

/**
 * This method will read the schema file from a given path
 *
 * @param schemaFilePath
 * @return
 */
public static org.apache.carbondata.format.TableInfo inferSchema(String carbonDataFilePath, AbsoluteTableIdentifier absoluteTableIdentifier, boolean schemaExists) throws IOException {
    TBaseCreator createTBase = new ThriftReader.TBaseCreator() {

        public org.apache.thrift.TBase<org.apache.carbondata.format.TableInfo, org.apache.carbondata.format.TableInfo._Fields> create() {
            return new org.apache.carbondata.format.TableInfo();
        }
    };
    if (schemaExists == false) {
        List<String> filePaths = getFilePathExternalFilePath(carbonDataFilePath + "/Fact/Part0/Segment_null");
        String fistFilePath = null;
        try {
            fistFilePath = filePaths.get(0);
        } catch (Exception e) {
            LOGGER.error("CarbonData file is not present in the table location");
        }
        CarbonHeaderReader carbonHeaderReader = new CarbonHeaderReader(fistFilePath);
        FileHeader fileHeader = carbonHeaderReader.readHeader();
        List<ColumnSchema> columnSchemaList = new ArrayList<ColumnSchema>();
        List<org.apache.carbondata.format.ColumnSchema> table_columns = fileHeader.getColumn_schema();
        for (int i = 0; i < table_columns.size(); i++) {
            ColumnSchema col = thriftColumnSchmeaToWrapperColumnSchema(table_columns.get(i));
            col.setColumnReferenceId(col.getColumnUniqueId());
            columnSchemaList.add(col);
        }
        TableSchema tableSchema = new TableSchema();
        tableSchema.setTableName(absoluteTableIdentifier.getTableName());
        tableSchema.setBucketingInfo(null);
        tableSchema.setSchemaEvalution(null);
        tableSchema.setTableId(UUID.randomUUID().toString());
        tableSchema.setListOfColumns(columnSchemaList);
        ThriftWrapperSchemaConverterImpl thriftWrapperSchemaConverter = new ThriftWrapperSchemaConverterImpl();
        SchemaEvolutionEntry schemaEvolutionEntry = new SchemaEvolutionEntry();
        schemaEvolutionEntry.setTimeStamp(System.currentTimeMillis());
        SchemaEvolution schemaEvol = new SchemaEvolution();
        List<SchemaEvolutionEntry> schEntryList = new ArrayList<>();
        schEntryList.add(schemaEvolutionEntry);
        schemaEvol.setSchemaEvolutionEntryList(schEntryList);
        tableSchema.setSchemaEvalution(schemaEvol);
        org.apache.carbondata.format.TableSchema thriftFactTable = thriftWrapperSchemaConverter.fromWrapperToExternalTableSchema(tableSchema);
        org.apache.carbondata.format.TableInfo tableInfo = new org.apache.carbondata.format.TableInfo(thriftFactTable, new ArrayList<org.apache.carbondata.format.TableSchema>());
        tableInfo.setDataMapSchemas(null);
        return tableInfo;
    } else {
        ThriftReader thriftReader = new ThriftReader(carbonDataFilePath, createTBase);
        thriftReader.open();
        org.apache.carbondata.format.TableInfo tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
        thriftReader.close();
        return tableInfo;
    }
}

Example 5 with ThriftWrapperSchemaConverterImpl

use of org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl in project carbondata by apache.

the class SchemaReader method readCarbonTableFromStore.

public static CarbonTable readCarbonTableFromStore(AbsoluteTableIdentifier identifier) throws IOException {
    String schemaFilePath = CarbonTablePath.getSchemaFilePath(identifier.getTablePath());
    if (FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.LOCAL) || FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.HDFS) || FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.S3) || FileFactory.isFileExist(schemaFilePath, FileFactory.FileType.VIEWFS)) {
        String tableName = identifier.getCarbonTableIdentifier().getTableName();
        org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil.readSchemaFile(CarbonTablePath.getSchemaFilePath(identifier.getTablePath()));
        SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
        TableInfo wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(tableInfo, identifier.getCarbonTableIdentifier().getDatabaseName(), tableName, identifier.getTablePath());
        CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo);
        return CarbonMetadata.getInstance().getCarbonTable(identifier.getCarbonTableIdentifier().getTableUniqueName());
    } else {
        throw new IOException("File does not exist: " + schemaFilePath);
    }
}