Examples with SchemaConverter org.apache.carbondata.core.metadata.converter.SchemaConverter used on opensource projects

Example 6 with SchemaConverter

use of org.apache.carbondata.core.metadata.converter.SchemaConverter in project carbondata by apache.

the class CarbonWriterBuilder method persistSchemaFile.

/**
 * Save the schema of the {@param table} to {@param persistFilePath}
 * @param table table object containing schema
 * @param persistFilePath absolute file path with file name
 */
private void persistSchemaFile(CarbonTable table, String persistFilePath) throws IOException {
    TableInfo tableInfo = table.getTableInfo();
    String schemaMetadataPath = CarbonTablePath.getFolderContainingFile(persistFilePath);
    CarbonMetadata.getInstance().loadTableMetadata(tableInfo);
    SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
    org.apache.carbondata.format.TableInfo thriftTableInfo = schemaConverter.fromWrapperToExternalTableInfo(tableInfo, tableInfo.getDatabaseName(), tableInfo.getFactTable().getTableName());
    org.apache.carbondata.format.SchemaEvolutionEntry schemaEvolutionEntry = new org.apache.carbondata.format.SchemaEvolutionEntry(tableInfo.getLastUpdatedTime());
    thriftTableInfo.getFact_table().getSchema_evolution().getSchema_evolution_history().add(schemaEvolutionEntry);
    FileFactory.FileType fileType = FileFactory.getFileType(schemaMetadataPath);
    if (!FileFactory.isFileExist(schemaMetadataPath, fileType)) {
        FileFactory.mkdirs(schemaMetadataPath, fileType);
    }
    ThriftWriter thriftWriter = new ThriftWriter(persistFilePath, false);
    thriftWriter.open();
    thriftWriter.write(thriftTableInfo);
    thriftWriter.close();
}

Example 7 with SchemaConverter

use of org.apache.carbondata.core.metadata.converter.SchemaConverter in project carbondata by apache.

the class AbstractFactDataWriter method getColumnSchemaListAndCardinality.

public static List<org.apache.carbondata.format.ColumnSchema> getColumnSchemaListAndCardinality(List<Integer> cardinality, int[] dictionaryColumnCardinality, List<ColumnSchema> wrapperColumnSchemaList) {
    List<org.apache.carbondata.format.ColumnSchema> columnSchemaList = new ArrayList<org.apache.carbondata.format.ColumnSchema>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
    int counter = 0;
    for (int i = 0; i < wrapperColumnSchemaList.size(); i++) {
        columnSchemaList.add(schemaConverter.fromWrapperToExternalColumnSchema(wrapperColumnSchemaList.get(i)));
        if (CarbonUtil.hasEncoding(wrapperColumnSchemaList.get(i).getEncodingList(), org.apache.carbondata.core.metadata.encoder.Encoding.DICTIONARY)) {
            cardinality.add(dictionaryColumnCardinality[counter]);
            counter++;
        } else if (!wrapperColumnSchemaList.get(i).isDimensionColumn()) {
            continue;
        } else {
            cardinality.add(-1);
        }
    }
    return columnSchemaList;
}

Example 8 with SchemaConverter

use of org.apache.carbondata.core.metadata.converter.SchemaConverter in project carbondata by apache.

the class SchemaReader method inferSchema.

public static TableInfo inferSchema(AbsoluteTableIdentifier identifier) throws IOException {
    // This routine is going to infer schema from the carbondata file footer
    // Convert the ColumnSchema -> TableSchema -> TableInfo.
    // Return the TableInfo.
    org.apache.carbondata.format.TableInfo tableInfo = CarbonUtil.inferSchema(identifier.getTablePath(), identifier, false);
    SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
    TableInfo wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(tableInfo, identifier.getDatabaseName(), identifier.getTableName(), identifier.getTablePath());
    return wrapperTableInfo;
}

Example 9 with SchemaConverter

use of org.apache.carbondata.core.metadata.converter.SchemaConverter in project carbondata by apache.

the class CarbonTableReader method parseCarbonMetadata.

/**
 * Read the metadata of the given table and cache it in this.carbonCache (CarbonTableReader cache).
 *
 * @param table name of the given table.
 * @return the CarbonTable instance which contains all the needed metadata for a table.
 */
private CarbonTable parseCarbonMetadata(SchemaTableName table) {
    CarbonTable result = null;
    try {
        CarbonTableCacheModel cache = carbonCache.get().get(table);
        if (cache == null) {
            cache = new CarbonTableCacheModel();
        }
        if (cache.isValid())
            return cache.carbonTable;
        // If table is not previously cached, then:
        // Step 1: get store path of the table and cache it.
        // create table identifier. the table id is randomly generated.
        CarbonTableIdentifier carbonTableIdentifier = new CarbonTableIdentifier(table.getSchemaName(), table.getTableName(), UUID.randomUUID().toString());
        String storePath = config.getStorePath();
        String tablePath = storePath + "/" + carbonTableIdentifier.getDatabaseName() + "/" + carbonTableIdentifier.getTableName();
        // Step 2: read the metadata (tableInfo) of the table.
        ThriftReader.TBaseCreator createTBase = new ThriftReader.TBaseCreator() {

            // TBase is used to read and write thrift objects.
            // TableInfo is a kind of TBase used to read and write table information.
            // TableInfo is generated by thrift, see schema.thrift under format/src/main/thrift for details.
            public TBase create() {
                return new org.apache.carbondata.format.TableInfo();
            }
        };
        ThriftReader thriftReader = new ThriftReader(CarbonTablePath.getSchemaFilePath(tablePath), createTBase);
        thriftReader.open();
        org.apache.carbondata.format.TableInfo tableInfo = (org.apache.carbondata.format.TableInfo) thriftReader.read();
        thriftReader.close();
        // Step 3: convert format level TableInfo to code level TableInfo
        SchemaConverter schemaConverter = new ThriftWrapperSchemaConverterImpl();
        // wrapperTableInfo is the code level information of a table in carbondata core, different from the Thrift TableInfo.
        TableInfo wrapperTableInfo = schemaConverter.fromExternalToWrapperTableInfo(tableInfo, table.getSchemaName(), table.getTableName(), tablePath);
        // Step 4: Load metadata info into CarbonMetadata
        CarbonMetadata.getInstance().loadTableMetadata(wrapperTableInfo);
        cache.carbonTable = CarbonMetadata.getInstance().getCarbonTable(table.getSchemaName(), table.getTableName());
        // cache the table
        carbonCache.get().put(table, cache);
        result = cache.carbonTable;
    } catch (Exception ex) {
        throw new RuntimeException(ex);
    }
    return result;
}