Examples with CarbonTable org.apache.carbondata.core.metadata.schema.table.CarbonTable used on opensource projects

Example 76 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class CarbonTableOutputFormat method getLoadModel.

public static CarbonLoadModel getLoadModel(Configuration conf) throws IOException {
    CarbonLoadModel model;
    String encodedString = conf.get(LOAD_MODEL);
    if (encodedString != null) {
        model = (CarbonLoadModel) ObjectSerializationUtil.convertStringToObject(encodedString);
        return model;
    }
    model = new CarbonLoadModel();
    CarbonProperties carbonProperty = CarbonProperties.getInstance();
    model.setDatabaseName(CarbonTableOutputFormat.getDatabaseName(conf));
    model.setTableName(CarbonTableOutputFormat.getTableName(conf));
    model.setCarbonTransactionalTable(true);
    model.setMetrics(new DataLoadMetrics());
    CarbonTable carbonTable = getCarbonTable(conf);
    // global dictionary is not supported since 2.0
    if (carbonTable.getTableInfo().getFactTable().getTableProperties().containsKey(CarbonCommonConstants.DICTIONARY_INCLUDE)) {
        DeprecatedFeatureException.globalDictNotSupported();
    }
    String columnCompressor = carbonTable.getTableInfo().getFactTable().getTableProperties().get(CarbonCommonConstants.COMPRESSOR);
    if (null == columnCompressor) {
        columnCompressor = CompressorFactory.getInstance().getCompressor().getName();
    }
    model.setColumnCompressor(columnCompressor);
    model.setCarbonDataLoadSchema(new CarbonDataLoadSchema(carbonTable));
    model.setTablePath(getTablePath(conf));
    setFileHeader(conf, model);
    model.setSerializationNullFormat(conf.get(SERIALIZATION_NULL_FORMAT, "\\N"));
    model.setBadRecordsLoggerEnable(conf.get(BAD_RECORDS_LOGGER_ENABLE, carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE, CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORDS_LOGGER_ENABLE_DEFAULT)));
    model.setBadRecordsAction(conf.get(BAD_RECORDS_LOGGER_ACTION, carbonProperty.getProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION_DEFAULT)));
    model.setIsEmptyDataBadRecord(conf.get(IS_EMPTY_DATA_BAD_RECORD, carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD, CarbonLoadOptionConstants.CARBON_OPTIONS_IS_EMPTY_DATA_BAD_RECORD_DEFAULT)));
    model.setSkipEmptyLine(conf.get(SKIP_EMPTY_LINE, carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_SKIP_EMPTY_LINE)));
    String complexDelim = conf.get(COMPLEX_DELIMITERS);
    if (null == complexDelim) {
        complexDelim = ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_1.value() + "," + ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_2.value() + "," + ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_3.value() + "," + ComplexDelimitersEnum.COMPLEX_DELIMITERS_LEVEL_4.value();
    }
    String[] split = complexDelim.split(",");
    model.setComplexDelimiter(split[0]);
    if (split.length > 3) {
        model.setComplexDelimiter(split[1]);
        model.setComplexDelimiter(split[2]);
        model.setComplexDelimiter(split[3]);
    } else if (split.length > 2) {
        model.setComplexDelimiter(split[1]);
        model.setComplexDelimiter(split[2]);
    } else if (split.length > 1) {
        model.setComplexDelimiter(split[1]);
    }
    model.setDateFormat(conf.get(DATE_FORMAT, carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT, CarbonLoadOptionConstants.CARBON_OPTIONS_DATEFORMAT_DEFAULT)));
    model.setTimestampFormat(conf.get(TIMESTAMP_FORMAT, carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT, CarbonLoadOptionConstants.CARBON_OPTIONS_TIMESTAMPFORMAT_DEFAULT)));
    model.setGlobalSortPartitions(conf.get(GLOBAL_SORT_PARTITIONS, carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_GLOBAL_SORT_PARTITIONS, null)));
    String badRecordsPath = conf.get(BAD_RECORD_PATH);
    if (StringUtils.isEmpty(badRecordsPath)) {
        badRecordsPath = carbonTable.getTableInfo().getFactTable().getTableProperties().get("bad_record_path");
        if (StringUtils.isEmpty(badRecordsPath)) {
            badRecordsPath = carbonProperty.getProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BAD_RECORD_PATH, carbonProperty.getProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC, CarbonCommonConstants.CARBON_BADRECORDS_LOC_DEFAULT_VAL));
        }
    }
    model.setBadRecordsLocation(badRecordsPath);
    return model;
}

Example 77 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class CarbonTableOutputFormat method getCarbonTable.

public static CarbonTable getCarbonTable(Configuration configuration) throws IOException {
    CarbonTable carbonTable = null;
    String encodedString = configuration.get(TABLE);
    if (encodedString != null) {
        byte[] bytes = (byte[]) ObjectSerializationUtil.convertStringToObject(encodedString);
        TableInfo tableInfo = TableInfo.deserialize(bytes);
        carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
    }
    return carbonTable;
}

Example 78 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class CarbonInputFormat method getOrCreateCarbonTable.

/**
 * Get the cached CarbonTable or create it by TableInfo in `configuration`
 */
public CarbonTable getOrCreateCarbonTable(Configuration configuration) throws IOException {
    if (carbonTable == null) {
        // carbon table should be created either from deserialized table info (schema saved in
        // hive metastore) or by reading schema in HDFS (schema saved in HDFS)
        TableInfo tableInfo = getTableInfo(configuration);
        CarbonTable carbonTable;
        if (tableInfo != null) {
            carbonTable = CarbonTable.buildFromTableInfo(tableInfo);
        } else {
            carbonTable = SchemaReader.readCarbonTableFromStore(getAbsoluteTableIdentifier(configuration));
        }
        this.carbonTable = carbonTable;
        return carbonTable;
    } else {
        return this.carbonTable;
    }
}

Example 79 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class CarbonInputFormat method getFilterPredicates.

public IndexFilter getFilterPredicates(Configuration configuration) {
    try {
        String filterExprString = configuration.get(FILTER_PREDICATE);
        if (filterExprString == null) {
            return null;
        }
        IndexFilter filter = (IndexFilter) ObjectSerializationUtil.convertStringToObject(filterExprString);
        if (filter != null) {
            CarbonTable carbonTable = getOrCreateCarbonTable(configuration);
            filter.setTable(carbonTable);
        }
        return filter;
    } catch (IOException e) {
        throw new RuntimeException("Error while reading filter expression", e);
    }
}

Example 80 with CarbonTable

use of org.apache.carbondata.core.metadata.schema.table.CarbonTable in project carbondata by apache.

the class CarbonFileInputFormat method getOrCreateCarbonTable.

public CarbonTable getOrCreateCarbonTable(Configuration configuration) throws IOException {
    CarbonTable carbonTableTemp;
    if (carbonTable == null) {
        // carbon table should be created either from deserialized table info (schema saved in
        // hive metastore) or by reading schema in HDFS (schema saved in HDFS)
        TableInfo tableInfo = getTableInfo(configuration);
        CarbonTable localCarbonTable;
        if (tableInfo != null) {
            localCarbonTable = CarbonTable.buildFromTableInfo(tableInfo);
        } else {
            String schemaPath = CarbonTablePath.getSchemaFilePath(getAbsoluteTableIdentifier(configuration).getTablePath());
            if (!FileFactory.isFileExist(schemaPath)) {
                TableInfo tableInfoInfer = SchemaReader.inferSchema(getAbsoluteTableIdentifier(configuration), true);
                localCarbonTable = CarbonTable.buildFromTableInfo(tableInfoInfer);
            } else {
                localCarbonTable = SchemaReader.readCarbonTableFromStore(getAbsoluteTableIdentifier(configuration));
            }
        }
        this.carbonTable = localCarbonTable;
        return localCarbonTable;
    } else {
        carbonTableTemp = this.carbonTable;
        return carbonTableTemp;
    }
}