Examples with BadRecordLogHolder org.apache.carbondata.processing.loading.converter.BadRecordLogHolder used on opensource projects

Example 1 with BadRecordLogHolder

use of org.apache.carbondata.processing.loading.converter.BadRecordLogHolder in project carbondata by apache.

the class DataConverterProcessorStepImpl method convertFakeRow.

// only convert sort column fields
private void convertFakeRow(CarbonRow fakeRow, SortColumnRangeInfo sortColumnRangeInfo) {
    FieldConverter[] fieldConverters = converters.get(0).getFieldConverters();
    BadRecordLogHolder logHolder = new BadRecordLogHolder();
    logHolder.setLogged(false);
    for (int colIdx : sortColumnRangeInfo.getSortColumnIndex()) {
        fieldConverters[colIdx].convert(fakeRow, logHolder);
    }
}

Example 2 with BadRecordLogHolder

use of org.apache.carbondata.processing.loading.converter.BadRecordLogHolder in project carbondata by apache.

the class BloomCoarseGrainIndex method initIndexColumnConverters.

/**
 * init field converters for index columns
 */
public void initIndexColumnConverters(CarbonTable carbonTable, List<CarbonColumn> indexedColumn) {
    this.name2Col = new HashMap<>(indexedColumn.size());
    for (CarbonColumn col : indexedColumn) {
        this.name2Col.put(col.getColName(), col);
    }
    this.name2Converters = new HashMap<>(indexedColumn.size());
    String nullFormat = "\\N";
    for (int i = 0; i < indexedColumn.size(); i++) {
        DataField dataField = new DataField(indexedColumn.get(i));
        String dateFormat = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT);
        dataField.setDateFormat(dateFormat);
        String tsFormat = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT);
        dataField.setTimestampFormat(tsFormat);
        FieldConverter fieldConverter = FieldEncoderFactory.getInstance().createFieldEncoder(dataField, i, nullFormat, false, false, carbonTable.getTablePath(), null);
        this.name2Converters.put(indexedColumn.get(i).getColName(), fieldConverter);
    }
    this.badRecordLogHolder = new BadRecordLogHolder();
    this.badRecordLogHolder.setLogged(false);
}

Example 3 with BadRecordLogHolder

use of org.apache.carbondata.processing.loading.converter.BadRecordLogHolder in project carbondata by apache.

the class RowConverterImpl method initialize.

@Override
public void initialize() {
    String nullFormat = configuration.getDataLoadProperty(DataLoadProcessorConstants.SERIALIZATION_NULL_FORMAT).toString();
    boolean isEmptyBadRecord = Boolean.parseBoolean(configuration.getDataLoadProperty(DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD).toString());
    List<FieldConverter> fieldConverterList = new ArrayList<>();
    List<FieldConverter> nonSchemaFieldConverterList = new ArrayList<>();
    long lruCacheStartTime = System.currentTimeMillis();
    Map<String, String> properties = configuration.getTableSpec().getCarbonTable().getTableInfo().getFactTable().getTableProperties();
    String spatialProperty = properties.get(CarbonCommonConstants.SPATIAL_INDEX);
    for (int i = 0; i < fields.length; i++) {
        FieldConverter fieldConverter = FieldEncoderFactory.getInstance().createFieldEncoder(fields[i], i, nullFormat, isEmptyBadRecord, isConvertToBinary, (String) configuration.getDataLoadProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BINARY_DECODER), configuration);
        if (spatialProperty != null && fields[i].getColumn().getColName().equalsIgnoreCase(spatialProperty.trim())) {
            nonSchemaFieldConverterList.add(fieldConverter);
        } else {
            fieldConverterList.add(fieldConverter);
        }
    }
    fieldConverterList.addAll(nonSchemaFieldConverterList);
    CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordLruCacheLoadTime((System.currentTimeMillis() - lruCacheStartTime) / 1000.0);
    fieldConverters = fieldConverterList.toArray(new FieldConverter[0]);
    logHolder = new BadRecordLogHolder();
}

Example 4 with BadRecordLogHolder

use of org.apache.carbondata.processing.loading.converter.BadRecordLogHolder in project carbondata by apache.

the class RowConverterImpl method createCopyForNewThread.

@Override
public RowConverter createCopyForNewThread() {
    RowConverterImpl converter = new RowConverterImpl(this.fields, this.configuration, this.badRecordLogger, this.isConvertToBinary);
    List<FieldConverter> fieldConverterList = new ArrayList<>();
    List<FieldConverter> nonSchemaFieldConverterList = new ArrayList<>();
    String nullFormat = configuration.getDataLoadProperty(DataLoadProcessorConstants.SERIALIZATION_NULL_FORMAT).toString();
    boolean isEmptyBadRecord = Boolean.parseBoolean(configuration.getDataLoadProperty(DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD).toString());
    Map<String, String> properties = configuration.getTableSpec().getCarbonTable().getTableInfo().getFactTable().getTableProperties();
    String spatialProperty = properties.get(CarbonCommonConstants.SPATIAL_INDEX);
    for (int i = 0; i < fields.length; i++) {
        FieldConverter fieldConverter = FieldEncoderFactory.getInstance().createFieldEncoder(fields[i], i, nullFormat, isEmptyBadRecord, isConvertToBinary, (String) configuration.getDataLoadProperty(CarbonLoadOptionConstants.CARBON_OPTIONS_BINARY_DECODER), configuration);
        if (spatialProperty != null && fields[i].getColumn().getColName().equalsIgnoreCase(spatialProperty.trim())) {
            nonSchemaFieldConverterList.add(fieldConverter);
        } else {
            fieldConverterList.add(fieldConverter);
        }
    }
    fieldConverterList.addAll(nonSchemaFieldConverterList);
    converter.fieldConverters = fieldConverterList.toArray(new FieldConverter[0]);
    converter.logHolder = new BadRecordLogHolder();
    return converter;
}