Examples with RowConverterImpl org.apache.carbondata.processing.loading.converter.impl.RowConverterImpl used on opensource projects

Example 1 with RowConverterImpl

use of org.apache.carbondata.processing.loading.converter.impl.RowConverterImpl in project carbondata by apache.

the class InputProcessorStepForPartitionImpl method initialize.

@Override
public void initialize() throws IOException {
    super.initialize();
    // if logger is enabled then raw data will be required.
    RowConverterImpl rowConverter = new RowConverterImpl(configuration.getDataFields(), configuration, null);
    rowConverter.initialize();
    configuration.setCardinalityFinder(rowConverter);
    noDictionaryMapping = CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
    dataTypes = new DataType[configuration.getDataFields().length];
    for (int i = 0; i < dataTypes.length; i++) {
        if (configuration.getDataFields()[i].getColumn().hasEncoding(Encoding.DICTIONARY)) {
            dataTypes[i] = DataTypes.INT;
        } else {
            dataTypes[i] = configuration.getDataFields()[i].getColumn().getDataType();
        }
    }
    orderOfData = arrangeData(configuration.getDataFields(), configuration.getHeader());
}

Example 2 with RowConverterImpl

use of org.apache.carbondata.processing.loading.converter.impl.RowConverterImpl in project carbondata by apache.

the class InputProcessorStepWithNoConverterImpl method initialize.

@Override
public void initialize() throws IOException {
    super.initialize();
    // if logger is enabled then raw data will be required.
    rowConverter = new RowConverterImpl(configuration.getDataFields(), configuration, null);
    rowConverter.initialize();
    if (!withoutReArrange) {
        noDictionaryMapping = CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
    }
    dataFieldsWithComplexDataType = new HashMap<>();
    convertComplexDataType(dataFieldsWithComplexDataType);
    dataTypes = new DataType[configuration.getDataFields().length];
    for (int i = 0; i < dataTypes.length; i++) {
        if (configuration.getDataFields()[i].getColumn().getDataType() == DataTypes.DATE) {
            dataTypes[i] = DataTypes.INT;
        } else {
            dataTypes[i] = configuration.getDataFields()[i].getColumn().getDataType();
        }
    }
    if (!withoutReArrange) {
        orderOfData = arrangeData(configuration.getDataFields(), configuration.getHeader());
    }
    if (null != configuration.getBucketingInfo()) {
        this.isBucketColumnEnabled = true;
        initializeBucketColumnPartitioner();
    }
}

Example 3 with RowConverterImpl

use of org.apache.carbondata.processing.loading.converter.impl.RowConverterImpl in project carbondata by apache.

the class DataConverterProcessorStepImpl method initialize.

@Override
public void initialize() throws IOException {
    super.initialize();
    child.initialize();
    converters = new ArrayList<>();
    badRecordLogger = BadRecordsLoggerProvider.createBadRecordLogger(configuration);
    RowConverter converter = new RowConverterImpl(child.getOutput(), configuration, badRecordLogger);
    converters.add(converter);
    converter.initialize();
    if (null != configuration.getBucketingInfo()) {
        this.isBucketColumnEnabled = true;
        initializeBucketColumnPartitioner();
    } else if (null != configuration.getSortColumnRangeInfo()) {
        this.isSortColumnRangeEnabled = true;
        initializeSortColumnRangesPartitioner();
    }
}

Example 4 with RowConverterImpl

use of org.apache.carbondata.processing.loading.converter.impl.RowConverterImpl in project carbondata by apache.

the class CarbonStreamRecordWriter method initializeAtFirstRow.

private void initializeAtFirstRow() throws IOException {
    // initialize parser and converter
    rowParser = new RowParserImpl(dataFields, configuration);
    badRecordLogger = BadRecordsLoggerProvider.createBadRecordLogger(configuration);
    converter = new RowConverterImpl(configuration.getDataFields(), configuration, badRecordLogger, true);
    converter.initialize();
    // initialize data writer and compressor
    String filePath = segmentDir + File.separator + fileName;
    CarbonFile carbonFile = FileFactory.getCarbonFile(filePath);
    if (carbonFile.exists()) {
        // if the file is existed, use the append api
        outputStream = FileFactory.getDataOutputStreamUsingAppend(filePath);
        // get the compressor from the file header. In legacy store,
        // the compressor name is not set and it use snappy compressor
        FileHeader header = new CarbonHeaderReader(filePath).readHeader();
        if (header.isSetCompressor_name()) {
            compressorName = header.getCompressor_name();
        } else {
            compressorName = CarbonProperties.getInstance().getDefaultCompressor();
        }
    } else {
        // IF the file is not existed, use the create api
        outputStream = FileFactory.getDataOutputStream(filePath);
        compressorName = carbonTable.getTableInfo().getFactTable().getTableProperties().get(CarbonCommonConstants.COMPRESSOR);
        if (null == compressorName) {
            compressorName = CarbonProperties.getInstance().getDefaultCompressor();
        }
        writeFileHeader();
    }
    // initialize encoder
    nullBitSet = new BitSet(dataFields.length);
    int rowBufferSize = hadoopConf.getInt(CarbonStreamOutputFormat.CARBON_ENCODER_ROW_BUFFER_SIZE, CarbonStreamOutputFormat.CARBON_ENCODER_ROW_BUFFER_SIZE_DEFAULT);
    output = new StreamBlockletWriter(maxCacheSize, maxRowNums, rowBufferSize, isNoDictionaryDimensionColumn.length, measureCount, measureDataTypes, compressorName);
    isFirstRow = false;
}