use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.
the class ValueCompressionUtilTest method testToGetValueCompressionModelForFloatAndDataTypeSelected1.
@Test
public void testToGetValueCompressionModelForFloatAndDataTypeSelected1() {
Object[] maxValues = { 32725566.00 };
Object[] minValues = { 32744.0 };
int[] decimalLength = { 1 };
Object[] uniqueValues = { 5 };
DataType[] types = { DataType.DOUBLE };
byte[] dataTypeSelected = { 1 };
MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected);
WriterCompressModel writerCompressModel = ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.ADAPTIVE, writerCompressModel.getCompType(0));
}
use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.
the class ValueCompressionUtilTest method testToGetValueCompressionModelForShortAndByte.
@Test
public void testToGetValueCompressionModelForShortAndByte() {
Object[] maxValues = { 32600.00 };
Object[] minValues = { 32500.00 };
int[] decimalLength = { 0 };
Object[] uniqueValues = { 5 };
DataType[] types = { DataType.DOUBLE };
byte[] dataTypeSelected = { 1 };
MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected);
WriterCompressModel writerCompressModel = ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, writerCompressModel.getCompType(0));
}
use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.
the class ValueCompressionUtilTest method testToGetValueCompressionModelForByteAndIntAndDecimal1.
@Test
public void testToGetValueCompressionModelForByteAndIntAndDecimal1() {
Object[] maxValues = { -32766.00 };
Object[] minValues = { 32744.0 };
int[] decimalLength = { 1 };
Object[] uniqueValues = { 5 };
DataType[] types = { DataType.DOUBLE };
byte[] dataTypeSelected = { 1 };
MeasureMetaDataModel measureMetaDataModel = new MeasureMetaDataModel(maxValues, minValues, decimalLength, 1, uniqueValues, types, dataTypeSelected);
WriterCompressModel writerCompressModel = ValueCompressionUtil.getWriterCompressModel(measureMetaDataModel);
assertEquals(ValueCompressionUtil.COMPRESSION_TYPE.DELTA_DOUBLE, writerCompressModel.getCompType(0));
}
use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.
the class CarbonFactDataHandlerModel method getCarbonFactDataHandlerModel.
/**
* This method will create a model object for carbon fact data handler
*
* @param loadModel
* @return
*/
public static CarbonFactDataHandlerModel getCarbonFactDataHandlerModel(CarbonLoadModel loadModel, CarbonTable carbonTable, SegmentProperties segmentProperties, String tableName, String tempStoreLocation) {
CarbonFactDataHandlerModel carbonFactDataHandlerModel = new CarbonFactDataHandlerModel();
carbonFactDataHandlerModel.setSchemaUpdatedTimeStamp(carbonTable.getTableLastUpdatedTime());
carbonFactDataHandlerModel.setDatabaseName(loadModel.getDatabaseName());
carbonFactDataHandlerModel.setTableName(tableName);
carbonFactDataHandlerModel.setMeasureCount(segmentProperties.getMeasures().size());
carbonFactDataHandlerModel.setStoreLocation(tempStoreLocation);
carbonFactDataHandlerModel.setDimLens(segmentProperties.getDimColumnsCardinality());
carbonFactDataHandlerModel.setSegmentProperties(segmentProperties);
carbonFactDataHandlerModel.setNoDictionaryCount(segmentProperties.getNumberOfNoDictionaryDimension());
carbonFactDataHandlerModel.setDimensionCount(segmentProperties.getDimensions().size() - carbonFactDataHandlerModel.getNoDictionaryCount());
List<ColumnSchema> wrapperColumnSchema = CarbonUtil.getColumnSchemaList(carbonTable.getDimensionByTableName(tableName), carbonTable.getMeasureByTableName(tableName));
carbonFactDataHandlerModel.setWrapperColumnSchema(wrapperColumnSchema);
// get the cardinality for all all the columns including no dictionary columns
int[] formattedCardinality = CarbonUtil.getFormattedCardinality(segmentProperties.getDimColumnsCardinality(), wrapperColumnSchema);
carbonFactDataHandlerModel.setColCardinality(formattedCardinality);
//TO-DO Need to handle complex types here .
Map<Integer, GenericDataType> complexIndexMap = new HashMap<Integer, GenericDataType>(segmentProperties.getComplexDimensions().size());
carbonFactDataHandlerModel.setComplexIndexMap(complexIndexMap);
DataType[] aggType = new DataType[segmentProperties.getMeasures().size()];
int i = 0;
for (CarbonMeasure msr : segmentProperties.getMeasures()) {
aggType[i++] = msr.getDataType();
}
carbonFactDataHandlerModel.setMeasureDataType(aggType);
String carbonDataDirectoryPath = CarbonDataProcessorUtil.checkAndCreateCarbonStoreLocation(loadModel.getStorePath(), loadModel.getDatabaseName(), tableName, loadModel.getPartitionId(), loadModel.getSegmentId());
carbonFactDataHandlerModel.setCarbonDataDirectoryPath(carbonDataDirectoryPath);
List<CarbonDimension> dimensionByTableName = loadModel.getCarbonDataLoadSchema().getCarbonTable().getDimensionByTableName(tableName);
boolean[] isUseInvertedIndexes = new boolean[dimensionByTableName.size()];
int index = 0;
for (CarbonDimension dimension : dimensionByTableName) {
isUseInvertedIndexes[index++] = dimension.isUseInvertedIndex();
}
carbonFactDataHandlerModel.setIsUseInvertedIndex(isUseInvertedIndexes);
carbonFactDataHandlerModel.setPrimitiveDimLens(segmentProperties.getDimColumnsCardinality());
carbonFactDataHandlerModel.setBlockSizeInMB(carbonTable.getBlockSizeInMB());
return carbonFactDataHandlerModel;
}
use of org.apache.carbondata.core.metadata.datatype.DataType in project carbondata by apache.
the class SortParameters method createSortParameters.
public static SortParameters createSortParameters(CarbonDataLoadConfiguration configuration) {
SortParameters parameters = new SortParameters();
CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
CarbonProperties carbonProperties = CarbonProperties.getInstance();
parameters.setDatabaseName(tableIdentifier.getDatabaseName());
parameters.setTableName(tableIdentifier.getTableName());
parameters.setPartitionID(configuration.getPartitionId());
parameters.setSegmentId(configuration.getSegmentId());
parameters.setTaskNo(configuration.getTaskNo());
parameters.setMeasureColCount(configuration.getMeasureCount());
parameters.setDimColCount(configuration.getDimensionCount() - configuration.getComplexColumnCount());
parameters.setNoDictionaryCount(configuration.getNoDictionaryCount());
parameters.setComplexDimColCount(configuration.getComplexColumnCount());
parameters.setNoDictionaryDimnesionColumn(CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields()));
parameters.setBatchSortSizeinMb(CarbonDataProcessorUtil.getBatchSortSizeinMb(configuration));
parameters.setNumberOfSortColumns(configuration.getNumberOfSortColumns());
parameters.setNumberOfNoDictSortColumns(configuration.getNumberOfNoDictSortColumns());
setNoDictionarySortColumnMapping(parameters);
parameters.setObserver(new SortObserver());
// get sort buffer size
parameters.setSortBufferSize(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.SORT_SIZE, CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL)));
LOGGER.info("Sort size for table: " + parameters.getSortBufferSize());
// set number of intermedaite file to merge
parameters.setNumberOfIntermediateFileToBeMerged(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT, CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE)));
LOGGER.info("Number of intermediate file to be merged: " + parameters.getNumberOfIntermediateFileToBeMerged());
// get file buffer size
parameters.setFileBufferSize(CarbonDataProcessorUtil.getFileBufferSize(parameters.getNumberOfIntermediateFileToBeMerged(), carbonProperties, CarbonCommonConstants.CONSTANT_SIZE_TEN));
LOGGER.info("File Buffer Size: " + parameters.getFileBufferSize());
String carbonDataDirectoryPath = CarbonDataProcessorUtil.getLocalDataFolderLocation(tableIdentifier.getDatabaseName(), tableIdentifier.getTableName(), configuration.getTaskNo(), configuration.getPartitionId(), configuration.getSegmentId(), false);
parameters.setTempFileLocation(carbonDataDirectoryPath + File.separator + CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
LOGGER.info("temp file location" + parameters.getTempFileLocation());
int numberOfCores;
try {
numberOfCores = Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.NUM_CORES_LOADING, CarbonCommonConstants.NUM_CORES_DEFAULT_VAL));
numberOfCores = numberOfCores / 2;
} catch (NumberFormatException exc) {
numberOfCores = Integer.parseInt(CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
}
parameters.setNumberOfCores(numberOfCores > 0 ? numberOfCores : 1);
parameters.setFileWriteBufferSize(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE, CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE)));
parameters.setSortFileCompressionEnabled(Boolean.parseBoolean(carbonProperties.getProperty(CarbonCommonConstants.IS_SORT_TEMP_FILE_COMPRESSION_ENABLED, CarbonCommonConstants.IS_SORT_TEMP_FILE_COMPRESSION_ENABLED_DEFAULTVALUE)));
int sortTempFileNoOFRecordsInCompression;
try {
sortTempFileNoOFRecordsInCompression = Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION, CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE));
if (sortTempFileNoOFRecordsInCompression < 1) {
LOGGER.error("Invalid value for: " + CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION + ":Only Positive Integer value(greater than zero) is allowed.Default value will " + "be used");
sortTempFileNoOFRecordsInCompression = Integer.parseInt(CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE);
}
} catch (NumberFormatException e) {
LOGGER.error("Invalid value for: " + CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION + ", only Positive Integer value is allowed. Default value will be used");
sortTempFileNoOFRecordsInCompression = Integer.parseInt(CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE);
}
parameters.setSortTempFileNoOFRecordsInCompression(sortTempFileNoOFRecordsInCompression);
if (parameters.isSortFileCompressionEnabled()) {
LOGGER.info("Compression will be used for writing the sort temp File");
}
parameters.setPrefetch(CarbonCommonConstants.CARBON_PREFETCH_IN_MERGE_VALUE);
parameters.setBufferSize(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE, CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE_DEFAULT)));
DataType[] measureDataType = configuration.getMeasureDataType();
parameters.setMeasureDataType(measureDataType);
return parameters;
}
Aggregations