Examples with SortObserver org.apache.carbondata.processing.schema.metadata.SortObserver used on opensource projects

Example 1 with SortObserver

use of org.apache.carbondata.processing.schema.metadata.SortObserver in project carbondata by apache.

the class SortParameters method createSortParameters.

public static SortParameters createSortParameters(CarbonTable carbonTable, String databaseName, String tableName, int dimColCount, int complexDimColCount, int measureColCount, int noDictionaryCount, String partitionID, String segmentId, String taskNo, boolean[] noDictionaryColMaping, boolean isCompactionFlow) {
    SortParameters parameters = new SortParameters();
    CarbonProperties carbonProperties = CarbonProperties.getInstance();
    parameters.setDatabaseName(databaseName);
    parameters.setTableName(tableName);
    parameters.setPartitionID(partitionID);
    parameters.setSegmentId(segmentId);
    parameters.setTaskNo(taskNo);
    parameters.setMeasureColCount(measureColCount);
    parameters.setDimColCount(dimColCount - complexDimColCount);
    parameters.setNumberOfSortColumns(carbonTable.getNumberOfSortColumns());
    parameters.setNoDictionaryCount(noDictionaryCount);
    parameters.setNumberOfNoDictSortColumns(carbonTable.getNumberOfNoDictSortColumns());
    parameters.setComplexDimColCount(complexDimColCount);
    parameters.setNoDictionaryDimnesionColumn(noDictionaryColMaping);
    parameters.setObserver(new SortObserver());
    // get sort buffer size
    parameters.setSortBufferSize(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.SORT_SIZE, CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL)));
    LOGGER.info("Sort size for table: " + parameters.getSortBufferSize());
    // set number of intermedaite file to merge
    parameters.setNumberOfIntermediateFileToBeMerged(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT, CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE)));
    LOGGER.info("Number of intermediate file to be merged: " + parameters.getNumberOfIntermediateFileToBeMerged());
    // get file buffer size
    parameters.setFileBufferSize(CarbonDataProcessorUtil.getFileBufferSize(parameters.getNumberOfIntermediateFileToBeMerged(), carbonProperties, CarbonCommonConstants.CONSTANT_SIZE_TEN));
    LOGGER.info("File Buffer Size: " + parameters.getFileBufferSize());
    String carbonDataDirectoryPath = CarbonDataProcessorUtil.getLocalDataFolderLocation(databaseName, tableName, taskNo, partitionID, segmentId, isCompactionFlow);
    parameters.setTempFileLocation(carbonDataDirectoryPath + File.separator + CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
    LOGGER.info("temp file location" + parameters.getTempFileLocation());
    int numberOfCores;
    try {
        numberOfCores = Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.NUM_CORES_LOADING, CarbonCommonConstants.NUM_CORES_DEFAULT_VAL));
        numberOfCores = numberOfCores / 2;
    } catch (NumberFormatException exc) {
        numberOfCores = Integer.parseInt(CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
    }
    parameters.setNumberOfCores(numberOfCores > 0 ? numberOfCores : 1);
    parameters.setFileWriteBufferSize(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE, CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE)));
    parameters.setSortFileCompressionEnabled(Boolean.parseBoolean(carbonProperties.getProperty(CarbonCommonConstants.IS_SORT_TEMP_FILE_COMPRESSION_ENABLED, CarbonCommonConstants.IS_SORT_TEMP_FILE_COMPRESSION_ENABLED_DEFAULTVALUE)));
    int sortTempFileNoOFRecordsInCompression;
    try {
        sortTempFileNoOFRecordsInCompression = Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION, CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE));
        if (sortTempFileNoOFRecordsInCompression < 1) {
            LOGGER.error("Invalid value for: " + CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION + ":Only Positive Integer value(greater than zero) is allowed.Default value will " + "be used");
            sortTempFileNoOFRecordsInCompression = Integer.parseInt(CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE);
        }
    } catch (NumberFormatException e) {
        LOGGER.error("Invalid value for: " + CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION + ", only Positive Integer value is allowed. Default value will be used");
        sortTempFileNoOFRecordsInCompression = Integer.parseInt(CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE);
    }
    parameters.setSortTempFileNoOFRecordsInCompression(sortTempFileNoOFRecordsInCompression);
    if (parameters.isSortFileCompressionEnabled()) {
        LOGGER.info("Compression will be used for writing the sort temp File");
    }
    parameters.setPrefetch(CarbonCommonConstants.CARBON_PREFETCH_IN_MERGE_VALUE);
    parameters.setBufferSize(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE, CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE_DEFAULT)));
    DataType[] type = CarbonDataProcessorUtil.getMeasureDataType(parameters.getMeasureColCount(), parameters.getDatabaseName(), parameters.getTableName());
    parameters.setMeasureDataType(type);
    setNoDictionarySortColumnMapping(parameters);
    return parameters;
}

Example 2 with SortObserver

use of org.apache.carbondata.processing.schema.metadata.SortObserver in project carbondata by apache.

the class SortParameters method createSortParameters.

public static SortParameters createSortParameters(CarbonDataLoadConfiguration configuration) {
    SortParameters parameters = new SortParameters();
    CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    CarbonProperties carbonProperties = CarbonProperties.getInstance();
    parameters.setDatabaseName(tableIdentifier.getDatabaseName());
    parameters.setTableName(tableIdentifier.getTableName());
    parameters.setPartitionID(configuration.getPartitionId());
    parameters.setSegmentId(configuration.getSegmentId());
    parameters.setTaskNo(configuration.getTaskNo());
    parameters.setMeasureColCount(configuration.getMeasureCount());
    parameters.setDimColCount(configuration.getDimensionCount() - configuration.getComplexColumnCount());
    parameters.setNoDictionaryCount(configuration.getNoDictionaryCount());
    parameters.setComplexDimColCount(configuration.getComplexColumnCount());
    parameters.setNoDictionaryDimnesionColumn(CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields()));
    parameters.setBatchSortSizeinMb(CarbonDataProcessorUtil.getBatchSortSizeinMb(configuration));
    parameters.setNumberOfSortColumns(configuration.getNumberOfSortColumns());
    parameters.setNumberOfNoDictSortColumns(configuration.getNumberOfNoDictSortColumns());
    setNoDictionarySortColumnMapping(parameters);
    parameters.setObserver(new SortObserver());
    // get sort buffer size
    parameters.setSortBufferSize(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.SORT_SIZE, CarbonCommonConstants.SORT_SIZE_DEFAULT_VAL)));
    LOGGER.info("Sort size for table: " + parameters.getSortBufferSize());
    // set number of intermedaite file to merge
    parameters.setNumberOfIntermediateFileToBeMerged(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT, CarbonCommonConstants.SORT_INTERMEDIATE_FILES_LIMIT_DEFAULT_VALUE)));
    LOGGER.info("Number of intermediate file to be merged: " + parameters.getNumberOfIntermediateFileToBeMerged());
    // get file buffer size
    parameters.setFileBufferSize(CarbonDataProcessorUtil.getFileBufferSize(parameters.getNumberOfIntermediateFileToBeMerged(), carbonProperties, CarbonCommonConstants.CONSTANT_SIZE_TEN));
    LOGGER.info("File Buffer Size: " + parameters.getFileBufferSize());
    String carbonDataDirectoryPath = CarbonDataProcessorUtil.getLocalDataFolderLocation(tableIdentifier.getDatabaseName(), tableIdentifier.getTableName(), configuration.getTaskNo(), configuration.getPartitionId(), configuration.getSegmentId(), false);
    parameters.setTempFileLocation(carbonDataDirectoryPath + File.separator + CarbonCommonConstants.SORT_TEMP_FILE_LOCATION);
    LOGGER.info("temp file location" + parameters.getTempFileLocation());
    int numberOfCores;
    try {
        numberOfCores = Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.NUM_CORES_LOADING, CarbonCommonConstants.NUM_CORES_DEFAULT_VAL));
        numberOfCores = numberOfCores / 2;
    } catch (NumberFormatException exc) {
        numberOfCores = Integer.parseInt(CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
    }
    parameters.setNumberOfCores(numberOfCores > 0 ? numberOfCores : 1);
    parameters.setFileWriteBufferSize(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE, CarbonCommonConstants.CARBON_SORT_FILE_WRITE_BUFFER_SIZE_DEFAULT_VALUE)));
    parameters.setSortFileCompressionEnabled(Boolean.parseBoolean(carbonProperties.getProperty(CarbonCommonConstants.IS_SORT_TEMP_FILE_COMPRESSION_ENABLED, CarbonCommonConstants.IS_SORT_TEMP_FILE_COMPRESSION_ENABLED_DEFAULTVALUE)));
    int sortTempFileNoOFRecordsInCompression;
    try {
        sortTempFileNoOFRecordsInCompression = Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION, CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE));
        if (sortTempFileNoOFRecordsInCompression < 1) {
            LOGGER.error("Invalid value for: " + CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION + ":Only Positive Integer value(greater than zero) is allowed.Default value will " + "be used");
            sortTempFileNoOFRecordsInCompression = Integer.parseInt(CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE);
        }
    } catch (NumberFormatException e) {
        LOGGER.error("Invalid value for: " + CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORDS_FOR_COMPRESSION + ", only Positive Integer value is allowed. Default value will be used");
        sortTempFileNoOFRecordsInCompression = Integer.parseInt(CarbonCommonConstants.SORT_TEMP_FILE_NO_OF_RECORD_FOR_COMPRESSION_DEFAULTVALUE);
    }
    parameters.setSortTempFileNoOFRecordsInCompression(sortTempFileNoOFRecordsInCompression);
    if (parameters.isSortFileCompressionEnabled()) {
        LOGGER.info("Compression will be used for writing the sort temp File");
    }
    parameters.setPrefetch(CarbonCommonConstants.CARBON_PREFETCH_IN_MERGE_VALUE);
    parameters.setBufferSize(Integer.parseInt(carbonProperties.getProperty(CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE, CarbonCommonConstants.CARBON_PREFETCH_BUFFERSIZE_DEFAULT)));
    DataType[] measureDataType = configuration.getMeasureDataType();
    parameters.setMeasureDataType(measureDataType);
    return parameters;
}