Examples with CarbonDataLoadingException org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException used on opensource projects

Example 21 with CarbonDataLoadingException

use of org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException in project carbondata by apache.

the class UnsafeParallelReadMergeSorterWithColumnRangeImpl method processRowToNextStep.

/**
 * Below method will be used to process data to next step
 */
private boolean processRowToNextStep(UnsafeSortDataRows[] sortDataRows, SortParameters parameters) throws CarbonDataLoadingException {
    if (null == sortDataRows || sortDataRows.length == 0) {
        LOGGER.info("Record Processed For table: " + parameters.getTableName());
        LOGGER.info("Number of Records was Zero");
        String logMessage = "Summary: Carbon Sort Key Step: Read: " + 0 + ": Write: " + 0;
        LOGGER.info(logMessage);
        return false;
    }
    try {
        for (int i = 0; i < sortDataRows.length; i++) {
            // start sorting
            sortDataRows[i].startSorting();
        }
        // check any more rows are present
        LOGGER.info("Record Processed For table: " + parameters.getTableName());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordSortRowsStepTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValuesTotalTime(parameters.getPartitionID(), System.currentTimeMillis());
        return false;
    } catch (Exception e) {
        throw new CarbonDataLoadingException(e);
    }
}

Example 22 with CarbonDataLoadingException

use of org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException in project carbondata by apache.

the class DataLoadExecutor method execute.

public void execute(CarbonLoadModel loadModel, String[] storeLocation, CarbonIterator<Object[]>[] inputIterators) throws Exception {
    try {
        loadProcessorStep = new DataLoadProcessBuilder().build(loadModel, storeLocation, inputIterators);
        // 1. initialize
        loadProcessorStep.initialize();
        LOGGER.info("Data Loading is started for table " + loadModel.getTableName());
        // 2. execute the step
        loadProcessorStep.execute();
        // check and remove any bad record key from bad record entry logger static map
        if (CarbonBadRecordUtil.hasBadRecord(loadModel)) {
            LOGGER.error("Data Load is partially success for table " + loadModel.getTableName());
        } else {
            LOGGER.info("Data loading is successful for table " + loadModel.getTableName());
        }
    } catch (CarbonDataLoadingException e) {
        if (e instanceof BadRecordFoundException) {
            throw new NoRetryException(e.getMessage());
        } else {
            throw e;
        }
    } catch (Exception e) {
        LOGGER.error(e, "Data Loading failed for table " + loadModel.getTableName());
        throw new CarbonDataLoadingException("Data Loading failed for table " + loadModel.getTableName(), e);
    }
}

Example 23 with CarbonDataLoadingException

use of org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException in project carbondata by apache.

the class BadRecordsLogger method writeBadRecordsToFile.

/**
 */
private synchronized void writeBadRecordsToFile(StringBuilder logStrings) throws CarbonDataLoadingException {
    if (null == logFilePath) {
        logFilePath = this.storePath + File.separator + this.fileName + CarbonCommonConstants.LOG_FILE_EXTENSION + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
    }
    try {
        if (null == bufferedWriter) {
            FileType fileType = FileFactory.getFileType(storePath);
            if (!FileFactory.isFileExist(this.storePath, fileType)) {
                // create the folders if not exist
                FileFactory.mkdirs(this.storePath, fileType);
                // create the files
                FileFactory.createNewFile(logFilePath, fileType);
            }
            outStream = FileFactory.getDataOutputStream(logFilePath, fileType);
            bufferedWriter = new BufferedWriter(new OutputStreamWriter(outStream, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
        }
        bufferedWriter.write(logStrings.toString());
        bufferedWriter.newLine();
    } catch (FileNotFoundException e) {
        LOGGER.error("Bad Log Files not found");
        throw new CarbonDataLoadingException("Bad Log Files not found", e);
    } catch (IOException e) {
        LOGGER.error("Error While writing bad record log File");
        throw new CarbonDataLoadingException("Error While writing bad record log File", e);
    } finally {
        // if the Bad record file is created means it partially success
        // if any entry present with key that means its have bad record for
        // that key
        badRecordEntry.put(taskKey, "Partially");
    }
}

Example 24 with CarbonDataLoadingException

use of org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException in project carbondata by apache.

the class BadRecordsLogger method writeBadRecordsToCSVFile.

/**
 * method will write the row having bad record in the csv file.
 *
 * @param logStrings
 */
private synchronized void writeBadRecordsToCSVFile(StringBuilder logStrings) throws CarbonDataLoadingException {
    if (null == csvFilePath) {
        csvFilePath = this.storePath + File.separator + this.fileName + CarbonCommonConstants.CSV_FILE_EXTENSION + CarbonCommonConstants.FILE_INPROGRESS_STATUS;
    }
    try {
        if (null == bufferedCSVWriter) {
            FileType fileType = FileFactory.getFileType(storePath);
            if (!FileFactory.isFileExist(this.storePath, fileType)) {
                // create the folders if not exist
                FileFactory.mkdirs(this.storePath, fileType);
                // create the files
                FileFactory.createNewFile(csvFilePath, fileType);
            }
            outCSVStream = FileFactory.getDataOutputStream(csvFilePath, fileType);
            bufferedCSVWriter = new BufferedWriter(new OutputStreamWriter(outCSVStream, Charset.forName(CarbonCommonConstants.DEFAULT_CHARSET)));
        }
        bufferedCSVWriter.write(logStrings.toString());
        bufferedCSVWriter.newLine();
    } catch (FileNotFoundException e) {
        LOGGER.error("Bad record csv Files not found");
        throw new CarbonDataLoadingException("Bad record csv Files not found", e);
    } catch (IOException e) {
        LOGGER.error("Error While writing bad record csv File");
        throw new CarbonDataLoadingException("Error While writing bad record csv File", e);
    } finally {
        badRecordEntry.put(taskKey, "Partially");
    }
}