Examples with CarbonDataLoadingException org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException used on opensource projects

Example 11 with CarbonDataLoadingException

use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.

the class DataLoadExecutor method execute.

public void execute(CarbonLoadModel loadModel, String storeLocation, CarbonIterator<Object[]>[] inputIterators) throws Exception {
    AbstractDataLoadProcessorStep loadProcessorStep = null;
    try {
        loadProcessorStep = new DataLoadProcessBuilder().build(loadModel, storeLocation, inputIterators);
        // 1. initialize
        loadProcessorStep.initialize();
        LOGGER.info("Data Loading is started for table " + loadModel.getTableName());
        // 2. execute the step
        loadProcessorStep.execute();
    } catch (CarbonDataLoadingException e) {
        throw e;
    } catch (Exception e) {
        LOGGER.error(e, "Data Loading failed for table " + loadModel.getTableName());
        throw new CarbonDataLoadingException("Data Loading failed for table " + loadModel.getTableName(), e);
    } finally {
        if (loadProcessorStep != null) {
            // 3. Close the step
            loadProcessorStep.close();
        }
    }
    String key = new CarbonTableIdentifier(loadModel.getDatabaseName(), loadModel.getTableName(), null).getBadRecordLoggerKey();
    if (null != BadRecordsLogger.hasBadRecord(key)) {
        LOGGER.error("Data Load is partially success for table " + loadModel.getTableName());
        throw new BadRecordFoundException("Bad records found during load");
    } else {
        LOGGER.info("Data loading is successful for table " + loadModel.getTableName());
    }
}

Example 12 with CarbonDataLoadingException

use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.

the class DictionaryFieldConverterImpl method convert.

@Override
public void convert(CarbonRow row, BadRecordLogHolder logHolder) throws CarbonDataLoadingException {
    try {
        String parsedValue = null;
        String dimensionValue = row.getString(index);
        if (dimensionValue == null || dimensionValue.equals(nullFormat)) {
            parsedValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
        } else {
            parsedValue = DataTypeUtil.parseValue(dimensionValue, carbonDimension);
        }
        if (null == parsedValue) {
            if ((dimensionValue.length() > 0) || (dimensionValue.length() == 0 && isEmptyBadRecord)) {
                String dataType = carbonDimension.getDataType().getName();
                logHolder.setReason("The value " + " \"" + dimensionValue + "\"" + " with column name " + carbonDimension.getColName() + " and column data type " + dataType + " is not a valid " + dataType + " type.");
            }
            row.update(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY, index);
        } else {
            row.update(dictionaryGenerator.getOrGenerateKey(parsedValue), index);
        }
    } catch (DictionaryGenerationException e) {
        throw new CarbonDataLoadingException(e);
    }
}

Example 13 with CarbonDataLoadingException

use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.

the class RowConverterImpl method convert.

@Override
public CarbonRow convert(CarbonRow row) throws CarbonDataLoadingException {
    //TODO: only copy if it is bad record
    CarbonRow copy = row.getCopy();
    logHolder.setLogged(false);
    logHolder.clear();
    for (int i = 0; i < fieldConverters.length; i++) {
        fieldConverters[i].convert(row, logHolder);
        if (!logHolder.isLogged() && logHolder.isBadRecordNotAdded()) {
            if (badRecordLogger.isDataLoadFail()) {
                String error = "Data load failed due to bad record: " + logHolder.getReason();
                throw new CarbonDataLoadingException(error);
            }
            badRecordLogger.addBadRecordsToBuilder(copy.getData(), logHolder.getReason());
            logHolder.clear();
            logHolder.setLogged(true);
            if (badRecordLogger.isBadRecordConvertNullDisable()) {
                return null;
            }
        }
    }
    return row;
}

Example 14 with CarbonDataLoadingException

use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.

the class CSVRecordReaderIterator method next.

@Override
public Object[] next() {
    try {
        String[] data = recordReader.getCurrentValue().get();
        isConsumed = false;
        return data;
    } catch (Exception e) {
        throw new CarbonDataLoadingException(e);
    }
}

Example 15 with CarbonDataLoadingException

use of org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException in project carbondata by apache.

the class CarbonRowDataWriterProcessorStepImpl method processBatch.

private void processBatch(CarbonRowBatch batch, CarbonFactHandler dataHandler, int iteratorIndex) throws CarbonDataLoadingException {
    try {
        while (batch.hasNext()) {
            CarbonRow row = batch.next();
            CarbonRow converted = convertRow(row);
            dataHandler.addDataToStore(converted);
            readCounter[iteratorIndex]++;
        }
        writeCounter[iteratorIndex] += batch.getSize();
    } catch (Exception e) {
        throw new CarbonDataLoadingException("unable to generate the mdkey", e);
    }
    rowCounter.getAndAdd(batch.getSize());
}