Examples with CarbonDataLoadingException org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException used on opensource projects

Example 1 with CarbonDataLoadingException

use of org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException in project carbondata by apache.

the class CarbondataPageSource method getNextPage.

@Override
public Page getNextPage() {
    if (nanoStart == 0) {
        nanoStart = System.nanoTime();
    }
    CarbonVectorBatch columnarBatch = null;
    int batchSize = 0;
    try {
        batchId++;
        if (vectorReader.nextKeyValue()) {
            Object vectorBatch = vectorReader.getCurrentValue();
            if (vectorBatch != null && vectorBatch instanceof CarbonVectorBatch) {
                columnarBatch = (CarbonVectorBatch) vectorBatch;
                batchSize = columnarBatch.numRows();
                if (batchSize == 0) {
                    close();
                    return null;
                }
            }
        } else {
            close();
            return null;
        }
        if (columnarBatch == null) {
            return null;
        }
        Block[] blocks = new Block[types.size()];
        for (int column = 0; column < blocks.length; column++) {
            Type type = types.get(column);
            readers[column].setBatchSize(columnarBatch.numRows());
            readers[column].setVectorReader(true);
            readers[column].setVector(columnarBatch.column(column));
            blocks[column] = new LazyBlock(batchSize, new CarbondataBlockLoader(column, type));
        }
        Page page = new Page(batchSize, blocks);
        sizeOfData += columnarBatch.capacity();
        return page;
    } catch (PrestoException e) {
        closeWithSuppression(e);
        throw e;
    } catch (RuntimeException | InterruptedException | IOException e) {
        closeWithSuppression(e);
        throw new CarbonDataLoadingException("Exception when creating the Carbon data Block", e);
    }
}

Example 2 with CarbonDataLoadingException

use of org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException in project carbondata by apache.

the class DataLoadProcessBuilder method setSortColumnInfo.

/**
 * set sort column info in configuration
 * @param carbonTable carbon table
 * @param loadModel load model
 * @param configuration configuration
 */
private static void setSortColumnInfo(CarbonTable carbonTable, CarbonLoadModel loadModel, CarbonDataLoadConfiguration configuration) {
    List<String> sortCols = carbonTable.getSortColumns(carbonTable.getTableName());
    SortScopeOptions.SortScope sortScope = SortScopeOptions.getSortScope(loadModel.getSortScope());
    if (!SortScopeOptions.SortScope.LOCAL_SORT.equals(sortScope) || sortCols.size() == 0 || StringUtils.isBlank(loadModel.getSortColumnsBoundsStr())) {
        if (!StringUtils.isBlank(loadModel.getSortColumnsBoundsStr())) {
            LOGGER.warn("sort column bounds will be ignored");
        }
        configuration.setSortColumnRangeInfo(null);
        return;
    }
    // column index for sort columns
    int[] sortColIndex = new int[sortCols.size()];
    boolean[] isSortColNoDict = new boolean[sortCols.size()];
    DataField[] outFields = configuration.getDataFields();
    int j = 0;
    boolean columnExist;
    for (String sortCol : sortCols) {
        columnExist = false;
        for (int i = 0; !columnExist && i < outFields.length; i++) {
            if (outFields[i].getColumn().getColName().equalsIgnoreCase(sortCol)) {
                columnExist = true;
                sortColIndex[j] = i;
                isSortColNoDict[j] = !outFields[i].hasDictionaryEncoding();
                j++;
            }
        }
        if (!columnExist) {
            throw new CarbonDataLoadingException("Field " + sortCol + " does not exist.");
        }
    }
    String[] sortColumnBounds = StringUtils.splitPreserveAllTokens(loadModel.getSortColumnsBoundsStr(), CarbonLoadOptionConstants.SORT_COLUMN_BOUNDS_ROW_DELIMITER, -1);
    for (String bound : sortColumnBounds) {
        String[] fieldInBounds = StringUtils.splitPreserveAllTokens(bound, CarbonLoadOptionConstants.SORT_COLUMN_BOUNDS_FIELD_DELIMITER, -1);
        if (fieldInBounds.length != sortCols.size()) {
            String msg = new StringBuilder("The number of field in bounds should be equal to that in sort columns.").append(" Expected ").append(sortCols.size()).append(", actual ").append(String.valueOf(fieldInBounds.length)).append(".").append(" The illegal bound is '").append(bound).append("'.").toString();
            throw new CarbonDataLoadingException(msg);
        }
    }
    SortColumnRangeInfo sortColumnRangeInfo = new SortColumnRangeInfo(sortColIndex, isSortColNoDict, sortColumnBounds, CarbonLoadOptionConstants.SORT_COLUMN_BOUNDS_FIELD_DELIMITER);
    configuration.setSortColumnRangeInfo(sortColumnRangeInfo);
}

Example 3 with CarbonDataLoadingException

use of org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException in project carbondata by apache.

the class ComplexFieldConverterImpl method convert.

@Override
public void convert(CarbonRow row, BadRecordLogHolder logHolder) {
    Object object = row.getObject(index);
    // TODO Its temporary, needs refactor here.
    ByteArrayOutputStream byteArray = new ByteArrayOutputStream();
    DataOutputStream dataOutputStream = new DataOutputStream(byteArray);
    try {
        genericDataType.writeByteArray(object, dataOutputStream);
        dataOutputStream.close();
        row.update(byteArray.toByteArray(), index);
    } catch (Exception e) {
        throw new CarbonDataLoadingException(object + "", e);
    }
}

Example 4 with CarbonDataLoadingException

use of org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException in project carbondata by apache.

the class DictionaryFieldConverterImpl method convert.

@Override
public void convert(CarbonRow row, BadRecordLogHolder logHolder) throws CarbonDataLoadingException {
    try {
        String parsedValue = null;
        String dimensionValue = row.getString(index);
        if (dimensionValue == null || dimensionValue.equals(nullFormat)) {
            parsedValue = CarbonCommonConstants.MEMBER_DEFAULT_VAL;
        } else {
            parsedValue = DataTypeUtil.parseValue(dimensionValue, carbonDimension);
        }
        if (null == parsedValue) {
            if ((dimensionValue.length() > 0) || (dimensionValue.length() == 0 && isEmptyBadRecord)) {
                String message = logHolder.getColumnMessageMap().get(carbonDimension.getColName());
                if (null == message) {
                    message = CarbonDataProcessorUtil.prepareFailureReason(carbonDimension.getColName(), carbonDimension.getDataType());
                    logHolder.getColumnMessageMap().put(carbonDimension.getColName(), message);
                }
                logHolder.setReason(message);
            }
            row.update(CarbonCommonConstants.MEMBER_DEFAULT_VAL_SURROGATE_KEY, index);
        } else {
            row.update(dictionaryGenerator.getOrGenerateKey(parsedValue), index);
        }
    } catch (DictionaryGenerationException e) {
        throw new CarbonDataLoadingException(e);
    }
}

Example 5 with CarbonDataLoadingException

use of org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException in project carbondata by apache.

the class NonDictionaryFieldConverterImpl method convert.

@Override
public void convert(CarbonRow row, BadRecordLogHolder logHolder) {
    String dimensionValue = row.getString(index);
    if (null == dimensionValue && column.getDataType() != DataTypes.STRING) {
        logHolder.setReason(CarbonDataProcessorUtil.prepareFailureReason(column.getColName(), column.getDataType()));
        updateWithNullValue(row);
    } else if (dimensionValue == null || dimensionValue.equals(nullformat)) {
        updateWithNullValue(row);
    } else {
        String dateFormat = null;
        if (dataType == DataTypes.DATE) {
            dateFormat = dataField.getDateFormat();
        } else if (dataType == DataTypes.TIMESTAMP) {
            dateFormat = dataField.getTimestampFormat();
        }
        try {
            if (!dataField.isUseActualData()) {
                byte[] value = DataTypeUtil.getBytesBasedOnDataTypeForNoDictionaryColumn(dimensionValue, dataType, dateFormat);
                if (dataType == DataTypes.STRING && value.length > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) {
                    throw new CarbonDataLoadingException("Dataload failed, String size cannot exceed " + CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT + " bytes");
                }
                row.update(value, index);
            } else {
                Object value = DataTypeUtil.getDataDataTypeForNoDictionaryColumn(dimensionValue, dataType, dateFormat);
                if (dataType == DataTypes.STRING && value.toString().length() > CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT) {
                    throw new CarbonDataLoadingException("Dataload failed, String size cannot exceed " + CarbonCommonConstants.MAX_CHARS_PER_COLUMN_DEFAULT + " bytes");
                }
                row.update(value, index);
            }
        } catch (CarbonDataLoadingException e) {
            throw e;
        } catch (Throwable ex) {
            if (dimensionValue.length() > 0 || (dimensionValue.length() == 0 && isEmptyBadRecord)) {
                String message = logHolder.getColumnMessageMap().get(column.getColName());
                if (null == message) {
                    message = CarbonDataProcessorUtil.prepareFailureReason(column.getColName(), column.getDataType());
                    logHolder.getColumnMessageMap().put(column.getColName(), message);
                }
                logHolder.setReason(message);
                updateWithNullValue(row);
            } else {
                updateWithNullValue(row);
            }
        }
    }
}