Examples with BadRecordFoundException org.apache.carbondata.processing.loading.exception.BadRecordFoundException used on opensource projects

Example 1 with BadRecordFoundException

use of org.apache.carbondata.processing.loading.exception.BadRecordFoundException in project carbondata by apache.

the class CarbonRowDataWriterProcessorStepImpl method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    final Iterator<CarbonRowBatch>[] iterators = child.execute();
    tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    tableName = tableIdentifier.getTableName();
    ExecutorService executorService = null;
    try {
        readCounter = new long[iterators.length];
        writeCounter = new long[iterators.length];
        dimensionWithComplexCount = configuration.getDimensionCount();
        noDictWithComplextCount = configuration.getNoDictionaryCount() + configuration.getComplexColumnCount();
        dimensionCount = configuration.getDimensionCount() - noDictWithComplextCount;
        isNoDictionaryDimensionColumn = CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
        measureDataType = configuration.getMeasureDataType();
        measureCount = configuration.getMeasureCount();
        outputLength = measureCount + (this.noDictWithComplextCount > 0 ? 1 : 0) + 1;
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis());
        if (iterators.length == 1) {
            doExecute(iterators[0], 0);
        } else {
            executorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("NoSortDataWriterPool:" + configuration.getTableIdentifier().getCarbonTableIdentifier().getTableName()));
            Future[] futures = new Future[iterators.length];
            for (int i = 0; i < iterators.length; i++) {
                futures[i] = executorService.submit(new DataWriterRunnable(iterators[i], i));
            }
            for (Future future : futures) {
                future.get();
            }
        }
    } catch (CarbonDataWriterException e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        throw new CarbonDataLoadingException("Error while initializing data handler : " + e.getMessage());
    } catch (Exception e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        if (e instanceof BadRecordFoundException) {
            throw new BadRecordFoundException(e.getMessage(), e);
        }
        throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage(), e);
    } finally {
        if (null != executorService && executorService.isShutdown()) {
            executorService.shutdownNow();
        }
    }
    return null;
}

Example 2 with BadRecordFoundException

use of org.apache.carbondata.processing.loading.exception.BadRecordFoundException in project carbondata by apache.

the class DataWriterBatchProcessorStepImpl method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    Iterator<CarbonRowBatch>[] iterators = child.execute();
    CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    String tableName = tableIdentifier.getTableName();
    try {
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis());
        int i = 0;
        String[] storeLocation = getStoreLocation(tableIdentifier);
        CarbonDataProcessorUtil.createLocations(storeLocation);
        for (Iterator<CarbonRowBatch> iterator : iterators) {
            int k = 0;
            while (iterator.hasNext()) {
                CarbonRowBatch next = iterator.next();
                // If no rows from merge sorter, then don't create a file in fact column handler
                if (next.hasNext()) {
                    CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration, storeLocation, 0, k++);
                    CarbonFactHandler dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
                    dataHandler.initialise();
                    processBatch(next, dataHandler);
                    finish(tableName, dataHandler);
                }
            }
            i++;
        }
    } catch (Exception e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterBatchProcessorStepImpl");
        if (e.getCause() instanceof BadRecordFoundException) {
            throw new BadRecordFoundException(e.getCause().getMessage());
        }
        throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage());
    }
    return null;
}

Example 3 with BadRecordFoundException

use of org.apache.carbondata.processing.loading.exception.BadRecordFoundException in project carbondata by apache.

the class DataLoadExecutor method execute.

public void execute(CarbonLoadModel loadModel, String[] storeLocation, CarbonIterator<Object[]>[] inputIterators) throws Exception {
    try {
        loadProcessorStep = new DataLoadProcessBuilder().build(loadModel, storeLocation, inputIterators);
        // 1. initialize
        loadProcessorStep.initialize();
        LOGGER.info("Data Loading is started for table " + loadModel.getTableName());
        // 2. execute the step
        loadProcessorStep.execute();
        // check and remove any bad record key from bad record entry logger static map
        if (CarbonBadRecordUtil.hasBadRecord(loadModel)) {
            LOGGER.error("Data Load is partially success for table " + loadModel.getTableName());
        } else {
            LOGGER.info("Data loading is successful for table " + loadModel.getTableName());
        }
    } catch (CarbonDataLoadingException e) {
        if (e instanceof BadRecordFoundException) {
            throw new NoRetryException(e.getMessage());
        } else {
            throw e;
        }
    } catch (Exception e) {
        LOGGER.error(e, "Data Loading failed for table " + loadModel.getTableName());
        throw new CarbonDataLoadingException("Data Loading failed for table " + loadModel.getTableName(), e);
    }
}