Examples with CarbonDataWriterException org.apache.carbondata.core.datastore.exception.CarbonDataWriterException used on opensource projects

Example 16 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class DataWriterProcessorStepImpl method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    Iterator<CarbonRowBatch>[] iterators = child.execute();
    CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    String tableName = tableIdentifier.getTableName();
    try {
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PARTITION_ID, System.currentTimeMillis());
        rangeExecutorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("WriterForwardPool: " + tableName, true));
        List<Future<Void>> rangeExecutorServiceSubmitList = new ArrayList<>(iterators.length);
        int i = 0;
        // do this concurrently
        for (Iterator<CarbonRowBatch> iterator : iterators) {
            rangeExecutorServiceSubmitList.add(rangeExecutorService.submit(new WriterForwarder(iterator, i)));
            i++;
        }
        try {
            rangeExecutorService.shutdown();
            rangeExecutorService.awaitTermination(2, TimeUnit.DAYS);
            for (int j = 0; j < rangeExecutorServiceSubmitList.size(); j++) {
                rangeExecutorServiceSubmitList.get(j).get();
            }
        } catch (InterruptedException e) {
            throw new CarbonDataWriterException(e);
        } catch (ExecutionException e) {
            throw new CarbonDataWriterException(e.getCause());
        }
    } catch (CarbonDataWriterException e) {
        LOGGER.error(e);
        throw new CarbonDataLoadingException("Error while initializing writer: " + e.getMessage(), e);
    } catch (Exception e) {
        throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage(), e);
    }
    return null;
}

Example 17 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class UnsafeSingleThreadFinalSortFilesMerger method startSorting.

/**
 * Below method will be used to start storing process This method will get
 * all the temp files present in sort temp folder then it will create the
 * record holder heap and then it will read first record from each file and
 * initialize the heap
 */
private void startSorting(UnsafeCarbonRowPage[] rowPages, List<UnsafeInMemoryIntermediateDataMerger> merges) throws CarbonDataWriterException {
    try {
        List<File> filesToMergeSort = getFilesToMergeSort();
        this.fileCounter = rowPages.length + filesToMergeSort.size() + merges.size();
        if (fileCounter == 0) {
            LOGGER.info("No files to merge sort");
            return;
        }
        LOGGER.info(String.format("Starting final merge of %d pages, including row pages: %d" + ", sort temp files: %d, intermediate merges: %d", this.fileCounter, rowPages.length, filesToMergeSort.size(), merges.size()));
        // create record holder heap
        createRecordHolderQueue();
        TableFieldStat tableFieldStat = new TableFieldStat(parameters);
        // iterate over file list and create chunk holder and add to heap
        LOGGER.info("Started adding first record from each page");
        for (final UnsafeCarbonRowPage rowPage : rowPages) {
            SortTempChunkHolder sortTempFileChunkHolder = new UnsafeInmemoryHolder(rowPage);
            // initialize
            sortTempFileChunkHolder.readRow();
            recordHolderHeapLocal.add(sortTempFileChunkHolder);
        }
        for (final UnsafeInMemoryIntermediateDataMerger merger : merges) {
            SortTempChunkHolder sortTempFileChunkHolder = new UnsafeFinalMergePageHolder(merger, tableFieldStat);
            // initialize
            sortTempFileChunkHolder.readRow();
            recordHolderHeapLocal.add(sortTempFileChunkHolder);
        }
        for (final File file : filesToMergeSort) {
            SortTempChunkHolder sortTempFileChunkHolder = new UnsafeSortTempFileChunkHolder(file, parameters, true, tableFieldStat);
            // initialize
            sortTempFileChunkHolder.readRow();
            recordHolderHeapLocal.add(sortTempFileChunkHolder);
        }
        LOGGER.info("Heap Size: " + this.recordHolderHeapLocal.size());
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        throw new CarbonDataWriterException(e);
    }
}

Example 18 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class CarbonRowDataWriterProcessorStepImpl method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    final Iterator<CarbonRowBatch>[] iterators = child.execute();
    tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    tableName = tableIdentifier.getTableName();
    try {
        readCounter = new long[iterators.length];
        writeCounter = new long[iterators.length];
        dimensionWithComplexCount = configuration.getDimensionCount();
        noDictWithComplextCount = configuration.getNoDictionaryCount() + configuration.getComplexDictionaryColumnCount() + configuration.getComplexNonDictionaryColumnCount();
        directDictionaryDimensionCount = configuration.getDimensionCount() - noDictWithComplextCount;
        isNoDictionaryDimensionColumn = CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
        measureCount = configuration.getMeasureCount();
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PARTITION_ID, System.currentTimeMillis());
        if (configuration.getDataLoadProperty(DataLoadProcessorConstants.NO_REARRANGE_OF_ROWS) != null) {
            initializeNoReArrangeIndexes();
        }
        if (iterators.length == 1) {
            doExecute(iterators[0], 0);
        } else {
            executorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("NoSortDataWriterPool:" + configuration.getTableIdentifier().getCarbonTableIdentifier().getTableName(), true));
            Future[] futures = new Future[iterators.length];
            for (int i = 0; i < iterators.length; i++) {
                futures[i] = executorService.submit(new DataWriterRunnable(iterators[i], i));
            }
            for (Future future : futures) {
                future.get();
            }
        }
    } catch (CarbonDataWriterException e) {
        LOGGER.error("Failed for table: " + tableName + " in DataWriterProcessorStepImpl", e);
        throw new CarbonDataLoadingException("Error while initializing data handler : " + e.getMessage(), e);
    } catch (Exception e) {
        LOGGER.error("Failed for table: " + tableName + " in DataWriterProcessorStepImpl", e);
        if (e instanceof BadRecordFoundException) {
            throw new BadRecordFoundException(e.getMessage(), e);
        }
        throw new CarbonDataLoadingException(e.getMessage(), e);
    }
    return null;
}

Example 19 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class CompactionResultSortProcessor method readAndLoadDataFromSortTempFiles.

/**
 * This method will read sort temp files, perform merge sort and add it to store for data loading
 */
private void readAndLoadDataFromSortTempFiles(List<RawResultIterator> sortedRawResultIterator) throws Exception {
    try {
        intermediateFileMerger.finish();
        finalMerger.startFinalMerge();
        if (sortedRawResultIterator != null && sortedRawResultIterator.size() > 0) {
            finalMerger.addInMemoryRawResultIterator(sortedRawResultIterator, segmentProperties, noDicAndComplexColumns, dataTypes);
        }
        while (finalMerger.hasNext()) {
            Object[] row = finalMerger.next();
            dataHandler.addDataToStore(new CarbonRow(row));
        }
        dataHandler.finish();
    } catch (CarbonDataWriterException e) {
        LOGGER.error(e.getMessage(), e);
        throw new Exception("Problem loading data during compaction.", e);
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        throw new Exception("Problem loading data during compaction.", e);
    } finally {
        if (null != dataHandler) {
            try {
                dataHandler.closeHandler();
            } catch (CarbonDataWriterException e) {
                LOGGER.error("Error in close data handler", e);
                throw new Exception("Error in close data handler", e);
            }
        }
    }
}

Example 20 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class RowResultMergerProcessor method execute.

/**
 * Merge function
 */
public boolean execute(List<RawResultIterator> unsortedResultIteratorList, List<RawResultIterator> sortedResultIteratorList) throws Exception {
    List<RawResultIterator> finalIteratorList = new ArrayList<>(unsortedResultIteratorList);
    finalIteratorList.addAll(sortedResultIteratorList);
    initRecordHolderHeap(finalIteratorList);
    boolean mergeStatus = false;
    int index = 0;
    boolean isDataPresent = false;
    try {
        // add all iterators to the queue
        for (RawResultIterator leaftTupleIterator : finalIteratorList) {
            if (leaftTupleIterator.hasNext()) {
                this.recordHolderHeap.add(leaftTupleIterator);
                index++;
            }
        }
        RawResultIterator iterator = null;
        while (index > 1) {
            // iterator the top record
            iterator = this.recordHolderHeap.peek();
            Object[] convertedRow = iterator.next();
            if (null == convertedRow) {
                index--;
                iterator.close();
                this.recordHolderHeap.poll();
                continue;
            }
            if (!isDataPresent) {
                dataHandler.initialise();
                isDataPresent = true;
            }
            // get the mdkey
            addRow(convertedRow);
            // index
            if (!iterator.hasNext()) {
                index--;
                iterator.close();
                this.recordHolderHeap.poll();
                continue;
            }
            // maintain heap
            this.recordHolderHeap.siftTopDown();
        }
        // if record holder is not empty then iterator the slice holder from
        // heap
        iterator = this.recordHolderHeap.poll();
        if (null != iterator) {
            while (true) {
                Object[] convertedRow = iterator.next();
                if (null == convertedRow) {
                    iterator.close();
                    break;
                }
                // do it only once
                if (!isDataPresent) {
                    dataHandler.initialise();
                    isDataPresent = true;
                }
                addRow(convertedRow);
                // check if leaf contains no record
                if (!iterator.hasNext()) {
                    break;
                }
            }
        }
        if (isDataPresent) {
            this.dataHandler.finish();
        }
        mergeStatus = true;
    } catch (Exception e) {
        mergeStatus = false;
        LOGGER.error(e.getLocalizedMessage(), e);
        throw e;
    } finally {
        try {
            if (isDataPresent) {
                this.dataHandler.closeHandler();
            }
            boolean isMergeIndex = Boolean.parseBoolean(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT, CarbonCommonConstants.CARBON_MERGE_INDEX_IN_SEGMENT_DEFAULT));
            // and will be written during merging index
            if (partitionSpec != null && !isMergeIndex) {
                // By default carbon.merge.index.in.segment is true and this code will be used for
                // developer debugging purpose.
                SegmentFileStore.writeSegmentFileForPartitionTable(loadModel.getTablePath(), loadModel.getTaskNo(), partitionSpec.getLocation().toString(), loadModel.getFactTimeStamp() + "", partitionSpec.getPartitions());
            }
        } catch (CarbonDataWriterException | IOException e) {
            mergeStatus = false;
            throw e;
        }
    }
    return mergeStatus;
}