Examples with CarbonDataWriterException org.apache.carbondata.core.datastore.exception.CarbonDataWriterException used on opensource projects

Example 1 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class CarbonFactDataWriterImplV3 method writeBlockletToFile.

/**
 * Write the collect blocklet data (blockletDataHolder) to file
 */
private void writeBlockletToFile() {
    // get the list of all encoded table page
    List<EncodedTablePage> encodedTablePageList = blockletDataHolder.getEncodedTablePages();
    int numDimensions = encodedTablePageList.get(0).getNumDimensions();
    int numMeasures = encodedTablePageList.get(0).getNumMeasures();
    // get data chunks for all the column
    byte[][] dataChunkBytes = new byte[numDimensions + numMeasures][];
    long metadataSize = fillDataChunk(encodedTablePageList, dataChunkBytes);
    // calculate the total size of data to be written
    long blockletSize = blockletDataHolder.getSize() + metadataSize;
    // to check if data size will exceed the block size then create a new file
    createNewFileIfReachThreshold(blockletSize);
    // write data to file
    try {
        if (currentOffsetInFile == 0) {
            // write the header if file is empty
            writeHeaderToFile();
        }
        writeBlockletToFile(dataChunkBytes);
        if (listener != null) {
            listener.onBlockletEnd(blockletId++);
        }
        pageId = 0;
    } catch (IOException e) {
        LOGGER.error(e, "Problem while writing file");
        throw new CarbonDataWriterException("Problem while writing file", e);
    } finally {
        // clear the data holder
        blockletDataHolder.clear();
    }
}

Example 2 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class CarbonFactDataWriterImplV3 method writeBlockletInfoToFile.

@Override
protected void writeBlockletInfoToFile() throws CarbonDataWriterException {
    try {
        // get the current file position
        long currentPosition = currentOffsetInFile;
        // get thrift file footer instance
        FileFooter3 convertFileMeta = CarbonMetadataUtil.convertFileFooterVersion3(blockletMetadata, blockletIndex, localCardinality, thriftColumnSchemaList.size());
        // fill the carbon index details
        fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFileName, currentPosition);
        // write the footer
        byte[] byteArray = CarbonUtil.getByteArray(convertFileMeta);
        ByteBuffer buffer = ByteBuffer.allocate(byteArray.length + CarbonCommonConstants.LONG_SIZE_IN_BYTE);
        buffer.put(byteArray);
        buffer.putLong(currentPosition);
        buffer.flip();
        currentOffsetInFile += fileChannel.write(buffer);
    } catch (IOException e) {
        LOGGER.error(e, "Problem while writing the carbon file");
        throw new CarbonDataWriterException("Problem while writing the carbon file: ", e);
    }
}

Example 3 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class CarbonRowDataWriterProcessorStepImpl method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    final Iterator<CarbonRowBatch>[] iterators = child.execute();
    tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    tableName = tableIdentifier.getTableName();
    ExecutorService executorService = null;
    try {
        readCounter = new long[iterators.length];
        writeCounter = new long[iterators.length];
        dimensionWithComplexCount = configuration.getDimensionCount();
        noDictWithComplextCount = configuration.getNoDictionaryCount() + configuration.getComplexColumnCount();
        dimensionCount = configuration.getDimensionCount() - noDictWithComplextCount;
        isNoDictionaryDimensionColumn = CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
        measureDataType = configuration.getMeasureDataType();
        measureCount = configuration.getMeasureCount();
        outputLength = measureCount + (this.noDictWithComplextCount > 0 ? 1 : 0) + 1;
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis());
        if (iterators.length == 1) {
            doExecute(iterators[0], 0);
        } else {
            executorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("NoSortDataWriterPool:" + configuration.getTableIdentifier().getCarbonTableIdentifier().getTableName()));
            Future[] futures = new Future[iterators.length];
            for (int i = 0; i < iterators.length; i++) {
                futures[i] = executorService.submit(new DataWriterRunnable(iterators[i], i));
            }
            for (Future future : futures) {
                future.get();
            }
        }
    } catch (CarbonDataWriterException e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        throw new CarbonDataLoadingException("Error while initializing data handler : " + e.getMessage());
    } catch (Exception e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        if (e instanceof BadRecordFoundException) {
            throw new BadRecordFoundException(e.getMessage(), e);
        }
        throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage(), e);
    } finally {
        if (null != executorService && executorService.isShutdown()) {
            executorService.shutdownNow();
        }
    }
    return null;
}

Example 4 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class DataWriterProcessorStepImpl method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    Iterator<CarbonRowBatch>[] iterators = child.execute();
    CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    String tableName = tableIdentifier.getTableName();
    try {
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis());
        ExecutorService rangeExecutorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("WriterForwardPool: " + tableName));
        List<Future<Void>> rangeExecutorServiceSubmitList = new ArrayList<>(iterators.length);
        int i = 0;
        // do this concurrently
        for (Iterator<CarbonRowBatch> iterator : iterators) {
            String[] storeLocation = getStoreLocation(tableIdentifier);
            CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration, storeLocation, i, 0);
            CarbonFactHandler dataHandler = null;
            boolean rowsNotExist = true;
            while (iterator.hasNext()) {
                if (rowsNotExist) {
                    rowsNotExist = false;
                    dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
                    dataHandler.initialise();
                }
                processBatch(iterator.next(), dataHandler);
            }
            if (!rowsNotExist) {
                finish(dataHandler);
            }
            rangeExecutorServiceSubmitList.add(rangeExecutorService.submit(new WriterForwarder(iterator, tableIdentifier, i)));
            i++;
        }
        try {
            rangeExecutorService.shutdown();
            rangeExecutorService.awaitTermination(2, TimeUnit.DAYS);
            for (int j = 0; j < rangeExecutorServiceSubmitList.size(); j++) {
                rangeExecutorServiceSubmitList.get(j).get();
            }
        } catch (InterruptedException | ExecutionException e) {
            throw new CarbonDataWriterException(e);
        }
    } catch (CarbonDataWriterException e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        throw new CarbonDataLoadingException("Error while initializing data handler : " + e.getMessage());
    } catch (Exception e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage(), e);
    }
    return null;
}

Example 5 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class CarbonFactDataHandlerColumnar method finish.

/**
 * below method will be used to finish the data handler
 *
 * @throws CarbonDataWriterException
 */
public void finish() throws CarbonDataWriterException {
    // than 0
    if (null == dataWriter) {
        return;
    }
    if (producerExecutorService.isShutdown()) {
        return;
    }
    LOGGER.info("Started Finish Operation");
    try {
        semaphore.acquire();
        producerExecutorServiceTaskList.add(producerExecutorService.submit(new Producer(tablePageList, dataRows, ++writerTaskSequenceCounter, true)));
        blockletProcessingCount.incrementAndGet();
        processedDataCount += entryCount;
        LOGGER.info("Total Number Of records added to store: " + processedDataCount);
        closeWriterExecutionService(producerExecutorService);
        processWriteTaskSubmitList(producerExecutorServiceTaskList);
        processingComplete = true;
    } catch (InterruptedException e) {
        LOGGER.error(e, e.getMessage());
        throw new CarbonDataWriterException(e.getMessage(), e);
    }
}