Examples with CarbonDataWriterException org.apache.carbondata.core.datastore.exception.CarbonDataWriterException used on opensource projects

Example 26 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class UnsafeSingleThreadFinalSortFilesMerger method getSortedRecordFromFile.

/**
 * This method will be used to get the sorted record from file
 *
 * @return sorted record sorted record
 */
private IntermediateSortTempRow getSortedRecordFromFile() throws CarbonDataWriterException {
    IntermediateSortTempRow row = null;
    // poll the top object from heap
    // heap maintains binary tree which is based on heap condition that will
    // be based on comparator we are passing the heap
    // when will call poll it will always delete root of the tree and then
    // it does trickel down operation complexity is log(n)
    SortTempChunkHolder poll = this.recordHolderHeapLocal.peek();
    // get the row from chunk
    row = poll.getRow();
    // check if there no entry present
    if (!poll.hasNext()) {
        // if chunk is empty then close the stream
        poll.close();
        recordHolderHeapLocal.poll();
        // change the file counter
        --this.fileCounter;
        // return row
        return row;
    }
    // read new row
    try {
        poll.readRow();
    } catch (Exception e) {
        throw new CarbonDataWriterException(e);
    }
    // maintain heap
    this.recordHolderHeapLocal.siftTopDown();
    // return row
    return row;
}

Example 27 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class SecondaryIndexQueryResultProcessor method readAndLoadDataFromSortTempFiles.

/**
 * This method will read sort temp files, perform merge sort and add it to store for data loading
 */
private void readAndLoadDataFromSortTempFiles() throws SecondaryIndexException {
    Throwable throwable = null;
    try {
        Object[] previousRow = null;
        // comparator for grouping the similar data, means every record
        // should be unique in index table
        RowComparator comparator = new RowComparator(noDictionaryColMapping, SecondaryIndexUtil.getNoDictDataTypes(indexTable));
        intermediateFileMerger.finish();
        sortDataRows = null;
        finalMerger.startFinalMerge();
        while (finalMerger.hasNext()) {
            Object[] rowRead = finalMerger.next();
            if (null == previousRow) {
                previousRow = rowRead;
            } else {
                int compareResult = comparator.compare(previousRow, rowRead);
                if (0 == compareResult) {
                    // skip similar data rows
                    continue;
                } else {
                    previousRow = rowRead;
                }
            }
            CarbonRow row = new CarbonRow(rowRead);
            dataHandler.addDataToStore(row);
        }
        dataHandler.finish();
    } catch (CarbonDataWriterException e) {
        LOGGER.error(e);
        throw new SecondaryIndexException("Problem loading data while creating secondary index: ", e);
    } catch (CarbonSortKeyAndGroupByException e) {
        LOGGER.error(e);
        throw new SecondaryIndexException("Problem in merging intermediate files while creating secondary index: ", e);
    } catch (Throwable t) {
        LOGGER.error(t);
        throw new SecondaryIndexException("Problem while creating secondary index: ", t);
    } finally {
        if (null != dataHandler) {
            try {
                dataHandler.closeHandler();
            } catch (CarbonDataWriterException e) {
                LOGGER.error(e);
                throwable = e;
            }
        }
    }
    if (null != throwable) {
        throw new SecondaryIndexException("Problem closing data handler while creating secondary index: ", throwable);
    }
    dataHandler = null;
}

Example 28 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class CarbonUtil method copyCarbonDataFileToCarbonStorePath.

/**
 * This method will copy the given file to carbon store location
 *
 * @param localFilePath local file name with full path
 * @throws CarbonDataWriterException
 * @return the file size
 */
public static long copyCarbonDataFileToCarbonStorePath(String localFilePath, String carbonDataDirectoryPath, long fileSizeInBytes) throws CarbonDataWriterException {
    long copyStartTime = System.currentTimeMillis();
    LOGGER.info(String.format("Copying %s to %s, operation id %d", localFilePath, carbonDataDirectoryPath, copyStartTime));
    long targetSize = 0;
    try {
        CarbonFile localCarbonFile = FileFactory.getCarbonFile(localFilePath);
        long localFileSize = localCarbonFile.getSize();
        // the size of local carbon file must be greater than 0
        if (localFileSize == 0L) {
            LOGGER.error("The size of local carbon file: " + localFilePath + " is 0.");
            throw new CarbonDataWriterException("The size of local carbon file is 0.");
        }
        String carbonFilePath = carbonDataDirectoryPath + localFilePath.substring(localFilePath.lastIndexOf(File.separator));
        copyLocalFileToCarbonStore(carbonFilePath, localFilePath, CarbonCommonConstants.BYTEBUFFER_SIZE, getMaxOfBlockAndFileSize(fileSizeInBytes, localFileSize));
        CarbonFile targetCarbonFile = FileFactory.getCarbonFile(carbonFilePath);
        // the size of carbon file must be greater than 0
        // and the same as the size of local carbon file
        targetSize = targetCarbonFile.getSize();
        if (targetSize == 0L || targetSize != localFileSize) {
            LOGGER.error("The size of carbon file: " + carbonFilePath + " is 0 " + "or is not the same as the size of local carbon file: (" + "carbon file size=" + targetSize + ", local carbon file size=" + localFileSize + ")");
            throw new CarbonDataWriterException("The size of carbon file is 0 " + "or is not the same as the size of local carbon file.");
        }
    } catch (Exception e) {
        throw new CarbonDataWriterException("Problem while copying file from local store to carbon store", e);
    }
    LOGGER.info(String.format("Total copy time is %d ms, operation id %d", System.currentTimeMillis() - copyStartTime, copyStartTime));
    return targetSize;
}

Example 29 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class CarbonFactDataWriterImplV3 method closeWriter.

/**
 * Method will be used to close the open file channel
 *
 * @throws CarbonDataWriterException
 */
public void closeWriter() throws CarbonDataWriterException {
    CarbonDataWriterException exception = null;
    try {
        commitCurrentFile(true);
        writeIndexFile();
    } catch (Exception e) {
        LOGGER.error("Problem while writing the index file", e);
        exception = new CarbonDataWriterException("Problem while writing the index file", e);
    } finally {
        try {
            closeExecutorService();
        } catch (CarbonDataWriterException e) {
            if (null == exception) {
                exception = e;
            }
        }
    }
    if (null != exception) {
        throw exception;
    }
}

Example 30 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class CarbonFactDataWriterImplV3 method writeFooterToFile.

@Override
protected void writeFooterToFile() throws CarbonDataWriterException {
    try {
        // get the current file position
        long footerOffset = currentOffsetInFile;
        // get thrift file footer instance
        FileFooter3 convertFileMeta = CarbonMetadataUtil.convertFileFooterVersion3(blockletMetadata, blockletIndex, thriftColumnSchemaList.size());
        convertFileMeta.setIs_sort(isSorted);
        String appName = CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME);
        if (appName == null) {
            throw new CarbonDataWriterException("DataLoading failed as CARBON_WRITTEN_BY_APPNAME is null");
        }
        convertFileMeta.putToExtra_info(CarbonCommonConstants.CARBON_WRITTEN_BY_FOOTER_INFO, appName);
        convertFileMeta.putToExtra_info(CarbonCommonConstants.CARBON_WRITTEN_VERSION, CarbonVersionConstants.CARBONDATA_VERSION);
        // write the footer
        byte[] byteArray = CarbonUtil.getByteArray(convertFileMeta);
        ByteBuffer buffer = ByteBuffer.allocate(byteArray.length + CarbonCommonConstants.LONG_SIZE_IN_BYTE);
        buffer.put(byteArray);
        buffer.putLong(footerOffset);
        buffer.flip();
        currentOffsetInFile += fileChannel.write(buffer);
        // fill the carbon index details
        fillBlockIndexInfoDetails(convertFileMeta.getNum_rows(), carbonDataFileName, footerOffset, currentOffsetInFile);
    } catch (IOException e) {
        LOGGER.error("Problem while writing the carbon file", e);
        throw new CarbonDataWriterException("Problem while writing the carbon file: ", e);
    }
}