Examples with CarbonDataWriterException org.apache.carbondata.core.datastore.exception.CarbonDataWriterException used on opensource projects

Example 31 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class AbstractFactDataWriter method initializeWriter.

/**
 * This method will be used to initialize the channel
 *
 * @throws CarbonDataWriterException
 */
public void initializeWriter() throws CarbonDataWriterException {
    this.carbonDataFileName = CarbonTablePath.getCarbonDataFileName(fileCount, model.getCarbonDataFileAttributes().getTaskId(), model.getBucketId(), model.getTaskExtension(), "" + model.getCarbonDataFileAttributes().getFactTimeStamp(), model.getSegmentId(), model.getColumnCompressor());
    this.carbonDataFileStorePath = model.getCarbonDataDirectoryPath() + File.separator + carbonDataFileName;
    try {
        FileFactory.mkdirs(model.getCarbonDataDirectoryPath());
        if (enableDirectlyWriteDataToStorePath) {
            if (model.getTableSpec().getCarbonTable().isHivePartitionTable() && model.getCarbonDataDirectoryPath().endsWith(".tmp")) {
                // set carbonData file store path to partition path instead of tmp directory
                carbonDataFileStorePath = model.getCarbonDataDirectoryPath().substring(0, model.getCarbonDataDirectoryPath().lastIndexOf(File.separator)) + File.separator + carbonDataFileName;
            }
            // the block size will be twice the block_size specified by user to make sure that
            // one carbondata file only consists exactly one HDFS block.
            fileOutputStream = FileFactory.getDataOutputStream(carbonDataFileStorePath, CarbonCommonConstants.BYTEBUFFER_SIZE, fileSizeInBytes * 2);
        } else {
            // each time we initialize writer, we choose a local temp location randomly
            String[] tempFileLocations = model.getStoreLocation();
            String chosenTempLocation = tempFileLocations[new Random().nextInt(tempFileLocations.length)];
            LOGGER.info("Randomly choose factdata temp location: " + chosenTempLocation);
            carbonDataFileTempPath = chosenTempLocation + File.separator + carbonDataFileName;
            fileOutputStream = FileFactory.getDataOutputStream(carbonDataFileTempPath, CarbonCommonConstants.BYTEBUFFER_SIZE, true);
        }
        this.fileCount++;
        // open channel for new data file
        this.fileChannel = Channels.newChannel(fileOutputStream);
        this.currentOffsetInFile = 0;
    } catch (IOException ex) {
        throw new CarbonDataWriterException("Problem while getting the channel for fact data file", ex);
    }
    notifyBlockStart();
}

Example 32 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class SingleThreadFinalSortFilesMerger method getSortedRecordFromFile.

/**
 * This method will be used to get the sorted record from file
 *
 * @return sorted record sorted record
 * @throws CarbonSortKeyAndGroupByException
 */
private IntermediateSortTempRow getSortedRecordFromFile() throws CarbonDataWriterException {
    IntermediateSortTempRow row = null;
    // poll the top object from heap
    // heap maintains binary tree which is based on heap condition that will
    // be based on comparator we are passing the heap
    // when will call poll it will always delete root of the tree and then
    // it does trickel down operation complexity is log(n)
    SortTempFileChunkHolder poll = this.recordHolderHeapLocal.peek();
    // get the row from chunk
    row = poll.getRow();
    // check if there no entry present
    if (!poll.hasNext()) {
        // if chunk is empty then close the stream
        poll.closeStream();
        this.recordHolderHeapLocal.poll();
        // return row
        return row;
    }
    // read new row
    try {
        poll.readRow();
    } catch (CarbonSortKeyAndGroupByException e) {
        close();
        throw new CarbonDataWriterException(e);
    }
    // maintain heap
    this.recordHolderHeapLocal.siftTopDown();
    // return row
    return row;
}

Example 33 with CarbonDataWriterException

use of org.apache.carbondata.core.datastore.exception.CarbonDataWriterException in project carbondata by apache.

the class CarbonFactDataHandlerColumnar method setFlatCarbonRowForComplex.

private int setFlatCarbonRowForComplex(CarbonRow row) {
    int noDictTotalComplexChildDepth = 0;
    Object[] noDictAndComplexDimension = WriteStepRowUtil.getNoDictAndComplexDimension(row);
    for (int i = 0; i < noDictAndComplexDimension.length; i++) {
        // complex types starts after no dictionary dimensions
        if (i >= model.getNoDictionaryCount() && (model.getTableSpec().getNoDictionaryDimensionSpec().get(i).getSchemaDataType().isComplexType())) {
            // this is for depth of each complex column, model is having only total depth.
            GenericDataType genericDataType = complexIndexMapCopy.get(i - model.getNoDictionaryCount() + model.getSegmentProperties().getNumberOfPrimitiveDimensions());
            int depth = genericDataType.getDepth();
            // initialize flatComplexColumnList
            List<ArrayList<byte[]>> flatComplexColumnList = new ArrayList<>(depth);
            for (int k = 0; k < depth; k++) {
                flatComplexColumnList.add(new ArrayList<byte[]>());
            }
            // flatten the complex byteArray as per depth
            try {
                ByteBuffer byteArrayInput = ByteBuffer.wrap((byte[]) noDictAndComplexDimension[i]);
                ByteArrayOutputStream byteArrayOutput = new ByteArrayOutputStream();
                DataOutputStream dataOutputStream = new DataOutputStream(byteArrayOutput);
                genericDataType.parseComplexValue(byteArrayInput, dataOutputStream);
                genericDataType.getColumnarDataForComplexType(flatComplexColumnList, ByteBuffer.wrap(byteArrayOutput.toByteArray()));
                byteArrayOutput.close();
            } catch (IOException | KeyGenException e) {
                throw new CarbonDataWriterException("Problem in splitting and writing complex data", e);
            }
            noDictTotalComplexChildDepth += flatComplexColumnList.size();
            // update the complex column data with the flat data
            noDictAndComplexDimension[i] = flatComplexColumnList;
        }
    }
    return noDictTotalComplexChildDepth;
}