Examples with MemoryBlock org.apache.carbondata.core.memory.MemoryBlock used on opensource projects

Example 1 with MemoryBlock

use of org.apache.carbondata.core.memory.MemoryBlock in project carbondata by apache.

the class UnsafeSortDataRows method getMemoryBlock.

public static MemoryBlock getMemoryBlock(long size) throws CarbonSortKeyAndGroupByException {
    MemoryBlock baseBlock = null;
    int tries = 0;
    while (true && tries < 100) {
        baseBlock = UnsafeMemoryManager.INSTANCE.allocateMemory(size);
        if (baseBlock == null) {
            try {
                Thread.sleep(50);
            } catch (InterruptedException e) {
                throw new CarbonSortKeyAndGroupByException(e);
            }
        } else {
            break;
        }
        tries++;
    }
    if (baseBlock == null) {
        throw new CarbonSortKeyAndGroupByException("Not enough memory to create page");
    }
    return baseBlock;
}

Example 2 with MemoryBlock

use of org.apache.carbondata.core.memory.MemoryBlock in project carbondata by apache.

the class UnsafeSortDataRows method addRowBatch.

/**
   * This method will be used to add new row
   *
   * @param rowBatch new rowBatch
   * @throws CarbonSortKeyAndGroupByException problem while writing
   */
public void addRowBatch(Object[][] rowBatch, int size) throws CarbonSortKeyAndGroupByException {
    // sort the list and then write current list data to file
    synchronized (addRowsLock) {
        for (int i = 0; i < size; i++) {
            if (rowPage.canAdd()) {
                bytesAdded += rowPage.addRow(rowBatch[i]);
            } else {
                try {
                    if (enableInMemoryIntermediateMerge) {
                        unsafeInMemoryIntermediateFileMerger.startInmemoryMergingIfPossible();
                    }
                    unsafeInMemoryIntermediateFileMerger.startFileMergingIfPossible();
                    semaphore.acquire();
                    dataSorterAndWriterExecutorService.submit(new DataSorterAndWriter(rowPage));
                    MemoryBlock memoryBlock = getMemoryBlock(inMemoryChunkSize);
                    boolean saveToDisk = !UnsafeMemoryManager.INSTANCE.isMemoryAvailable();
                    rowPage = new UnsafeCarbonRowPage(parameters.getNoDictionaryDimnesionColumn(), parameters.getNoDictionarySortColumn(), parameters.getDimColCount() + parameters.getComplexDimColCount(), parameters.getMeasureColCount(), parameters.getMeasureDataType(), memoryBlock, saveToDisk);
                    bytesAdded += rowPage.addRow(rowBatch[i]);
                } catch (Exception e) {
                    LOGGER.error("exception occurred while trying to acquire a semaphore lock: " + e.getMessage());
                    throw new CarbonSortKeyAndGroupByException(e);
                }
            }
        }
    }
}

Example 3 with MemoryBlock

use of org.apache.carbondata.core.memory.MemoryBlock in project carbondata by apache.

the class UnsafeSortDataRows method addRow.

/**
   * This method will be used to add new row
   */
public void addRow(Object[] row) throws CarbonSortKeyAndGroupByException {
    // sort the list and then write current list data to file
    if (rowPage.canAdd()) {
        rowPage.addRow(row);
    } else {
        try {
            if (enableInMemoryIntermediateMerge) {
                unsafeInMemoryIntermediateFileMerger.startInmemoryMergingIfPossible();
            }
            unsafeInMemoryIntermediateFileMerger.startFileMergingIfPossible();
            semaphore.acquire();
            dataSorterAndWriterExecutorService.submit(new DataSorterAndWriter(rowPage));
            MemoryBlock memoryBlock = getMemoryBlock(inMemoryChunkSize);
            boolean saveToDisk = !UnsafeMemoryManager.INSTANCE.isMemoryAvailable();
            rowPage = new UnsafeCarbonRowPage(parameters.getNoDictionaryDimnesionColumn(), parameters.getNoDictionarySortColumn(), parameters.getDimColCount(), parameters.getMeasureColCount(), parameters.getMeasureDataType(), memoryBlock, saveToDisk);
            rowPage.addRow(row);
        } catch (Exception e) {
            LOGGER.error("exception occurred while trying to acquire a semaphore lock: " + e.getMessage());
            throw new CarbonSortKeyAndGroupByException(e);
        }
    }
}

Example 4 with MemoryBlock

use of org.apache.carbondata.core.memory.MemoryBlock in project carbondata by apache.

the class UnsafeSortDataRows method initialize.

/**
   * This method will be used to initialize
   */
public void initialize() throws CarbonSortKeyAndGroupByException {
    MemoryBlock baseBlock = getMemoryBlock(inMemoryChunkSize);
    this.rowPage = new UnsafeCarbonRowPage(parameters.getNoDictionaryDimnesionColumn(), parameters.getNoDictionarySortColumn(), parameters.getDimColCount() + parameters.getComplexDimColCount(), parameters.getMeasureColCount(), parameters.getMeasureDataType(), baseBlock, !UnsafeMemoryManager.INSTANCE.isMemoryAvailable());
    // Delete if any older file exists in sort temp folder
    deleteSortLocationIfExists();
    // create new sort temp directory
    if (!new File(parameters.getTempFileLocation()).mkdirs()) {
        LOGGER.info("Sort Temp Location Already Exists");
    }
    this.dataSorterAndWriterExecutorService = Executors.newFixedThreadPool(parameters.getNumberOfCores());
    semaphore = new Semaphore(parameters.getNumberOfCores());
}

Example 5 with MemoryBlock

use of org.apache.carbondata.core.memory.MemoryBlock in project carbondata by apache.

the class UnsafeFixLengthColumnPage method compress.

@Override
public byte[] compress(Compressor compressor) throws MemoryException, IOException {
    if (UnsafeMemoryManager.isOffHeap()) {
        // use raw compression and copy to byte[]
        int inputSize = pageSize * dataType.getSizeInBytes();
        int compressedMaxSize = compressor.maxCompressedLength(inputSize);
        MemoryBlock compressed = UnsafeMemoryManager.allocateMemoryWithRetry(taskId, compressedMaxSize);
        long outSize = compressor.rawCompress(baseOffset, inputSize, compressed.getBaseOffset());
        assert outSize < Integer.MAX_VALUE;
        byte[] output = new byte[(int) outSize];
        CarbonUnsafe.getUnsafe().copyMemory(compressed.getBaseObject(), compressed.getBaseOffset(), output, CarbonUnsafe.BYTE_ARRAY_OFFSET, outSize);
        UnsafeMemoryManager.INSTANCE.freeMemory(taskId, compressed);
        return output;
    } else {
        return super.compress(compressor);
    }
}