Examples with DeleteDeltaBlockDetails org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails used on opensource projects

Example 1 with DeleteDeltaBlockDetails

use of org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails in project carbondata by apache.

the class CarbonDeleteDeltaFileReaderImpl method readJson.

/**
   * Reads delete delta file (json file) and returns DeleteDeltaBlockDetails
   * @return DeleteDeltaBlockDetails
   * @throws IOException
   */
@Override
public DeleteDeltaBlockDetails readJson() throws IOException {
    Gson gsonObjectToRead = new Gson();
    DataInputStream dataInputStream = null;
    BufferedReader buffReader = null;
    InputStreamReader inStream = null;
    DeleteDeltaBlockDetails deleteDeltaBlockDetails;
    AtomicFileOperations fileOperation = new AtomicFileOperationsImpl(filePath, FileFactory.getFileType(filePath));
    try {
        if (!FileFactory.isFileExist(filePath, FileFactory.getFileType(filePath))) {
            return new DeleteDeltaBlockDetails("");
        }
        dataInputStream = fileOperation.openForRead();
        inStream = new InputStreamReader(dataInputStream, CarbonCommonConstants.CARBON_DEFAULT_STREAM_ENCODEFORMAT);
        buffReader = new BufferedReader(inStream);
        deleteDeltaBlockDetails = gsonObjectToRead.fromJson(buffReader, DeleteDeltaBlockDetails.class);
    } catch (IOException e) {
        return new DeleteDeltaBlockDetails("");
    } finally {
        CarbonUtil.closeStreams(buffReader, inStream, dataInputStream);
    }
    return deleteDeltaBlockDetails;
}

Example 2 with DeleteDeltaBlockDetails

use of org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails in project carbondata by apache.

the class CarbonDeleteFilesDataReader method getCompactedDeleteDeltaFileFromBlock.

/**
   * returns delete delta file details for the specified block name
   * @param deltaFiles
   * @param blockName
   * @return DeleteDeltaBlockDetails
   * @throws Exception
   */
public DeleteDeltaBlockDetails getCompactedDeleteDeltaFileFromBlock(List<String> deltaFiles, String blockName) throws Exception {
    // get the data.
    List<Future<DeleteDeltaBlockDetails>> taskSubmitList = new ArrayList<>(deltaFiles.size());
    ExecutorService executorService = Executors.newFixedThreadPool(thread_pool_size);
    for (final String deltaFile : deltaFiles) {
        taskSubmitList.add(executorService.submit(new Callable<DeleteDeltaBlockDetails>() {

            @Override
            public DeleteDeltaBlockDetails call() throws IOException {
                CarbonDeleteDeltaFileReaderImpl deltaFileReader = new CarbonDeleteDeltaFileReaderImpl(deltaFile, FileFactory.getFileType(deltaFile));
                return deltaFileReader.readJson();
            }
        }));
    }
    try {
        executorService.shutdown();
        executorService.awaitTermination(30, TimeUnit.MINUTES);
    } catch (InterruptedException e) {
        LOGGER.error("Error while reading the delete delta files : " + e.getMessage());
    }
    // Get a new DeleteDeltaBlockDetails as result set where all the data will me merged
    // based on each Blocklet.
    DeleteDeltaBlockDetails deleteDeltaResultSet = new DeleteDeltaBlockDetails(blockName);
    for (int i = 0; i < taskSubmitList.size(); i++) {
        try {
            List<DeleteDeltaBlockletDetails> blockletDetails = taskSubmitList.get(i).get().getBlockletDetails();
            for (DeleteDeltaBlockletDetails blocklet : blockletDetails) {
                deleteDeltaResultSet.addBlockletDetails(blocklet);
            }
        } catch (Throwable e) {
            LOGGER.error(e.getMessage());
            throw new Exception(e.getMessage());
        }
    }
    return deleteDeltaResultSet;
}

Example 3 with DeleteDeltaBlockDetails

use of org.apache.carbondata.core.mutate.DeleteDeltaBlockDetails in project carbondata by apache.

the class CarbonDataMergerUtil method startCompactionDeleteDeltaFiles.

/**
   * this method compact the delete delta files.
   * @param deleteDeltaFiles
   * @param blockName
   * @param fullBlockFilePath
   * @return
   */
public static Boolean startCompactionDeleteDeltaFiles(List<String> deleteDeltaFiles, String blockName, String fullBlockFilePath) throws IOException {
    DeleteDeltaBlockDetails deleteDeltaBlockDetails = null;
    CarbonDeleteFilesDataReader dataReader = new CarbonDeleteFilesDataReader();
    try {
        deleteDeltaBlockDetails = dataReader.getCompactedDeleteDeltaFileFromBlock(deleteDeltaFiles, blockName);
    } catch (Exception e) {
        String blockFilePath = fullBlockFilePath.substring(0, fullBlockFilePath.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR));
        LOGGER.error("Error while getting the delete delta blocks in path " + blockFilePath);
        throw new IOException();
    }
    CarbonDeleteDeltaWriterImpl carbonDeleteWriter = new CarbonDeleteDeltaWriterImpl(fullBlockFilePath, FileFactory.getFileType(fullBlockFilePath));
    try {
        carbonDeleteWriter.write(deleteDeltaBlockDetails);
    } catch (IOException e) {
        LOGGER.error("Error while writing compacted delete delta file " + fullBlockFilePath);
        throw new IOException();
    }
    return true;
}