Examples with DeleteDeltaBlockletDetails org.apache.carbondata.core.mutate.DeleteDeltaBlockletDetails used on opensource projects

Example 1 with DeleteDeltaBlockletDetails

use of org.apache.carbondata.core.mutate.DeleteDeltaBlockletDetails in project carbondata by apache.

the class CarbonDeleteFilesDataReader method getDeleteDataFromAllFiles.

/**
   * Returns all deleted records from all specified delta files
   *
   * @param deltaFiles
   * @return
   * @throws Exception
   */
public int[] getDeleteDataFromAllFiles(List<String> deltaFiles, String blockletId) throws Exception {
    List<Future<DeleteDeltaBlockDetails>> taskSubmitList = new ArrayList<>();
    ExecutorService executorService = Executors.newFixedThreadPool(thread_pool_size);
    for (final String deltaFile : deltaFiles) {
        taskSubmitList.add(executorService.submit(new Callable<DeleteDeltaBlockDetails>() {

            @Override
            public DeleteDeltaBlockDetails call() throws IOException {
                CarbonDeleteDeltaFileReaderImpl deltaFileReader = new CarbonDeleteDeltaFileReaderImpl(deltaFile, FileFactory.getFileType(deltaFile));
                return deltaFileReader.readJson();
            }
        }));
    }
    try {
        executorService.shutdown();
        executorService.awaitTermination(30, TimeUnit.MINUTES);
    } catch (InterruptedException e) {
        LOGGER.error("Error while reading the delete delta files : " + e.getMessage());
    }
    Set<Integer> result = new TreeSet<Integer>();
    for (int i = 0; i < taskSubmitList.size(); i++) {
        try {
            List<DeleteDeltaBlockletDetails> blockletDetails = taskSubmitList.get(i).get().getBlockletDetails();
            result.addAll(blockletDetails.get(blockletDetails.indexOf(new DeleteDeltaBlockletDetails(blockletId))).getDeletedRows());
        } catch (Throwable e) {
            LOGGER.error(e.getMessage());
            throw new Exception(e.getMessage());
        }
    }
    return ArrayUtils.toPrimitive(result.toArray(new Integer[result.size()]));
}

Example 2 with DeleteDeltaBlockletDetails

use of org.apache.carbondata.core.mutate.DeleteDeltaBlockletDetails in project carbondata by apache.

the class CarbonDeleteFilesDataReader method getCompactedDeleteDeltaFileFromBlock.

/**
   * returns delete delta file details for the specified block name
   * @param deltaFiles
   * @param blockName
   * @return DeleteDeltaBlockDetails
   * @throws Exception
   */
public DeleteDeltaBlockDetails getCompactedDeleteDeltaFileFromBlock(List<String> deltaFiles, String blockName) throws Exception {
    // get the data.
    List<Future<DeleteDeltaBlockDetails>> taskSubmitList = new ArrayList<>(deltaFiles.size());
    ExecutorService executorService = Executors.newFixedThreadPool(thread_pool_size);
    for (final String deltaFile : deltaFiles) {
        taskSubmitList.add(executorService.submit(new Callable<DeleteDeltaBlockDetails>() {

            @Override
            public DeleteDeltaBlockDetails call() throws IOException {
                CarbonDeleteDeltaFileReaderImpl deltaFileReader = new CarbonDeleteDeltaFileReaderImpl(deltaFile, FileFactory.getFileType(deltaFile));
                return deltaFileReader.readJson();
            }
        }));
    }
    try {
        executorService.shutdown();
        executorService.awaitTermination(30, TimeUnit.MINUTES);
    } catch (InterruptedException e) {
        LOGGER.error("Error while reading the delete delta files : " + e.getMessage());
    }
    // Get a new DeleteDeltaBlockDetails as result set where all the data will me merged
    // based on each Blocklet.
    DeleteDeltaBlockDetails deleteDeltaResultSet = new DeleteDeltaBlockDetails(blockName);
    for (int i = 0; i < taskSubmitList.size(); i++) {
        try {
            List<DeleteDeltaBlockletDetails> blockletDetails = taskSubmitList.get(i).get().getBlockletDetails();
            for (DeleteDeltaBlockletDetails blocklet : blockletDetails) {
                deleteDeltaResultSet.addBlockletDetails(blocklet);
            }
        } catch (Throwable e) {
            LOGGER.error(e.getMessage());
            throw new Exception(e.getMessage());
        }
    }
    return deleteDeltaResultSet;
}