Examples with CarbonThreadFactory org.apache.carbondata.core.util.CarbonThreadFactory used on opensource projects

Example 6 with CarbonThreadFactory

use of org.apache.carbondata.core.util.CarbonThreadFactory in project carbondata by apache.

the class CarbonRowDataWriterProcessorStepImpl method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    final Iterator<CarbonRowBatch>[] iterators = child.execute();
    tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    tableName = tableIdentifier.getTableName();
    try {
        readCounter = new long[iterators.length];
        writeCounter = new long[iterators.length];
        dimensionWithComplexCount = configuration.getDimensionCount();
        noDictWithComplextCount = configuration.getNoDictionaryCount() + configuration.getComplexDictionaryColumnCount() + configuration.getComplexNonDictionaryColumnCount();
        directDictionaryDimensionCount = configuration.getDimensionCount() - noDictWithComplextCount;
        isNoDictionaryDimensionColumn = CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
        measureCount = configuration.getMeasureCount();
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PARTITION_ID, System.currentTimeMillis());
        if (configuration.getDataLoadProperty(DataLoadProcessorConstants.NO_REARRANGE_OF_ROWS) != null) {
            initializeNoReArrangeIndexes();
        }
        if (iterators.length == 1) {
            doExecute(iterators[0], 0);
        } else {
            executorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("NoSortDataWriterPool:" + configuration.getTableIdentifier().getCarbonTableIdentifier().getTableName(), true));
            Future[] futures = new Future[iterators.length];
            for (int i = 0; i < iterators.length; i++) {
                futures[i] = executorService.submit(new DataWriterRunnable(iterators[i], i));
            }
            for (Future future : futures) {
                future.get();
            }
        }
    } catch (CarbonDataWriterException e) {
        LOGGER.error("Failed for table: " + tableName + " in DataWriterProcessorStepImpl", e);
        throw new CarbonDataLoadingException("Error while initializing data handler : " + e.getMessage(), e);
    } catch (Exception e) {
        LOGGER.error("Failed for table: " + tableName + " in DataWriterProcessorStepImpl", e);
        if (e instanceof BadRecordFoundException) {
            throw new BadRecordFoundException(e.getMessage(), e);
        }
        throw new CarbonDataLoadingException(e.getMessage(), e);
    }
    return null;
}

Example 7 with CarbonThreadFactory

use of org.apache.carbondata.core.util.CarbonThreadFactory in project carbondata by apache.

the class InputProcessorStepImpl method initialize.

@Override
public void initialize() throws IOException {
    super.initialize();
    rowParser = new RowParserImpl(getOutput(), configuration);
    executorService = Executors.newCachedThreadPool(new CarbonThreadFactory("InputProcessorPool:" + configuration.getTableIdentifier().getCarbonTableIdentifier().getTableName(), true));
    // if logger is enabled then raw data will be required.
    this.isRawDataRequired = CarbonDataProcessorUtil.isRawDataRequired(configuration);
}

Example 8 with CarbonThreadFactory

use of org.apache.carbondata.core.util.CarbonThreadFactory in project carbondata by apache.

the class CarbonTableOutputFormat method getRecordWriter.

@Override
public RecordWriter<NullWritable, ObjectArrayWritable> getRecordWriter(final TaskAttemptContext taskAttemptContext) throws IOException {
    final CarbonLoadModel loadModel = getLoadModel(taskAttemptContext.getConfiguration());
    loadModel.setMetrics(new DataLoadMetrics());
    String appName = taskAttemptContext.getConfiguration().get(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME);
    if (null != appName) {
        CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_WRITTEN_BY_APPNAME, appName);
    }
    // if loadModel having taskNo already(like in SDK) then no need to overwrite
    short sdkWriterCores = loadModel.getSdkWriterCores();
    int itrSize = (sdkWriterCores > 0) ? sdkWriterCores : 1;
    final CarbonOutputIteratorWrapper[] iterators = new CarbonOutputIteratorWrapper[itrSize];
    for (int i = 0; i < itrSize; i++) {
        iterators[i] = new CarbonOutputIteratorWrapper();
    }
    if (null == loadModel.getTaskNo() || loadModel.getTaskNo().isEmpty()) {
        loadModel.setTaskNo(taskAttemptContext.getConfiguration().get("carbon.outputformat.taskno", String.valueOf(DEFAULT_TASK_NO.getAndIncrement())));
    }
    loadModel.setDataWritePath(taskAttemptContext.getConfiguration().get("carbon.outputformat.writepath"));
    final String[] tempStoreLocations = getTempStoreLocations(taskAttemptContext);
    DataTypeUtil.clearFormatter();
    final DataLoadExecutor dataLoadExecutor = new DataLoadExecutor();
    final ExecutorService executorService = Executors.newFixedThreadPool(1, new CarbonThreadFactory("CarbonRecordWriter:" + loadModel.getTableName(), true));
    // It should be started in new thread as the underlying iterator uses blocking queue.
    Future future = executorService.submit(() -> {
        ThreadLocalSessionInfo.getOrCreateCarbonSessionInfo().getNonSerializableExtraInfo().put("carbonConf", taskAttemptContext.getConfiguration());
        try {
            dataLoadExecutor.execute(loadModel, tempStoreLocations, iterators);
        } catch (Exception e) {
            executorService.shutdownNow();
            for (CarbonOutputIteratorWrapper iterator : iterators) {
                iterator.closeWriter(true);
            }
            try {
                dataLoadExecutor.close();
            } catch (Exception ex) {
                // As already exception happened before close() send that exception.
                throw new RuntimeException(e);
            }
            throw new RuntimeException(e);
        } finally {
            ThreadLocalSessionInfo.unsetAll();
        }
    });
    if (sdkWriterCores > 0) {
        // CarbonMultiRecordWriter handles the load balancing of the write rows in round robin.
        return new CarbonMultiRecordWriter(iterators, dataLoadExecutor, loadModel, future, executorService);
    } else {
        return new CarbonRecordWriter(iterators[0], dataLoadExecutor, loadModel, future, executorService);
    }
}

Example 9 with CarbonThreadFactory

use of org.apache.carbondata.core.util.CarbonThreadFactory in project carbondata by apache.

the class TableProcessingOperations method deleteLocalDataLoadFolderLocation.

/**
 * This method will delete the local data load folder location after data load is complete
 *
 * @param tempLocationKey temporary location set in carbon properties
 * @param tableName
 */
public static void deleteLocalDataLoadFolderLocation(String tempLocationKey, String tableName) {
    // form local store location
    final String localStoreLocations = CarbonProperties.getInstance().getProperty(tempLocationKey);
    if (localStoreLocations == null) {
        throw new RuntimeException("Store location not set for the key " + tempLocationKey);
    }
    // submit local folder clean up in another thread so that main thread execution is not blocked
    ExecutorService localFolderDeletionService = Executors.newFixedThreadPool(1, new CarbonThreadFactory("LocalFolderDeletionPool:" + tableName, true));
    try {
        localFolderDeletionService.submit(new Callable<Void>() {

            @Override
            public Void call() {
                long startTime = System.currentTimeMillis();
                String[] locArray = StringUtils.split(localStoreLocations, File.pathSeparator);
                for (String loc : locArray) {
                    try {
                        CarbonUtil.deleteFoldersAndFiles(new File(loc));
                    } catch (IOException | InterruptedException e) {
                        LOGGER.error("Failed to delete local data load folder location: " + loc, e);
                    }
                }
                LOGGER.info("Deleted the local store location: " + localStoreLocations + " : Time taken: " + (System.currentTimeMillis() - startTime));
                return null;
            }
        });
    } finally {
        CarbonProperties.getInstance().removeProperty(tempLocationKey);
        if (null != localFolderDeletionService) {
            localFolderDeletionService.shutdown();
        }
    }
}

Example 10 with CarbonThreadFactory

use of org.apache.carbondata.core.util.CarbonThreadFactory in project carbondata by apache.

the class ExtendedBlockletWrapperContainer method getExtendedBlocklets.

public List<ExtendedBlocklet> getExtendedBlocklets(IndexInputFormat indexInputFormat) throws IOException {
    if (!isFallbackJob) {
        int numOfThreads = CarbonProperties.getNumOfThreadsForPruning();
        ExecutorService executorService = Executors.newFixedThreadPool(numOfThreads, new CarbonThreadFactory("SplitDeseralizerPool", true));
        int numberOfWrapperPerThread = extendedBlockletWrappers.length / numOfThreads;
        int leftOver = extendedBlockletWrappers.length % numOfThreads;
        int[] split = null;
        if (numberOfWrapperPerThread > 0) {
            split = new int[numOfThreads];
        } else {
            split = new int[leftOver];
        }
        Arrays.fill(split, numberOfWrapperPerThread);
        for (int i = 0; i < leftOver; i++) {
            split[i] += 1;
        }
        int start = 0;
        int end = 0;
        List<Future<List<ExtendedBlocklet>>> futures = new ArrayList<>();
        for (int value : split) {
            end += value;
            futures.add(executorService.submit(new ExtendedBlockletDeserializerThread(start, end, indexInputFormat.getCarbonTable().getTablePath(), indexInputFormat.getQueryId(), indexInputFormat.isCountStarJob(), indexInputFormat.getCdcVO())));
            start += value;
        }
        executorService.shutdown();
        try {
            executorService.awaitTermination(1, TimeUnit.HOURS);
        } catch (InterruptedException e) {
            LOGGER.error(e);
            throw new RuntimeException(e);
        }
        List<ExtendedBlocklet> extendedBlocklets = new ArrayList<>();
        for (int i = 0; i < futures.size(); i++) {
            try {
                extendedBlocklets.addAll(futures.get(i).get());
            } catch (InterruptedException | ExecutionException e) {
                LOGGER.error(e);
                throw new RuntimeException(e);
            }
        }
        return extendedBlocklets;
    } else {
        List<ExtendedBlocklet> extendedBlocklets = new ArrayList<>();
        for (ExtendedBlockletWrapper extendedBlockletWrapper : extendedBlockletWrappers) {
            extendedBlocklets.addAll(extendedBlockletWrapper.readBlocklet(indexInputFormat.getCarbonTable().getTablePath(), indexInputFormat.getQueryId(), indexInputFormat.isCountStarJob(), indexInputFormat.getCdcVO()));
        }
        return extendedBlocklets;
    }
}