Examples with CarbonThreadFactory org.apache.carbondata.core.util.CarbonThreadFactory used on opensource projects

Example 1 with CarbonThreadFactory

use of org.apache.carbondata.core.util.CarbonThreadFactory in project carbondata by apache.

the class CarbonRowDataWriterProcessorStepImpl method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    final Iterator<CarbonRowBatch>[] iterators = child.execute();
    tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    tableName = tableIdentifier.getTableName();
    ExecutorService executorService = null;
    try {
        readCounter = new long[iterators.length];
        writeCounter = new long[iterators.length];
        dimensionWithComplexCount = configuration.getDimensionCount();
        noDictWithComplextCount = configuration.getNoDictionaryCount() + configuration.getComplexColumnCount();
        dimensionCount = configuration.getDimensionCount() - noDictWithComplextCount;
        isNoDictionaryDimensionColumn = CarbonDataProcessorUtil.getNoDictionaryMapping(configuration.getDataFields());
        measureDataType = configuration.getMeasureDataType();
        measureCount = configuration.getMeasureCount();
        outputLength = measureCount + (this.noDictWithComplextCount > 0 ? 1 : 0) + 1;
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis());
        if (iterators.length == 1) {
            doExecute(iterators[0], 0);
        } else {
            executorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("NoSortDataWriterPool:" + configuration.getTableIdentifier().getCarbonTableIdentifier().getTableName()));
            Future[] futures = new Future[iterators.length];
            for (int i = 0; i < iterators.length; i++) {
                futures[i] = executorService.submit(new DataWriterRunnable(iterators[i], i));
            }
            for (Future future : futures) {
                future.get();
            }
        }
    } catch (CarbonDataWriterException e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        throw new CarbonDataLoadingException("Error while initializing data handler : " + e.getMessage());
    } catch (Exception e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        if (e instanceof BadRecordFoundException) {
            throw new BadRecordFoundException(e.getMessage(), e);
        }
        throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage(), e);
    } finally {
        if (null != executorService && executorService.isShutdown()) {
            executorService.shutdownNow();
        }
    }
    return null;
}

Example 2 with CarbonThreadFactory

use of org.apache.carbondata.core.util.CarbonThreadFactory in project carbondata by apache.

the class DataWriterProcessorStepImpl method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    Iterator<CarbonRowBatch>[] iterators = child.execute();
    CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    String tableName = tableIdentifier.getTableName();
    try {
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis());
        ExecutorService rangeExecutorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("WriterForwardPool: " + tableName));
        List<Future<Void>> rangeExecutorServiceSubmitList = new ArrayList<>(iterators.length);
        int i = 0;
        // do this concurrently
        for (Iterator<CarbonRowBatch> iterator : iterators) {
            String[] storeLocation = getStoreLocation(tableIdentifier);
            CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration, storeLocation, i, 0);
            CarbonFactHandler dataHandler = null;
            boolean rowsNotExist = true;
            while (iterator.hasNext()) {
                if (rowsNotExist) {
                    rowsNotExist = false;
                    dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
                    dataHandler.initialise();
                }
                processBatch(iterator.next(), dataHandler);
            }
            if (!rowsNotExist) {
                finish(dataHandler);
            }
            rangeExecutorServiceSubmitList.add(rangeExecutorService.submit(new WriterForwarder(iterator, tableIdentifier, i)));
            i++;
        }
        try {
            rangeExecutorService.shutdown();
            rangeExecutorService.awaitTermination(2, TimeUnit.DAYS);
            for (int j = 0; j < rangeExecutorServiceSubmitList.size(); j++) {
                rangeExecutorServiceSubmitList.get(j).get();
            }
        } catch (InterruptedException | ExecutionException e) {
            throw new CarbonDataWriterException(e);
        }
    } catch (CarbonDataWriterException e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        throw new CarbonDataLoadingException("Error while initializing data handler : " + e.getMessage());
    } catch (Exception e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage(), e);
    }
    return null;
}

Example 3 with CarbonThreadFactory

use of org.apache.carbondata.core.util.CarbonThreadFactory in project carbondata by apache.

the class InputProcessorStepImpl method initialize.

@Override
public void initialize() throws IOException {
    super.initialize();
    rowParser = new RowParserImpl(getOutput(), configuration);
    executorService = Executors.newCachedThreadPool(new CarbonThreadFactory("InputProcessorPool:" + configuration.getTableIdentifier().getCarbonTableIdentifier().getTableName()));
    // if logger is enabled then raw data will be required.
    this.isRawDataRequired = CarbonDataProcessorUtil.isRawDataRequired(configuration);
}

Example 4 with CarbonThreadFactory

use of org.apache.carbondata.core.util.CarbonThreadFactory in project carbondata by apache.

the class CarbonFactDataHandlerColumnar method initParameters.

private void initParameters(CarbonFactDataHandlerModel model) {
    SortScopeOptions.SortScope sortScope = model.getSortScope();
    this.colGrpModel = model.getSegmentProperties().getColumnGroupModel();
    // need to convert it to byte array.
    if (model.isCompactionFlow()) {
        try {
            numberOfCores = Integer.parseInt(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.NUM_CORES_COMPACTING, CarbonCommonConstants.NUM_CORES_DEFAULT_VAL));
        } catch (NumberFormatException exc) {
            LOGGER.error("Configured value for property " + CarbonCommonConstants.NUM_CORES_COMPACTING + "is wrong.Falling back to the default value " + CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
            numberOfCores = Integer.parseInt(CarbonCommonConstants.NUM_CORES_DEFAULT_VAL);
        }
    } else {
        numberOfCores = CarbonProperties.getInstance().getNumberOfCores();
    }
    if (sortScope != null && sortScope.equals(SortScopeOptions.SortScope.GLOBAL_SORT)) {
        numberOfCores = 1;
    }
    // Overriding it to the task specified cores.
    if (model.getWritingCoresCount() > 0) {
        numberOfCores = model.getWritingCoresCount();
    }
    blockletProcessingCount = new AtomicInteger(0);
    producerExecutorService = Executors.newFixedThreadPool(numberOfCores, new CarbonThreadFactory("ProducerPool:" + model.getTableName() + ", range: " + model.getBucketId()));
    producerExecutorServiceTaskList = new ArrayList<>(CarbonCommonConstants.DEFAULT_COLLECTION_SIZE);
    LOGGER.info("Initializing writer executors");
    consumerExecutorService = Executors.newFixedThreadPool(1, new CarbonThreadFactory("ConsumerPool:" + model.getTableName() + ", range: " + model.getBucketId()));
    consumerExecutorServiceTaskList = new ArrayList<>(1);
    semaphore = new Semaphore(numberOfCores);
    tablePageList = new TablePageList();
    // Start the consumer which will take each blocklet/page in order and write to a file
    Consumer consumer = new Consumer(tablePageList);
    consumerExecutorServiceTaskList.add(consumerExecutorService.submit(consumer));
}

Example 5 with CarbonThreadFactory

use of org.apache.carbondata.core.util.CarbonThreadFactory in project carbondata by apache.

the class SortDataRows method initialize.

/**
 * This method will be used to initialize
 */
public void initialize() throws CarbonSortKeyAndGroupByException {
    // create holder list which will hold incoming rows
    // size of list will be sort buffer size + 1 to avoid creation of new
    // array in list array
    this.recordHolderList = new Object[sortBufferSize][];
    // Delete if any older file exists in sort temp folder
    deleteSortLocationIfExists();
    // create new sort temp directory
    CarbonDataProcessorUtil.createLocations(parameters.getTempFileLocation());
    this.dataSorterAndWriterExecutorService = Executors.newFixedThreadPool(parameters.getNumberOfCores(), new CarbonThreadFactory("SortDataRowPool:" + parameters.getTableName()));
    semaphore = new Semaphore(parameters.getNumberOfCores());
}