Search in sources :

Example 1 with CarbonFactHandler

use of org.apache.carbondata.processing.store.CarbonFactHandler in project carbondata by apache.

the class CarbonRowDataWriterProcessorStepImpl method doExecute.

private void doExecute(Iterator<CarbonRowBatch> iterator, int iteratorIndex) {
    String[] storeLocation = getStoreLocation(tableIdentifier);
    CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration, storeLocation, 0, iteratorIndex);
    CarbonFactHandler dataHandler = null;
    boolean rowsNotExist = true;
    while (iterator.hasNext()) {
        if (rowsNotExist) {
            rowsNotExist = false;
            dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
            dataHandler.initialise();
        }
        processBatch(iterator.next(), dataHandler, iteratorIndex);
    }
    if (!rowsNotExist) {
        finish(dataHandler, iteratorIndex);
    }
}
Also used : CarbonFactDataHandlerModel(org.apache.carbondata.processing.store.CarbonFactDataHandlerModel) CarbonFactHandler(org.apache.carbondata.processing.store.CarbonFactHandler)

Example 2 with CarbonFactHandler

use of org.apache.carbondata.processing.store.CarbonFactHandler in project carbondata by apache.

the class DataWriterBatchProcessorStepImpl method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    Iterator<CarbonRowBatch>[] iterators = child.execute();
    CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    String tableName = tableIdentifier.getTableName();
    try {
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis());
        int i = 0;
        String[] storeLocation = getStoreLocation(tableIdentifier);
        CarbonDataProcessorUtil.createLocations(storeLocation);
        for (Iterator<CarbonRowBatch> iterator : iterators) {
            int k = 0;
            while (iterator.hasNext()) {
                CarbonRowBatch next = iterator.next();
                // If no rows from merge sorter, then don't create a file in fact column handler
                if (next.hasNext()) {
                    CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration, storeLocation, 0, k++);
                    CarbonFactHandler dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
                    dataHandler.initialise();
                    processBatch(next, dataHandler);
                    finish(tableName, dataHandler);
                }
            }
            i++;
        }
    } catch (Exception e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterBatchProcessorStepImpl");
        if (e.getCause() instanceof BadRecordFoundException) {
            throw new BadRecordFoundException(e.getCause().getMessage());
        }
        throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage());
    }
    return null;
}
Also used : CarbonRowBatch(org.apache.carbondata.processing.loading.row.CarbonRowBatch) CarbonFactDataHandlerModel(org.apache.carbondata.processing.store.CarbonFactDataHandlerModel) CarbonDataLoadingException(org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException) CarbonFactHandler(org.apache.carbondata.processing.store.CarbonFactHandler) IOException(java.io.IOException) CarbonDataLoadingException(org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException) BadRecordFoundException(org.apache.carbondata.processing.loading.exception.BadRecordFoundException) CarbonTableIdentifier(org.apache.carbondata.core.metadata.CarbonTableIdentifier) Iterator(java.util.Iterator) BadRecordFoundException(org.apache.carbondata.processing.loading.exception.BadRecordFoundException)

Example 3 with CarbonFactHandler

use of org.apache.carbondata.processing.store.CarbonFactHandler in project carbondata by apache.

the class DataWriterProcessorStepImpl method processRange.

private void processRange(Iterator<CarbonRowBatch> insideRangeIterator, CarbonTableIdentifier tableIdentifier, int rangeId) {
    String[] storeLocation = getStoreLocation(tableIdentifier);
    CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration, storeLocation, rangeId, 0);
    CarbonFactHandler dataHandler = null;
    boolean rowsNotExist = true;
    while (insideRangeIterator.hasNext()) {
        if (rowsNotExist) {
            rowsNotExist = false;
            dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
            dataHandler.initialise();
        }
        processBatch(insideRangeIterator.next(), dataHandler);
    }
    if (!rowsNotExist) {
        finish(dataHandler);
    }
}
Also used : CarbonFactDataHandlerModel(org.apache.carbondata.processing.store.CarbonFactDataHandlerModel) CarbonFactHandler(org.apache.carbondata.processing.store.CarbonFactHandler)

Example 4 with CarbonFactHandler

use of org.apache.carbondata.processing.store.CarbonFactHandler in project carbondata by apache.

the class DataWriterProcessorStepImpl method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    Iterator<CarbonRowBatch>[] iterators = child.execute();
    CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    String tableName = tableIdentifier.getTableName();
    try {
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(CarbonTablePath.DEPRECATED_PATITION_ID, System.currentTimeMillis());
        ExecutorService rangeExecutorService = Executors.newFixedThreadPool(iterators.length, new CarbonThreadFactory("WriterForwardPool: " + tableName));
        List<Future<Void>> rangeExecutorServiceSubmitList = new ArrayList<>(iterators.length);
        int i = 0;
        // do this concurrently
        for (Iterator<CarbonRowBatch> iterator : iterators) {
            String[] storeLocation = getStoreLocation(tableIdentifier);
            CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration, storeLocation, i, 0);
            CarbonFactHandler dataHandler = null;
            boolean rowsNotExist = true;
            while (iterator.hasNext()) {
                if (rowsNotExist) {
                    rowsNotExist = false;
                    dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
                    dataHandler.initialise();
                }
                processBatch(iterator.next(), dataHandler);
            }
            if (!rowsNotExist) {
                finish(dataHandler);
            }
            rangeExecutorServiceSubmitList.add(rangeExecutorService.submit(new WriterForwarder(iterator, tableIdentifier, i)));
            i++;
        }
        try {
            rangeExecutorService.shutdown();
            rangeExecutorService.awaitTermination(2, TimeUnit.DAYS);
            for (int j = 0; j < rangeExecutorServiceSubmitList.size(); j++) {
                rangeExecutorServiceSubmitList.get(j).get();
            }
        } catch (InterruptedException | ExecutionException e) {
            throw new CarbonDataWriterException(e);
        }
    } catch (CarbonDataWriterException e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        throw new CarbonDataLoadingException("Error while initializing data handler : " + e.getMessage());
    } catch (Exception e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage(), e);
    }
    return null;
}
Also used : CarbonRowBatch(org.apache.carbondata.processing.loading.row.CarbonRowBatch) CarbonFactDataHandlerModel(org.apache.carbondata.processing.store.CarbonFactDataHandlerModel) CarbonDataLoadingException(org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException) ArrayList(java.util.ArrayList) CarbonFactHandler(org.apache.carbondata.processing.store.CarbonFactHandler) CarbonDataWriterException(org.apache.carbondata.core.datastore.exception.CarbonDataWriterException) KeyGenException(org.apache.carbondata.core.keygenerator.KeyGenException) CarbonDataWriterException(org.apache.carbondata.core.datastore.exception.CarbonDataWriterException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) CarbonDataLoadingException(org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException) CarbonTableIdentifier(org.apache.carbondata.core.metadata.CarbonTableIdentifier) Iterator(java.util.Iterator) ExecutorService(java.util.concurrent.ExecutorService) CarbonThreadFactory(org.apache.carbondata.core.util.CarbonThreadFactory) Future(java.util.concurrent.Future) ExecutionException(java.util.concurrent.ExecutionException)

Example 5 with CarbonFactHandler

use of org.apache.carbondata.processing.store.CarbonFactHandler in project carbondata by apache.

the class CarbonRowDataWriterProcessorStepImpl method doExecute.

private void doExecute(Iterator<CarbonRowBatch> iterator, int partitionId, int iteratorIndex) {
    String storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId));
    CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, iteratorIndex);
    CarbonFactHandler dataHandler = null;
    boolean rowsNotExist = true;
    while (iterator.hasNext()) {
        if (rowsNotExist) {
            rowsNotExist = false;
            dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
            dataHandler.initialise();
        }
        processBatch(iterator.next(), dataHandler, iteratorIndex);
    }
    if (!rowsNotExist) {
        finish(dataHandler, iteratorIndex);
    }
}
Also used : CarbonFactDataHandlerModel(org.apache.carbondata.processing.store.CarbonFactDataHandlerModel) CarbonFactHandler(org.apache.carbondata.processing.store.CarbonFactHandler)

Aggregations

CarbonFactDataHandlerModel (org.apache.carbondata.processing.store.CarbonFactDataHandlerModel)7 CarbonFactHandler (org.apache.carbondata.processing.store.CarbonFactHandler)7 IOException (java.io.IOException)4 Iterator (java.util.Iterator)4 CarbonTableIdentifier (org.apache.carbondata.core.metadata.CarbonTableIdentifier)4 CarbonDataLoadingException (org.apache.carbondata.processing.loading.exception.CarbonDataLoadingException)2 CarbonRowBatch (org.apache.carbondata.processing.loading.row.CarbonRowBatch)2 CarbonDataLoadingException (org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException)2 CarbonRowBatch (org.apache.carbondata.processing.newflow.row.CarbonRowBatch)2 ArrayList (java.util.ArrayList)1 ExecutionException (java.util.concurrent.ExecutionException)1 ExecutorService (java.util.concurrent.ExecutorService)1 Future (java.util.concurrent.Future)1 CarbonDataWriterException (org.apache.carbondata.core.datastore.exception.CarbonDataWriterException)1 KeyGenException (org.apache.carbondata.core.keygenerator.KeyGenException)1 CarbonThreadFactory (org.apache.carbondata.core.util.CarbonThreadFactory)1 BadRecordFoundException (org.apache.carbondata.processing.loading.exception.BadRecordFoundException)1 CarbonDataWriterException (org.apache.carbondata.processing.store.writer.exception.CarbonDataWriterException)1