Examples with CarbonFactHandler org.apache.carbondata.processing.store.CarbonFactHandler used on opensource projects

Example 6 with CarbonFactHandler

use of org.apache.carbondata.processing.store.CarbonFactHandler in project carbondata by apache.

the class CarbonRowDataWriterProcessorStepImpl method doExecute.

private void doExecute(Iterator<CarbonRowBatch> iterator, int partitionId, int iteratorIndex) {
    String storeLocation = getStoreLocation(tableIdentifier, String.valueOf(partitionId));
    CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration, storeLocation, partitionId, iteratorIndex);
    CarbonFactHandler dataHandler = null;
    boolean rowsNotExist = true;
    while (iterator.hasNext()) {
        if (rowsNotExist) {
            rowsNotExist = false;
            dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
            dataHandler.initialise();
        }
        processBatch(iterator.next(), dataHandler, iteratorIndex);
    }
    if (!rowsNotExist) {
        finish(dataHandler, iteratorIndex);
    }
}

Example 7 with CarbonFactHandler

use of org.apache.carbondata.processing.store.CarbonFactHandler in project carbondata by apache.

the class DataWriterBatchProcessorStepImpl method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    Iterator<CarbonRowBatch>[] iterators = child.execute();
    CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    String tableName = tableIdentifier.getTableName();
    try {
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(), System.currentTimeMillis());
        int i = 0;
        for (Iterator<CarbonRowBatch> iterator : iterators) {
            String storeLocation = getStoreLocation(tableIdentifier, String.valueOf(i));
            int k = 0;
            while (iterator.hasNext()) {
                CarbonRowBatch next = iterator.next();
                CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration, storeLocation, i, k++);
                CarbonFactHandler dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
                dataHandler.initialise();
                processBatch(next, dataHandler);
                finish(tableName, dataHandler);
            }
            i++;
        }
    } catch (Exception e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterBatchProcessorStepImpl");
        throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage());
    }
    return null;
}

Example 8 with CarbonFactHandler

use of org.apache.carbondata.processing.store.CarbonFactHandler in project carbondata by apache.

the class DataWriterProcessorStepImpl method execute.

@Override
public Iterator<CarbonRowBatch>[] execute() throws CarbonDataLoadingException {
    Iterator<CarbonRowBatch>[] iterators = child.execute();
    CarbonTableIdentifier tableIdentifier = configuration.getTableIdentifier().getCarbonTableIdentifier();
    String tableName = tableIdentifier.getTableName();
    try {
        CarbonTimeStatisticsFactory.getLoadStatisticsInstance().recordDictionaryValue2MdkAdd2FileTime(configuration.getPartitionId(), System.currentTimeMillis());
        int i = 0;
        for (Iterator<CarbonRowBatch> iterator : iterators) {
            String storeLocation = getStoreLocation(tableIdentifier, String.valueOf(i));
            CarbonFactDataHandlerModel model = CarbonFactDataHandlerModel.createCarbonFactDataHandlerModel(configuration, storeLocation, i, 0);
            CarbonFactHandler dataHandler = null;
            boolean rowsNotExist = true;
            while (iterator.hasNext()) {
                if (rowsNotExist) {
                    rowsNotExist = false;
                    dataHandler = CarbonFactHandlerFactory.createCarbonFactHandler(model, CarbonFactHandlerFactory.FactHandlerType.COLUMNAR);
                    dataHandler.initialise();
                }
                processBatch(iterator.next(), dataHandler);
            }
            if (!rowsNotExist) {
                finish(tableName, dataHandler);
            }
            i++;
        }
    } catch (CarbonDataWriterException e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        throw new CarbonDataLoadingException("Error while initializing data handler : " + e.getMessage());
    } catch (Exception e) {
        LOGGER.error(e, "Failed for table: " + tableName + " in DataWriterProcessorStepImpl");
        throw new CarbonDataLoadingException("There is an unexpected error: " + e.getMessage(), e);
    }
    return null;
}

Example 9 with CarbonFactHandler

use of org.apache.carbondata.processing.store.CarbonFactHandler in project carbondata by apache.

the class DataWriterProcessorStepImpl method close.

@Override
public void close() {
    if (!closed) {
        super.close();
        if (listener != null) {
            try {
                LOGGER.debug("closing all the Index writers registered to index writer listener");
                listener.finish();
            } catch (IOException e) {
                LOGGER.error("error while closing the index writers", e);
            // ignoring the exception
            }
        }
        if (null != rangeExecutorService) {
            rangeExecutorService.shutdownNow();
        }
        if (null != this.carbonFactHandlers && !this.carbonFactHandlers.isEmpty()) {
            for (CarbonFactHandler carbonFactHandler : this.carbonFactHandlers) {
                carbonFactHandler.finish();
                carbonFactHandler.closeHandler();
            }
        }
        if (configuration.getMetrics() != null) {
            configuration.getMetrics().addOutputRows(rowCounter.get());
        }
    }
}