Search in sources :

Example 1 with CarbonLoadModel

use of org.apache.carbondata.processing.loading.model.CarbonLoadModel in project carbondata by apache.

the class StoreCreator method createCarbonStore.

/**
 * Create store without any restructure
 */
public static void createCarbonStore() {
    try {
        String factFilePath = new File("../hadoop/src/test/resources/data.csv").getCanonicalPath();
        File storeDir = new File(storePath);
        CarbonUtil.deleteFoldersAndFiles(storeDir);
        CarbonProperties.getInstance().addProperty(CarbonCommonConstants.STORE_LOCATION_HDFS, storePath);
        CarbonTable table = createTable();
        writeDictionary(factFilePath, table);
        CarbonDataLoadSchema schema = new CarbonDataLoadSchema(table);
        CarbonLoadModel loadModel = new CarbonLoadModel();
        loadModel.setCarbonDataLoadSchema(schema);
        loadModel.setDatabaseName(identifier.getCarbonTableIdentifier().getDatabaseName());
        loadModel.setTableName(identifier.getCarbonTableIdentifier().getTableName());
        loadModel.setTableName(identifier.getCarbonTableIdentifier().getTableName());
        loadModel.setFactFilePath(factFilePath);
        loadModel.setLoadMetadataDetails(new ArrayList<LoadMetadataDetails>());
        loadModel.setTablePath(identifier.getTablePath());
        loadModel.setDateFormat(null);
        loadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT));
        loadModel.setDefaultDateFormat(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT));
        loadModel.setSerializationNullFormat(TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName() + "," + "\\N");
        loadModel.setBadRecordsLoggerEnable(TableOptionConstant.BAD_RECORDS_LOGGER_ENABLE.getName() + "," + "false");
        loadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + "," + "FORCE");
        loadModel.setIsEmptyDataBadRecord(DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD + "," + "false");
        loadModel.setCsvHeader("ID,date,country,name,phonetype,serialname,salary");
        loadModel.setCsvHeaderColumns(loadModel.getCsvHeader().split(","));
        loadModel.setTaskNo("0");
        loadModel.setSegmentId("0");
        loadModel.setFactTimeStamp(System.currentTimeMillis());
        loadModel.setMaxColumns("10");
        loadData(loadModel, storePath);
    } catch (Exception e) {
        e.printStackTrace();
    }
}
Also used : CarbonTable(org.apache.carbondata.core.metadata.schema.table.CarbonTable) LoadMetadataDetails(org.apache.carbondata.core.statusmanager.LoadMetadataDetails) CarbonLoadModel(org.apache.carbondata.processing.loading.model.CarbonLoadModel) CarbonDataLoadSchema(org.apache.carbondata.processing.loading.model.CarbonDataLoadSchema) File(java.io.File) IOException(java.io.IOException)

Example 2 with CarbonLoadModel

use of org.apache.carbondata.processing.loading.model.CarbonLoadModel in project carbondata by apache.

the class CarbonWriterBuilder method buildWriterForCSVInput.

/**
 * Build a {@link CarbonWriter}, which accepts row in CSV format
 */
public CarbonWriter buildWriterForCSVInput() throws IOException, InvalidLoadOptionException {
    Objects.requireNonNull(schema, "schema should not be null");
    Objects.requireNonNull(path, "path should not be null");
    CarbonLoadModel loadModel = createLoadModel();
    return new CSVCarbonWriter(loadModel);
}
Also used : CarbonLoadModel(org.apache.carbondata.processing.loading.model.CarbonLoadModel)

Example 3 with CarbonLoadModel

use of org.apache.carbondata.processing.loading.model.CarbonLoadModel in project carbondata by apache.

the class CarbonWriterBuilder method buildWriterForAvroInput.

/**
 * Build a {@link CarbonWriter}, which accepts Avro object
 * @return
 * @throws IOException
 */
public CarbonWriter buildWriterForAvroInput() throws IOException, InvalidLoadOptionException {
    Objects.requireNonNull(schema, "schema should not be null");
    Objects.requireNonNull(path, "path should not be null");
    CarbonLoadModel loadModel = createLoadModel();
    return new AvroCarbonWriter(loadModel);
}
Also used : CarbonLoadModel(org.apache.carbondata.processing.loading.model.CarbonLoadModel)

Example 4 with CarbonLoadModel

use of org.apache.carbondata.processing.loading.model.CarbonLoadModel in project carbondata by apache.

the class StoreCreator method buildCarbonLoadModel.

public static CarbonLoadModel buildCarbonLoadModel(CarbonTable table, String factFilePath, AbsoluteTableIdentifier absoluteTableIdentifier) {
    CarbonDataLoadSchema schema = new CarbonDataLoadSchema(table);
    CarbonLoadModel loadModel = new CarbonLoadModel();
    loadModel.setCarbonDataLoadSchema(schema);
    loadModel.setDatabaseName(absoluteTableIdentifier.getCarbonTableIdentifier().getDatabaseName());
    loadModel.setTableName(absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
    loadModel.setTableName(absoluteTableIdentifier.getCarbonTableIdentifier().getTableName());
    loadModel.setFactFilePath(factFilePath);
    loadModel.setLoadMetadataDetails(new ArrayList<LoadMetadataDetails>());
    loadModel.setTablePath(absoluteTableIdentifier.getTablePath());
    loadModel.setDateFormat(null);
    loadModel.setDefaultTimestampFormat(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS));
    loadModel.setDefaultDateFormat(CarbonProperties.getInstance().getProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT));
    loadModel.setSerializationNullFormat(TableOptionConstant.SERIALIZATION_NULL_FORMAT.getName() + "," + "\\N");
    loadModel.setBadRecordsLoggerEnable(TableOptionConstant.BAD_RECORDS_LOGGER_ENABLE.getName() + "," + "false");
    loadModel.setBadRecordsAction(TableOptionConstant.BAD_RECORDS_ACTION.getName() + "," + "FORCE");
    loadModel.setIsEmptyDataBadRecord(DataLoadProcessorConstants.IS_EMPTY_DATA_BAD_RECORD + "," + "false");
    loadModel.setCsvHeader("ID,date,country,name,phonetype,serialname,salary");
    loadModel.setCsvHeaderColumns(loadModel.getCsvHeader().split(","));
    loadModel.setTaskNo("0");
    loadModel.setSegmentId("0");
    loadModel.setFactTimeStamp(System.currentTimeMillis());
    loadModel.setMaxColumns("10");
    return loadModel;
}
Also used : LoadMetadataDetails(org.apache.carbondata.core.statusmanager.LoadMetadataDetails) CarbonLoadModel(org.apache.carbondata.processing.loading.model.CarbonLoadModel) CarbonDataLoadSchema(org.apache.carbondata.processing.loading.model.CarbonDataLoadSchema)

Example 5 with CarbonLoadModel

use of org.apache.carbondata.processing.loading.model.CarbonLoadModel in project carbondata by apache.

the class StoreCreator method createCarbonStore.

/**
 * Create store without any restructure
 */
public static void createCarbonStore() throws Exception {
    CarbonLoadModel loadModel = createTableAndLoadModel();
    loadData(loadModel, storePath);
}
Also used : CarbonLoadModel(org.apache.carbondata.processing.loading.model.CarbonLoadModel)

Aggregations

CarbonLoadModel (org.apache.carbondata.processing.loading.model.CarbonLoadModel)11 IOException (java.io.IOException)4 LoadMetadataDetails (org.apache.carbondata.core.statusmanager.LoadMetadataDetails)4 CarbonDataLoadSchema (org.apache.carbondata.processing.loading.model.CarbonDataLoadSchema)3 SegmentFileStore (org.apache.carbondata.core.metadata.SegmentFileStore)2 CarbonTable (org.apache.carbondata.core.metadata.schema.table.CarbonTable)2 File (java.io.File)1 HashSet (java.util.HashSet)1 ExecutionException (java.util.concurrent.ExecutionException)1 ExecutorService (java.util.concurrent.ExecutorService)1 Future (java.util.concurrent.Future)1 CarbonIterator (org.apache.carbondata.common.CarbonIterator)1 Segment (org.apache.carbondata.core.datamap.Segment)1 CarbonFile (org.apache.carbondata.core.datastore.filesystem.CarbonFile)1 SegmentStatusManager (org.apache.carbondata.core.statusmanager.SegmentStatusManager)1 CarbonProperties (org.apache.carbondata.core.util.CarbonProperties)1 CarbonThreadFactory (org.apache.carbondata.core.util.CarbonThreadFactory)1 OperationContext (org.apache.carbondata.events.OperationContext)1 DataLoadExecutor (org.apache.carbondata.processing.loading.DataLoadExecutor)1 LoadEvents (org.apache.carbondata.processing.loading.events.LoadEvents)1