Examples with CarbonTableInputFormat org.apache.carbondata.hadoop.api.CarbonTableInputFormat used on opensource projects

Example 6 with CarbonTableInputFormat

use of org.apache.carbondata.hadoop.api.CarbonTableInputFormat in project carbondata by apache.

the class CarbonTableInputFormatTest method testGetFilteredSplits.

@Test
public void testGetFilteredSplits() throws Exception {
    CarbonTableInputFormat carbonInputFormat = new CarbonTableInputFormat();
    JobConf jobConf = new JobConf(new Configuration());
    Job job = Job.getInstance(jobConf);
    job.getConfiguration().set("query.id", UUID.randomUUID().toString());
    String tblPath = StoreCreator.getAbsoluteTableIdentifier().getTablePath();
    FileInputFormat.addInputPath(job, new Path(tblPath));
    CarbonTableInputFormat.setDatabaseName(job.getConfiguration(), StoreCreator.getAbsoluteTableIdentifier().getDatabaseName());
    CarbonTableInputFormat.setTableName(job.getConfiguration(), StoreCreator.getAbsoluteTableIdentifier().getTableName());
    Expression expression = new EqualToExpression(new ColumnExpression("country", DataTypes.STRING), new LiteralExpression("china", DataTypes.STRING));
    CarbonTableInputFormat.setFilterPredicates(job.getConfiguration(), expression);
    List splits = carbonInputFormat.getSplits(job);
    Assert.assertTrue(splits != null);
    Assert.assertTrue(!splits.isEmpty());
}

Example 7 with CarbonTableInputFormat

use of org.apache.carbondata.hadoop.api.CarbonTableInputFormat in project carbondata by apache.

the class CarbonTableReader method getInputSplits2.

public List<CarbonLocalInputSplit> getInputSplits2(CarbonTableCacheModel tableCacheModel, Expression filters) {
    List<CarbonLocalInputSplit> result = new ArrayList<>();
    if (config.getUnsafeMemoryInMb() != null) {
        CarbonProperties.getInstance().addProperty(CarbonCommonConstants.UNSAFE_WORKING_MEMORY_IN_MB, config.getUnsafeMemoryInMb());
    }
    CarbonTable carbonTable = tableCacheModel.carbonTable;
    TableInfo tableInfo = tableCacheModel.carbonTable.getTableInfo();
    Configuration config = new Configuration();
    config.set(CarbonTableInputFormat.INPUT_SEGMENT_NUMBERS, "");
    String carbonTablePath = carbonTable.getAbsoluteTableIdentifier().getTablePath();
    config.set(CarbonTableInputFormat.INPUT_DIR, carbonTablePath);
    config.set(CarbonTableInputFormat.DATABASE_NAME, carbonTable.getDatabaseName());
    config.set(CarbonTableInputFormat.TABLE_NAME, carbonTable.getTableName());
    try {
        CarbonTableInputFormat.setTableInfo(config, tableInfo);
        CarbonTableInputFormat carbonTableInputFormat = createInputFormat(config, carbonTable.getAbsoluteTableIdentifier(), filters);
        JobConf jobConf = new JobConf(config);
        Job job = Job.getInstance(jobConf);
        List<InputSplit> splits = carbonTableInputFormat.getSplits(job);
        CarbonInputSplit carbonInputSplit = null;
        Gson gson = new Gson();
        if (splits != null && splits.size() > 0) {
            for (InputSplit inputSplit : splits) {
                carbonInputSplit = (CarbonInputSplit) inputSplit;
                result.add(new CarbonLocalInputSplit(carbonInputSplit.getSegmentId(), carbonInputSplit.getPath().toString(), carbonInputSplit.getStart(), carbonInputSplit.getLength(), Arrays.asList(carbonInputSplit.getLocations()), carbonInputSplit.getNumberOfBlocklets(), carbonInputSplit.getVersion().number(), carbonInputSplit.getDeleteDeltaFiles(), gson.toJson(carbonInputSplit.getDetailInfo())));
            }
        }
    } catch (IOException e) {
        throw new RuntimeException("Error creating Splits from CarbonTableInputFormat", e);
    }
    return result;
}

Example 8 with CarbonTableInputFormat

use of org.apache.carbondata.hadoop.api.CarbonTableInputFormat in project carbondata by apache.

the class CarbonTableReader method createInputFormat.

private CarbonTableInputFormat<Object> createInputFormat(Configuration conf, AbsoluteTableIdentifier identifier, Expression filterExpression) throws IOException {
    CarbonTableInputFormat format = new CarbonTableInputFormat<Object>();
    CarbonTableInputFormat.setTablePath(conf, identifier.appendWithLocalPrefix(identifier.getTablePath()));
    CarbonTableInputFormat.setFilterPredicates(conf, filterExpression);
    return format;
}

Example 9 with CarbonTableInputFormat

use of org.apache.carbondata.hadoop.api.CarbonTableInputFormat in project carbondata by apache.

the class CarbondataRecordSetProvider method createInputFormat.

private CarbonTableInputFormat<Object> createInputFormat(Configuration conf, CarbonTable carbonTable, Expression filterExpression, CarbonProjection projection) {
    AbsoluteTableIdentifier identifier = carbonTable.getAbsoluteTableIdentifier();
    CarbonTableInputFormat format = new CarbonTableInputFormat<Object>();
    CarbonTableInputFormat.setTablePath(conf, identifier.appendWithLocalPrefix(identifier.getTablePath()));
    CarbonTableInputFormat.setDatabaseName(conf, identifier.getCarbonTableIdentifier().getDatabaseName());
    CarbonTableInputFormat.setTableName(conf, identifier.getCarbonTableIdentifier().getTableName());
    CarbonTableInputFormat.setFilterPredicates(conf, filterExpression);
    CarbonTableInputFormat.setColumnProjection(conf, projection);
    return format;
}