Examples with Field org.apache.carbondata.core.metadata.datatype.Field used on opensource projects

Example 56 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class PaginationCarbonReaderTest method testSDKPaginationInsertData.

@Test
public void testSDKPaginationInsertData() throws IOException, InvalidLoadOptionException, InterruptedException {
    List<String[]> data1 = new ArrayList<String[]>();
    String[] row1 = { "1", "AAA", "3", "3444345.66", "true", "1979-12-09", "2011-2-10 1:00:20", "Pune", "IT" };
    String[] row2 = { "2", "BBB", "2", "543124.66", "false", "1987-2-19", "2017-1-1 12:00:20", "Bangalore", "DATA" };
    String[] row3 = { "3", "CCC", "1", "787878.888", "false", "1982-05-12", "2015-12-1 2:20:20", "Pune", "DATA" };
    String[] row4 = { "4", "DDD", "1", "99999.24", "true", "1981-04-09", "2000-1-15 7:00:20", "Delhi", "MAINS" };
    String[] row5 = { "5", "EEE", "3", "545656.99", "true", "1987-12-09", "2017-11-25 04:00:20", "Delhi", "IT" };
    data1.add(row1);
    data1.add(row2);
    data1.add(row3);
    data1.add(row4);
    data1.add(row5);
    String path = "./testWriteFiles";
    FileUtils.deleteDirectory(new File(path));
    Field[] fields = new Field[9];
    fields[0] = new Field("id", DataTypes.INT);
    fields[1] = new Field("name", DataTypes.STRING);
    fields[2] = new Field("rank", DataTypes.SHORT);
    fields[3] = new Field("salary", DataTypes.DOUBLE);
    fields[4] = new Field("active", DataTypes.BOOLEAN);
    fields[5] = new Field("dob", DataTypes.DATE);
    fields[6] = new Field("doj", DataTypes.TIMESTAMP);
    fields[7] = new Field("city", DataTypes.STRING);
    fields[8] = new Field("dept", DataTypes.STRING);
    CarbonWriterBuilder builder = CarbonWriter.builder().outputPath(path).withBlockletSize(1).withBlockSize(2);
    CarbonWriter writer = builder.withCsvInput(new Schema(fields)).writtenBy("TestUtil").build();
    for (int i = 0; i < 5; i++) {
        writer.write(data1.get(i));
    }
    writer.close();
    String[] row = { "222", "Daisy", "3", "334.456", "true", "1956-11-08", "2013-12-10 12:00:20", "Pune", "IT" };
    writer = CarbonWriter.builder().outputPath(path).withBlockletSize(1).withBlockSize(2).withCsvInput(new Schema(fields)).writtenBy("TestUtil").build();
    writer.write(row);
    writer.close();
    // configure cache size = 4 blocklet
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_MAX_PAGINATION_LRU_CACHE_SIZE_IN_MB, "4");
    PaginationCarbonReader<Object> paginationCarbonReader = (PaginationCarbonReader<Object>) CarbonReader.builder(path, "_temp").withPaginationSupport().projection(new String[] { "id", "name", "rank", "salary", "active", "dob", "doj", "city", "dept" }).build();
    assert (paginationCarbonReader.getTotalRows() == 6);
    Object[] rows = paginationCarbonReader.read(1, 6);
    assert (rows.length == 6);
    CarbonIUD.getInstance().delete(path, "name", "AAA").commit();
    CarbonReaderBuilder carbonReaderBuilder = CarbonReader.builder(path, "_temp").withPaginationSupport().projection(new String[] { "id", "name", "rank", "salary", "active", "dob", "doj", "city", "dept" });
    paginationCarbonReader = (PaginationCarbonReader<Object>) carbonReaderBuilder.build();
    assert (paginationCarbonReader.getTotalRows() == 5);
    rows = paginationCarbonReader.read(1, 5);
    assert (rows.length == 5);
    paginationCarbonReader.close();
    CarbonIUD.getInstance().update(path, "name", "AAA", "name", "nihal").commit();
    paginationCarbonReader = (PaginationCarbonReader<Object>) CarbonReader.builder(path, "_temp").withPaginationSupport().projection(new String[] { "id", "name", "rank", "salary", "active", "dob", "doj", "city", "dept" }).build();
    assert (paginationCarbonReader.getTotalRows() == 5);
    rows = paginationCarbonReader.read(1, 5);
    assert (rows.length == 5);
    paginationCarbonReader.close();
    FileUtils.deleteDirectory(new File(path));
}

Example 57 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class PaginationCarbonReaderTest method testDataCorrectness.

@Test
public void testDataCorrectness() throws IOException {
    String path = "./testWriteFiles";
    FileUtils.deleteDirectory(new File(path));
    Field[] fields = new Field[3];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("data", DataTypes.VARCHAR);
    fields[2] = new Field("id", DataTypes.LONG);
    String data = RandomStringUtils.randomAlphabetic(1024);
    // create more than one blocklet
    try {
        CarbonWriterBuilder builder = CarbonWriter.builder().outputPath(path).withBlockletSize(1).withBlockSize(2).withTableProperty("local_dictionary_enable", "false");
        CarbonWriter writer = builder.withCsvInput(new Schema(fields)).writtenBy("TestUtil").build();
        for (int i = 1; i <= 100000; i++) {
            writer.write(new String[] { "robot" + i, data, String.valueOf(i) });
        }
        writer.close();
    } catch (Exception ex) {
        assert (false);
    }
    // configure cache size = 4 blocklet
    CarbonProperties.getInstance().addProperty(CarbonCommonConstants.CARBON_MAX_PAGINATION_LRU_CACHE_SIZE_IN_MB, "4");
    try {
        CarbonReaderBuilder carbonReaderBuilder = new CarbonReaderBuilder(path, "temptest").withPaginationSupport();
        PaginationCarbonReader<Object> paginationCarbonReader = (PaginationCarbonReader<Object>) carbonReaderBuilder.build();
        assert (paginationCarbonReader.getTotalRows() == 100000);
        // test case creates 4 blocklets and total rows are split as shown below
        // 0 = 1 - 32000
        // 1 = 32001 - 64000
        // 2 = 64001 - 96000
        // 3 = 96001 - 100000
        Object[] rows;
        // so test for all combination of ranges
        // 1. from resides in one blocklet, to resides in another blocklet
        // a. from and to exist beside each other (from - 0, to - 1)
        rows = paginationCarbonReader.read(31999, 32005);
        // note length is (from - to + 1)
        assert (rows.length == 7);
        int index = 31999;
        for (Object row : rows) {
            // verify the result
            assert (((Object[]) row)[0].equals("robot" + (index)));
            index++;
        }
        // b. from and to exit with some gap (from - 0, to - 3)
        rows = paginationCarbonReader.read(31999, 64005);
        // (from - to + 1)
        assert (rows.length == 32007);
        index = 31999;
        for (Object row : rows) {
            // verify the result
            assert (((Object[]) row)[0].equals("robot" + (index)));
            index++;
        }
        // 2. from and to resides in the same blocklet
        // a. whole blocklet
        rows = paginationCarbonReader.read(64001, 96000);
        // (from - to + 1)
        assert (rows.length == 32000);
        index = 64001;
        for (Object row : rows) {
            // verify the result
            assert (((Object[]) row)[0].equals("robot" + (index)));
            index++;
        }
        // b. some rows in blocklet
        rows = paginationCarbonReader.read(100, 300);
        // (from - to + 1)
        assert (rows.length == 201);
        index = 100;
        for (Object row : rows) {
            // verify the result
            assert (((Object[]) row)[0].equals("robot" + (index)));
            index++;
        }
        // read one row
        rows = paginationCarbonReader.read(10, 10);
        // (from - to + 1)
        assert (rows.length == 1);
        index = 10;
        for (Object row : rows) {
            // verify the result
            assert (((Object[]) row)[0].equals("robot" + (index)));
            index++;
        }
        // close the reader
        paginationCarbonReader.close();
    } catch (Exception ex) {
        Assert.fail(ex.getMessage());
    }
    FileUtils.deleteDirectory(new File(path));
}

Example 58 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class LocalCarbonStoreTest method testWriteAndReadFiles.

// TODO: complete this testcase
// Currently result rows are empty, because SDK is not writing table status file
// so that reader does not find any segment.
// Complete this testcase after flat folder reader is done.
@Test
public void testWriteAndReadFiles() throws IOException {
    String path = "./testWriteFiles";
    FileUtils.deleteDirectory(new File(path));
    Field[] fields = new Field[2];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    TestUtil.writeFilesAndVerify(100, new Schema(fields), path);
    CarbonStore store = new LocalCarbonStore();
    Iterator<CarbonRow> rows = store.scan(AbsoluteTableIdentifier.from(path, "", ""), new String[] { "name, age" }, null);
    while (rows.hasNext()) {
        CarbonRow row = rows.next();
        System.out.println(row.toString());
    }
    FileUtils.deleteDirectory(new File(path));
}

Example 59 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class AvroCarbonWriterTest method testExceptionForDuplicateColumns.

@Test
public void testExceptionForDuplicateColumns() throws IOException, InvalidLoadOptionException {
    Field[] field = new Field[2];
    field[0] = new Field("name", DataTypes.STRING);
    field[1] = new Field("name", DataTypes.STRING);
    CarbonWriterBuilder writer = CarbonWriter.builder().uniqueIdentifier(System.currentTimeMillis()).outputPath(path);
    try {
        writer.withCsvInput(new org.apache.carbondata.sdk.file.Schema(field)).writtenBy("AvroCarbonWriterTest").build();
        Assert.fail();
    } catch (Exception e) {
        assert (e.getMessage().contains("Duplicate column name found in table schema"));
    }
    FileUtils.deleteDirectory(new File(path));
}

Example 60 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CarbonIUDTest method testDeleteWithAndFilter.

@Test
public void testDeleteWithAndFilter() throws Exception {
    String path = "./testWriteFiles";
    FileUtils.deleteDirectory(new File(path));
    Field[] fields = new Field[3];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    fields[2] = new Field("doubleField", DataTypes.DOUBLE);
    TestUtil.writeFilesAndVerify(20, new Schema(fields), path);
    ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE);
    LessThanExpression lessThanExpression1 = new LessThanExpression(columnExpression, new LiteralExpression("3.5", DataTypes.DOUBLE));
    ColumnExpression columnExpression2 = new ColumnExpression("age", DataTypes.INT);
    LessThanExpression lessThanExpression2 = new LessThanExpression(columnExpression2, new LiteralExpression("4", DataTypes.INT));
    AndExpression andExpression = new AndExpression(lessThanExpression1, lessThanExpression2);
    CarbonIUD.getInstance().delete(path, andExpression);
    CarbonReader reader = CarbonReader.builder(path).projection(new String[] { "name", "age", "doubleField" }).build();
    int i = 0;
    while (reader.hasNext()) {
        Object[] row = (Object[]) reader.readNextRow();
        assert (!((int) row[1] < 4));
        i++;
    }
    Assert.assertEquals(i, 16);
    reader.close();
    FileUtils.deleteDirectory(new File(path));
}