Examples with Field org.apache.carbondata.core.metadata.datatype.Field used on opensource projects

Example 6 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CarbonReaderTest method testReadSchemaFromDataFile.

@Test
public void testReadSchemaFromDataFile() throws IOException {
    String path = "./testWriteFiles";
    FileUtils.deleteDirectory(new File(path));
    Field[] fields = new Field[2];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    TestUtil.writeFilesAndVerify(100, new Schema(fields), path);
    File[] dataFiles = new File(path).listFiles(new FilenameFilter() {

        @Override
        public boolean accept(File dir, String name) {
            return name.endsWith("carbondata");
        }
    });
    Assert.assertTrue(dataFiles != null);
    Assert.assertTrue(dataFiles.length > 0);
    Schema schema = CarbonSchemaReader.readSchema(dataFiles[0].getAbsolutePath());
    Assert.assertTrue(schema.getFields().length == 2);
    Assert.assertEquals("name", (schema.getFields())[0].getFieldName());
    Assert.assertEquals("age", (schema.getFields())[1].getFieldName());
    Assert.assertEquals(DataTypes.STRING, (schema.getFields())[0].getDataType());
    Assert.assertEquals(DataTypes.INT, (schema.getFields())[1].getDataType());
    FileUtils.deleteDirectory(new File(path));
}

Example 7 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CarbonReaderTest method testReadingNullValues.

@Test
public void testReadingNullValues() {
    String path = "./testWriteFiles";
    try {
        FileUtils.deleteDirectory(new File(path));
        Field[] fields = new Field[2];
        fields[0] = new Field("stringField", DataTypes.STRING);
        fields[1] = new Field("booleanField", DataTypes.BOOLEAN);
        CarbonWriter writer = CarbonWriter.builder().outputPath(path).withCsvInput(new Schema(fields)).writtenBy("CarbonReaderTest").build();
        for (int i = 0; i < 2; i++) {
            String[] row2 = new String[] { "robot" + (i % 10), "" };
            writer.write(row2);
        }
        writer.close();
        // Read data
        CarbonReader reader = CarbonReader.builder(path, "_temp").build();
        int i = 0;
        while (reader.hasNext()) {
            reader.readNextRow();
            i++;
        }
        assert (i == 2);
        reader.close();
    } catch (Throwable e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    } finally {
        try {
            FileUtils.deleteDirectory(new File(path));
        } catch (IOException e) {
            e.printStackTrace();
            Assert.fail(e.getMessage());
        }
    }
}

Example 8 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CarbonReaderTest method testReadColumnTwice.

@Test
public void testReadColumnTwice() throws IOException, InterruptedException {
    String path = "./testWriteFiles";
    FileUtils.deleteDirectory(new File(path));
    IndexStoreManager.getInstance().clearIndexCache(AbsoluteTableIdentifier.from(path), false);
    Field[] fields = new Field[2];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    TestUtil.writeFilesAndVerify(100, new Schema(fields), path);
    CarbonReader reader = CarbonReader.builder(path, "_temp").projection(new String[] { "name", "age", "age", "name" }).build();
    int i = 0;
    while (reader.hasNext()) {
        Object[] row = (Object[]) reader.readNextRow();
        // Default sort column is applied for dimensions. So, need  to validate accordingly
        Assert.assertEquals(("robot" + (i % 10)), row[0]);
        Assert.assertEquals(i, row[1]);
        Assert.assertEquals(i, row[2]);
        Assert.assertEquals("robot" + (i % 10), row[3]);
        i++;
    }
    Assert.assertEquals(i, 100);
    reader.close();
    FileUtils.deleteDirectory(new File(path));
}

Example 9 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CarbonReaderTest method testReadWithFilterOfNonTransactionalLessThan.

@Test
public void testReadWithFilterOfNonTransactionalLessThan() throws IOException, InterruptedException {
    String path = "./testWriteFiles";
    FileUtils.deleteDirectory(new File(path));
    IndexStoreManager.getInstance().clearIndexCache(AbsoluteTableIdentifier.from(path), false);
    Field[] fields = new Field[3];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    fields[2] = new Field("doubleField", DataTypes.DOUBLE);
    TestUtil.writeFilesAndVerify(200, new Schema(fields), path);
    ColumnExpression columnExpression = new ColumnExpression("doubleField", DataTypes.DOUBLE);
    LessThanExpression lessThanExpression = new LessThanExpression(columnExpression, new LiteralExpression("13.5", DataTypes.DOUBLE));
    ColumnExpression columnExpression2 = new ColumnExpression("name", DataTypes.STRING);
    EqualToExpression equalToExpression2 = new EqualToExpression(columnExpression2, new LiteralExpression("robot7", DataTypes.STRING));
    AndExpression andExpression = new AndExpression(lessThanExpression, equalToExpression2);
    CarbonReader reader = CarbonReader.builder(path, "_temp").projection(new String[] { "name", "age", "doubleField" }).filter(andExpression).build();
    int i = 0;
    while (reader.hasNext()) {
        Object[] row = (Object[]) reader.readNextRow();
        assert (((String) row[0]).contains("robot7"));
        assert (7 == ((int) (row[1]) % 10));
        assert ((double) row[2] < 13.5);
        i++;
    }
    Assert.assertEquals(i, 2);
    reader.close();
    FileUtils.deleteDirectory(new File(path));
}

Example 10 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CarbonReaderTest method testValidateEscapeCharWithProperValue.

@Test
public void testValidateEscapeCharWithProperValue() throws IOException {
    String path = "./testValidateEscapeCharWithProperValue";
    Field[] fields = new Field[2];
    fields[0] = new Field("stringField", DataTypes.STRING);
    fields[1] = new Field("varcharField", DataTypes.VARCHAR);
    Schema schema = new Schema(fields);
    Map map = new HashMap();
    map.put("escapechar", "#");
    try {
        CarbonWriter.builder().outputPath(path).withLoadOptions(map).withCsvInput(schema).enableLocalDictionary(false).writtenBy("CarbonReaderTest").build();
    } catch (IllegalArgumentException e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    } finally {
        FileUtils.deleteDirectory(new File(path));
    }
}