Examples with Field org.apache.carbondata.core.metadata.datatype.Field used on opensource projects

Example 21 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CarbonReaderTest method testReadDateAndTimestampColumnInArray.

@Test
public void testReadDateAndTimestampColumnInArray() {
    String path = "./testWriteFiles";
    try {
        FileUtils.deleteDirectory(new File(path));
        Field[] fields = new Field[7];
        fields[0] = new Field("stringField", DataTypes.STRING);
        fields[1] = new Field("shortField", DataTypes.SHORT);
        fields[2] = new Field("dateField", DataTypes.DATE);
        fields[3] = new Field("timeField", DataTypes.TIMESTAMP);
        fields[4] = new Field("varcharField", DataTypes.VARCHAR);
        fields[5] = new Field("arrayFieldDate", DataTypes.createArrayType(DataTypes.DATE));
        fields[6] = new Field("arrayFieldTimestamp", DataTypes.createArrayType(DataTypes.TIMESTAMP));
        Map<String, String> map = new HashMap<>();
        map.put("complex_delimiter_level_1", "#");
        CarbonWriter writer = CarbonWriter.builder().outputPath(path).withLoadOptions(map).withCsvInput(new Schema(fields)).writtenBy("CarbonReaderTest").build();
        for (int i = 0; i < 10; i++) {
            String[] row2 = new String[] { "robot" + (i % 10), String.valueOf(i % 10000), "2019-03-02", "2019-02-12 03:03:34", "varchar", "2019-03-02#2019-03-03#2019-03-04#2019-03-05", "2019-02-12 03:03:34#2019-02-12 03:03:38#2019-02-12 03:03:41#2019-02-12 03:12:34" };
            writer.write(row2);
        }
        writer.close();
        File[] dataFiles = new File(path).listFiles(new FilenameFilter() {

            @Override
            public boolean accept(File dir, String name) {
                if (name == null) {
                    return false;
                }
                return name.endsWith("carbondata");
            }
        });
        if (dataFiles == null || dataFiles.length < 1) {
            throw new RuntimeException("Carbon data file not exists.");
        }
        Schema schema = CarbonSchemaReader.readSchema(dataFiles[0].getAbsolutePath()).asOriginOrder();
        // Transform the schema
        String[] strings = new String[schema.getFields().length];
        for (int i = 0; i < schema.getFields().length; i++) {
            strings[i] = (schema.getFields())[i].getFieldName();
        }
        // Read data
        CarbonReader reader = CarbonReader.builder(path).projection(strings).build();
        Object[] arrDate = new Object[] { "2019-03-02", "2019-03-03", "2019-03-04", "2019-03-05" };
        Object[] arrTimestamp = new Object[] { "2019-02-12 03:03:34", "2019-02-12 03:03:38", "2019-02-12 03:03:41", "2019-02-12 03:12:34" };
        int i = 0;
        while (reader.hasNext()) {
            Object[] row = (Object[]) reader.readNextRow();
            assert (row[0].equals("robot" + i));
            assert (row[2].equals("2019-03-02"));
            assert (row[3].equals("2019-02-12 03:03:34"));
            Assert.assertArrayEquals(arrDate, (Object[]) row[5]);
            Assert.assertArrayEquals(arrTimestamp, (Object[]) row[6]);
            i++;
        }
        Assert.assertEquals(i, 10);
        reader.close();
        FileUtils.deleteDirectory(new File(path));
    } catch (Throwable e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 22 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CSVCarbonWriterTest method testCsvLoadAndCarbonReadWithComplexType.

@Test
public void testCsvLoadAndCarbonReadWithComplexType() throws IOException, InterruptedException, InvalidLoadOptionException {
    String filePath = "../../examples/spark/src/main/resources/data.csv";
    Field[] fields = new Field[11];
    fields[0] = new Field("shortField", "SHORT");
    fields[1] = new Field("intField", "INT");
    fields[2] = new Field("bigintField", "LONG");
    fields[3] = new Field("doubleField", "DOUBLE");
    fields[4] = new Field("stringField", "STRING");
    fields[5] = new Field("timestampfield", "TIMESTAMP");
    fields[6] = new Field("decimalField", "DECIMAL");
    fields[7] = new Field("datefield", "DATE");
    fields[8] = new Field("charField", "VARCHAR");
    fields[9] = new Field("floatField", "FLOAT");
    StructField[] structFields = new StructField[3];
    structFields[0] = new StructField("fooField", DataTypes.STRING);
    structFields[1] = new StructField("barField", DataTypes.STRING);
    structFields[2] = new StructField("worldField", DataTypes.STRING);
    Field structType = new Field("structField", "struct", Arrays.asList(structFields));
    fields[10] = structType;
    Map<String, String> options = new HashMap<>();
    options.put("timestampformat", "yyyy/MM/dd HH:mm:ss");
    options.put("dateformat", "yyyy/MM/dd");
    options.put("complex_delimiter_level_1", "#");
    Schema schema = new Schema(fields);
    loadCsvFile(filePath, schema, options);
    CarbonReader reader = CarbonReader.builder("./testCsvFileLoad", "_temp").projection(new String[] { "structfield" }).build();
    int rowCount = 0;
    while (reader.hasNext()) {
        Object[] row = (Object[]) reader.readNextRow();
        Object[] structCol = (Object[]) row[0];
        assert (structCol.length == 3);
        Assert.assertEquals(structCol[0], "'foo'");
        Assert.assertEquals(structCol[1], "'bar'");
        Assert.assertEquals(structCol[2], "'world'");
        rowCount++;
    }
    assert (rowCount == 10);
    reader.close();
}

Example 23 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CSVCarbonWriterTest method testWhenWriterthrowsError.

@Test(expected = IOException.class)
public void testWhenWriterthrowsError() throws IOException {
    CarbonWriter carbonWriter = null;
    String path = "./testWriteFiles";
    FileUtils.deleteDirectory(new File(path));
    Field[] fields = new Field[2];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    try {
        carbonWriter = CarbonWriter.builder().outputPath(path).withCsvInput(new Schema(fields)).writtenBy("CSVCarbonWriterTest").build();
    } catch (InvalidLoadOptionException e) {
        e.printStackTrace();
        Assert.assertTrue(false);
    }
    carbonWriter.write("babu,1");
    carbonWriter.close();
}

Example 24 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CSVCarbonWriterTest method testSortColumns.

@Test
public void testSortColumns() throws IOException {
    String path = "./testWriteFiles";
    FileUtils.deleteDirectory(new File(path));
    Field[] fields = new Field[2];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    TestUtil.writeFilesAndVerify(new Schema(fields), path, new String[] { "name" });
    // TODO: implement reader and verify the data is sorted
    FileUtils.deleteDirectory(new File(path));
}

Example 25 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CSVCarbonWriterTest method testCsvLoadAndCarbonReadWithPrimitiveType.

@Test
public void testCsvLoadAndCarbonReadWithPrimitiveType() throws IOException, InvalidLoadOptionException, InterruptedException {
    String filePath = "./src/test/resources/file/csv_files/primitive_data.csv";
    Field[] fields = new Field[4];
    fields[0] = new Field("id", "INT");
    fields[1] = new Field("country", "STRING");
    fields[2] = new Field("name", "STRING");
    fields[3] = new Field("salary", "INT");
    Schema schema = new Schema(fields);
    loadCsvFile(filePath, schema, null);
    CarbonReader reader = CarbonReader.builder("./testCsvFileLoad", "_temp").projection(new String[] { "id", "country", "name", "salary" }).build();
    int rowCount = 0;
    while (reader.hasNext()) {
        Object[] row = (Object[]) reader.readNextRow();
        rowCount++;
        Assert.assertEquals(row[0], rowCount);
        Assert.assertEquals(row[1], "china");
        Assert.assertEquals(row[2], "aaa" + rowCount);
        Assert.assertEquals(row[3], 14999 + rowCount);
    }
    assert (rowCount == 10);
    reader.close();
}