Examples with Field org.apache.carbondata.core.metadata.datatype.Field used on opensource projects

Example 81 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CarbonReaderTest method testReadDateAndTimestampColumnInStruct.

@Test
public void testReadDateAndTimestampColumnInStruct() {
    String path = "./testWriteFiles";
    try {
        FileUtils.deleteDirectory(new File(path));
        Field[] fields = new Field[3];
        fields[0] = new Field("name", DataTypes.STRING);
        fields[1] = new Field("age", DataTypes.INT);
        ArrayList<StructField> structFields = new ArrayList<>();
        structFields.add(new StructField("dateField", DataTypes.DATE));
        structFields.add(new StructField("timestampField", DataTypes.TIMESTAMP));
        fields[2] = new Field("structField", DataTypes.createStructType(structFields));
        Map<String, String> map = new HashMap<>();
        map.put("complex_delimiter_level_1", "#");
        CarbonWriter writer = CarbonWriter.builder().outputPath(path).withLoadOptions(map).withCsvInput(new Schema(fields)).writtenBy("CarbonReaderTest").build();
        for (int i = 0; i < 10; i++) {
            String[] row2 = new String[] { "robot" + (i % 10), String.valueOf(i % 10000), "2019-03-02#2019-02-12 03:12:34" };
            writer.write(row2);
        }
        writer.close();
        File[] dataFiles = new File(path).listFiles(new FilenameFilter() {

            @Override
            public boolean accept(File dir, String name) {
                if (name == null) {
                    return false;
                }
                return name.endsWith("carbondata");
            }
        });
        if (dataFiles == null || dataFiles.length < 1) {
            throw new RuntimeException("Carbon data file not exists.");
        }
        Schema schema = CarbonSchemaReader.readSchema(dataFiles[0].getAbsolutePath()).asOriginOrder();
        // Transform the schema
        String[] strings = new String[schema.getFields().length];
        for (int i = 0; i < schema.getFields().length; i++) {
            strings[i] = (schema.getFields())[i].getFieldName();
        }
        // Read data
        CarbonReader reader = CarbonReader.builder(path).projection(strings).build();
        int i = 0;
        while (reader.hasNext()) {
            Object[] row = (Object[]) reader.readNextRow();
            assert (row[0].equals("robot" + i));
            Object[] arr = (Object[]) row[2];
            assert (arr[0].equals("2019-03-02"));
            assert (arr[1].equals("2019-02-12 03:12:34"));
            i++;
        }
        Assert.assertEquals(i, 10);
        reader.close();
        FileUtils.deleteDirectory(new File(path));
    } catch (Throwable e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 82 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CarbonReaderTest method testValidateBadRecordsActionWithProperValue.

@Test
public void testValidateBadRecordsActionWithProperValue() throws IOException {
    String path = "./testValidateBadRecordsActionValue";
    Field[] fields = new Field[2];
    fields[0] = new Field("stringField", DataTypes.STRING);
    fields[1] = new Field("varcharField", DataTypes.VARCHAR);
    Schema schema = new Schema(fields);
    Map map = new HashMap();
    map.put("BAD_RECORDS_ACTION", "FAIL");
    try {
        CarbonWriter.builder().outputPath(path).withLoadOptions(map).withCsvInput(schema).enableLocalDictionary(false).writtenBy("CarbonReaderTest").build();
    } catch (IllegalArgumentException e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    } finally {
        FileUtils.deleteDirectory(new File(path));
    }
}

Example 83 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CarbonReaderTest method testValidateBadRecordsActionWithImproperValue.

@Test
public void testValidateBadRecordsActionWithImproperValue() throws IOException {
    String path = "./testValidateBadRecordsActionValue";
    Field[] fields = new Field[2];
    fields[0] = new Field("stringField", DataTypes.STRING);
    fields[1] = new Field("varcharField", DataTypes.VARCHAR);
    Schema schema = new Schema(fields);
    Map map = new HashMap();
    map.put("BAD_RECORDS_ACTION", "FAL");
    try {
        CarbonWriter.builder().outputPath(path).withLoadOptions(map).withCsvInput(schema).enableLocalDictionary(false).writtenBy("CarbonReaderTest").build();
        Assert.fail();
    } catch (IllegalArgumentException e) {
        Assert.assertTrue(e.getMessage().contains("option BAD_RECORDS_ACTION can have only either " + "FORCE or IGNORE or REDIRECT or FAIL. It shouldn't be FAL"));
    } catch (Exception e) {
        Assert.fail();
    } finally {
        FileUtils.deleteDirectory(new File(path));
    }
}

Example 84 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CarbonReaderTest method testReadBatchWithZeroBatchSize.

@Test
public void testReadBatchWithZeroBatchSize() throws Exception {
    String path = "./testWriteFiles";
    FileUtils.deleteDirectory(new File(path));
    IndexStoreManager.getInstance().clearIndexCache(AbsoluteTableIdentifier.from(path), false);
    Field[] fields = new Field[2];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    TestUtil.writeFilesAndVerify(10, new Schema(fields), path);
    CarbonReader reader;
    reader = CarbonReader.builder(path).withRowRecordReader().withBatch(0).build();
    int i = 0;
    while (reader.hasNext()) {
        Object[] row = reader.readNextBatchRow();
        Assert.assertEquals(row.length, 10);
        i++;
    }
    Assert.assertEquals(i, 1);
    FileUtils.deleteDirectory(new File(path));
}

Example 85 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CarbonReaderTest method testValidateBadRecordsLoggerEnableWithProperValue.

@Test
public void testValidateBadRecordsLoggerEnableWithProperValue() throws IOException {
    String path = "./testValidateBadRecordsLoggerEnableValue";
    Field[] fields = new Field[2];
    fields[0] = new Field("stringField", DataTypes.STRING);
    fields[1] = new Field("varcharField", DataTypes.VARCHAR);
    Schema schema = new Schema(fields);
    Map map = new HashMap();
    map.put("bad_records_logger_enable", "FALSE");
    try {
        CarbonWriter.builder().outputPath(path).withLoadOptions(map).withCsvInput(schema).enableLocalDictionary(false).writtenBy("CarbonReaderTest").build();
    } catch (IllegalArgumentException e) {
        e.printStackTrace();
        Assert.fail();
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    } finally {
        FileUtils.deleteDirectory(new File(path));
    }
}