Examples with Field org.apache.carbondata.core.metadata.datatype.Field used on opensource projects

Example 51 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CarbonReaderTest method testReadFilesWithNullProjection.

@Test
public void testReadFilesWithNullProjection() throws IOException, InterruptedException {
    String path = "./testWriteFiles";
    FileUtils.deleteDirectory(new File(path));
    Field[] fields = new Field[2];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    TestUtil.writeFilesAndVerify(100, new Schema(fields), path);
    try {
        CarbonReader reader = CarbonReader.builder(path, "_temp").projection(new String[] {}).build();
        assert (false);
    } catch (RuntimeException e) {
        assert (e.getMessage().equalsIgnoreCase("Projection can't be empty"));
    }
}

Example 52 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class ConcurrentSdkWriterTest method testWriteFiles.

@Test
public void testWriteFiles() throws IOException {
    String path = "./testWriteFiles";
    FileUtils.deleteDirectory(new File(path));
    Field[] fields = new Field[2];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    ExecutorService executorService = Executors.newFixedThreadPool(numOfThreads);
    try {
        CarbonWriterBuilder builder = CarbonWriter.builder().outputPath(path).withThreadSafe(numOfThreads);
        CarbonWriter writer = builder.withCsvInput(new Schema(fields)).writtenBy("ConcurrentSdkWriterTest").build();
        // write in multi-thread
        for (int i = 0; i < numOfThreads; i++) {
            executorService.submit(new WriteLogic(writer));
        }
        executorService.shutdown();
        executorService.awaitTermination(2, TimeUnit.HOURS);
        writer.close();
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
    // read the files and verify the count
    CarbonReader reader;
    try {
        reader = CarbonReader.builder(path, "_temp1121").projection(new String[] { "name", "age" }).build();
        int i = 0;
        while (reader.hasNext()) {
            Object[] row = (Object[]) reader.readNextRow();
            i++;
        }
        Assert.assertEquals(i, numOfThreads * recordsPerItr);
        reader.close();
    } catch (InterruptedException e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
    FileUtils.deleteDirectory(new File(path));
}

Example 53 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class JSONCarbonWriterTest method buildSchema.

private Schema buildSchema() {
    Field[] fields = new Field[9];
    fields[0] = new Field("stringField", "STRING");
    fields[1] = new Field("intField", "INT");
    fields[2] = new Field("shortField", "SHORT");
    fields[3] = new Field("longField", "LONG");
    fields[4] = new Field("doubleField", "DOUBLE");
    fields[5] = new Field("boolField", "BOOLEAN");
    fields[6] = new Field("dateField", "DATE");
    fields[7] = new Field("timeField", "TIMESTAMP");
    fields[8] = new Field("decimalField", "DECIMAL");
    return new Schema(fields);
}

Example 54 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class JSONCarbonWriterTest method testJsonFileLoadWithComplexSchema.

@Test
public void testJsonFileLoadWithComplexSchema() throws IOException, InvalidLoadOptionException, InterruptedException {
    String filePath = "../../integration/spark/src/test/resources/jsonFiles/" + "data/arrayOfarrayOfarrayOfStruct.json";
    Field[] fields = new Field[3];
    fields[0] = new Field("name", "STRING");
    fields[1] = new Field("age", "INT");
    StructField[] structFields = new StructField[2];
    structFields[0] = new StructField("street", DataTypes.STRING);
    structFields[1] = new StructField("city", DataTypes.STRING);
    Field structType = new Field("structField", "struct", Arrays.asList(structFields));
    List<StructField> arrayField3 = new ArrayList<>();
    arrayField3.add(new StructField("structField", structType.getDataType(), Arrays.asList(structFields)));
    Field arrayLevel3 = new Field("arr3", "array", arrayField3);
    List<StructField> arrayField2 = new ArrayList<>();
    arrayField2.add(new StructField("arrayField3", arrayLevel3.getDataType(), arrayField3));
    Field arrayLevel2 = new Field("arr2", "array", arrayField2);
    List<StructField> arrayField = new ArrayList<>();
    arrayField.add(new StructField("arrayField2", arrayLevel2.getDataType(), arrayField2));
    Field arrayLevel1 = new Field("BuildNum", "array", arrayField);
    fields[2] = arrayLevel1;
    Schema schema = new Schema(fields);
    loadJsonFile(filePath, schema);
    CarbonReader reader = CarbonReader.builder("./testLoadJsonFIle", "_temp").projection(new String[] { "name", "age", "BuildNum" }).build();
    int count = 0;
    while (reader.hasNext()) {
        Object[] row = (Object[]) reader.readNextRow();
        Assert.assertEquals(row[0], "ajantha");
        Assert.assertEquals(row[1], 26);
        Object[] outerArray = (Object[]) row[2];
        Assert.assertEquals(outerArray.length, 2);
        Object[] middleArray = (Object[]) outerArray[0];
        Assert.assertEquals(middleArray.length, 2);
        Object[] innerArray1 = (Object[]) middleArray[0];
        Assert.assertEquals(innerArray1.length, 3);
        Object[] structField1 = (Object[]) innerArray1[0];
        Assert.assertEquals(structField1.length, 2);
        Assert.assertEquals(structField1[0], "abc");
        Assert.assertEquals(structField1[1], "city1");
        Object[] structField2 = (Object[]) innerArray1[1];
        Assert.assertEquals(structField2.length, 2);
        Assert.assertEquals(structField2[0], "def");
        Assert.assertEquals(structField2[1], "city2");
        count++;
    }
    assert (count == 1);
    reader.close();
}

Example 55 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class AvroCarbonWriterTest method testExceptionForInvalidDate.

@Test
public void testExceptionForInvalidDate() throws IOException, InvalidLoadOptionException {
    Field[] field = new Field[2];
    field[0] = new Field("name", DataTypes.STRING);
    field[1] = new Field("date", DataTypes.DATE);
    CarbonWriterBuilder writer = CarbonWriter.builder().uniqueIdentifier(System.currentTimeMillis()).outputPath(path);
    try {
        Map<String, String> loadOptions = new HashMap<String, String>();
        loadOptions.put("bad_records_action", "fail");
        CarbonWriter carbonWriter = writer.withLoadOptions(loadOptions).withCsvInput(new org.apache.carbondata.sdk.file.Schema(field)).writtenBy("AvroCarbonWriterTest").build();
        carbonWriter.write(new String[] { "k", "20-02-2233" });
        carbonWriter.close();
        Assert.fail();
    } catch (Exception e) {
        assert (e.getMessage().contains("Data load failed due to bad record"));
    }
    FileUtils.deleteDirectory(new File(path));
}