Examples with Field org.apache.carbondata.core.metadata.datatype.Field used on opensource projects

Example 96 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class ImageTest method writeCarbonFile.

public void writeCarbonFile(String path, int num) throws Exception {
    try {
        FileUtils.deleteDirectory(new File(path));
    } catch (IOException e) {
        e.printStackTrace();
    }
    Field[] fields = new Field[5];
    fields[0] = new Field("imageId", DataTypes.INT);
    fields[1] = new Field("imageName", DataTypes.STRING);
    fields[2] = new Field("imageBinary", DataTypes.BINARY);
    fields[3] = new Field("txtName", DataTypes.STRING);
    fields[4] = new Field("txtContent", DataTypes.STRING);
    String imageFolder = "./src/test/resources/image/flowers";
    byte[] originBinary = null;
    // read and write image data
    for (int j = 0; j < num; j++) {
        CarbonWriter writer = CarbonWriter.builder().outputPath(path).withCsvInput(new Schema(fields)).writtenBy("SDKS3Example").withPageSizeInMb(1).build();
        ArrayList files = listFiles(imageFolder, ".jpg");
        if (null != files) {
            for (int i = 0; i < files.size(); i++) {
                // read image and encode to Hex
                BufferedInputStream bis = new BufferedInputStream(new FileInputStream(files.get(i).toString()));
                char[] hexValue = null;
                originBinary = new byte[bis.available()];
                while ((bis.read(originBinary)) != -1) {
                    hexValue = Hex.encodeHex(originBinary);
                }
                String txtFileName = files.get(i).toString().split(".jpg")[0] + ".txt";
                BufferedInputStream txtBis = new BufferedInputStream(new FileInputStream(txtFileName));
                String txtValue = null;
                byte[] txtBinary = null;
                txtBinary = new byte[txtBis.available()];
                while ((txtBis.read(txtBinary)) != -1) {
                    txtValue = new String(txtBinary, "UTF-8");
                }
                // write data
                System.out.println(files.get(i).toString());
                writer.write(new String[] { String.valueOf(i), files.get(i).toString(), String.valueOf(hexValue), txtFileName, txtValue });
                bis.close();
            }
        }
        writer.close();
    }
}

Example 97 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class ImageTest method testWriteBinaryWithSort.

@Test
public void testWriteBinaryWithSort() {
    int num = 1;
    String path = "./target/binary";
    try {
        FileUtils.deleteDirectory(new File(path));
    } catch (IOException e) {
        e.printStackTrace();
    }
    Field[] fields = new Field[5];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    fields[2] = new Field("image1", DataTypes.BINARY);
    fields[3] = new Field("image2", DataTypes.BINARY);
    fields[4] = new Field("image3", DataTypes.BINARY);
    // read and write image data
    for (int j = 0; j < num; j++) {
        try {
            CarbonWriter.builder().outputPath(path).withCsvInput(new Schema(fields)).writtenBy("SDKS3Example").withPageSizeInMb(1).withTableProperty("sort_columns", "image1").build();
            assert (false);
        } catch (Exception e) {
            assert (e.getMessage().contains("sort columns not supported for array, struct, map, double, float, decimal, varchar, binary"));
        }
    }
}

Example 98 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class ImageTest method testWriteBinaryWithLong_string_columns.

@Test
public void testWriteBinaryWithLong_string_columns() {
    int num = 1;
    String path = "./target/binary";
    try {
        FileUtils.deleteDirectory(new File(path));
    } catch (IOException e) {
        e.printStackTrace();
    }
    Field[] fields = new Field[5];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    fields[2] = new Field("image1", DataTypes.BINARY);
    fields[3] = new Field("image2", DataTypes.BINARY);
    fields[4] = new Field("image3", DataTypes.BINARY);
    // read and write image data
    for (int j = 0; j < num; j++) {
        try {
            CarbonWriter.builder().outputPath(path).withCsvInput(new Schema(fields)).writtenBy("SDKS3Example").withPageSizeInMb(1).withTableProperty("long_string_columns", "image1").build();
            assert (false);
        } catch (Exception e) {
            assert (e.getMessage().contains("long string column : image1 is not supported for data type: BINARY"));
        }
    }
}

Example 99 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class ImageTest method testWriteBinaryWithInverted_index.

@Test
public void testWriteBinaryWithInverted_index() {
    int num = 1;
    String path = "./target/binary";
    try {
        FileUtils.deleteDirectory(new File(path));
    } catch (IOException e) {
        e.printStackTrace();
    }
    Field[] fields = new Field[5];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    fields[2] = new Field("image1", DataTypes.BINARY);
    fields[3] = new Field("image2", DataTypes.BINARY);
    fields[4] = new Field("image3", DataTypes.BINARY);
    // read and write image data
    for (int j = 0; j < num; j++) {
        try {
            CarbonWriter.builder().outputPath(path).withCsvInput(new Schema(fields)).writtenBy("SDKS3Example").withPageSizeInMb(1).withTableProperty("inverted_index", "image1").build();
        // TODO: should throw exception
        // assert(false);
        } catch (Exception e) {
            System.out.println(e.getMessage());
            assert (e.getMessage().contains("INVERTED_INDEX column: image1 should be present in SORT_COLUMNS"));
        }
    }
}

Example 100 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class ImageTest method testBinaryWithComplexType.

@Test
public void testBinaryWithComplexType() throws IOException, InvalidLoadOptionException, InterruptedException {
    int num = 1;
    int rows = 1;
    String path = "./target/binary";
    try {
        FileUtils.deleteDirectory(new File(path));
    } catch (IOException e) {
        e.printStackTrace();
    }
    Field[] fields = new Field[4];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    fields[2] = new Field("arrayField", DataTypes.createArrayType(DataTypes.BINARY));
    ArrayList<StructField> structFields = new ArrayList<>();
    structFields.add(new StructField("b", DataTypes.BINARY));
    fields[3] = new Field("structField", DataTypes.createStructType(structFields));
    // read and write image data
    for (int j = 0; j < num; j++) {
        CarbonWriter writer = CarbonWriter.builder().outputPath(path).withCsvInput(new Schema(fields)).writtenBy("BinaryExample").withPageSizeInMb(1).build();
        for (int i = 0; i < rows; i++) {
            // write data
            writer.write(new String[] { "robot" + (i % 10), String.valueOf(i), "binary1", "binary2" });
        }
        writer.close();
    }
    CarbonReader reader = CarbonReader.builder(path, "_temp").build();
    while (reader.hasNext()) {
        Object[] row = (Object[]) reader.readNextRow();
        Object[] arrayResult = (Object[]) row[1];
        Object[] structResult = (Object[]) row[2];
        assert (new String((byte[]) arrayResult[0]).equalsIgnoreCase("binary1"));
        assert (new String((byte[]) structResult[0]).equalsIgnoreCase("binary2"));
    }
    reader.close();
}