Examples with Field org.apache.carbondata.core.metadata.datatype.Field used on opensource projects

Example 36 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class ImageTest method testWriteTwoImageColumn.

public void testWriteTwoImageColumn() throws Exception {
    String imagePath = "./src/test/resources/image/vocForSegmentationClass";
    String path = "./target/vocForSegmentationClass";
    int num = 1;
    Field[] fields = new Field[4];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    fields[2] = new Field("rawImage", DataTypes.BINARY);
    fields[3] = new Field("segmentationClass", DataTypes.BINARY);
    byte[] originBinary = null;
    byte[] originBinary2 = null;
    Object[] files = listFiles(imagePath, ".jpg").toArray();
    // read and write image data
    for (int j = 0; j < num; j++) {
        CarbonWriter writer = CarbonWriter.builder().outputPath(path).withCsvInput(new Schema(fields)).writtenBy("SDKS3Example").withPageSizeInMb(1).build();
        for (int i = 0; i < files.length; i++) {
            // read image and encode to Hex
            String filePath = (String) files[i];
            BufferedInputStream bis = new BufferedInputStream(new FileInputStream(filePath));
            originBinary = new byte[bis.available()];
            while ((bis.read(originBinary)) != -1) {
            }
            BufferedInputStream bis2 = new BufferedInputStream(new FileInputStream(filePath.replace(".jpg", ".png")));
            originBinary2 = new byte[bis2.available()];
            while ((bis2.read(originBinary2)) != -1) {
            }
            // write data
            writer.write(new Object[] { "robot" + (i % 10), i, originBinary, originBinary2 });
            bis.close();
            bis2.close();
        }
        writer.close();
    }
    CarbonReader reader = CarbonReader.builder(path, "_temp").build();
    System.out.println("\nData:");
    int i = 0;
    while (i < 20 && reader.hasNext()) {
        Object[] row = (Object[]) reader.readNextRow();
        byte[] outputBinary = (byte[]) row[1];
        byte[] outputBinary2 = (byte[]) row[2];
        System.out.println(row[0] + " " + row[3] + " image1 size:" + outputBinary.length + " image2 size:" + outputBinary2.length);
        for (int k = 0; k < 2; k++) {
            byte[] originBinaryTemp = (byte[]) row[1 + k];
            // save image, user can compare the save image and original image
            String destString = null;
            if (k == 0) {
                destString = "./target/vocForSegmentationClass/image" + k + "_" + i + ".jpg";
            } else {
                destString = "./target/vocForSegmentationClass/image" + k + "_" + i + ".png";
            }
            BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(destString));
            bos.write(originBinaryTemp);
            bos.close();
        }
        i++;
    }
    System.out.println("\nFinished");
    reader.close();
}

Example 37 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class ImageTest method testBinaryWithOrWithoutFilter.

@Test
public void testBinaryWithOrWithoutFilter() throws IOException, InvalidLoadOptionException, InterruptedException, DecoderException {
    String imagePath = "./src/test/resources/image/carbondatalogo.jpg";
    int num = 1;
    int rows = 1;
    String path = "./target/binary";
    try {
        FileUtils.deleteDirectory(new File(path));
    } catch (IOException e) {
        e.printStackTrace();
    }
    Field[] fields = new Field[3];
    fields[0] = new Field("name", DataTypes.STRING);
    fields[1] = new Field("age", DataTypes.INT);
    fields[2] = new Field("image", DataTypes.BINARY);
    byte[] originBinary = null;
    // read and write image data
    String binaryValue = null;
    for (int j = 0; j < num; j++) {
        CarbonWriter writer = CarbonWriter.builder().outputPath(path).withCsvInput(new Schema(fields)).writtenBy("SDKS3Example").withPageSizeInMb(1).build();
        for (int i = 0; i < rows; i++) {
            // read image and encode to Hex
            BufferedInputStream bis = new BufferedInputStream(new FileInputStream(imagePath));
            char[] hexValue = null;
            originBinary = new byte[bis.available()];
            while ((bis.read(originBinary)) != -1) {
                hexValue = Hex.encodeHex(originBinary);
            }
            // write data
            binaryValue = String.valueOf(hexValue);
            writer.write(new String[] { "robot" + (i % 10), String.valueOf(i), binaryValue });
            bis.close();
        }
        writer.close();
    }
    // Read data with filter
    EqualToExpression equalToExpression = new EqualToExpression(new ColumnExpression("name", DataTypes.STRING), new LiteralExpression("robot0", DataTypes.STRING));
    CarbonReader reader = CarbonReader.builder(path, "_temp").filter(equalToExpression).build();
    System.out.println("\nData:");
    int i = 0;
    while (i < 20 && reader.hasNext()) {
        Object[] row = (Object[]) reader.readNextRow();
        byte[] outputBinary = Hex.decodeHex(new String((byte[]) row[1]).toCharArray());
        System.out.println(row[0] + " " + row[2] + " image size:" + outputBinary.length);
        // validate output binary data and origin binary data
        assert (originBinary.length == outputBinary.length);
        for (int j = 0; j < originBinary.length; j++) {
            assert (originBinary[j] == outputBinary[j]);
        }
        String value = new String(outputBinary);
        Assert.assertTrue(value.startsWith("�PNG"));
        // save image, user can compare the save image and original image
        String destString = "./target/binary/image" + i + ".jpg";
        BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(destString));
        bos.write(outputBinary);
        bos.close();
        i++;
    }
    System.out.println("\nFinished");
    reader.close();
    CarbonReader reader2 = CarbonReader.builder(path, "_temp").build();
    System.out.println("\nData:");
    i = 0;
    while (i < 20 && reader2.hasNext()) {
        Object[] row = (Object[]) reader2.readNextRow();
        byte[] outputBinary = Hex.decodeHex(new String((byte[]) row[1]).toCharArray());
        System.out.println(row[0] + " " + row[2] + " image size:" + outputBinary.length);
        // validate output binary data and origin binary data
        assert (originBinary.length == outputBinary.length);
        for (int j = 0; j < originBinary.length; j++) {
            assert (originBinary[j] == outputBinary[j]);
        }
        // save image, user can compare the save image and original image
        String destString = "./target/binary/image" + i + ".jpg";
        BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(destString));
        bos.write(outputBinary);
        bos.close();
        i++;
    }
    reader2.close();
    // Read data with filter for binary
    CarbonReader reader3 = CarbonReader.builder(path, "_temp").filter(prepareEqualToExpression("image", "binary", binaryValue)).build();
    System.out.println("\nData:");
    i = 0;
    while (i < 20 && reader3.hasNext()) {
        Object[] row = (Object[]) reader3.readNextRow();
        byte[] outputBinary = Hex.decodeHex(new String((byte[]) row[1]).toCharArray());
        System.out.println(row[0] + " " + row[2] + " image size:" + outputBinary.length);
        // validate output binary data and origin binary data
        assert (originBinary.length == outputBinary.length);
        for (int j = 0; j < originBinary.length; j++) {
            assert (originBinary[j] == outputBinary[j]);
        }
        String value = new String(outputBinary);
        Assert.assertTrue(value.startsWith("�PNG"));
        // save image, user can compare the save image and original image
        String destString = "./target/binary/image" + i + ".jpg";
        BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(destString));
        bos.write(outputBinary);
        bos.close();
        i++;
    }
    assert (1 == i);
    System.out.println("\nFinished");
    reader3.close();
    CarbonReader reader4 = CarbonReader.builder(path, "_temp").filter(prepareEqualToExpression("image", "binary", "hello")).build();
    System.out.println("\nData:");
    i = 0;
    while (i < 20 && reader4.hasNext()) {
        Object[] row = (Object[]) reader4.readNextRow();
        assert (null == row[1]);
    }
    System.out.println("\nFinished");
    reader4.close();
    try {
        FileUtils.deleteDirectory(new File(path));
    } catch (IOException e) {
        e.printStackTrace();
    }
    System.out.println("\nFinished");
}

Example 38 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class ImageTest method testBinaryWithManyImages.

@Test
public void testBinaryWithManyImages() throws IOException, InvalidLoadOptionException, InterruptedException {
    int num = 1;
    String path = "./target/flowers";
    Field[] fields = new Field[5];
    fields[0] = new Field("binaryId", DataTypes.INT);
    fields[1] = new Field("binaryName", DataTypes.STRING);
    fields[2] = new Field("binary", "Binary");
    fields[3] = new Field("labelName", DataTypes.STRING);
    fields[4] = new Field("labelContent", DataTypes.STRING);
    String imageFolder = "./src/test/resources/image/flowers";
    byte[] originBinary = null;
    // read and write image data
    for (int j = 0; j < num; j++) {
        CarbonWriter writer = CarbonWriter.builder().outputPath(path).withCsvInput(new Schema(fields)).writtenBy("SDKS3Example").withPageSizeInMb(1).build();
        File file = new File(imageFolder);
        File[] files = file.listFiles(new FilenameFilter() {

            @Override
            public boolean accept(File dir, String name) {
                if (name == null) {
                    return false;
                }
                return name.endsWith(".jpg");
            }
        });
        if (null != files) {
            for (int i = 0; i < files.length; i++) {
                // read image and encode to Hex
                BufferedInputStream bis = new BufferedInputStream(new FileInputStream(files[i]));
                char[] hexValue = null;
                originBinary = new byte[bis.available()];
                while ((bis.read(originBinary)) != -1) {
                    hexValue = Hex.encodeHex(originBinary);
                }
                String txtFileName = files[i].getCanonicalPath().split(".jpg")[0] + ".txt";
                BufferedInputStream txtBis = new BufferedInputStream(new FileInputStream(txtFileName));
                String txtValue = null;
                byte[] txtBinary = null;
                txtBinary = new byte[txtBis.available()];
                while ((txtBis.read(txtBinary)) != -1) {
                    txtValue = new String(txtBinary, "UTF-8");
                }
                // write data
                System.out.println(files[i].getCanonicalPath());
                writer.write(new String[] { String.valueOf(i), files[i].getCanonicalPath(), String.valueOf(hexValue), txtFileName, txtValue });
                bis.close();
            }
        }
        writer.close();
    }
    CarbonReader reader = CarbonReader.builder(path).build();
    System.out.println("\nData:");
    int i = 0;
    while (i < 20 && reader.hasNext()) {
        Object[] row = (Object[]) reader.readNextRow();
        byte[] outputBinary = (byte[]) row[1];
        System.out.println(row[0] + " " + row[2] + " image size:" + outputBinary.length);
        // save image, user can compare the save image and original image
        String destString = "./target/flowers/image" + i + ".jpg";
        BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(destString));
        bos.write(outputBinary);
        bos.close();
        i++;
    }
    System.out.println("\nFinished");
    reader.close();
}

Example 39 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class MinMaxTest method testMinMax.

@Test
public void testMinMax() throws IOException {
    Field[] fields = new Field[6];
    fields[0] = new Field("byteField", DataTypes.BYTE);
    fields[1] = new Field("shortField", DataTypes.SHORT);
    fields[2] = new Field("infField", DataTypes.INT);
    fields[3] = new Field("longField", DataTypes.LONG);
    fields[4] = new Field("floatField", DataTypes.FLOAT);
    fields[5] = new Field("doubleField", DataTypes.DOUBLE);
    writeFilesAndVerify(new Schema(fields), path);
    readIndexAndVerify(path);
}

Example 40 with Field

use of org.apache.carbondata.core.metadata.datatype.Field in project carbondata by apache.

the class CarbonReaderTest method testValidateEscapeCharWithImproperValue.

@Test
public void testValidateEscapeCharWithImproperValue() throws IOException {
    String path = "./testValidateEscapeCharWithImproperValue";
    Field[] fields = new Field[2];
    fields[0] = new Field("stringField", DataTypes.STRING);
    fields[1] = new Field("varcharField", DataTypes.VARCHAR);
    Schema schema = new Schema(fields);
    Map map = new HashMap();
    map.put("escapechar", "##");
    try {
        CarbonWriter.builder().outputPath(path).withLoadOptions(map).withCsvInput(schema).enableLocalDictionary(false).writtenBy("CarbonReaderTest").build();
        Assert.fail();
    } catch (IllegalArgumentException e) {
        Assert.assertTrue(e.getMessage().contains("ESCAPECHAR cannot be more than one character."));
    } catch (Exception e) {
        Assert.fail(e.getMessage());
    } finally {
        FileUtils.deleteDirectory(new File(path));
    }
}