Examples with PojoTypeInfo org.apache.flink.api.java.typeutils.PojoTypeInfo used on opensource projects

Example 1 with PojoTypeInfo

use of org.apache.flink.api.java.typeutils.PojoTypeInfo in project flink by apache.

the class CsvInputFormatTest method testPojoTypeWithInvalidFieldMapping.

@Test
public void testPojoTypeWithInvalidFieldMapping() throws Exception {
    File tempFile = File.createTempFile("CsvReaderPojoType", "tmp");
    tempFile.deleteOnExit();
    tempFile.setWritable(true);
    @SuppressWarnings("unchecked") PojoTypeInfo<PojoItem> typeInfo = (PojoTypeInfo<PojoItem>) TypeExtractor.createTypeInfo(PojoItem.class);
    try {
        new PojoCsvInputFormat<PojoItem>(new Path(tempFile.toURI().toString()), typeInfo, new String[] { "field1", "field2" });
        fail("The number of POJO fields cannot be same as that of selected CSV fields");
    } catch (IllegalArgumentException e) {
    // success
    }
    try {
        new PojoCsvInputFormat<PojoItem>(new Path(tempFile.toURI().toString()), typeInfo, new String[] { "field1", "field2", null, "field4" });
        fail("Fields mapping cannot contain null.");
    } catch (NullPointerException e) {
    // success
    }
    try {
        new PojoCsvInputFormat<PojoItem>(new Path(tempFile.toURI().toString()), typeInfo, new String[] { "field1", "field2", "field3", "field5" });
        fail("Invalid field name");
    } catch (IllegalArgumentException e) {
    // success
    }
}

Example 2 with PojoTypeInfo

use of org.apache.flink.api.java.typeutils.PojoTypeInfo in project flink by apache.

the class CsvInputFormatTest method testPojoTypeWithMappingInfoAndPartialField.

@Test
public void testPojoTypeWithMappingInfoAndPartialField() throws Exception {
    File tempFile = File.createTempFile("CsvReaderPojoType", "tmp");
    tempFile.deleteOnExit();
    tempFile.setWritable(true);
    OutputStreamWriter wrt = new OutputStreamWriter(new FileOutputStream(tempFile));
    wrt.write("123,3.123,AAA,BBB\n");
    wrt.write("456,1.123,BBB,AAA\n");
    wrt.close();
    @SuppressWarnings("unchecked") PojoTypeInfo<PojoItem> typeInfo = (PojoTypeInfo<PojoItem>) TypeExtractor.createTypeInfo(PojoItem.class);
    CsvInputFormat<PojoItem> inputFormat = new PojoCsvInputFormat<PojoItem>(new Path(tempFile.toURI().toString()), typeInfo, new String[] { "field1", "field4" }, new boolean[] { true, false, false, true });
    inputFormat.configure(new Configuration());
    FileInputSplit[] splits = inputFormat.createInputSplits(1);
    inputFormat.open(splits[0]);
    PojoItem item = new PojoItem();
    inputFormat.nextRecord(item);
    assertEquals(123, item.field1);
    assertEquals("BBB", item.field4);
}

Example 3 with PojoTypeInfo

use of org.apache.flink.api.java.typeutils.PojoTypeInfo in project flink by apache.

the class CsvInputFormatTest method testPojoTypeWithPartialFieldInCSV.

@Test
public void testPojoTypeWithPartialFieldInCSV() throws Exception {
    File tempFile = File.createTempFile("CsvReaderPojoType", "tmp");
    tempFile.deleteOnExit();
    tempFile.setWritable(true);
    OutputStreamWriter wrt = new OutputStreamWriter(new FileOutputStream(tempFile));
    wrt.write("123,NODATA,AAA,NODATA,3.123,BBB\n");
    wrt.write("456,NODATA,BBB,NODATA,1.123,AAA\n");
    wrt.close();
    @SuppressWarnings("unchecked") PojoTypeInfo<PojoItem> typeInfo = (PojoTypeInfo<PojoItem>) TypeExtractor.createTypeInfo(PojoItem.class);
    CsvInputFormat<PojoItem> inputFormat = new PojoCsvInputFormat<PojoItem>(new Path(tempFile.toURI().toString()), typeInfo, new boolean[] { true, false, true, false, true, true });
    inputFormat.configure(new Configuration());
    FileInputSplit[] splits = inputFormat.createInputSplits(1);
    inputFormat.open(splits[0]);
    validatePojoItem(inputFormat);
}

Example 4 with PojoTypeInfo

use of org.apache.flink.api.java.typeutils.PojoTypeInfo in project flink by apache.

the class AvroInputFormatTypeExtractionTest method testTypeExtraction.

@Test
public void testTypeExtraction() {
    try {
        InputFormat<MyAvroType, ?> format = new AvroInputFormat<MyAvroType>(new Path("file:///ignore/this/file"), MyAvroType.class);
        TypeInformation<?> typeInfoDirect = TypeExtractor.getInputFormatTypes(format);
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<MyAvroType> input = env.createInput(format);
        TypeInformation<?> typeInfoDataSet = input.getType();
        Assert.assertTrue(typeInfoDirect instanceof PojoTypeInfo);
        Assert.assertTrue(typeInfoDataSet instanceof PojoTypeInfo);
        Assert.assertEquals(MyAvroType.class, typeInfoDirect.getTypeClass());
        Assert.assertEquals(MyAvroType.class, typeInfoDataSet.getTypeClass());
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    }
}

Example 5 with PojoTypeInfo

use of org.apache.flink.api.java.typeutils.PojoTypeInfo in project flink by apache.

the class PojoSerializerTest method testTuplePojoTestEquality.

/**
	 * This tests if the hashes returned by the pojo and tuple comparators are the same
	 */
@SuppressWarnings({ "rawtypes", "unchecked" })
@Test
public void testTuplePojoTestEquality() {
    // test with a simple, string-key first.
    PojoTypeInfo<TestUserClass> pType = (PojoTypeInfo<TestUserClass>) type;
    List<FlatFieldDescriptor> result = new ArrayList<FlatFieldDescriptor>();
    pType.getFlatFields("nestedClass.dumm2", 0, result);
    // see below
    int[] fields = new int[1];
    fields[0] = result.get(0).getPosition();
    TypeComparator<TestUserClass> pojoComp = pType.createComparator(fields, new boolean[] { true }, 0, new ExecutionConfig());
    TestUserClass pojoTestRecord = new TestUserClass(0, "abc", 3d, new int[] { 1, 2, 3 }, new Date(), new NestedTestUserClass(1, "haha", 4d, new int[] { 5, 4, 3 }));
    int pHash = pojoComp.hash(pojoTestRecord);
    Tuple1<String> tupleTest = new Tuple1<String>("haha");
    TupleTypeInfo<Tuple1<String>> tType = (TupleTypeInfo<Tuple1<String>>) TypeExtractor.getForObject(tupleTest);
    TypeComparator<Tuple1<String>> tupleComp = tType.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
    int tHash = tupleComp.hash(tupleTest);
    Assert.assertTrue("The hashing for tuples and pojos must be the same, so that they are mixable", pHash == tHash);
    // its important here to use the same values.
    Tuple3<Integer, String, Double> multiTupleTest = new Tuple3<Integer, String, Double>(1, "haha", 4d);
    TupleTypeInfo<Tuple3<Integer, String, Double>> multiTupleType = (TupleTypeInfo<Tuple3<Integer, String, Double>>) TypeExtractor.getForObject(multiTupleTest);
    ExpressionKeys fieldKey = new ExpressionKeys(new int[] { 1, 0, 2 }, multiTupleType);
    ExpressionKeys expressKey = new ExpressionKeys(new String[] { "nestedClass.dumm2", "nestedClass.dumm1", "nestedClass.dumm3" }, pType);
    try {
        Assert.assertTrue("Expecting the keys to be compatible", fieldKey.areCompatible(expressKey));
    } catch (IncompatibleKeysException e) {
        e.printStackTrace();
        Assert.fail("Keys must be compatible: " + e.getMessage());
    }
    TypeComparator<TestUserClass> multiPojoComp = pType.createComparator(expressKey.computeLogicalKeyPositions(), new boolean[] { true, true, true }, 0, new ExecutionConfig());
    int multiPojoHash = multiPojoComp.hash(pojoTestRecord);
    // pojo order is: dumm2 (str), dumm1 (int), dumm3 (double).
    TypeComparator<Tuple3<Integer, String, Double>> multiTupleComp = multiTupleType.createComparator(fieldKey.computeLogicalKeyPositions(), new boolean[] { true, true, true }, 0, new ExecutionConfig());
    int multiTupleHash = multiTupleComp.hash(multiTupleTest);
    Assert.assertTrue("The hashing for tuples and pojos must be the same, so that they are mixable. Also for those with multiple key fields", multiPojoHash == multiTupleHash);
}