Examples with BinaryStringData org.apache.flink.table.data.binary.BinaryStringData used on opensource projects

Example 11 with BinaryStringData

use of org.apache.flink.table.data.binary.BinaryStringData in project flink by apache.

the class OrcBulkRowDataWriterTest method initInput.

@Before
public void initInput() {
    input = new ArrayList<>();
    fieldTypes = new LogicalType[4];
    fieldTypes[0] = new VarCharType();
    fieldTypes[1] = new IntType();
    List<RowType.RowField> arrayRowFieldList = Collections.singletonList(new RowType.RowField("_col2_col0", new VarCharType()));
    fieldTypes[2] = new ArrayType(new RowType(arrayRowFieldList));
    List<RowType.RowField> mapRowFieldList = Arrays.asList(new RowType.RowField("_col3_col0", new VarCharType()), new RowType.RowField("_col3_col1", new TimestampType()));
    fieldTypes[3] = new MapType(new VarCharType(), new RowType(mapRowFieldList));
    {
        GenericRowData rowData = new GenericRowData(4);
        rowData.setField(0, new BinaryStringData("_col_0_string_1"));
        rowData.setField(1, 1);
        GenericRowData arrayValue1 = new GenericRowData(1);
        arrayValue1.setField(0, new BinaryStringData("_col_2_row_0_string_1"));
        GenericRowData arrayValue2 = new GenericRowData(1);
        arrayValue2.setField(0, new BinaryStringData("_col_2_row_1_string_1"));
        GenericArrayData arrayData = new GenericArrayData(new Object[] { arrayValue1, arrayValue2 });
        rowData.setField(2, arrayData);
        GenericRowData mapValue1 = new GenericRowData(2);
        mapValue1.setField(0, new BinaryStringData(("_col_3_map_value_string_1")));
        mapValue1.setField(1, TimestampData.fromTimestamp(new Timestamp(3600000)));
        Map<StringData, RowData> mapDataMap = new HashMap<>();
        mapDataMap.put(new BinaryStringData("_col_3_map_key_1"), mapValue1);
        GenericMapData mapData = new GenericMapData(mapDataMap);
        rowData.setField(3, mapData);
        input.add(rowData);
    }
    {
        GenericRowData rowData = new GenericRowData(4);
        rowData.setField(0, new BinaryStringData("_col_0_string_2"));
        rowData.setField(1, 2);
        GenericRowData arrayValue1 = new GenericRowData(1);
        arrayValue1.setField(0, new BinaryStringData("_col_2_row_0_string_2"));
        GenericRowData arrayValue2 = new GenericRowData(1);
        arrayValue2.setField(0, new BinaryStringData("_col_2_row_1_string_2"));
        GenericArrayData arrayData = new GenericArrayData(new Object[] { arrayValue1, arrayValue2 });
        rowData.setField(2, arrayData);
        GenericRowData mapValue1 = new GenericRowData(2);
        mapValue1.setField(0, new BinaryStringData(("_col_3_map_value_string_2")));
        mapValue1.setField(1, TimestampData.fromTimestamp(new Timestamp(3600000)));
        Map<StringData, RowData> mapDataMap = new HashMap<>();
        mapDataMap.put(new BinaryStringData("_col_3_map_key_2"), mapValue1);
        GenericMapData mapData = new GenericMapData(mapDataMap);
        rowData.setField(3, mapData);
        input.add(rowData);
    }
}

Example 12 with BinaryStringData

use of org.apache.flink.table.data.binary.BinaryStringData in project flink by apache.

the class BinaryStringDataTest method testEmptyString.

@Test
public void testEmptyString() {
    BinaryStringData str2 = fromString("hahahahah");
    BinaryStringData str3;
    {
        MemorySegment[] segments = new MemorySegment[2];
        segments[0] = MemorySegmentFactory.wrap(new byte[10]);
        segments[1] = MemorySegmentFactory.wrap(new byte[10]);
        str3 = BinaryStringData.fromAddress(segments, 15, 0);
    }
    assertTrue(BinaryStringData.EMPTY_UTF8.compareTo(str2) < 0);
    assertTrue(str2.compareTo(BinaryStringData.EMPTY_UTF8) > 0);
    assertEquals(0, BinaryStringData.EMPTY_UTF8.compareTo(str3));
    assertEquals(0, str3.compareTo(BinaryStringData.EMPTY_UTF8));
    assertNotEquals(BinaryStringData.EMPTY_UTF8, str2);
    assertNotEquals(str2, BinaryStringData.EMPTY_UTF8);
    assertEquals(BinaryStringData.EMPTY_UTF8, str3);
    assertEquals(str3, BinaryStringData.EMPTY_UTF8);
}

Example 13 with BinaryStringData

use of org.apache.flink.table.data.binary.BinaryStringData in project flink by apache.

the class BinaryStringDataTest method testMultiSegments.

@Test
public void testMultiSegments() {
    // prepare
    MemorySegment[] segments1 = new MemorySegment[2];
    segments1[0] = MemorySegmentFactory.wrap(new byte[10]);
    segments1[1] = MemorySegmentFactory.wrap(new byte[10]);
    segments1[0].put(5, "abcde".getBytes(UTF_8), 0, 5);
    segments1[1].put(0, "aaaaa".getBytes(UTF_8), 0, 5);
    MemorySegment[] segments2 = new MemorySegment[2];
    segments2[0] = MemorySegmentFactory.wrap(new byte[5]);
    segments2[1] = MemorySegmentFactory.wrap(new byte[5]);
    segments2[0].put(0, "abcde".getBytes(UTF_8), 0, 5);
    segments2[1].put(0, "b".getBytes(UTF_8), 0, 1);
    // test go ahead both
    BinaryStringData binaryString1 = BinaryStringData.fromAddress(segments1, 5, 10);
    BinaryStringData binaryString2 = BinaryStringData.fromAddress(segments2, 0, 6);
    assertEquals("abcdeaaaaa", binaryString1.toString());
    assertEquals("abcdeb", binaryString2.toString());
    assertEquals(-1, binaryString1.compareTo(binaryString2));
    // test needCompare == len
    binaryString1 = BinaryStringData.fromAddress(segments1, 5, 5);
    binaryString2 = BinaryStringData.fromAddress(segments2, 0, 5);
    assertEquals("abcde", binaryString1.toString());
    assertEquals("abcde", binaryString2.toString());
    assertEquals(0, binaryString1.compareTo(binaryString2));
    // test find the first segment of this string
    binaryString1 = BinaryStringData.fromAddress(segments1, 10, 5);
    binaryString2 = BinaryStringData.fromAddress(segments2, 0, 5);
    assertEquals("aaaaa", binaryString1.toString());
    assertEquals("abcde", binaryString2.toString());
    assertEquals(-1, binaryString1.compareTo(binaryString2));
    assertEquals(1, binaryString2.compareTo(binaryString1));
    // test go ahead single
    segments2 = new MemorySegment[] { MemorySegmentFactory.wrap(new byte[10]) };
    segments2[0].put(4, "abcdeb".getBytes(UTF_8), 0, 6);
    binaryString1 = BinaryStringData.fromAddress(segments1, 5, 10);
    binaryString2 = BinaryStringData.fromAddress(segments2, 4, 6);
    assertEquals("abcdeaaaaa", binaryString1.toString());
    assertEquals("abcdeb", binaryString2.toString());
    assertEquals(-1, binaryString1.compareTo(binaryString2));
    assertEquals(1, binaryString2.compareTo(binaryString1));
}

Example 14 with BinaryStringData

use of org.apache.flink.table.data.binary.BinaryStringData in project flink by apache.

the class BinaryStringDataTest method concatWsTest.

@Test
public void concatWsTest() {
    // Returns empty if the separator is null
    assertEquals(null, concatWs(null, (BinaryStringData) null));
    assertEquals(null, concatWs(null, fromString("a")));
    // If separator is null, concatWs should skip all null inputs and never return null.
    BinaryStringData sep = fromString("哈哈");
    assertEquals(empty, concatWs(sep, empty));
    assertEquals(fromString("ab"), concatWs(sep, fromString("ab")));
    assertEquals(fromString("a哈哈b"), concatWs(sep, fromString("a"), fromString("b")));
    assertEquals(fromString("a哈哈b哈哈c"), concatWs(sep, fromString("a"), fromString("b"), fromString("c")));
    assertEquals(fromString("a哈哈c"), concatWs(sep, fromString("a"), null, fromString("c")));
    assertEquals(fromString("a"), concatWs(sep, fromString("a"), null, null));
    assertEquals(empty, concatWs(sep, null, null, null));
    assertEquals(fromString("数据哈哈砖头"), concatWs(sep, fromString("数据"), fromString("砖头")));
}

Example 15 with BinaryStringData

use of org.apache.flink.table.data.binary.BinaryStringData in project flink by apache.

the class SortUtilTest method testNormalizedKey.

@Test
public void testNormalizedKey() {
    int len = 10;
    Random random = new Random();
    MemorySegment[] segments = new MemorySegment[len];
    MemorySegment[] compareSegs = new MemorySegment[len];
    for (int i = 0; i < len; i++) {
        segments[i] = MemorySegmentFactory.allocateUnpooledSegment(20);
        compareSegs[i] = MemorySegmentFactory.allocateUnpooledSegment(20);
    }
    {
        SortUtil.minNormalizedKey(segments[0], 0, 20);
        SortUtil.maxNormalizedKey(segments[1], 0, 20);
        for (int i = 0; i < len; i++) {
            byte[] rndBytes = new byte[20];
            random.nextBytes(rndBytes);
            segments[2].put(0, rndBytes);
            Assert.assertTrue(segments[0].compare(segments[2], 0, 0, 20) <= 0);
            Assert.assertTrue(segments[1].compare(segments[2], 0, 0, 20) >= 0);
        }
    }
    {
        DecimalData[] arr = new DecimalData[len];
        for (int i = 0; i < len; i++) {
            arr[i] = DecimalData.fromBigDecimal(new BigDecimal(random.nextInt()), 18, 0);
            SortUtil.putDecimalNormalizedKey(arr[i], segments[i], 0, 8);
        }
        Arrays.sort(arr, DecimalData::compareTo);
        for (int i = 0; i < len; i++) {
            SortUtil.putDecimalNormalizedKey(arr[i], compareSegs[i], 0, 8);
        }
        Arrays.sort(segments, (o1, o2) -> o1.compare(o2, 0, 0, 8));
        for (int i = 0; i < len; i++) {
            Assert.assertTrue(compareSegs[i].equalTo(segments[i], 0, 0, 8));
        }
    }
    {
        Float[] arr = new Float[len];
        for (int i = 0; i < len; i++) {
            arr[i] = random.nextFloat();
            SortUtil.putFloatNormalizedKey(arr[i], segments[i], 0, 4);
        }
        Arrays.sort(arr, Float::compareTo);
        for (int i = 0; i < len; i++) {
            SortUtil.putFloatNormalizedKey(arr[i], compareSegs[i], 0, 4);
        }
        Arrays.sort(segments, (o1, o2) -> o1.compare(o2, 0, 0, 4));
        for (int i = 0; i < len; i++) {
            Assert.assertTrue(compareSegs[i].equalTo(segments[i], 0, 0, 4));
        }
    }
    {
        Double[] arr = new Double[len];
        for (int i = 0; i < len; i++) {
            arr[i] = random.nextDouble();
            SortUtil.putDoubleNormalizedKey(arr[i], segments[i], 0, 8);
        }
        Arrays.sort(arr, Double::compareTo);
        for (int i = 0; i < len; i++) {
            SortUtil.putDoubleNormalizedKey(arr[i], compareSegs[i], 0, 8);
        }
        Arrays.sort(segments, (o1, o2) -> o1.compare(o2, 0, 0, 8));
        for (int i = 0; i < len; i++) {
            Assert.assertTrue(compareSegs[i].equalTo(segments[i], 0, 0, 8));
        }
    }
    {
        BinaryStringData[] arr = new BinaryStringData[len];
        for (int i = 0; i < len; i++) {
            arr[i] = BinaryStringData.fromString(String.valueOf(random.nextLong()));
            SortUtil.putStringNormalizedKey(arr[i], segments[i], 0, 8);
        }
        Arrays.sort(arr, StringData::compareTo);
        for (int i = 0; i < len; i++) {
            SortUtil.putStringNormalizedKey(arr[i], compareSegs[i], 0, 8);
        }
        Arrays.sort(segments, (o1, o2) -> o1.compare(o2, 0, 0, 8));
        for (int i = 0; i < len; i++) {
            Assert.assertTrue(compareSegs[i].equalTo(segments[i], 0, 0, 8));
        }
    }
}