Examples with GenericRowData org.apache.flink.table.data.GenericRowData used on opensource projects

Example 91 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class NonBufferOverWindowOperatorTest method test.

private void test(boolean[] resetAccumulators, GenericRowData[] expect) throws Exception {
    operator = new NonBufferOverWindowOperator(functions, comparator, resetAccumulators) {

        {
            output = new ConsumerOutput(new Consumer<RowData>() {

                @Override
                public void accept(RowData r) {
                    collect.add(GenericRowData.of(r.getInt(0), r.getLong(1), r.getLong(2), r.getLong(3), r.getLong(4)));
                }
            });
        }

        @Override
        public ClassLoader getUserCodeClassloader() {
            return Thread.currentThread().getContextClassLoader();
        }

        @Override
        public StreamConfig getOperatorConfig() {
            StreamConfig conf = mock(StreamConfig.class);
            when(conf.<RowData>getTypeSerializerIn1(getUserCodeClassloader())).thenReturn(inputSer);
            return conf;
        }

        @Override
        public StreamingRuntimeContext getRuntimeContext() {
            return mock(StreamingRuntimeContext.class);
        }
    };
    operator.setProcessingTimeService(new TestProcessingTimeService());
    operator.open();
    addRow(0, 1L, 4L);
    addRow(0, 1L, 1L);
    addRow(1, 5L, 2L);
    addRow(2, 5L, 4L);
    addRow(2, 6L, 2L);
    GenericRowData[] outputs = this.collect.toArray(new GenericRowData[0]);
    Assert.assertArrayEquals(expect, outputs);
}

Example 92 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class CodeSplitTest method testRecordComparator.

@Test
public void testRecordComparator() {
    int numFields = 600;
    RowType rowType = getIntRowType(numFields);
    SortSpec.SortSpecBuilder builder = SortSpec.builder();
    for (int i = 0; i < numFields; i++) {
        builder.addField(i, true, true);
    }
    SortSpec sortSpec = builder.build();
    GenericRowData rowData1 = new GenericRowData(numFields);
    GenericRowData rowData2 = new GenericRowData(numFields);
    Random random = new Random();
    for (int i = 0; i < numFields; i++) {
        int x = random.nextInt(100);
        rowData1.setField(i, x);
        rowData2.setField(i, x);
    }
    int result = random.nextInt(3) - 1;
    if (result == -1) {
        rowData1.setField(random.nextInt(numFields), -1);
    } else if (result == 1) {
        rowData1.setField(random.nextInt(numFields), 100);
    }
    Consumer<TableConfig> consumer = tableConfig -> {
        RecordComparator instance = ComparatorCodeGenerator.gen(tableConfig, "", rowType, sortSpec).newInstance(classLoader);
        for (int i = 0; i < 100; i++) {
            Assert.assertEquals(result, instance.compare(rowData1, rowData2));
        }
    };
    runTest(consumer);
}

Example 93 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class CodeSplitTest method testProjection.

@SuppressWarnings("unchecked")
@Test
public void testProjection() {
    int numFields = 1000;
    RowType rowType = getIntRowType(numFields);
    List<Integer> order = new ArrayList<>();
    for (int i = 0; i < numFields; i++) {
        order.add(i);
    }
    Collections.shuffle(order);
    GenericRowData input = new GenericRowData(numFields);
    for (int i = 0; i < numFields; i++) {
        input.setField(i, i);
    }
    BinaryRowData output = new BinaryRowData(numFields);
    BinaryRowWriter outputWriter = new BinaryRowWriter(output);
    for (int i = 0; i < numFields; i++) {
        outputWriter.writeInt(i, order.get(i));
    }
    outputWriter.complete();
    Consumer<TableConfig> consumer = tableConfig -> {
        Projection instance = ProjectionCodeGenerator.generateProjection(new CodeGeneratorContext(tableConfig), "", rowType, rowType, order.stream().mapToInt(i -> i).toArray()).newInstance(classLoader);
        for (int i = 0; i < 100; i++) {
            Assert.assertEquals(output, instance.apply(input));
        }
    };
    runTest(consumer);
}

Example 94 with GenericRowData

use of org.apache.flink.table.data.GenericRowData in project flink by apache.

the class HashJoinOperator method open.

@Override
public void open() throws Exception {
    super.open();
    ClassLoader cl = getContainingTask().getUserCodeClassLoader();
    final AbstractRowDataSerializer buildSerializer = (AbstractRowDataSerializer) getOperatorConfig().getTypeSerializerIn1(getUserCodeClassloader());
    final AbstractRowDataSerializer probeSerializer = (AbstractRowDataSerializer) getOperatorConfig().getTypeSerializerIn2(getUserCodeClassloader());
    boolean hashJoinUseBitMaps = getContainingTask().getEnvironment().getTaskConfiguration().getBoolean(AlgorithmOptions.HASH_JOIN_BLOOM_FILTERS);
    int parallel = getRuntimeContext().getNumberOfParallelSubtasks();
    this.condition = parameter.condFuncCode.newInstance(cl);
    condition.setRuntimeContext(getRuntimeContext());
    condition.open(new Configuration());
    this.table = new BinaryHashTable(getContainingTask().getJobConfiguration(), getContainingTask(), buildSerializer, probeSerializer, parameter.buildProjectionCode.newInstance(cl), parameter.probeProjectionCode.newInstance(cl), getContainingTask().getEnvironment().getMemoryManager(), computeMemorySize(), getContainingTask().getEnvironment().getIOManager(), parameter.buildRowSize, parameter.buildRowCount / parallel, hashJoinUseBitMaps, type, condition, reverseJoinFunction, parameter.filterNullKeys, parameter.tryDistinctBuildRow);
    this.collector = new StreamRecordCollector<>(output);
    this.buildSideNullRow = new GenericRowData(buildSerializer.getArity());
    this.probeSideNullRow = new GenericRowData(probeSerializer.getArity());
    this.joinedRow = new JoinedRowData();
    this.buildEnd = false;
    getMetricGroup().gauge("memoryUsedSizeInBytes", table::getUsedMemoryInBytes);
    getMetricGroup().gauge("numSpillFiles", table::getNumSpillFiles);
    getMetricGroup().gauge("spillInBytes", table::getSpillInBytes);
    parameter.condFuncCode = null;
    parameter.buildProjectionCode = null;
    parameter.probeProjectionCode = null;
}