Examples with StringValue org.apache.flink.types.StringValue used on opensource projects

Example 71 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class DistinctTranslationTest method translateDistinctKeySelector.

@Test
public void translateDistinctKeySelector() {
    try {
        final int parallelism = 8;
        ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(parallelism);
        DataSet<Tuple3<Double, StringValue, LongValue>> initialData = getSourceDataSet(env);
        initialData.distinct(new KeySelector<Tuple3<Double, StringValue, LongValue>, StringValue>() {

            public StringValue getKey(Tuple3<Double, StringValue, LongValue> value) {
                return value.f1;
            }
        }).setParallelism(4).output(new DiscardingOutputFormat<Tuple3<Double, StringValue, LongValue>>());
        Plan p = env.createProgramPlan();
        GenericDataSinkBase<?> sink = p.getDataSinks().iterator().next();
        MapOperatorBase<?, ?, ?> keyRemover = (MapOperatorBase<?, ?, ?>) sink.getInput();
        PlanUnwrappingReduceOperator<?, ?> reducer = (PlanUnwrappingReduceOperator<?, ?>) keyRemover.getInput();
        MapOperatorBase<?, ?, ?> keyExtractor = (MapOperatorBase<?, ?, ?>) reducer.getInput();
        // check the parallelisms
        assertEquals(1, keyExtractor.getParallelism());
        assertEquals(4, reducer.getParallelism());
        // check types
        TypeInformation<?> keyValueInfo = new TupleTypeInfo<Tuple2<StringValue, Tuple3<Double, StringValue, LongValue>>>(new ValueTypeInfo<StringValue>(StringValue.class), initialData.getType());
        assertEquals(initialData.getType(), keyExtractor.getOperatorInfo().getInputType());
        assertEquals(keyValueInfo, keyExtractor.getOperatorInfo().getOutputType());
        assertEquals(keyValueInfo, reducer.getOperatorInfo().getInputType());
        assertEquals(keyValueInfo, reducer.getOperatorInfo().getOutputType());
        assertEquals(keyValueInfo, keyRemover.getOperatorInfo().getInputType());
        assertEquals(initialData.getType(), keyRemover.getOperatorInfo().getOutputType());
        // check keys
        assertEquals(KeyExtractingMapper.class, keyExtractor.getUserCodeWrapper().getUserCodeClass());
        assertTrue(keyExtractor.getInput() instanceof GenericDataSourceBase<?, ?>);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test caused an error: " + e.getMessage());
    }
}

Example 72 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class UnionTranslationTest method translateUnion3SortedGroup.

@Test
public void translateUnion3SortedGroup() {
    try {
        final int parallelism = 4;
        ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(parallelism);
        DataSet<Tuple3<Double, StringValue, LongValue>> dataset1 = getSourceDataSet(env, 2);
        DataSet<Tuple3<Double, StringValue, LongValue>> dataset2 = getSourceDataSet(env, 3);
        DataSet<Tuple3<Double, StringValue, LongValue>> dataset3 = getSourceDataSet(env, -1);
        dataset1.union(dataset2).union(dataset3).groupBy((KeySelector<Tuple3<Double, StringValue, LongValue>, String>) value -> "").sortGroup((KeySelector<Tuple3<Double, StringValue, LongValue>, String>) value -> "", Order.ASCENDING).reduceGroup((GroupReduceFunction<Tuple3<Double, StringValue, LongValue>, String>) (values, out) -> {
        }).returns(String.class).output(new DiscardingOutputFormat<>());
        Plan p = env.createProgramPlan();
        // The plan should look like the following one.
        // 
        // DataSet1(2) - MapOperator(2)-+
        // |- Union(-1) -+
        // DataSet2(3) - MapOperator(3)-+             |- Union(-1) - SingleInputOperator - Sink
        // |
        // DataSet3(-1) - MapOperator(-1)-+
        GenericDataSinkBase<?> sink = p.getDataSinks().iterator().next();
        Union secondUnionOperator = (Union) ((SingleInputOperator) sink.getInput()).getInput();
        // The first input of the second union should be the first union.
        Union firstUnionOperator = (Union) secondUnionOperator.getFirstInput();
        // The key mapper should be added to the second input stream of the second union.
        assertTrue(secondUnionOperator.getSecondInput() instanceof MapOperatorBase<?, ?, ?>);
        // The key mappers should be added to both of the two input streams for the first union.
        assertTrue(firstUnionOperator.getFirstInput() instanceof MapOperatorBase<?, ?, ?>);
        assertTrue(firstUnionOperator.getSecondInput() instanceof MapOperatorBase<?, ?, ?>);
        // The parallelisms of the key mappers should be equal to those of their inputs.
        assertEquals(firstUnionOperator.getFirstInput().getParallelism(), 2);
        assertEquals(firstUnionOperator.getSecondInput().getParallelism(), 3);
        assertEquals(secondUnionOperator.getSecondInput().getParallelism(), -1);
        // The union should always have the default parallelism.
        assertEquals(secondUnionOperator.getParallelism(), ExecutionConfig.PARALLELISM_DEFAULT);
        assertEquals(firstUnionOperator.getParallelism(), ExecutionConfig.PARALLELISM_DEFAULT);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test caused an error: " + e.getMessage());
    }
}

Example 73 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class AggregateTranslationTest method translateAggregate.

@Test
public void translateAggregate() {
    try {
        final int parallelism = 8;
        ExecutionEnvironment env = ExecutionEnvironment.createLocalEnvironment(parallelism);
        @SuppressWarnings("unchecked") DataSet<Tuple3<Double, StringValue, Long>> initialData = env.fromElements(new Tuple3<Double, StringValue, Long>(3.141592, new StringValue("foobar"), Long.valueOf(77)));
        initialData.groupBy(0).aggregate(Aggregations.MIN, 1).and(Aggregations.SUM, 2).output(new DiscardingOutputFormat<Tuple3<Double, StringValue, Long>>());
        Plan p = env.createProgramPlan();
        GenericDataSinkBase<?> sink = p.getDataSinks().iterator().next();
        GroupReduceOperatorBase<?, ?, ?> reducer = (GroupReduceOperatorBase<?, ?, ?>) sink.getInput();
        // check keys
        assertEquals(1, reducer.getKeyColumns(0).length);
        assertEquals(0, reducer.getKeyColumns(0)[0]);
        assertEquals(-1, reducer.getParallelism());
        assertTrue(reducer.isCombinable());
        assertTrue(reducer.getInput() instanceof GenericDataSourceBase<?, ?>);
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        fail("Test caused an error: " + e.getMessage());
    }
}

Example 74 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class GraphKeyTypeTransformTest method testFromStringValue.

// StringValue
@Test
public void testFromStringValue() throws Exception {
    TranslateFunction<StringValue, LongValueWithProperHashCode> translator = new StringValueToLongValueWithProperHashCode();
    Assert.assertEquals(new LongValueWithProperHashCode(0L), translator.translate(new StringValue("0"), longValueWithProperHashCode));
    Assert.assertEquals(new LongValueWithProperHashCode(Long.MIN_VALUE), translator.translate(new StringValue("-9223372036854775808"), longValueWithProperHashCode));
    Assert.assertEquals(new LongValueWithProperHashCode(Long.MAX_VALUE), translator.translate(new StringValue("9223372036854775807"), longValueWithProperHashCode));
}

Example 75 with StringValue

use of org.apache.flink.types.StringValue in project flink by apache.

the class EventWithAggregatorsTest method testSerializationOfEventWithAggregateValues.

@Test
public void testSerializationOfEventWithAggregateValues() {
    StringValue stringValue = new StringValue("test string");
    LongValue longValue = new LongValue(68743254);
    String stringValueName = "stringValue";
    String longValueName = "longValue";
    Aggregator<StringValue> stringAgg = new TestAggregator<StringValue>(stringValue);
    Aggregator<LongValue> longAgg = new TestAggregator<LongValue>(longValue);
    Map<String, Aggregator<?>> aggMap = new HashMap<String, Aggregator<?>>();
    aggMap.put(stringValueName, stringAgg);
    aggMap.put(longValueName, longAgg);
    Set<String> allNames = new HashSet<String>();
    allNames.add(stringValueName);
    allNames.add(longValueName);
    Set<Value> allVals = new HashSet<Value>();
    allVals.add(stringValue);
    allVals.add(longValue);
    // run the serialization
    AllWorkersDoneEvent e = new AllWorkersDoneEvent(aggMap);
    IterationEventWithAggregators deserialized = pipeThroughSerialization(e);
    // verify the result
    String[] names = deserialized.getAggregatorNames();
    Value[] aggregates = deserialized.getAggregates(cl);
    Assert.assertEquals(allNames.size(), names.length);
    Assert.assertEquals(allVals.size(), aggregates.length);
    // check that all the correct names and values are returned
    for (String s : names) {
        allNames.remove(s);
    }
    for (Value v : aggregates) {
        allVals.remove(v);
    }
    Assert.assertTrue(allNames.isEmpty());
    Assert.assertTrue(allVals.isEmpty());
}