Examples with DataSet org.apache.flink.api.java.DataSet used on opensource projects

Example 6 with DataSet

use of org.apache.flink.api.java.DataSet in project flink by apache.

the class PythonPlanBinder method createHashPartitionOperation.

@SuppressWarnings("unchecked")
private void createHashPartitionOperation(PythonOperationInfo info) throws IOException {
    DataSet op1 = (DataSet) sets.get(info.parentID);
    sets.put(info.setID, op1.partitionByHash(info.keys).setParallelism(getParallelism(info)).map(new KeyDiscarder()).setParallelism(getParallelism(info)).name("HashPartitionPostStep"));
}

Example 7 with DataSet

use of org.apache.flink.api.java.DataSet in project flink by apache.

the class PythonPlanBinder method createCrossOperation.

@SuppressWarnings("unchecked")
private void createCrossOperation(DatasizeHint mode, PythonOperationInfo info) {
    DataSet op1 = (DataSet) sets.get(info.parentID);
    DataSet op2 = (DataSet) sets.get(info.otherID);
    DefaultCross defaultResult;
    switch(mode) {
        case NONE:
            defaultResult = op1.cross(op2);
            break;
        case HUGE:
            defaultResult = op1.crossWithHuge(op2);
            break;
        case TINY:
            defaultResult = op1.crossWithTiny(op2);
            break;
        default:
            throw new IllegalArgumentException("Invalid Cross mode specified: " + mode);
    }
    defaultResult.setParallelism(getParallelism(info));
    if (info.usesUDF) {
        sets.put(info.setID, defaultResult.mapPartition(new PythonMapPartition(info.setID, info.types)).setParallelism(getParallelism(info)).name(info.name));
    } else {
        sets.put(info.setID, defaultResult.name("DefaultCross"));
    }
}

Example 8 with DataSet

use of org.apache.flink.api.java.DataSet in project flink by apache.

the class PythonPlanBinder method createCoGroupOperation.

@SuppressWarnings("unchecked")
private void createCoGroupOperation(PythonOperationInfo info) {
    DataSet op1 = (DataSet) sets.get(info.parentID);
    DataSet op2 = (DataSet) sets.get(info.otherID);
    Keys.ExpressionKeys<?> key1 = new Keys.ExpressionKeys(info.keys1, op1.getType());
    Keys.ExpressionKeys<?> key2 = new Keys.ExpressionKeys(info.keys2, op2.getType());
    PythonCoGroup pcg = new PythonCoGroup(info.setID, info.types);
    sets.put(info.setID, new CoGroupRawOperator(op1, op2, key1, key2, pcg, info.types, info.name).setParallelism(getParallelism(info)));
}

Example 9 with DataSet

use of org.apache.flink.api.java.DataSet in project flink by apache.

the class PythonPlanBinder method createMapPartitionOperation.

@SuppressWarnings("unchecked")
private void createMapPartitionOperation(PythonOperationInfo info) {
    DataSet op1 = (DataSet) sets.get(info.parentID);
    sets.put(info.setID, op1.mapPartition(new PythonMapPartition(info.setID, info.types)).setParallelism(getParallelism(info)).name(info.name));
}

Example 10 with DataSet

use of org.apache.flink.api.java.DataSet in project flink by apache.

the class PythonPlanBinder method createGroupOperation.

private void createGroupOperation(PythonOperationInfo info) throws IOException {
    DataSet op1 = (DataSet) sets.get(info.parentID);
    sets.put(info.setID, op1.groupBy(info.keys));
}