Examples with Tuple4 org.apache.flink.api.java.tuple.Tuple4 used on opensource projects

Example 31 with Tuple4

use of org.apache.flink.api.java.tuple.Tuple4 in project flink by apache.

the class SortPartitionTest method testSortPartitionWithKeySelector4.

@Test
public void testSortPartitionWithKeySelector4() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs = env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);
    // should work
    try {
        tupleDs.sortPartition(new KeySelector<Tuple4<Integer, Long, CustomType, Long[]>, Tuple2<Integer, Long>>() {

            @Override
            public Tuple2<Integer, Long> getKey(Tuple4<Integer, Long, CustomType, Long[]> value) throws Exception {
                return new Tuple2<>(value.f0, value.f1);
            }
        }, Order.ASCENDING);
    } catch (Exception e) {
        Assert.fail();
    }
}

Example 32 with Tuple4

use of org.apache.flink.api.java.tuple.Tuple4 in project flink by apache.

the class SortPartitionTest method testSortPartitionWithKeySelector2.

@Test(expected = InvalidProgramException.class)
public void testSortPartitionWithKeySelector2() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs = env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);
    // must not work
    tupleDs.sortPartition(new KeySelector<Tuple4<Integer, Long, CustomType, Long[]>, Long[]>() {

        @Override
        public Long[] getKey(Tuple4<Integer, Long, CustomType, Long[]> value) throws Exception {
            return value.f3;
        }
    }, Order.ASCENDING);
}

Example 33 with Tuple4

use of org.apache.flink.api.java.tuple.Tuple4 in project flink by apache.

the class SortPartitionTest method testSortPartitionWithExpressionKeys4.

@Test(expected = InvalidProgramException.class)
public void testSortPartitionWithExpressionKeys4() {
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    DataSet<Tuple4<Integer, Long, CustomType, Long[]>> tupleDs = env.fromCollection(tupleWithCustomData, tupleWithCustomInfo);
    // must not work
    tupleDs.sortPartition("f3", Order.ASCENDING);
}

Example 34 with Tuple4

use of org.apache.flink.api.java.tuple.Tuple4 in project flink by apache.

the class GroupingTupleTranslationTest method testCustomPartitioningTupleGroupReduceSorted2.

@Test
public void testCustomPartitioningTupleGroupReduceSorted2() {
    try {
        ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
        DataSet<Tuple4<Integer, Integer, Integer, Integer>> data = env.fromElements(new Tuple4<Integer, Integer, Integer, Integer>(0, 0, 0, 0)).rebalance().setParallelism(4);
        data.groupBy(0).withPartitioner(new TestPartitionerInt()).sortGroup(1, Order.ASCENDING).sortGroup(2, Order.DESCENDING).reduceGroup(new IdentityGroupReducerCombinable<Tuple4<Integer, Integer, Integer, Integer>>()).output(new DiscardingOutputFormat<Tuple4<Integer, Integer, Integer, Integer>>());
        Plan p = env.createProgramPlan();
        OptimizedPlan op = compileNoStats(p);
        SinkPlanNode sink = op.getDataSinks().iterator().next();
        SingleInputPlanNode reducer = (SingleInputPlanNode) sink.getInput().getSource();
        SingleInputPlanNode combiner = (SingleInputPlanNode) reducer.getInput().getSource();
        assertEquals(ShipStrategyType.FORWARD, sink.getInput().getShipStrategy());
        assertEquals(ShipStrategyType.PARTITION_CUSTOM, reducer.getInput().getShipStrategy());
        assertEquals(ShipStrategyType.FORWARD, combiner.getInput().getShipStrategy());
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 35 with Tuple4

use of org.apache.flink.api.java.tuple.Tuple4 in project flink by apache.

the class FileCache method createTmpFile.

// ------------------------------------------------------------------------
/**
	 * If the file doesn't exists locally, it will copy the file to the temp directory.
	 *
	 * @param name  The name under which the file is registered.
	 * @param entry The cache entry descriptor (path, executable flag)
	 * @param jobID The ID of the job for which the file is copied.
	 * @return The handle to the task that copies the file.
	 */
public Future<Path> createTmpFile(String name, DistributedCacheEntry entry, JobID jobID) {
    synchronized (lock) {
        Map<String, Tuple4<Integer, File, Path, Future<Path>>> jobEntries = entries.get(jobID);
        if (jobEntries == null) {
            jobEntries = new HashMap<String, Tuple4<Integer, File, Path, Future<Path>>>();
            entries.put(jobID, jobEntries);
        }
        // tuple is (ref-count, parent-temp-dir, cached-file-path, copy-process)
        Tuple4<Integer, File, Path, Future<Path>> fileEntry = jobEntries.get(name);
        if (fileEntry != null) {
            // file is already in the cache. return a future that
            // immediately returns the file
            fileEntry.f0 = fileEntry.f0 + 1;
            // return the future. may be that the copy is still in progress
            return fileEntry.f3;
        } else {
            // need to copy the file
            // create the target path
            File tempDirToUse = new File(storageDirectories[nextDirectory++], jobID.toString());
            if (nextDirectory >= storageDirectories.length) {
                nextDirectory = 0;
            }
            String sourceFile = entry.filePath;
            int posOfSep = sourceFile.lastIndexOf("/");
            if (posOfSep > 0) {
                sourceFile = sourceFile.substring(posOfSep + 1);
            }
            Path target = new Path(tempDirToUse.getAbsolutePath() + "/" + sourceFile);
            // kick off the copying
            CopyProcess cp = new CopyProcess(entry, target);
            FutureTask<Path> copyTask = new FutureTask<Path>(cp);
            executorService.submit(copyTask);
            // store our entry
            jobEntries.put(name, new Tuple4<Integer, File, Path, Future<Path>>(1, tempDirToUse, target, copyTask));
            return copyTask;
        }
    }
}