Examples with FileReader java.io.FileReader used on opensource projects

Example 6 with FileReader

use of java.io.FileReader in project flink by apache.

the class DataSinkTaskTest method testSortingDataSinkTask.

@Test
@SuppressWarnings("unchecked")
public void testSortingDataSinkTask() {
    int keyCnt = 100;
    int valCnt = 20;
    double memoryFraction = 1.0;
    super.initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
    super.addInput(new UniformRecordGenerator(keyCnt, valCnt, true), 0);
    DataSinkTask<Record> testTask = new DataSinkTask<>();
    // set sorting
    super.getTaskConfig().setInputLocalStrategy(0, LocalStrategy.SORT);
    super.getTaskConfig().setInputComparator(new RecordComparatorFactory(new int[] { 1 }, (new Class[] { IntValue.class })), 0);
    super.getTaskConfig().setRelativeMemoryInput(0, memoryFraction);
    super.getTaskConfig().setFilehandlesInput(0, 8);
    super.getTaskConfig().setSpillingThresholdInput(0, 0.8f);
    super.registerFileOutputTask(testTask, MockOutputFormat.class, new File(tempTestPath).toURI().toString());
    try {
        testTask.invoke();
    } catch (Exception e) {
        LOG.debug("Exception while invoking the test task.", e);
        Assert.fail("Invoke method caused exception.");
    }
    File tempTestFile = new File(this.tempTestPath);
    Assert.assertTrue("Temp output file does not exist", tempTestFile.exists());
    FileReader fr = null;
    BufferedReader br = null;
    try {
        fr = new FileReader(tempTestFile);
        br = new BufferedReader(fr);
        Set<Integer> keys = new HashSet<>();
        int curVal = -1;
        while (br.ready()) {
            String line = br.readLine();
            Integer key = Integer.parseInt(line.substring(0, line.indexOf("_")));
            Integer val = Integer.parseInt(line.substring(line.indexOf("_") + 1, line.length()));
            // check that values are in correct order
            Assert.assertTrue("Values not in ascending order", val >= curVal);
            // next value hit
            if (val > curVal) {
                if (curVal != -1) {
                    // check that we saw 100 distinct keys for this values
                    Assert.assertTrue("Keys missing for value", keys.size() == 100);
                }
                // empty keys set
                keys.clear();
                // update current value
                curVal = val;
            }
            Assert.assertTrue("Duplicate key for value", keys.add(key));
        }
    } catch (FileNotFoundException e) {
        Assert.fail("Out file got lost...");
    } catch (IOException ioe) {
        Assert.fail("Caught IOE while reading out file");
    } finally {
        if (br != null) {
            try {
                br.close();
            } catch (Throwable t) {
            }
        }
        if (fr != null) {
            try {
                fr.close();
            } catch (Throwable t) {
            }
        }
    }
}

Example 7 with FileReader

use of java.io.FileReader in project flink by apache.

the class DataSinkTaskTest method testUnionDataSinkTask.

@Test
public void testUnionDataSinkTask() {
    int keyCnt = 10;
    int valCnt = 20;
    super.initEnvironment(MEMORY_MANAGER_SIZE, NETWORK_BUFFER_SIZE);
    final IteratorWrappingTestSingleInputGate<?>[] readers = new IteratorWrappingTestSingleInputGate[4];
    readers[0] = super.addInput(new UniformRecordGenerator(keyCnt, valCnt, 0, 0, false), 0, false);
    readers[1] = super.addInput(new UniformRecordGenerator(keyCnt, valCnt, keyCnt, 0, false), 0, false);
    readers[2] = super.addInput(new UniformRecordGenerator(keyCnt, valCnt, keyCnt * 2, 0, false), 0, false);
    readers[3] = super.addInput(new UniformRecordGenerator(keyCnt, valCnt, keyCnt * 3, 0, false), 0, false);
    DataSinkTask<Record> testTask = new DataSinkTask<>();
    super.registerFileOutputTask(testTask, MockOutputFormat.class, new File(tempTestPath).toURI().toString());
    try {
        // which checks forwards existing notifications on registerListener calls.
        for (IteratorWrappingTestSingleInputGate<?> reader : readers) {
            reader.notifyNonEmpty();
        }
        testTask.invoke();
    } catch (Exception e) {
        LOG.debug("Exception while invoking the test task.", e);
        Assert.fail("Invoke method caused exception.");
    }
    File tempTestFile = new File(this.tempTestPath);
    Assert.assertTrue("Temp output file does not exist", tempTestFile.exists());
    FileReader fr = null;
    BufferedReader br = null;
    try {
        fr = new FileReader(tempTestFile);
        br = new BufferedReader(fr);
        HashMap<Integer, HashSet<Integer>> keyValueCountMap = new HashMap<>(keyCnt);
        while (br.ready()) {
            String line = br.readLine();
            Integer key = Integer.parseInt(line.substring(0, line.indexOf("_")));
            Integer val = Integer.parseInt(line.substring(line.indexOf("_") + 1, line.length()));
            if (!keyValueCountMap.containsKey(key)) {
                keyValueCountMap.put(key, new HashSet<Integer>());
            }
            keyValueCountMap.get(key).add(val);
        }
        Assert.assertTrue("Invalid key count in out file. Expected: " + keyCnt + " Actual: " + keyValueCountMap.keySet().size(), keyValueCountMap.keySet().size() == keyCnt * 4);
        for (Integer key : keyValueCountMap.keySet()) {
            Assert.assertTrue("Invalid value count for key: " + key + ". Expected: " + valCnt + " Actual: " + keyValueCountMap.get(key).size(), keyValueCountMap.get(key).size() == valCnt);
        }
    } catch (FileNotFoundException e) {
        Assert.fail("Out file got lost...");
    } catch (IOException ioe) {
        Assert.fail("Caught IOE while reading out file");
    } finally {
        if (br != null) {
            try {
                br.close();
            } catch (Throwable t) {
            }
        }
        if (fr != null) {
            try {
                fr.close();
            } catch (Throwable t) {
            }
        }
    }
}

Example 8 with FileReader

use of java.io.FileReader in project flink by apache.

the class TestBaseUtils method getResultReader.

public static BufferedReader[] getResultReader(String resultPath, String[] excludePrefixes, boolean inOrderOfFiles) throws IOException {
    File[] files = getAllInvolvedFiles(resultPath, excludePrefixes);
    if (inOrderOfFiles) {
        // sort the files after their name (1, 2, 3, 4)...
        // we cannot sort by path, because strings sort by prefix
        Arrays.sort(files, new Comparator<File>() {

            @Override
            public int compare(File o1, File o2) {
                try {
                    int f1 = Integer.parseInt(o1.getName());
                    int f2 = Integer.parseInt(o2.getName());
                    return f1 < f2 ? -1 : (f1 > f2 ? 1 : 0);
                } catch (NumberFormatException e) {
                    throw new RuntimeException("The file names are no numbers and cannot be ordered: " + o1.getName() + "/" + o2.getName());
                }
            }
        });
    }
    BufferedReader[] readers = new BufferedReader[files.length];
    for (int i = 0; i < files.length; i++) {
        readers[i] = new BufferedReader(new FileReader(files[i]));
    }
    return readers;
}

Example 9 with FileReader

use of java.io.FileReader in project flink by apache.

the class AggregatorsITCase method testDistributedCacheWithIterations.

@Test
public void testDistributedCacheWithIterations() throws Exception {
    File tempFile = new File(testPath);
    try (FileWriter writer = new FileWriter(tempFile)) {
        writer.write(testString);
    }
    final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
    env.registerCachedFile(resultPath, testName);
    IterativeDataSet<Long> solution = env.fromElements(1L).iterate(2);
    solution.closeWith(env.generateSequence(1, 2).filter(new RichFilterFunction<Long>() {

        @Override
        public void open(Configuration parameters) throws Exception {
            File file = getRuntimeContext().getDistributedCache().getFile(testName);
            BufferedReader reader = new BufferedReader(new FileReader(file));
            String output = reader.readLine();
            reader.close();
            assertEquals(output, testString);
        }

        @Override
        public boolean filter(Long value) throws Exception {
            return false;
        }
    }).withBroadcastSet(solution, "SOLUTION")).output(new DiscardingOutputFormat<Long>());
    env.execute();
    // this will be a useless verification now.
    expected = testString;
}

Example 10 with FileReader

use of java.io.FileReader in project hadoop by apache.

the class DataGenerator method genFiles.

/** Read file structure file under the input directory.
   * Create each file under the specified root.
   * The file names are relative to the root.
   */
private void genFiles() throws IOException {
    BufferedReader in = new BufferedReader(new FileReader(new File(inDir, StructureGenerator.FILE_STRUCTURE_FILE_NAME)));
    String line;
    while ((line = in.readLine()) != null) {
        String[] tokens = line.split(" ");
        if (tokens.length != 2) {
            throw new IOException("Expect at most 2 tokens per line: " + line);
        }
        String fileName = root + tokens[0];
        long fileSize = (long) (BLOCK_SIZE * Double.parseDouble(tokens[1]));
        genFile(new Path(fileName), fileSize);
    }
}