Examples with DummyInvokable org.apache.flink.runtime.operators.testutils.DummyInvokable used on opensource projects

Example 31 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class MassiveStringValueSorting method testStringValueSorting.

public void testStringValueSorting() {
    File input = null;
    File sorted = null;
    try {
        // the source file
        input = generateFileWithStrings(300000, "http://some-uri.com/that/is/a/common/prefix/to/all");
        // the sorted file
        sorted = File.createTempFile("sorted_strings", "txt");
        String[] command = { "/bin/bash", "-c", "export LC_ALL=\"C\" && cat \"" + input.getAbsolutePath() + "\" | sort > \"" + sorted.getAbsolutePath() + "\"" };
        Process p = null;
        try {
            p = Runtime.getRuntime().exec(command);
            int retCode = p.waitFor();
            if (retCode != 0) {
                throw new Exception("Command failed with return code " + retCode);
            }
            p = null;
        } finally {
            if (p != null) {
                p.destroy();
            }
        }
        // sort the data
        UnilateralSortMerger<StringValue> sorter = null;
        BufferedReader reader = null;
        BufferedReader verifyReader = null;
        try {
            MemoryManager mm = new MemoryManager(1024 * 1024, 1);
            IOManager ioMan = new IOManagerAsync();
            TypeSerializer<StringValue> serializer = new CopyableValueSerializer<StringValue>(StringValue.class);
            TypeComparator<StringValue> comparator = new CopyableValueComparator<StringValue>(true, StringValue.class);
            reader = new BufferedReader(new FileReader(input));
            MutableObjectIterator<StringValue> inputIterator = new StringValueReaderMutableObjectIterator(reader);
            sorter = new UnilateralSortMerger<StringValue>(mm, ioMan, inputIterator, new DummyInvokable(), new RuntimeSerializerFactory<StringValue>(serializer, StringValue.class), comparator, 1.0, 4, 0.8f, true, /* use large record handler */
            true);
            MutableObjectIterator<StringValue> sortedData = sorter.getIterator();
            reader.close();
            // verify
            verifyReader = new BufferedReader(new FileReader(sorted));
            String nextVerify;
            StringValue nextFromFlinkSort = new StringValue();
            while ((nextVerify = verifyReader.readLine()) != null) {
                nextFromFlinkSort = sortedData.next(nextFromFlinkSort);
                Assert.assertNotNull(nextFromFlinkSort);
                Assert.assertEquals(nextVerify, nextFromFlinkSort.getValue());
            }
        } finally {
            if (reader != null) {
                reader.close();
            }
            if (verifyReader != null) {
                verifyReader.close();
            }
            if (sorter != null) {
                sorter.close();
            }
        }
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        Assert.fail(e.getMessage());
    } finally {
        if (input != null) {
            //noinspection ResultOfMethodCallIgnored
            input.delete();
        }
        if (sorted != null) {
            //noinspection ResultOfMethodCallIgnored
            sorted.delete();
        }
    }
}

Example 32 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class MemoryManagerLazyAllocationTest method allocateAllMulti.

@Test
public void allocateAllMulti() {
    try {
        final AbstractInvokable mockInvoke = new DummyInvokable();
        final List<MemorySegment> segments = new ArrayList<MemorySegment>();
        try {
            for (int i = 0; i < NUM_PAGES / 2; i++) {
                segments.addAll(this.memoryManager.allocatePages(mockInvoke, 2));
            }
        } catch (MemoryAllocationException e) {
            Assert.fail("Unable to allocate memory");
        }
        this.memoryManager.release(segments);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 33 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class MemoryManagerLazyAllocationTest method allocateMultipleOwners.

@Test
public void allocateMultipleOwners() {
    final int NUM_OWNERS = 17;
    try {
        AbstractInvokable[] owners = new AbstractInvokable[NUM_OWNERS];
        @SuppressWarnings("unchecked") List<MemorySegment>[] mems = (List<MemorySegment>[]) new List<?>[NUM_OWNERS];
        for (int i = 0; i < NUM_OWNERS; i++) {
            owners[i] = new DummyInvokable();
            mems[i] = new ArrayList<MemorySegment>(64);
        }
        // allocate all memory to the different owners
        for (int i = 0; i < NUM_PAGES; i++) {
            final int owner = this.random.nextInt(NUM_OWNERS);
            mems[owner].addAll(this.memoryManager.allocatePages(owners[owner], 1));
        }
        // free one owner at a time
        for (int i = 0; i < NUM_OWNERS; i++) {
            this.memoryManager.releaseAll(owners[i]);
            owners[i] = null;
            Assert.assertTrue("Released memory segments have not been destroyed.", allMemorySegmentsFreed(mems[i]));
            mems[i] = null;
            // check that the owner owners were not affected
            for (int k = i + 1; k < NUM_OWNERS; k++) {
                Assert.assertTrue("Non-released memory segments are accidentaly destroyed.", allMemorySegmentsValid(mems[k]));
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 34 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class MemoryManagerLazyAllocationTest method allocateTooMuch.

@Test
public void allocateTooMuch() {
    try {
        final AbstractInvokable mockInvoke = new DummyInvokable();
        List<MemorySegment> segs = this.memoryManager.allocatePages(mockInvoke, NUM_PAGES);
        try {
            this.memoryManager.allocatePages(mockInvoke, 1);
            Assert.fail("Expected MemoryAllocationException.");
        } catch (MemoryAllocationException maex) {
        // expected
        }
        Assert.assertTrue("The previously allocated segments were not valid any more.", allMemorySegmentsValid(segs));
        this.memoryManager.releaseAll(mockInvoke);
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}

Example 35 with DummyInvokable

use of org.apache.flink.runtime.operators.testutils.DummyInvokable in project flink by apache.

the class MemoryManagerTest method allocateMultipleOwners.

@Test
public void allocateMultipleOwners() {
    final int NUM_OWNERS = 17;
    try {
        AbstractInvokable[] owners = new AbstractInvokable[NUM_OWNERS];
        @SuppressWarnings("unchecked") List<MemorySegment>[] mems = (List<MemorySegment>[]) new List<?>[NUM_OWNERS];
        for (int i = 0; i < NUM_OWNERS; i++) {
            owners[i] = new DummyInvokable();
            mems[i] = new ArrayList<MemorySegment>(64);
        }
        // allocate all memory to the different owners
        for (int i = 0; i < NUM_PAGES; i++) {
            final int owner = this.random.nextInt(NUM_OWNERS);
            mems[owner].addAll(this.memoryManager.allocatePages(owners[owner], 1));
        }
        // free one owner at a time
        for (int i = 0; i < NUM_OWNERS; i++) {
            this.memoryManager.releaseAll(owners[i]);
            owners[i] = null;
            Assert.assertTrue("Released memory segments have not been destroyed.", allMemorySegmentsFreed(mems[i]));
            mems[i] = null;
            // check that the owner owners were not affected
            for (int k = i + 1; k < NUM_OWNERS; k++) {
                Assert.assertTrue("Non-released memory segments are accidentaly destroyed.", allMemorySegmentsValid(mems[k]));
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        fail(e.getMessage());
    }
}