Examples with MemoryManager org.apache.flink.runtime.memory.MemoryManager used on opensource projects

Example 21 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class SortOperator method open.

@Override
public void open() throws Exception {
    super.open();
    LOG.info("Opening SortOperator");
    ClassLoader cl = getContainingTask().getUserCodeClassLoader();
    AbstractRowDataSerializer inputSerializer = (AbstractRowDataSerializer) getOperatorConfig().getTypeSerializerIn1(getUserCodeClassloader());
    this.binarySerializer = new BinaryRowDataSerializer(inputSerializer.getArity());
    NormalizedKeyComputer computer = gComputer.newInstance(cl);
    RecordComparator comparator = gComparator.newInstance(cl);
    gComputer = null;
    gComparator = null;
    MemoryManager memManager = getContainingTask().getEnvironment().getMemoryManager();
    this.sorter = new BinaryExternalSorter(this.getContainingTask(), memManager, computeMemorySize(), this.getContainingTask().getEnvironment().getIOManager(), inputSerializer, binarySerializer, computer, comparator, getContainingTask().getJobConfiguration());
    this.sorter.startThreads();
    collector = new StreamRecordCollector<>(output);
    // register the metrics.
    getMetricGroup().gauge("memoryUsedSizeInBytes", (Gauge<Long>) sorter::getUsedMemoryInBytes);
    getMetricGroup().gauge("numSpillFiles", (Gauge<Long>) sorter::getNumSpillFiles);
    getMetricGroup().gauge("spillInBytes", (Gauge<Long>) sorter::getSpillInBytes);
}

Example 22 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class MassiveStringSorting method testStringTuplesSorting.

@SuppressWarnings("unchecked")
public void testStringTuplesSorting() {
    final int numStrings = 300000;
    File input = null;
    File sorted = null;
    try {
        // the source file
        input = generateFileWithStringTuples(numStrings, "http://some-uri.com/that/is/a/common/prefix/to/all");
        // the sorted file
        sorted = File.createTempFile("sorted_strings", "txt");
        String[] command = { "/bin/bash", "-c", "export LC_ALL=\"C\" && cat \"" + input.getAbsolutePath() + "\" | sort > \"" + sorted.getAbsolutePath() + "\"" };
        Process p = null;
        try {
            p = Runtime.getRuntime().exec(command);
            int retCode = p.waitFor();
            if (retCode != 0) {
                throw new Exception("Command failed with return code " + retCode);
            }
            p = null;
        } finally {
            if (p != null) {
                p.destroy();
            }
        }
        // sort the data
        Sorter<Tuple2<String, String[]>> sorter = null;
        BufferedReader reader = null;
        BufferedReader verifyReader = null;
        MemoryManager mm = null;
        try (IOManager ioMan = new IOManagerAsync()) {
            mm = MemoryManagerBuilder.newBuilder().setMemorySize(1024 * 1024).build();
            TupleTypeInfo<Tuple2<String, String[]>> typeInfo = (TupleTypeInfo<Tuple2<String, String[]>>) new TypeHint<Tuple2<String, String[]>>() {
            }.getTypeInfo();
            TypeSerializer<Tuple2<String, String[]>> serializer = typeInfo.createSerializer(new ExecutionConfig());
            TypeComparator<Tuple2<String, String[]>> comparator = typeInfo.createComparator(new int[] { 0 }, new boolean[] { true }, 0, new ExecutionConfig());
            reader = new BufferedReader(new FileReader(input));
            MutableObjectIterator<Tuple2<String, String[]>> inputIterator = new StringTupleReaderMutableObjectIterator(reader);
            sorter = ExternalSorter.newBuilder(mm, new DummyInvokable(), serializer, comparator).maxNumFileHandles(4).enableSpilling(ioMan, 0.8f).memoryFraction(1.0).objectReuse(false).largeRecords(true).build(inputIterator);
            // use this part to verify that all if good when sorting in memory
            // List<MemorySegment> memory = mm.allocatePages(new DummyInvokable(),
            // mm.computeNumberOfPages(1024*1024*1024));
            // NormalizedKeySorter<Tuple2<String, String[]>> nks = new
            // NormalizedKeySorter<Tuple2<String,String[]>>(serializer, comparator, memory);
            // 
            // {
            // Tuple2<String, String[]> wi = new Tuple2<String, String[]>("", new
            // String[0]);
            // while ((wi = inputIterator.next(wi)) != null) {
            // Assert.assertTrue(nks.write(wi));
            // }
            // 
            // new QuickSort().sort(nks);
            // }
            // 
            // MutableObjectIterator<Tuple2<String, String[]>> sortedData =
            // nks.getIterator();
            MutableObjectIterator<Tuple2<String, String[]>> sortedData = sorter.getIterator();
            reader.close();
            // verify
            verifyReader = new BufferedReader(new FileReader(sorted));
            MutableObjectIterator<Tuple2<String, String[]>> verifyIterator = new StringTupleReaderMutableObjectIterator(verifyReader);
            Tuple2<String, String[]> next = new Tuple2<String, String[]>("", new String[0]);
            Tuple2<String, String[]> nextFromStratoSort = new Tuple2<String, String[]>("", new String[0]);
            int num = 0;
            while ((next = verifyIterator.next(next)) != null) {
                num++;
                nextFromStratoSort = sortedData.next(nextFromStratoSort);
                Assert.assertNotNull(nextFromStratoSort);
                Assert.assertEquals(next.f0, nextFromStratoSort.f0);
                Assert.assertArrayEquals(next.f1, nextFromStratoSort.f1);
            }
            Assert.assertNull(sortedData.next(nextFromStratoSort));
            Assert.assertEquals(numStrings, num);
        } finally {
            if (reader != null) {
                reader.close();
            }
            if (verifyReader != null) {
                verifyReader.close();
            }
            if (sorter != null) {
                sorter.close();
            }
            if (mm != null) {
                mm.shutdown();
            }
        }
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        Assert.fail(e.getMessage());
    } finally {
        if (input != null) {
            input.delete();
        }
        if (sorted != null) {
            sorted.delete();
        }
    }
}

Example 23 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class MassiveStringSorting method testStringSorting.

public void testStringSorting() {
    File input = null;
    File sorted = null;
    try {
        // the source file
        input = generateFileWithStrings(300000, "http://some-uri.com/that/is/a/common/prefix/to/all");
        // the sorted file
        sorted = File.createTempFile("sorted_strings", "txt");
        String[] command = { "/bin/bash", "-c", "export LC_ALL=\"C\" && cat \"" + input.getAbsolutePath() + "\" | sort > \"" + sorted.getAbsolutePath() + "\"" };
        Process p = null;
        try {
            p = Runtime.getRuntime().exec(command);
            int retCode = p.waitFor();
            if (retCode != 0) {
                throw new Exception("Command failed with return code " + retCode);
            }
            p = null;
        } finally {
            if (p != null) {
                p.destroy();
            }
        }
        // sort the data
        Sorter<String> sorter = null;
        BufferedReader reader = null;
        BufferedReader verifyReader = null;
        MemoryManager mm = null;
        try (IOManager ioMan = new IOManagerAsync()) {
            mm = MemoryManagerBuilder.newBuilder().setMemorySize(1024 * 1024).build();
            TypeSerializer<String> serializer = StringSerializer.INSTANCE;
            TypeComparator<String> comparator = new StringComparator(true);
            reader = new BufferedReader(new FileReader(input));
            MutableObjectIterator<String> inputIterator = new StringReaderMutableObjectIterator(reader);
            sorter = ExternalSorter.newBuilder(mm, new DummyInvokable(), serializer, comparator).maxNumFileHandles(4).enableSpilling(ioMan, 0.8f).memoryFraction(1.0).objectReuse(false).largeRecords(true).build(inputIterator);
            MutableObjectIterator<String> sortedData = sorter.getIterator();
            reader.close();
            // verify
            verifyReader = new BufferedReader(new FileReader(sorted));
            String next;
            while ((next = verifyReader.readLine()) != null) {
                String nextFromStratoSort = sortedData.next("");
                Assert.assertNotNull(nextFromStratoSort);
                Assert.assertEquals(next, nextFromStratoSort);
            }
        } finally {
            if (reader != null) {
                reader.close();
            }
            if (verifyReader != null) {
                verifyReader.close();
            }
            if (sorter != null) {
                sorter.close();
            }
            if (mm != null) {
                mm.shutdown();
            }
        }
    } catch (Exception e) {
        System.err.println(e.getMessage());
        e.printStackTrace();
        Assert.fail(e.getMessage());
    } finally {
        if (input != null) {
            input.delete();
        }
        if (sorted != null) {
            sorted.delete();
        }
    }
}

Example 24 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class StreamTaskMailboxTestHarness method close.

@Override
public void close() throws Exception {
    if (streamTask.isRunning()) {
        streamTask.cancel();
        finishProcessing();
    }
    streamMockEnvironment.getIOManager().close();
    MemoryManager memMan = this.streamMockEnvironment.getMemoryManager();
    if (memMan != null) {
        assertTrue("Memory Manager managed memory was not completely freed.", memMan.verifyEmpty());
        memMan.shutdown();
    }
}

Example 25 with MemoryManager

use of org.apache.flink.runtime.memory.MemoryManager in project flink by apache.

the class BinaryHashTableTest method testInMemoryMutableHashTable.

@Test
public void testInMemoryMutableHashTable() throws IOException {
    final int numKeys = 100000;
    final int buildValsPerKey = 3;
    final int probeValsPerKey = 10;
    // create a build input that gives 3 million pairs with 3 values sharing the same key
    MutableObjectIterator<BinaryRowData> buildInput = new UniformBinaryRowGenerator(numKeys, buildValsPerKey, false);
    // create a probe input that gives 10 million pairs with 10 values sharing a key
    MutableObjectIterator<BinaryRowData> probeInput = new UniformBinaryRowGenerator(numKeys, probeValsPerKey, true);
    MemoryManager memManager = MemoryManagerBuilder.newBuilder().setMemorySize(896 * PAGE_SIZE).build();
    // ----------------------------------------------------------------------------------------
    final BinaryHashTable table = newBinaryHashTable(this.buildSideSerializer, this.probeSideSerializer, new MyProjection(), new MyProjection(), memManager, 100 * PAGE_SIZE, ioManager);
    int numRecordsInJoinResult = join(table, buildInput, probeInput);
    Assert.assertEquals("Wrong number of records in join result.", numKeys * buildValsPerKey * probeValsPerKey, numRecordsInJoinResult);
    table.close();
    table.free();
}