Search in sources :

Example 1 with ThrowingConsumer

use of org.apache.flink.util.function.ThrowingConsumer in project flink by apache.

the class SequentialChannelStateReaderImplTest method withResultPartitions.

private void withResultPartitions(ThrowingConsumer<BufferWritingResultPartition[], Exception> action) throws Exception {
    int segmentsToAllocate = parLevel * parLevel * buffersPerChannel;
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(segmentsToAllocate, bufferSize);
    BufferWritingResultPartition[] resultPartitions = range(0, parLevel).mapToObj(i -> new ResultPartitionBuilder().setResultPartitionIndex(i).setNumberOfSubpartitions(parLevel).setNetworkBufferPool(networkBufferPool).build()).toArray(BufferWritingResultPartition[]::new);
    try {
        for (ResultPartition resultPartition : resultPartitions) {
            resultPartition.setup();
        }
        action.accept(resultPartitions);
    } finally {
        for (ResultPartition resultPartition : resultPartitions) {
            resultPartition.close();
        }
        try {
            assertEquals(segmentsToAllocate, networkBufferPool.getNumberOfAvailableMemorySegments());
        } finally {
            networkBufferPool.destroyAllBufferPools();
            networkBufferPool.destroy();
        }
    }
}
Also used : BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog) ByteArrayOutputStream(java.io.ByteArrayOutputStream) Tuple2(org.apache.flink.api.java.tuple.Tuple2) IntStream.range(java.util.stream.IntStream.range) BiFunction(java.util.function.BiFunction) RunWith(org.junit.runner.RunWith) HashMap(java.util.HashMap) Random(java.util.Random) Function(java.util.function.Function) NetworkBuffer(org.apache.flink.runtime.io.network.buffer.NetworkBuffer) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) ArrayList(java.util.ArrayList) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) DataOutputStream(java.io.DataOutputStream) Collectors.toMap(java.util.stream.Collectors.toMap) Map(java.util.Map) ThrowingConsumer(org.apache.flink.util.function.ThrowingConsumer) Parameterized(org.junit.runners.Parameterized) SingleInputGate(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate) TaskStateSnapshot(org.apache.flink.runtime.checkpoint.TaskStateSnapshot) InputGate(org.apache.flink.runtime.io.network.partition.consumer.InputGate) MemorySegmentFactory(org.apache.flink.core.memory.MemorySegmentFactory) SingleInputGateBuilder(org.apache.flink.runtime.io.network.partition.consumer.SingleInputGateBuilder) ResultPartitionBuilder(org.apache.flink.runtime.io.network.partition.ResultPartitionBuilder) InputChannelStateHandle(org.apache.flink.runtime.state.InputChannelStateHandle) Closer(org.apache.flink.shaded.guava30.com.google.common.io.Closer) Assert.assertTrue(org.junit.Assert.assertTrue) FreeingBufferRecycler(org.apache.flink.runtime.io.network.buffer.FreeingBufferRecycler) Test(org.junit.Test) IOException(java.io.IOException) OperatorSubtaskState(org.apache.flink.runtime.checkpoint.OperatorSubtaskState) ResultSubpartitionStateHandle(org.apache.flink.runtime.state.ResultSubpartitionStateHandle) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) List(java.util.List) Collectors.toList(java.util.stream.Collectors.toList) ByteStreamStateHandle(org.apache.flink.runtime.state.memory.ByteStreamStateHandle) Function.identity(java.util.function.Function.identity) OperatorID(org.apache.flink.runtime.jobgraph.OperatorID) Optional(java.util.Optional) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) Collections(java.util.Collections) StateObjectCollection(org.apache.flink.runtime.checkpoint.StateObjectCollection) BufferOrEvent(org.apache.flink.runtime.io.network.partition.consumer.BufferOrEvent) Assert.assertEquals(org.junit.Assert.assertEquals) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition) ResultPartitionBuilder(org.apache.flink.runtime.io.network.partition.ResultPartitionBuilder) NetworkBufferPool(org.apache.flink.runtime.io.network.buffer.NetworkBufferPool) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) BufferWritingResultPartition(org.apache.flink.runtime.io.network.partition.BufferWritingResultPartition)

Example 2 with ThrowingConsumer

use of org.apache.flink.util.function.ThrowingConsumer in project flink by apache.

the class FineGrainedSlotManagerTest method testAccessMetricValueDuringItsUnregister.

private void testAccessMetricValueDuringItsUnregister(ThrowingConsumer<SlotManager, Exception> closeFn) throws Exception {
    final AtomicInteger registeredMetrics = new AtomicInteger();
    final MetricRegistry metricRegistry = TestingMetricRegistry.builder().setRegisterConsumer((a, b, c) -> registeredMetrics.incrementAndGet()).setUnregisterConsumer((a, b, c) -> registeredMetrics.decrementAndGet()).build();
    final Context context = new Context();
    context.setSlotManagerMetricGroup(SlotManagerMetricGroup.create(metricRegistry, "localhost"));
    context.runTest(() -> {
        // sanity check to ensure metrics were actually registered
        assertThat(registeredMetrics.get(), greaterThan(0));
        context.runInMainThreadAndWait(() -> {
            try {
                closeFn.accept(context.getSlotManager());
            } catch (Exception e) {
                fail("Error when closing slot manager.");
            }
        });
        assertThat(registeredMetrics.get(), is(0));
    });
}
Also used : Tuple2(org.apache.flink.api.java.tuple.Tuple2) Matchers.not(org.hamcrest.Matchers.not) Tuple6(org.apache.flink.api.java.tuple.Tuple6) ResourceRequirement(org.apache.flink.runtime.slots.ResourceRequirement) CompletableFuture(java.util.concurrent.CompletableFuture) TaskExecutorGateway(org.apache.flink.runtime.taskexecutor.TaskExecutorGateway) ArrayList(java.util.ArrayList) Assert.assertThat(org.junit.Assert.assertThat) BigDecimal(java.math.BigDecimal) TestingMetricRegistry(org.apache.flink.runtime.metrics.util.TestingMetricRegistry) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) MetricRegistry(org.apache.flink.runtime.metrics.MetricRegistry) SlotID(org.apache.flink.runtime.clusterframework.types.SlotID) Matchers.hasSize(org.hamcrest.Matchers.hasSize) Assert.fail(org.junit.Assert.fail) ResourceID(org.apache.flink.runtime.clusterframework.types.ResourceID) ResourceRequirements(org.apache.flink.runtime.slots.ResourceRequirements) ThrowingConsumer(org.apache.flink.util.function.ThrowingConsumer) Matchers.empty(org.hamcrest.Matchers.empty) Collection(java.util.Collection) ResourceManagerId(org.apache.flink.runtime.resourcemanager.ResourceManagerId) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) InstanceID(org.apache.flink.runtime.instance.InstanceID) SlotManagerMetricGroup(org.apache.flink.runtime.metrics.groups.SlotManagerMetricGroup) Acknowledge(org.apache.flink.runtime.messages.Acknowledge) ResourceProfile(org.apache.flink.runtime.clusterframework.types.ResourceProfile) Consumer(java.util.function.Consumer) List(java.util.List) JobID(org.apache.flink.api.common.JobID) TaskExecutorConnection(org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection) Assert.assertFalse(org.junit.Assert.assertFalse) TestingTaskExecutorGatewayBuilder(org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder) Matchers.equalTo(org.hamcrest.Matchers.equalTo) SlotStatus(org.apache.flink.runtime.taskexecutor.SlotStatus) Optional(java.util.Optional) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) Assume.assumeTrue(org.junit.Assume.assumeTrue) SlotReport(org.apache.flink.runtime.taskexecutor.SlotReport) Time(org.apache.flink.api.common.time.Time) AllocationID(org.apache.flink.runtime.clusterframework.types.AllocationID) Assert.assertEquals(org.junit.Assert.assertEquals) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) TestingMetricRegistry(org.apache.flink.runtime.metrics.util.TestingMetricRegistry) MetricRegistry(org.apache.flink.runtime.metrics.MetricRegistry)

Example 3 with ThrowingConsumer

use of org.apache.flink.util.function.ThrowingConsumer in project flink by apache.

the class DeclarativeSlotManagerTest method testAccessMetricValueDuringItsUnregister.

private void testAccessMetricValueDuringItsUnregister(ThrowingConsumer<SlotManager, Exception> closeFn) throws Exception {
    final AtomicInteger registeredMetrics = new AtomicInteger();
    final MetricRegistry metricRegistry = TestingMetricRegistry.builder().setRegisterConsumer((a, b, c) -> registeredMetrics.incrementAndGet()).setUnregisterConsumer((a, b, c) -> registeredMetrics.decrementAndGet()).build();
    final DeclarativeSlotManager slotManager = createDeclarativeSlotManagerBuilder().setSlotManagerMetricGroup(SlotManagerMetricGroup.create(metricRegistry, "localhost")).buildAndStartWithDirectExec();
    // sanity check to ensure metrics were actually registered
    assertThat(registeredMetrics.get(), greaterThan(0));
    closeFn.accept(slotManager);
    assertThat(registeredMetrics.get(), is(0));
}
Also used : ComponentMainThreadExecutorServiceAdapter(org.apache.flink.runtime.concurrent.ComponentMainThreadExecutorServiceAdapter) TestingTaskExecutorGateway(org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGateway) ManuallyTriggeredScheduledExecutor(org.apache.flink.util.concurrent.ManuallyTriggeredScheduledExecutor) Arrays(java.util.Arrays) CoreMatchers.hasItem(org.hamcrest.CoreMatchers.hasItem) Tuple2(org.apache.flink.api.java.tuple.Tuple2) Tuple6(org.apache.flink.api.java.tuple.Tuple6) ResourceRequirement(org.apache.flink.runtime.slots.ResourceRequirement) TimeoutException(java.util.concurrent.TimeoutException) TaskExecutorGateway(org.apache.flink.runtime.taskexecutor.TaskExecutorGateway) Assert.assertThat(org.junit.Assert.assertThat) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) MetricRegistry(org.apache.flink.runtime.metrics.MetricRegistry) FunctionUtils(org.apache.flink.util.function.FunctionUtils) TestLogger(org.apache.flink.util.TestLogger) SlotID(org.apache.flink.runtime.clusterframework.types.SlotID) SlotOccupiedException(org.apache.flink.runtime.taskexecutor.exceptions.SlotOccupiedException) ScheduledExecutor(org.apache.flink.util.concurrent.ScheduledExecutor) Collection(java.util.Collection) ResourceManagerId(org.apache.flink.runtime.resourcemanager.ResourceManagerId) Set(java.util.Set) BlockingQueue(java.util.concurrent.BlockingQueue) SlotManagerMetricGroup(org.apache.flink.runtime.metrics.groups.SlotManagerMetricGroup) Acknowledge(org.apache.flink.runtime.messages.Acknowledge) ResourceProfile(org.apache.flink.runtime.clusterframework.types.ResourceProfile) TestingUtils(org.apache.flink.testutils.TestingUtils) ArrayBlockingQueue(java.util.concurrent.ArrayBlockingQueue) List(java.util.List) TaskExecutorConnection(org.apache.flink.runtime.resourcemanager.registration.TaskExecutorConnection) Matchers.containsInAnyOrder(org.hamcrest.Matchers.containsInAnyOrder) Assert.assertFalse(org.junit.Assert.assertFalse) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Matchers.greaterThan(org.hamcrest.Matchers.greaterThan) Matchers.is(org.hamcrest.Matchers.is) SlotReport(org.apache.flink.runtime.taskexecutor.SlotReport) SlotAllocationException(org.apache.flink.runtime.taskexecutor.exceptions.SlotAllocationException) AllocationID(org.apache.flink.runtime.clusterframework.types.AllocationID) FlinkException(org.apache.flink.util.FlinkException) WorkerResourceSpec(org.apache.flink.runtime.resourcemanager.WorkerResourceSpec) CoreMatchers.not(org.hamcrest.CoreMatchers.not) CompletableFuture(java.util.concurrent.CompletableFuture) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Assert.assertSame(org.junit.Assert.assertSame) ManuallyTriggeredScheduledExecutorService(org.apache.flink.core.testutils.ManuallyTriggeredScheduledExecutorService) TestingMetricRegistry(org.apache.flink.runtime.metrics.util.TestingMetricRegistry) FutureUtils(org.apache.flink.util.concurrent.FutureUtils) Matchers.hasSize(org.hamcrest.Matchers.hasSize) ResourceID(org.apache.flink.runtime.clusterframework.types.ResourceID) ResourceRequirements(org.apache.flink.runtime.slots.ResourceRequirements) ThrowingConsumer(org.apache.flink.util.function.ThrowingConsumer) Matchers.empty(org.hamcrest.Matchers.empty) Iterator(java.util.Iterator) Executor(java.util.concurrent.Executor) Assert.assertNotNull(org.junit.Assert.assertNotNull) Assert.assertTrue(org.junit.Assert.assertTrue) SystemExitTrackingSecurityManager(org.apache.flink.runtime.testutils.SystemExitTrackingSecurityManager) Test(org.junit.Test) InstanceID(org.apache.flink.runtime.instance.InstanceID) Iterators(org.apache.flink.shaded.guava30.com.google.common.collect.Iterators) TimeUnit(java.util.concurrent.TimeUnit) JobID(org.apache.flink.api.common.JobID) TestingTaskExecutorGatewayBuilder(org.apache.flink.runtime.taskexecutor.TestingTaskExecutorGatewayBuilder) SlotStatus(org.apache.flink.runtime.taskexecutor.SlotStatus) Collections(java.util.Collections) Assert.assertEquals(org.junit.Assert.assertEquals) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) MetricRegistry(org.apache.flink.runtime.metrics.MetricRegistry) TestingMetricRegistry(org.apache.flink.runtime.metrics.util.TestingMetricRegistry)

Aggregations

ArrayList (java.util.ArrayList)3 List (java.util.List)3 Tuple2 (org.apache.flink.api.java.tuple.Tuple2)3 Collection (java.util.Collection)2 Collections (java.util.Collections)2 Optional (java.util.Optional)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)2 JobID (org.apache.flink.api.common.JobID)2 Tuple6 (org.apache.flink.api.java.tuple.Tuple6)2 AllocationID (org.apache.flink.runtime.clusterframework.types.AllocationID)2 ResourceID (org.apache.flink.runtime.clusterframework.types.ResourceID)2 ResourceProfile (org.apache.flink.runtime.clusterframework.types.ResourceProfile)2 SlotID (org.apache.flink.runtime.clusterframework.types.SlotID)2 InstanceID (org.apache.flink.runtime.instance.InstanceID)2 ThrowingConsumer (org.apache.flink.util.function.ThrowingConsumer)2 Assert.assertEquals (org.junit.Assert.assertEquals)2 Assert.assertTrue (org.junit.Assert.assertTrue)2 Test (org.junit.Test)2 ByteArrayOutputStream (java.io.ByteArrayOutputStream)1