Search in sources :

Example 1 with CHANGELOG_STORAGE_UPLOAD_QUEUE_SIZE

use of org.apache.flink.changelog.fs.ChangelogStorageMetricGroup.CHANGELOG_STORAGE_UPLOAD_QUEUE_SIZE in project flink by apache.

the class ChangelogStorageMetricsTest method testQueueSize.

@Test
public void testQueueSize() throws Exception {
    AtomicReference<Gauge<Integer>> queueSizeGauge = new AtomicReference<>();
    ChangelogStorageMetricGroup metrics = new ChangelogStorageMetricGroup(new TaskManagerJobMetricGroup(TestingMetricRegistry.builder().setRegisterConsumer((metric, name, unused) -> {
        if (name.equals(CHANGELOG_STORAGE_UPLOAD_QUEUE_SIZE)) {
            queueSizeGauge.set((Gauge<Integer>) metric);
        }
    }).build(), createUnregisteredTaskManagerMetricGroup(), new JobID(), "test"));
    Path path = Path.fromLocalFile(temporaryFolder.newFolder());
    StateChangeFsUploader delegate = new StateChangeFsUploader(path, path.getFileSystem(), false, 100, metrics);
    ManuallyTriggeredScheduledExecutorService scheduler = new ManuallyTriggeredScheduledExecutorService();
    BatchingStateChangeUploader batcher = new BatchingStateChangeUploader(Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, RetryPolicy.NONE, delegate, scheduler, new RetryingExecutor(1, metrics.getAttemptsPerUpload()), metrics);
    try (FsStateChangelogStorage storage = new FsStateChangelogStorage(batcher, Long.MAX_VALUE)) {
        FsStateChangelogWriter writer = storage.createWriter("writer", EMPTY_KEY_GROUP_RANGE);
        int numUploads = 11;
        for (int i = 0; i < numUploads; i++) {
            SequenceNumber from = writer.nextSequenceNumber();
            writer.append(0, new byte[] { 0 });
            writer.persist(from);
        }
        assertEquals(numUploads, (int) queueSizeGauge.get().getValue());
        scheduler.triggerScheduledTasks();
        assertEquals(0, (int) queueSizeGauge.get().getValue());
    }
}
Also used : HashMap(java.util.HashMap) AtomicReference(java.util.concurrent.atomic.AtomicReference) ManuallyTriggeredScheduledExecutorService(org.apache.flink.core.testutils.ManuallyTriggeredScheduledExecutorService) Executors.newSingleThreadScheduledExecutor(java.util.concurrent.Executors.newSingleThreadScheduledExecutor) CHANGELOG_STORAGE_UPLOAD_QUEUE_SIZE(org.apache.flink.changelog.fs.ChangelogStorageMetricGroup.CHANGELOG_STORAGE_UPLOAD_QUEUE_SIZE) TestingMetricRegistry(org.apache.flink.runtime.metrics.util.TestingMetricRegistry) Path(org.apache.flink.core.fs.Path) Map(java.util.Map) Gauge(org.apache.flink.metrics.Gauge) HistogramStatistics(org.apache.flink.metrics.HistogramStatistics) Assert.assertTrue(org.junit.Assert.assertTrue) TaskManagerJobMetricGroup(org.apache.flink.runtime.metrics.groups.TaskManagerJobMetricGroup) Test(org.junit.Test) IOException(java.io.IOException) File(java.io.File) JobID(org.apache.flink.api.common.JobID) Rule(org.junit.Rule) UnregisteredMetricGroups.createUnregisteredTaskManagerJobMetricGroup(org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups.createUnregisteredTaskManagerJobMetricGroup) SequenceNumber(org.apache.flink.runtime.state.changelog.SequenceNumber) EmptyStreamStateHandle(org.apache.flink.runtime.state.testutils.EmptyStreamStateHandle) EMPTY_KEY_GROUP_RANGE(org.apache.flink.runtime.state.KeyGroupRange.EMPTY_KEY_GROUP_RANGE) UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup(org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup) Collections(java.util.Collections) TemporaryFolder(org.junit.rules.TemporaryFolder) Assert.assertEquals(org.junit.Assert.assertEquals) Path(org.apache.flink.core.fs.Path) TaskManagerJobMetricGroup(org.apache.flink.runtime.metrics.groups.TaskManagerJobMetricGroup) UnregisteredMetricGroups.createUnregisteredTaskManagerJobMetricGroup(org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups.createUnregisteredTaskManagerJobMetricGroup) AtomicReference(java.util.concurrent.atomic.AtomicReference) Gauge(org.apache.flink.metrics.Gauge) ManuallyTriggeredScheduledExecutorService(org.apache.flink.core.testutils.ManuallyTriggeredScheduledExecutorService) SequenceNumber(org.apache.flink.runtime.state.changelog.SequenceNumber) JobID(org.apache.flink.api.common.JobID) Test(org.junit.Test)

Aggregations

File (java.io.File)1 IOException (java.io.IOException)1 Collections (java.util.Collections)1 HashMap (java.util.HashMap)1 Map (java.util.Map)1 Executors.newSingleThreadScheduledExecutor (java.util.concurrent.Executors.newSingleThreadScheduledExecutor)1 AtomicReference (java.util.concurrent.atomic.AtomicReference)1 JobID (org.apache.flink.api.common.JobID)1 CHANGELOG_STORAGE_UPLOAD_QUEUE_SIZE (org.apache.flink.changelog.fs.ChangelogStorageMetricGroup.CHANGELOG_STORAGE_UPLOAD_QUEUE_SIZE)1 Path (org.apache.flink.core.fs.Path)1 ManuallyTriggeredScheduledExecutorService (org.apache.flink.core.testutils.ManuallyTriggeredScheduledExecutorService)1 Gauge (org.apache.flink.metrics.Gauge)1 HistogramStatistics (org.apache.flink.metrics.HistogramStatistics)1 TaskManagerJobMetricGroup (org.apache.flink.runtime.metrics.groups.TaskManagerJobMetricGroup)1 UnregisteredMetricGroups.createUnregisteredTaskManagerJobMetricGroup (org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups.createUnregisteredTaskManagerJobMetricGroup)1 UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup (org.apache.flink.runtime.metrics.groups.UnregisteredMetricGroups.createUnregisteredTaskManagerMetricGroup)1 TestingMetricRegistry (org.apache.flink.runtime.metrics.util.TestingMetricRegistry)1 EMPTY_KEY_GROUP_RANGE (org.apache.flink.runtime.state.KeyGroupRange.EMPTY_KEY_GROUP_RANGE)1 SequenceNumber (org.apache.flink.runtime.state.changelog.SequenceNumber)1 EmptyStreamStateHandle (org.apache.flink.runtime.state.testutils.EmptyStreamStateHandle)1