Examples with SequenceNumber org.apache.flink.runtime.state.changelog.SequenceNumber used on opensource projects

Example 11 with SequenceNumber

use of org.apache.flink.runtime.state.changelog.SequenceNumber in project flink by apache.

the class ChangelogStorageMetricsTest method testUploadFailuresCounter.

@Test
public void testUploadFailuresCounter() throws Exception {
    // using file instead of folder will cause a failure
    File file = temporaryFolder.newFile();
    ChangelogStorageMetricGroup metrics = new ChangelogStorageMetricGroup(createUnregisteredTaskManagerJobMetricGroup());
    try (FsStateChangelogStorage storage = new FsStateChangelogStorage(Path.fromLocalFile(file), false, 100, metrics)) {
        FsStateChangelogWriter writer = storage.createWriter("writer", EMPTY_KEY_GROUP_RANGE);
        int numUploads = 5;
        for (int i = 0; i < numUploads; i++) {
            SequenceNumber from = writer.nextSequenceNumber();
            writer.append(0, new byte[] { 0, 1, 2, 3 });
            try {
                writer.persist(from).get();
            } catch (IOException e) {
            // ignore
            }
        }
        assertEquals(numUploads, metrics.getUploadFailuresCounter().getCount());
    }
}

Example 12 with SequenceNumber

use of org.apache.flink.runtime.state.changelog.SequenceNumber in project flink by apache.

the class ChangelogStorageMetricsTest method testUploadSizes.

@Test
public void testUploadSizes() throws Exception {
    ChangelogStorageMetricGroup metrics = new ChangelogStorageMetricGroup(createUnregisteredTaskManagerJobMetricGroup());
    try (FsStateChangelogStorage storage = new FsStateChangelogStorage(Path.fromLocalFile(temporaryFolder.newFolder()), false, 100, metrics)) {
        FsStateChangelogWriter writer = storage.createWriter("writer", EMPTY_KEY_GROUP_RANGE);
        // upload single byte to infer header size
        SequenceNumber from = writer.nextSequenceNumber();
        writer.append(0, new byte[] { 0 });
        writer.persist(from).get();
        long headerSize = metrics.getUploadSizes().getStatistics().getMin() - 1;
        byte[] upload = new byte[33];
        for (int i = 0; i < 5; i++) {
            from = writer.nextSequenceNumber();
            writer.append(0, upload);
            writer.persist(from).get();
        }
        long expected = upload.length + headerSize;
        assertEquals(expected, metrics.getUploadSizes().getStatistics().getMax());
    }
}

Example 13 with SequenceNumber

use of org.apache.flink.runtime.state.changelog.SequenceNumber in project flink by apache.

the class ChangelogStorageMetricsTest method testUploadBatchSizes.

@Test
public void testUploadBatchSizes() throws Exception {
    int numWriters = 5, numUploads = 5;
    ChangelogStorageMetricGroup metrics = new ChangelogStorageMetricGroup(createUnregisteredTaskManagerJobMetricGroup());
    Path basePath = Path.fromLocalFile(temporaryFolder.newFolder());
    StateChangeFsUploader uploader = new StateChangeFsUploader(basePath, basePath.getFileSystem(), false, 100, metrics);
    ManuallyTriggeredScheduledExecutorService scheduler = new ManuallyTriggeredScheduledExecutorService();
    BatchingStateChangeUploader batcher = new BatchingStateChangeUploader(Long.MAX_VALUE, Long.MAX_VALUE, Long.MAX_VALUE, RetryPolicy.NONE, uploader, scheduler, new RetryingExecutor(1, metrics.getAttemptsPerUpload()), metrics);
    FsStateChangelogStorage storage = new FsStateChangelogStorage(batcher, Integer.MAX_VALUE);
    FsStateChangelogWriter[] writers = new FsStateChangelogWriter[numWriters];
    for (int i = 0; i < numWriters; i++) {
        writers[i] = storage.createWriter(Integer.toString(i), EMPTY_KEY_GROUP_RANGE);
    }
    try {
        for (int upload = 0; upload < numUploads; upload++) {
            for (int writer = 0; writer < numWriters; writer++) {
                // with all thresholds on MAX and manually triggered executor, this shouldn't
                // cause actual uploads
                SequenceNumber from = writers[writer].nextSequenceNumber();
                writers[writer].append(0, new byte[] { 0, 1, 2, 3 });
                writers[writer].persist(from);
            }
            // now the uploads should be grouped and executed at once
            scheduler.triggerScheduledTasks();
        }
        assertEquals(numWriters, metrics.getUploadBatchSizes().getStatistics().getMin());
        assertEquals(numWriters, metrics.getUploadBatchSizes().getStatistics().getMax());
    } finally {
        storage.close();
    }
}

Example 14 with SequenceNumber

use of org.apache.flink.runtime.state.changelog.SequenceNumber in project flink by apache.

the class ChangelogStorageMetricsTest method testUploadsCounter.

@Test
public void testUploadsCounter() throws Exception {
    ChangelogStorageMetricGroup metrics = new ChangelogStorageMetricGroup(createUnregisteredTaskManagerJobMetricGroup());
    try (FsStateChangelogStorage storage = new FsStateChangelogStorage(Path.fromLocalFile(temporaryFolder.newFolder()), false, 100, metrics)) {
        FsStateChangelogWriter writer = storage.createWriter("writer", EMPTY_KEY_GROUP_RANGE);
        int numUploads = 5;
        for (int i = 0; i < numUploads; i++) {
            SequenceNumber from = writer.nextSequenceNumber();
            writer.append(0, new byte[] { 0, 1, 2, 3 });
            writer.persist(from).get();
        }
        assertEquals(numUploads, metrics.getUploadsCounter().getCount());
        assertTrue(metrics.getUploadLatenciesNanos().getStatistics().getMin() > 0);
    }
}

Example 15 with SequenceNumber

use of org.apache.flink.runtime.state.changelog.SequenceNumber in project flink by apache.

the class FsStateChangelogWriterTest method testPersistNonFailedChanges.

@Test
public void testPersistNonFailedChanges() throws Exception {
    withWriter((writer, uploader) -> {
        byte[] bytes = getBytes();
        SequenceNumber sqn1 = append(writer, bytes);
        // future result ignored
        writer.persist(sqn1);
        uploader.failUpload(new RuntimeException("test"));
        uploader.reset();
        SequenceNumber sqn2 = append(writer, bytes);
        CompletableFuture<ChangelogStateHandleStreamImpl> future = writer.persist(sqn2);
        uploader.completeUpload();
        future.get();
    });
}