Search in sources :

Example 1 with TestFs

use of org.apache.flink.runtime.state.filesystem.TestFs in project flink by apache.

the class FileSystemBlobStoreTest method fileSystemBlobStoreCallsSyncOnPut.

@Test
public void fileSystemBlobStoreCallsSyncOnPut(@TempDir Path storageDirectory) throws IOException {
    final Path blobStoreDirectory = storageDirectory.resolve("blobStore");
    final AtomicReference<TestingLocalDataOutputStream> createdOutputStream = new AtomicReference<>();
    final FunctionWithException<org.apache.flink.core.fs.Path, FSDataOutputStream, IOException> outputStreamFactory = value -> {
        final File file = new File(value.toString());
        FileUtils.createParentDirectories(file);
        final TestingLocalDataOutputStream outputStream = new TestingLocalDataOutputStream(file);
        createdOutputStream.compareAndSet(null, outputStream);
        return outputStream;
    };
    try (FileSystemBlobStore fileSystemBlobStore = new FileSystemBlobStore(new TestFs(outputStreamFactory), blobStoreDirectory.toString())) {
        final BlobKey blobKey = BlobKey.createKey(BlobKey.BlobType.PERMANENT_BLOB);
        final File localFile = storageDirectory.resolve("localFile").toFile();
        FileUtils.createParentDirectories(localFile);
        FileUtils.writeStringToFile(localFile, "foobar", StandardCharsets.UTF_8);
        fileSystemBlobStore.put(localFile, new JobID(), blobKey);
        assertThat(createdOutputStream.get().hasSyncBeenCalled()).isTrue();
    }
}
Also used : Path(java.nio.file.Path) TestFs(org.apache.flink.runtime.state.filesystem.TestFs) BeforeEach(org.junit.jupiter.api.BeforeEach) MessageDigest(java.security.MessageDigest) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) AtomicReference(java.util.concurrent.atomic.AtomicReference) TestLoggerExtension(org.apache.flink.util.TestLoggerExtension) FSDataOutputStream(org.apache.flink.core.fs.FSDataOutputStream) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) FunctionWithException(org.apache.flink.util.function.FunctionWithException) TestFileSystem(org.apache.flink.testutils.TestFileSystem) Path(java.nio.file.Path) Files(java.nio.file.Files) BufferedWriter(java.io.BufferedWriter) FileWriter(java.io.FileWriter) FileUtils(org.apache.commons.io.FileUtils) IOException(java.io.IOException) FileInputStream(java.io.FileInputStream) LocalDataOutputStream(org.apache.flink.core.fs.local.LocalDataOutputStream) Preconditions(org.apache.flink.util.Preconditions) File(java.io.File) FileNotFoundException(java.io.FileNotFoundException) StandardCharsets(java.nio.charset.StandardCharsets) Test(org.junit.jupiter.api.Test) AfterEach(org.junit.jupiter.api.AfterEach) JobID(org.apache.flink.api.common.JobID) TempDir(org.junit.jupiter.api.io.TempDir) InputStream(java.io.InputStream) TestFs(org.apache.flink.runtime.state.filesystem.TestFs) AtomicReference(java.util.concurrent.atomic.AtomicReference) IOException(java.io.IOException) FSDataOutputStream(org.apache.flink.core.fs.FSDataOutputStream) File(java.io.File) JobID(org.apache.flink.api.common.JobID) Test(org.junit.jupiter.api.Test)

Aggregations

BufferedWriter (java.io.BufferedWriter)1 File (java.io.File)1 FileInputStream (java.io.FileInputStream)1 FileNotFoundException (java.io.FileNotFoundException)1 FileWriter (java.io.FileWriter)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 StandardCharsets (java.nio.charset.StandardCharsets)1 Files (java.nio.file.Files)1 Path (java.nio.file.Path)1 MessageDigest (java.security.MessageDigest)1 AtomicReference (java.util.concurrent.atomic.AtomicReference)1 FileUtils (org.apache.commons.io.FileUtils)1 JobID (org.apache.flink.api.common.JobID)1 FSDataOutputStream (org.apache.flink.core.fs.FSDataOutputStream)1 LocalDataOutputStream (org.apache.flink.core.fs.local.LocalDataOutputStream)1 TestFs (org.apache.flink.runtime.state.filesystem.TestFs)1 TestFileSystem (org.apache.flink.testutils.TestFileSystem)1 Preconditions (org.apache.flink.util.Preconditions)1 TestLoggerExtension (org.apache.flink.util.TestLoggerExtension)1