Examples with InMemoryFileSystem com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem used on opensource projects

Example 1 with InMemoryFileSystem

use of com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem in project bazel by bazelbuild.

the class DigestUtilsTest method assertDigestCalculationConcurrency.

private static void assertDigestCalculationConcurrency(boolean expectConcurrent, final boolean fastDigest, final int fileSize1, final int fileSize2, HashFunction hf) throws Exception {
    // Used to block test threads.
    final CountDownLatch barrierLatch = new CountDownLatch(2);
    // Used to block main thread.
    final CountDownLatch readyLatch = new CountDownLatch(1);
    FileSystem myfs = new InMemoryFileSystem(BlazeClock.instance()) {

        @Override
        protected byte[] getMD5Digest(Path path) throws IOException {
            try {
                barrierLatch.countDown();
                readyLatch.countDown();
                // Either both threads will be inside getMD5Digest at the same time or they
                // both will be blocked.
                barrierLatch.await();
            } catch (Exception e) {
                throw new IOException(e);
            }
            return super.getMD5Digest(path);
        }

        @Override
        protected byte[] getSHA1Digest(Path path) throws IOException {
            try {
                barrierLatch.countDown();
                readyLatch.countDown();
                // Either both threads will be inside getSHA1Digest at the same time or they
                // both will be blocked.
                barrierLatch.await();
            } catch (Exception e) {
                throw new IOException(e);
            }
            return super.getSHA1Digest(path);
        }

        @Override
        protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
            return fastDigest ? super.getDigest(path, hashFunction) : null;
        }
    };
    FileSystem.setDigestFunctionForTesting(hf);
    final Path myFile1 = myfs.getPath("/f1.dat");
    final Path myFile2 = myfs.getPath("/f2.dat");
    FileSystemUtils.writeContentAsLatin1(myFile1, Strings.repeat("a", fileSize1));
    FileSystemUtils.writeContentAsLatin1(myFile2, Strings.repeat("b", fileSize2));
    TestThread thread1 = new TestThread() {

        @Override
        public void runTest() throws Exception {
            DigestUtils.getDigestOrFail(myFile1, fileSize1);
        }
    };
    TestThread thread2 = new TestThread() {

        @Override
        public void runTest() throws Exception {
            DigestUtils.getDigestOrFail(myFile2, fileSize2);
        }
    };
    thread1.start();
    thread2.start();
    if (!expectConcurrent) {
        // Synchronized case.
        // Wait until at least one thread reached getMD5Digest().
        assertTrue(readyLatch.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
        // Only 1 thread should be inside getMD5Digest().
        assertEquals(1, barrierLatch.getCount());
        // Release barrier latch, allowing both threads to proceed.
        barrierLatch.countDown();
    }
    // Test successful execution within 5 seconds.
    thread1.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
    thread2.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
}

Example 2 with InMemoryFileSystem

use of com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem in project bazel by bazelbuild.

the class DigestUtilsTest method assertRecoverFromMalformedDigest.

public void assertRecoverFromMalformedDigest(HashFunction... hashFunctions) throws Exception {
    final byte[] malformed = { 0, 0, 0 };
    FileSystem myFS = new InMemoryFileSystem(BlazeClock.instance()) {

        @Override
        protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
            // Digest functions have more than 3 bytes, usually at least 16.
            return malformed;
        }
    };
    Path path = myFS.getPath("/file");
    FileSystemUtils.writeContentAsLatin1(path, "a");
    for (HashFunction hf : hashFunctions) {
        FileSystem.setDigestFunctionForTesting(hf);
        byte[] result = DigestUtils.getDigestOrFail(path, 1);
        assertArrayEquals(path.getDigest(), result);
        assertNotSame(malformed, result);
        assertTrue(path.isValidDigest(result));
    }
}

Example 3 with InMemoryFileSystem

use of com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem in project bazel by bazelbuild.

the class DigestUtilsTest method testCache.

@Test
public void testCache() throws Exception {
    final AtomicInteger getFastDigestCounter = new AtomicInteger(0);
    final AtomicInteger getDigestCounter = new AtomicInteger(0);
    FileSystem tracingFileSystem = new InMemoryFileSystem(BlazeClock.instance()) {

        @Override
        protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
            getFastDigestCounter.incrementAndGet();
            return null;
        }

        @Override
        protected byte[] getDigest(Path path) throws IOException {
            getDigestCounter.incrementAndGet();
            return super.getDigest(path);
        }
    };
    DigestUtils.configureCache(2);
    final Path file1 = tracingFileSystem.getPath("/1.txt");
    final Path file2 = tracingFileSystem.getPath("/2.txt");
    final Path file3 = tracingFileSystem.getPath("/3.txt");
    FileSystemUtils.writeContentAsLatin1(file1, "some contents");
    FileSystemUtils.writeContentAsLatin1(file2, "some other contents");
    FileSystemUtils.writeContentAsLatin1(file3, "and something else");
    byte[] digest1 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
    assertEquals(1, getFastDigestCounter.get());
    assertEquals(1, getDigestCounter.get());
    new CacheStatsChecker().evictionCount(0).hitCount(0).missCount(1).check();
    byte[] digest2 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
    assertEquals(2, getFastDigestCounter.get());
    assertEquals(1, getDigestCounter.get());
    new CacheStatsChecker().evictionCount(0).hitCount(1).missCount(1).check();
    assertArrayEquals(digest1, digest2);
    // Evict the digest for the previous file.
    DigestUtils.getDigestOrFail(file2, file2.getFileSize());
    DigestUtils.getDigestOrFail(file3, file3.getFileSize());
    new CacheStatsChecker().evictionCount(1).hitCount(1).missCount(3).check();
    // And now try to recompute it.
    byte[] digest3 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
    new CacheStatsChecker().evictionCount(2).hitCount(1).missCount(4).check();
    assertArrayEquals(digest1, digest3);
}

Example 4 with InMemoryFileSystem

use of com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem in project bazel by bazelbuild.

the class TestFileOutErr method newInMemoryFile.

private static Path newInMemoryFile(File root, String name) {
    InMemoryFileSystem inMemFS = new InMemoryFileSystem();
    Path directory = inMemFS.getPath(root.getPath());
    try {
        FileSystemUtils.createDirectoryAndParents(directory);
    } catch (IOException e) {
        throw new IllegalStateException(e);
    }
    return directory.getRelative(name);
}

Example 5 with InMemoryFileSystem

use of com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem in project bazel by bazelbuild.

the class ParallelBuilderTest method testUpdateCacheError.

@Test
public void testUpdateCacheError() throws Exception {
    FileSystem fs = new InMemoryFileSystem() {

        @Override
        public FileStatus stat(Path path, boolean followSymlinks) throws IOException {
            final FileStatus stat = super.stat(path, followSymlinks);
            if (path.toString().endsWith("/out/foo")) {
                return new FileStatus() {

                    private final FileStatus original = stat;

                    @Override
                    public boolean isSymbolicLink() {
                        return original.isSymbolicLink();
                    }

                    @Override
                    public boolean isFile() {
                        return original.isFile();
                    }

                    @Override
                    public boolean isDirectory() {
                        return original.isDirectory();
                    }

                    @Override
                    public boolean isSpecialFile() {
                        return original.isSpecialFile();
                    }

                    @Override
                    public long getSize() throws IOException {
                        return original.getSize();
                    }

                    @Override
                    public long getNodeId() throws IOException {
                        return original.getNodeId();
                    }

                    @Override
                    public long getLastModifiedTime() throws IOException {
                        throw new IOException();
                    }

                    @Override
                    public long getLastChangeTime() throws IOException {
                        return original.getLastChangeTime();
                    }
                };
            }
            return stat;
        }
    };
    Artifact foo = createDerivedArtifact(fs, "foo");
    registerAction(new TestAction(TestAction.NO_EFFECT, emptySet, ImmutableList.of(foo)));
    reporter.removeHandler(failFastHandler);
    try {
        buildArtifacts(foo);
        fail("Expected to fail");
    } catch (BuildFailedException e) {
        assertContainsEvent("not all outputs were created or valid");
    }
}