Examples with HashFunction com.google.devtools.build.lib.vfs.FileSystem.HashFunction used on opensource projects

Example 1 with HashFunction

use of com.google.devtools.build.lib.vfs.FileSystem.HashFunction in project bazel by bazelbuild.

the class DigestUtilsTest method assertDigestCalculationConcurrency.

private static void assertDigestCalculationConcurrency(boolean expectConcurrent, final boolean fastDigest, final int fileSize1, final int fileSize2, HashFunction hf) throws Exception {
    // Used to block test threads.
    final CountDownLatch barrierLatch = new CountDownLatch(2);
    // Used to block main thread.
    final CountDownLatch readyLatch = new CountDownLatch(1);
    FileSystem myfs = new InMemoryFileSystem(BlazeClock.instance()) {

        @Override
        protected byte[] getMD5Digest(Path path) throws IOException {
            try {
                barrierLatch.countDown();
                readyLatch.countDown();
                // Either both threads will be inside getMD5Digest at the same time or they
                // both will be blocked.
                barrierLatch.await();
            } catch (Exception e) {
                throw new IOException(e);
            }
            return super.getMD5Digest(path);
        }

        @Override
        protected byte[] getSHA1Digest(Path path) throws IOException {
            try {
                barrierLatch.countDown();
                readyLatch.countDown();
                // Either both threads will be inside getSHA1Digest at the same time or they
                // both will be blocked.
                barrierLatch.await();
            } catch (Exception e) {
                throw new IOException(e);
            }
            return super.getSHA1Digest(path);
        }

        @Override
        protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
            return fastDigest ? super.getDigest(path, hashFunction) : null;
        }
    };
    FileSystem.setDigestFunctionForTesting(hf);
    final Path myFile1 = myfs.getPath("/f1.dat");
    final Path myFile2 = myfs.getPath("/f2.dat");
    FileSystemUtils.writeContentAsLatin1(myFile1, Strings.repeat("a", fileSize1));
    FileSystemUtils.writeContentAsLatin1(myFile2, Strings.repeat("b", fileSize2));
    TestThread thread1 = new TestThread() {

        @Override
        public void runTest() throws Exception {
            DigestUtils.getDigestOrFail(myFile1, fileSize1);
        }
    };
    TestThread thread2 = new TestThread() {

        @Override
        public void runTest() throws Exception {
            DigestUtils.getDigestOrFail(myFile2, fileSize2);
        }
    };
    thread1.start();
    thread2.start();
    if (!expectConcurrent) {
        // Synchronized case.
        // Wait until at least one thread reached getMD5Digest().
        assertTrue(readyLatch.await(TestUtils.WAIT_TIMEOUT_SECONDS, TimeUnit.SECONDS));
        // Only 1 thread should be inside getMD5Digest().
        assertEquals(1, barrierLatch.getCount());
        // Release barrier latch, allowing both threads to proceed.
        barrierLatch.countDown();
    }
    // Test successful execution within 5 seconds.
    thread1.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
    thread2.joinAndAssertState(TestUtils.WAIT_TIMEOUT_MILLISECONDS);
}

Example 2 with HashFunction

use of com.google.devtools.build.lib.vfs.FileSystem.HashFunction in project bazel by bazelbuild.

the class DigestUtilsTest method testCalculationConcurrency.

/**
   * Ensures that MD5 calculation is synchronized for files
   * greater than 4096 bytes if MD5 is not available cheaply,
   * so machines with rotating drives don't become unusable.
   */
@Test
public void testCalculationConcurrency() throws Exception {
    for (HashFunction hf : Arrays.asList(HashFunction.MD5, HashFunction.SHA1)) {
        assertDigestCalculationConcurrency(true, true, 4096, 4096, hf);
        assertDigestCalculationConcurrency(true, true, 4097, 4097, hf);
        assertDigestCalculationConcurrency(true, false, 4096, 4096, hf);
        assertDigestCalculationConcurrency(false, false, 4097, 4097, hf);
        assertDigestCalculationConcurrency(true, false, 1024, 4097, hf);
        assertDigestCalculationConcurrency(true, false, 1024, 1024, hf);
    }
}

Example 3 with HashFunction

use of com.google.devtools.build.lib.vfs.FileSystem.HashFunction in project bazel by bazelbuild.

the class DigestUtilsTest method assertRecoverFromMalformedDigest.

public void assertRecoverFromMalformedDigest(HashFunction... hashFunctions) throws Exception {
    final byte[] malformed = { 0, 0, 0 };
    FileSystem myFS = new InMemoryFileSystem(BlazeClock.instance()) {

        @Override
        protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
            // Digest functions have more than 3 bytes, usually at least 16.
            return malformed;
        }
    };
    Path path = myFS.getPath("/file");
    FileSystemUtils.writeContentAsLatin1(path, "a");
    for (HashFunction hf : hashFunctions) {
        FileSystem.setDigestFunctionForTesting(hf);
        byte[] result = DigestUtils.getDigestOrFail(path, 1);
        assertArrayEquals(path.getDigest(), result);
        assertNotSame(malformed, result);
        assertTrue(path.isValidDigest(result));
    }
}

Example 4 with HashFunction

use of com.google.devtools.build.lib.vfs.FileSystem.HashFunction in project bazel by bazelbuild.

the class DigestUtilsTest method testCache.

@Test
public void testCache() throws Exception {
    final AtomicInteger getFastDigestCounter = new AtomicInteger(0);
    final AtomicInteger getDigestCounter = new AtomicInteger(0);
    FileSystem tracingFileSystem = new InMemoryFileSystem(BlazeClock.instance()) {

        @Override
        protected byte[] getFastDigest(Path path, HashFunction hashFunction) throws IOException {
            getFastDigestCounter.incrementAndGet();
            return null;
        }

        @Override
        protected byte[] getDigest(Path path) throws IOException {
            getDigestCounter.incrementAndGet();
            return super.getDigest(path);
        }
    };
    DigestUtils.configureCache(2);
    final Path file1 = tracingFileSystem.getPath("/1.txt");
    final Path file2 = tracingFileSystem.getPath("/2.txt");
    final Path file3 = tracingFileSystem.getPath("/3.txt");
    FileSystemUtils.writeContentAsLatin1(file1, "some contents");
    FileSystemUtils.writeContentAsLatin1(file2, "some other contents");
    FileSystemUtils.writeContentAsLatin1(file3, "and something else");
    byte[] digest1 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
    assertEquals(1, getFastDigestCounter.get());
    assertEquals(1, getDigestCounter.get());
    new CacheStatsChecker().evictionCount(0).hitCount(0).missCount(1).check();
    byte[] digest2 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
    assertEquals(2, getFastDigestCounter.get());
    assertEquals(1, getDigestCounter.get());
    new CacheStatsChecker().evictionCount(0).hitCount(1).missCount(1).check();
    assertArrayEquals(digest1, digest2);
    // Evict the digest for the previous file.
    DigestUtils.getDigestOrFail(file2, file2.getFileSize());
    DigestUtils.getDigestOrFail(file3, file3.getFileSize());
    new CacheStatsChecker().evictionCount(1).hitCount(1).missCount(3).check();
    // And now try to recompute it.
    byte[] digest3 = DigestUtils.getDigestOrFail(file1, file1.getFileSize());
    new CacheStatsChecker().evictionCount(2).hitCount(1).missCount(4).check();
    assertArrayEquals(digest1, digest3);
}

Example 5 with HashFunction

use of com.google.devtools.build.lib.vfs.FileSystem.HashFunction in project bazel by bazelbuild.

the class FileFunctionTest method testUnreadableFileWithFastDigest.

@Test
public void testUnreadableFileWithFastDigest() throws Exception {
    final byte[] expectedDigest = MessageDigest.getInstance("md5").digest("blah".getBytes(StandardCharsets.UTF_8));
    createFsAndRoot(new CustomInMemoryFs(manualClock) {

        @Override
        protected byte[] getFastDigest(Path path, HashFunction hf) {
            return path.getBaseName().equals("unreadable") ? expectedDigest : null;
        }
    });
    Path p = file("unreadable");
    p.chmod(0);
    FileValue value = valueForPath(p);
    assertThat(value.exists()).isTrue();
    assertThat(value.getDigest()).isNotNull();
}