Examples with BorrowablePath com.facebook.buck.io.BorrowablePath used on opensource projects

Example 1 with BorrowablePath

use of com.facebook.buck.io.BorrowablePath in project buck by facebook.

the class AbstractNetworkCache method store.

@Override
public ListenableFuture<Void> store(final ArtifactInfo info, final BorrowablePath output) {
    if (!isStoreSupported()) {
        return Futures.immediateFuture(null);
    }
    final HttpArtifactCacheEvent.Scheduled scheduled = HttpArtifactCacheEvent.newStoreScheduledEvent(ArtifactCacheEvent.getTarget(info.getMetadata()), info.getRuleKeys());
    buckEventBus.post(scheduled);
    final Path tmp;
    try {
        tmp = getPathForArtifact(output);
    } catch (IOException e) {
        LOG.error(e, "Failed to store artifact in temp file: " + output.getPath().toString());
        return Futures.immediateFuture(null);
    }
    // HTTP Store operations are asynchronous.
    return httpWriteExecutorService.submit(() -> {
        HttpArtifactCacheEvent.Started startedEvent = HttpArtifactCacheEvent.newStoreStartedEvent(scheduled);
        buckEventBus.post(startedEvent);
        HttpArtifactCacheEvent.Finished.Builder finishedEventBuilder = HttpArtifactCacheEvent.newFinishedEventBuilder(startedEvent);
        finishedEventBuilder.getStoreBuilder().setRuleKeys(info.getRuleKeys());
        try {
            long artifactSizeBytes = projectFilesystem.getFileSize(tmp);
            finishedEventBuilder.getStoreBuilder().setArtifactSizeBytes(artifactSizeBytes).setRuleKeys(info.getRuleKeys());
            if (!isArtefactTooBigToBeStored(artifactSizeBytes, maxStoreSize)) {
                storeImpl(info, tmp, finishedEventBuilder);
            } else {
                LOG.info("Artifact too big so not storing it in the distributed cache. " + "file=[%s] buildTarget=[%s]", tmp, info.getBuildTarget());
            }
            buckEventBus.post(finishedEventBuilder.build());
        } catch (IOException e) {
            reportFailure(e, "store(%s): %s: %s", info.getRuleKeys(), e.getClass().getName(), e.getMessage());
            finishedEventBuilder.getStoreBuilder().setWasStoreSuccessful(false).setErrorMessage(e.toString());
            buckEventBus.post(finishedEventBuilder.build());
        }
        try {
            projectFilesystem.deleteFileAtPathIfExists(tmp);
        } catch (IOException e) {
            LOG.warn(e, "Failed to delete file %s", tmp);
        }
    }, /* result */
    null);
}

Example 2 with BorrowablePath

use of com.facebook.buck.io.BorrowablePath in project buck by facebook.

the class AbstractNetworkCache method getPathForArtifact.

/// depending on if we can borrow the output or not, we will either use output directly or
/// hold it temporary in hidden place
private Path getPathForArtifact(BorrowablePath output) throws IOException {
    Path tmp;
    if (output.canBorrow()) {
        tmp = output.getPath();
    } else {
        tmp = projectFilesystem.createTempFile("artifact", ".tmp");
        projectFilesystem.copyFile(output.getPath(), tmp);
    }
    return tmp;
}

Example 3 with BorrowablePath

use of com.facebook.buck.io.BorrowablePath in project buck by facebook.

the class DirArtifactCache method store.

@Override
public ListenableFuture<Void> store(ArtifactInfo info, BorrowablePath output) {
    if (!doStore) {
        return Futures.immediateFuture(null);
    }
    try {
        Optional<Path> borrowedAndStoredArtifactPath = Optional.empty();
        for (RuleKey ruleKey : info.getRuleKeys()) {
            Path artifactPath = getPathForRuleKey(ruleKey, Optional.empty());
            Path metadataPath = getPathForRuleKey(ruleKey, Optional.of(".metadata"));
            if (filesystem.exists(artifactPath) && filesystem.exists(metadataPath)) {
                continue;
            }
            filesystem.mkdirs(getParentDirForRuleKey(ruleKey));
            if (!output.canBorrow()) {
                storeArtifactOutput(output.getPath(), artifactPath);
            } else {
                // move it without copying.  This significantly optimizes the Disk I/O.
                if (!borrowedAndStoredArtifactPath.isPresent()) {
                    borrowedAndStoredArtifactPath = Optional.of(artifactPath);
                    filesystem.move(output.getPath(), artifactPath, StandardCopyOption.REPLACE_EXISTING);
                } else {
                    storeArtifactOutput(borrowedAndStoredArtifactPath.get(), artifactPath);
                }
            }
            bytesSinceLastDeleteOldFiles += filesystem.getFileSize(artifactPath);
            // Now, write the meta data artifact.
            Path tmp = filesystem.createTempFile(getPreparedTempFolder(), "metadata", TMP_EXTENSION);
            try {
                try (DataOutputStream out = new DataOutputStream(filesystem.newFileOutputStream(tmp))) {
                    out.writeInt(info.getMetadata().size());
                    for (Map.Entry<String, String> ent : info.getMetadata().entrySet()) {
                        out.writeUTF(ent.getKey());
                        byte[] val = ent.getValue().getBytes(Charsets.UTF_8);
                        out.writeInt(val.length);
                        out.write(val);
                    }
                }
                filesystem.move(tmp, metadataPath, StandardCopyOption.REPLACE_EXISTING);
                bytesSinceLastDeleteOldFiles += filesystem.getFileSize(metadataPath);
            } finally {
                filesystem.deleteFileAtPathIfExists(tmp);
            }
        }
    } catch (IOException e) {
        LOG.warn(e, "Artifact store(%s, %s) error", info.getRuleKeys(), output);
    }
    if (maxCacheSizeBytes.isPresent() && bytesSinceLastDeleteOldFiles > (maxCacheSizeBytes.get() * STORED_TO_MAX_BYTES_RATIO_TRIM_TRIGGER)) {
        bytesSinceLastDeleteOldFiles = 0L;
        deleteOldFiles();
    }
    return Futures.immediateFuture(null);
}

Example 4 with BorrowablePath

use of com.facebook.buck.io.BorrowablePath in project buck by facebook.

the class BuildInfoRecorderTest method testPerformUploadToArtifactCache.

@Test
public void testPerformUploadToArtifactCache() throws IOException, InterruptedException {
    FakeProjectFilesystem filesystem = new FakeProjectFilesystem();
    BuildInfoRecorder buildInfoRecorder = createBuildInfoRecorder(filesystem);
    BuckEventBus bus = new BuckEventBus(new FakeClock(0), new BuildId("BUILD"));
    final byte[] contents = "contents".getBytes();
    Path file = Paths.get("file");
    filesystem.writeBytesToPath(contents, file);
    buildInfoRecorder.recordArtifact(file);
    Path dir = Paths.get("dir");
    filesystem.mkdirs(dir);
    filesystem.writeBytesToPath(contents, dir.resolve("file"));
    buildInfoRecorder.recordArtifact(dir);
    // Record some metadata.
    buildInfoRecorder.addMetadata("metadata", "metadata");
    // Record some build metadata.
    buildInfoRecorder.addBuildMetadata("build-metadata", "build-metadata");
    buildInfoRecorder.writeMetadataToDisk(true);
    final AtomicBoolean stored = new AtomicBoolean(false);
    final ArtifactCache cache = new NoopArtifactCache() {

        @Override
        public boolean isStoreSupported() {
            return true;
        }

        @Override
        public ListenableFuture<Void> store(ArtifactInfo info, BorrowablePath output) {
            stored.set(true);
            // Verify the build metadata.
            assertThat(info.getMetadata().get("build-metadata"), Matchers.equalTo("build-metadata"));
            // Verify zip contents
            try (Zip zip = new Zip(output.getPath(), /* forWriting */
            false)) {
                assertEquals(ImmutableSet.of("", "dir/", "buck-out/", "buck-out/bin/", "buck-out/bin/foo/", "buck-out/bin/foo/.bar/", "buck-out/bin/foo/.bar/metadata/"), zip.getDirNames());
                assertEquals(ImmutableSet.of("dir/file", "file", "buck-out/bin/foo/.bar/metadata/metadata"), zip.getFileNames());
                assertArrayEquals(contents, zip.readFully("file"));
                assertArrayEquals(contents, zip.readFully("dir/file"));
            } catch (IOException e) {
                Throwables.throwIfUnchecked(e);
                throw new RuntimeException(e);
            }
            return Futures.immediateFuture(null);
        }
    };
    buildInfoRecorder.performUploadToArtifactCache(ImmutableSet.of(new RuleKey("aa")), cache, bus);
    assertTrue(stored.get());
}

Example 5 with BorrowablePath

use of com.facebook.buck.io.BorrowablePath in project buck by facebook.

the class LocalFsContentsProvider method writeFileAndGetInputStream.

public void writeFileAndGetInputStream(BuildJobStateFileHashEntry entry, Path absPath) throws IOException {
    RuleKey key = new RuleKey(entry.getHashCode());
    ArtifactInfo artifactInfo = ArtifactInfo.builder().setRuleKeys(ImmutableList.of(key)).build();
    BorrowablePath nonBorrowablePath = BorrowablePath.notBorrowablePath(absPath);
    try {
        dirCache.store(artifactInfo, nonBorrowablePath).get();
    } catch (InterruptedException | ExecutionException e) {
        throw new IOException("Failed to store artifact to DirCache.", e);
    }
}