Examples with HttpResponse com.facebook.buck.slb.HttpResponse used on opensource projects

Example 1 with HttpResponse

use of com.facebook.buck.slb.HttpResponse in project buck by facebook.

the class HttpArtifactCache method storeImpl.

@Override
protected void storeImpl(ArtifactInfo info, final Path file, final Finished.Builder eventBuilder) throws IOException {
    // Build the request, hitting the multi-key endpoint.
    Request.Builder builder = new Request.Builder();
    final HttpArtifactCacheBinaryProtocol.StoreRequest storeRequest = new HttpArtifactCacheBinaryProtocol.StoreRequest(info, new ByteSource() {

        @Override
        public InputStream openStream() throws IOException {
            return projectFilesystem.newFileInputStream(file);
        }
    });
    eventBuilder.getStoreBuilder().setRequestSizeBytes(storeRequest.getContentLength());
    // Wrap the file into a `RequestBody` which uses `ProjectFilesystem`.
    builder.put(new RequestBody() {

        @Override
        public MediaType contentType() {
            return OCTET_STREAM_CONTENT_TYPE;
        }

        @Override
        public long contentLength() throws IOException {
            return storeRequest.getContentLength();
        }

        @Override
        public void writeTo(BufferedSink bufferedSink) throws IOException {
            StoreWriteResult writeResult = storeRequest.write(bufferedSink.outputStream());
            eventBuilder.getStoreBuilder().setArtifactContentHash(writeResult.getArtifactContentHashCode().toString());
        }
    });
    // Dispatch the store operation and verify it succeeded.
    try (HttpResponse response = storeClient.makeRequest("/artifacts/key", builder)) {
        final boolean requestFailed = response.statusCode() != HttpURLConnection.HTTP_ACCEPTED;
        if (requestFailed) {
            reportFailure("store(%s, %s): unexpected response: [%d:%s].", response.requestUrl(), info.getRuleKeys(), response.statusCode(), response.statusMessage());
        }
        eventBuilder.getStoreBuilder().setWasStoreSuccessful(!requestFailed);
    }
}

Example 2 with HttpResponse

use of com.facebook.buck.slb.HttpResponse in project buck by facebook.

the class ThriftArtifactCache method fetchImpl.

@Override
public CacheResult fetchImpl(RuleKey ruleKey, LazyPath output, HttpArtifactCacheEvent.Finished.Builder eventBuilder) throws IOException {
    BuckCacheFetchRequest fetchRequest = new BuckCacheFetchRequest();
    com.facebook.buck.artifact_cache.thrift.RuleKey thriftRuleKey = new com.facebook.buck.artifact_cache.thrift.RuleKey();
    thriftRuleKey.setHashString(ruleKey.getHashCode().toString());
    fetchRequest.setRuleKey(thriftRuleKey);
    fetchRequest.setRepository(repository);
    fetchRequest.setScheduleType(scheduleType);
    fetchRequest.setDistributedBuildModeEnabled(distributedBuildModeEnabled);
    BuckCacheRequest cacheRequest = new BuckCacheRequest();
    cacheRequest.setType(BuckCacheRequestType.FETCH);
    cacheRequest.setFetchRequest(fetchRequest);
    LOG.verbose("Will fetch key %s", thriftRuleKey);
    final ThriftArtifactCacheProtocol.Request request = ThriftArtifactCacheProtocol.createRequest(PROTOCOL, cacheRequest);
    Request.Builder builder = toOkHttpRequest(request);
    try (HttpResponse httpResponse = fetchClient.makeRequest(hybridThriftEndpoint, builder)) {
        if (httpResponse.statusCode() != 200) {
            String message = String.format("Failed to fetch cache artifact with HTTP status code [%d:%s] " + " to url [%s] for rule key [%s].", httpResponse.statusCode(), httpResponse.statusMessage(), httpResponse.requestUrl(), ruleKey.toString());
            LOG.error(message);
            return CacheResult.error(name, message);
        }
        try (ThriftArtifactCacheProtocol.Response response = ThriftArtifactCacheProtocol.parseResponse(PROTOCOL, httpResponse.getBody())) {
            eventBuilder.getFetchBuilder().setResponseSizeBytes(httpResponse.contentLength());
            BuckCacheResponse cacheResponse = response.getThriftData();
            if (!cacheResponse.isWasSuccessful()) {
                LOG.warn("Request was unsuccessful: %s", cacheResponse.getErrorMessage());
                return CacheResult.error(name, cacheResponse.getErrorMessage());
            }
            BuckCacheFetchResponse fetchResponse = cacheResponse.getFetchResponse();
            if (LOG.isDebugEnabled()) {
                LOG.debug("Debug info for cache fetch request: request=[%s] response=[%s]", ThriftUtil.thriftToDebugJson(cacheRequest), ThriftUtil.thriftToDebugJson(cacheResponse));
            }
            if (!fetchResponse.isArtifactExists()) {
                LOG.verbose("Artifact did not exist.");
                return CacheResult.miss();
            }
            LOG.verbose("Got artifact.  Attempting to read payload.");
            Path tmp = createTempFileForDownload();
            ThriftArtifactCacheProtocol.Response.ReadPayloadInfo readResult;
            try (OutputStream tmpFile = projectFilesystem.newFileOutputStream(tmp)) {
                readResult = response.readPayload(tmpFile);
                LOG.verbose("Successfully read payload: %d bytes.", readResult.getBytesRead());
            }
            ArtifactMetadata metadata = fetchResponse.getMetadata();
            if (LOG.isVerboseEnabled()) {
                LOG.verbose(String.format("Fetched artifact with rule key [%s] contains the following metadata: [%s]", ruleKey, ThriftUtil.thriftToDebugJson(metadata)));
            }
            eventBuilder.setTarget(Optional.ofNullable(metadata.getBuildTarget())).getFetchBuilder().setAssociatedRuleKeys(toImmutableSet(metadata.getRuleKeys())).setArtifactSizeBytes(readResult.getBytesRead());
            if (!metadata.isSetArtifactPayloadMd5()) {
                String msg = "Fetched artifact is missing the MD5 hash.";
                LOG.warn(msg);
            } else {
                eventBuilder.getFetchBuilder().setArtifactContentHash(metadata.getArtifactPayloadMd5());
                if (!readResult.getMd5Hash().equals(fetchResponse.getMetadata().getArtifactPayloadMd5())) {
                    String msg = String.format("The artifact fetched from cache is corrupted. ExpectedMD5=[%s] ActualMD5=[%s]", fetchResponse.getMetadata().getArtifactPayloadMd5(), readResult.getMd5Hash());
                    LOG.error(msg);
                    return CacheResult.error(name, msg);
                }
            }
            // This makes sure we don't have 'half downloaded files' in the dir cache.
            projectFilesystem.move(tmp, output.get(), StandardCopyOption.REPLACE_EXISTING);
            return CacheResult.hit(name, ImmutableMap.copyOf(fetchResponse.getMetadata().getMetadata()), readResult.getBytesRead());
        }
    }
}

Example 3 with HttpResponse

use of com.facebook.buck.slb.HttpResponse in project buck by facebook.

the class DefaultDefectReporter method uploadReport.

private DefectSubmitResult uploadReport(final DefectReport defectReport, DefectSubmitResult.Builder defectSubmitResult, ClientSideSlb slb) throws IOException {
    long timeout = rageConfig.getHttpTimeout();
    OkHttpClient httpClient = new OkHttpClient.Builder().connectTimeout(timeout, TimeUnit.MILLISECONDS).readTimeout(timeout, TimeUnit.MILLISECONDS).writeTimeout(timeout, TimeUnit.MILLISECONDS).build();
    HttpService httpService = new RetryingHttpService(buckEventBus, new LoadBalancedService(slb, httpClient, buckEventBus), rageConfig.getMaxUploadRetries());
    try {
        Request.Builder requestBuilder = new Request.Builder();
        requestBuilder.addHeader(REQUEST_PROTOCOL_VERSION, rageConfig.getProtocolVersion().name().toLowerCase());
        requestBuilder.post(new RequestBody() {

            @Override
            public MediaType contentType() {
                return MediaType.parse("application/x-www-form-urlencoded");
            }

            @Override
            public void writeTo(BufferedSink bufferedSink) throws IOException {
                writeReport(defectReport, bufferedSink.outputStream());
            }
        });
        HttpResponse response = httpService.makeRequest(rageConfig.getReportUploadPath(), requestBuilder);
        String responseBody;
        try (InputStream inputStream = response.getBody()) {
            responseBody = CharStreams.toString(new InputStreamReader(inputStream, Charsets.UTF_8));
        }
        if (response.statusCode() == HTTP_SUCCESS_CODE) {
            defectSubmitResult.setIsRequestSuccessful(true);
            if (rageConfig.getProtocolVersion().equals(AbstractRageConfig.RageProtocolVersion.SIMPLE)) {
                return defectSubmitResult.setReportSubmitMessage(responseBody).setReportSubmitLocation(responseBody).build();
            } else {
                // Decode Json response.
                RageJsonResponse json = objectMapper.readValue(responseBody.getBytes(Charsets.UTF_8), RageJsonResponse.class);
                return defectSubmitResult.setIsRequestSuccessful(json.getRequestSuccessful()).setReportSubmitErrorMessage(json.getErrorMessage()).setReportSubmitMessage(json.getMessage()).setReportSubmitLocation(json.getRageUrl()).build();
            }
        } else {
            throw new IOException(String.format("Connection to %s returned code %d and message: %s", response.requestUrl(), response.statusCode(), responseBody));
        }
    } catch (IOException e) {
        throw new IOException(String.format("Failed uploading report because [%s].", e.getMessage()));
    } finally {
        httpService.close();
    }
}

Example 4 with HttpResponse

use of com.facebook.buck.slb.HttpResponse in project buck by facebook.

the class HttpArtifactCache method fetchImpl.

@Override
protected CacheResult fetchImpl(RuleKey ruleKey, LazyPath output, final Finished.Builder eventBuilder) throws IOException {
    Request.Builder requestBuilder = new Request.Builder().get();
    try (HttpResponse response = fetchClient.makeRequest("/artifacts/key/" + ruleKey.toString(), requestBuilder)) {
        eventBuilder.getFetchBuilder().setResponseSizeBytes(response.contentLength());
        try (DataInputStream input = new DataInputStream(new FullyReadOnCloseInputStream(response.getBody()))) {
            if (response.statusCode() == HttpURLConnection.HTTP_NOT_FOUND) {
                LOG.info("fetch(%s, %s): cache miss", response.requestUrl(), ruleKey);
                return CacheResult.miss();
            }
            if (response.statusCode() != HttpURLConnection.HTTP_OK) {
                String msg = String.format("unexpected server response: [%d:%s]", response.statusCode(), response.statusMessage());
                reportFailure("fetch(%s, %s): %s", response.requestUrl(), ruleKey, msg);
                eventBuilder.getFetchBuilder().setErrorMessage(msg);
                return CacheResult.error(name, msg);
            }
            // Setup a temporary file, which sits next to the destination, to write to and
            // make sure all parent dirs exist.
            Path file = output.get();
            projectFilesystem.createParentDirs(file);
            Path temp = projectFilesystem.createTempFile(file.getParent(), file.getFileName().toString(), ".tmp");
            FetchResponseReadResult fetchedData;
            try (OutputStream tempFileOutputStream = projectFilesystem.newFileOutputStream(temp)) {
                fetchedData = HttpArtifactCacheBinaryProtocol.readFetchResponse(input, tempFileOutputStream);
            }
            eventBuilder.setTarget(ArtifactCacheEvent.getTarget(fetchedData.getMetadata())).getFetchBuilder().setResponseSizeBytes(fetchedData.getResponseSizeBytes()).setArtifactContentHash(fetchedData.getArtifactOnlyHashCode().toString());
            // Verify that we were one of the rule keys that stored this artifact.
            if (!fetchedData.getRuleKeys().contains(ruleKey)) {
                String msg = "incorrect key name";
                reportFailure("fetch(%s, %s): %s", response.requestUrl(), ruleKey, msg);
                eventBuilder.getFetchBuilder().setErrorMessage(msg);
                return CacheResult.error(name, msg);
            }
            // the HTTP header.  If it's incorrect, log this and return a miss.
            if (!fetchedData.getExpectedHashCode().equals(fetchedData.getActualHashCode())) {
                String msg = "artifact had invalid checksum";
                reportFailure("fetch(%s, %s): %s", response.requestUrl(), ruleKey, msg);
                projectFilesystem.deleteFileAtPath(temp);
                eventBuilder.getFetchBuilder().setErrorMessage(msg);
                return CacheResult.error(name, msg);
            }
            // Finally, move the temp file into it's final place.
            projectFilesystem.move(temp, file, StandardCopyOption.REPLACE_EXISTING);
            LOG.info("fetch(%s, %s): cache hit", response.requestUrl(), ruleKey);
            return CacheResult.hit(name, fetchedData.getMetadata(), fetchedData.getResponseSizeBytes());
        }
    }
}

Example 5 with HttpResponse

use of com.facebook.buck.slb.HttpResponse in project buck by facebook.

the class ThriftArtifactCache method storeImpl.

@Override
protected void storeImpl(final ArtifactInfo info, final Path file, final HttpArtifactCacheEvent.Finished.Builder eventBuilder) throws IOException {
    final ByteSource artifact = new ByteSource() {

        @Override
        public InputStream openStream() throws IOException {
            return projectFilesystem.newFileInputStream(file);
        }
    };
    BuckCacheStoreRequest storeRequest = new BuckCacheStoreRequest();
    ArtifactMetadata artifactMetadata = infoToMetadata(info, artifact, repository, scheduleType, distributedBuildModeEnabled);
    storeRequest.setMetadata(artifactMetadata);
    PayloadInfo payloadInfo = new PayloadInfo();
    long artifactSizeBytes = artifact.size();
    payloadInfo.setSizeBytes(artifactSizeBytes);
    BuckCacheRequest cacheRequest = new BuckCacheRequest();
    cacheRequest.addToPayloads(payloadInfo);
    cacheRequest.setType(BuckCacheRequestType.STORE);
    cacheRequest.setStoreRequest(storeRequest);
    if (LOG.isVerboseEnabled()) {
        LOG.verbose(String.format("Storing artifact with metadata: [%s].", ThriftUtil.thriftToDebugJson(artifactMetadata)));
    }
    final ThriftArtifactCacheProtocol.Request request = ThriftArtifactCacheProtocol.createRequest(PROTOCOL, cacheRequest, artifact);
    Request.Builder builder = toOkHttpRequest(request);
    eventBuilder.getStoreBuilder().setRequestSizeBytes(request.getRequestLengthBytes());
    try (HttpResponse httpResponse = storeClient.makeRequest(hybridThriftEndpoint, builder)) {
        if (httpResponse.statusCode() != 200) {
            throw new IOException(String.format("Failed to store cache artifact with HTTP status code [%d:%s] " + " to url [%s] for build target [%s] that has size [%d] bytes.", httpResponse.statusCode(), httpResponse.statusMessage(), httpResponse.requestUrl(), info.getBuildTarget().orElse(null), artifactSizeBytes));
        }
        try (ThriftArtifactCacheProtocol.Response response = ThriftArtifactCacheProtocol.parseResponse(PROTOCOL, httpResponse.getBody())) {
            BuckCacheResponse cacheResponse = response.getThriftData();
            if (!cacheResponse.isWasSuccessful()) {
                reportFailure("Failed to store artifact with thriftErrorMessage=[%s] " + "url=[%s] artifactSizeBytes=[%d]", response.getThriftData().getErrorMessage(), httpResponse.requestUrl(), artifactSizeBytes);
            }
            eventBuilder.getStoreBuilder().setArtifactContentHash(storeRequest.getMetadata().artifactPayloadMd5);
            eventBuilder.getStoreBuilder().setWasStoreSuccessful(cacheResponse.isWasSuccessful());
            if (LOG.isDebugEnabled()) {
                LOG.debug("Debug info for cache store request: artifactMetadata=[%s] response=[%s]", ThriftUtil.thriftToDebugJson(artifactMetadata), ThriftUtil.thriftToDebugJson(cacheResponse));
            }
        }
    }
}