Search in sources :

Example 1 with Response

use of oasis.names.tc.xacml._3_0.core.schema.wd_17.Response in project buck by facebook.

the class HttpArtifactCacheTest method testFetchMetadata.

@Test
public void testFetchMetadata() throws Exception {
    Path output = Paths.get("output/file");
    final String data = "test";
    final RuleKey ruleKey = new RuleKey("00000000000000000000000000000000");
    final ImmutableMap<String, String> metadata = ImmutableMap.of("some", "metadata");
    argsBuilder.setFetchClient(withMakeRequest(((path, requestBuilder) -> {
        Request request = requestBuilder.url(SERVER + path).build();
        Response response = new Response.Builder().request(request).protocol(Protocol.HTTP_1_1).code(HttpURLConnection.HTTP_OK).body(createResponseBody(ImmutableSet.of(ruleKey), metadata, ByteSource.wrap(data.getBytes(Charsets.UTF_8)), data)).build();
        return new OkHttpResponseWrapper(response);
    })));
    HttpArtifactCache cache = new HttpArtifactCache(argsBuilder.build());
    CacheResult result = cache.fetch(ruleKey, LazyPath.ofInstance(output));
    assertEquals(CacheResultType.HIT, result.getType());
    assertEquals(metadata, result.getMetadata());
    cache.close();
}
Also used : Path(java.nio.file.Path) LazyPath(com.facebook.buck.io.LazyPath) Response(okhttp3.Response) HttpResponse(com.facebook.buck.slb.HttpResponse) RuleKey(com.facebook.buck.rules.RuleKey) Request(okhttp3.Request) OkHttpResponseWrapper(com.facebook.buck.slb.OkHttpResponseWrapper) Test(org.junit.Test)

Example 2 with Response

use of oasis.names.tc.xacml._3_0.core.schema.wd_17.Response in project buck by facebook.

the class HttpArtifactCacheTest method errorTextReplaced.

@Test
public void errorTextReplaced() throws InterruptedException {
    FakeProjectFilesystem filesystem = new FakeProjectFilesystem();
    final String cacheName = "http cache";
    final RuleKey ruleKey = new RuleKey("00000000000000000000000000000000");
    final RuleKey otherRuleKey = new RuleKey("11111111111111111111111111111111");
    final String data = "data";
    final AtomicBoolean consoleEventReceived = new AtomicBoolean(false);
    argsBuilder.setCacheName(cacheName).setProjectFilesystem(filesystem).setBuckEventBus(new BuckEventBus(new IncrementingFakeClock(), new BuildId()) {

        @Override
        public void post(BuckEvent event) {
            if (event instanceof ConsoleEvent) {
                consoleEventReceived.set(true);
                ConsoleEvent consoleEvent = (ConsoleEvent) event;
                assertThat(consoleEvent.getMessage(), Matchers.containsString(cacheName));
                assertThat(consoleEvent.getMessage(), Matchers.containsString("incorrect key name"));
            }
        }
    }).setFetchClient(withMakeRequest((path, requestBuilder) -> {
        Request request = requestBuilder.url(SERVER + path).build();
        Response response = new Response.Builder().request(request).protocol(Protocol.HTTP_1_1).code(HttpURLConnection.HTTP_OK).body(createResponseBody(ImmutableSet.of(otherRuleKey), ImmutableMap.of(), ByteSource.wrap(data.getBytes(Charsets.UTF_8)), data)).build();
        return new OkHttpResponseWrapper(response);
    }));
    HttpArtifactCache cache = new HttpArtifactCache(argsBuilder.build());
    Path output = Paths.get("output/file");
    CacheResult result = cache.fetch(ruleKey, LazyPath.ofInstance(output));
    assertEquals(CacheResultType.ERROR, result.getType());
    assertEquals(Optional.empty(), filesystem.readFileIfItExists(output));
    assertTrue(consoleEventReceived.get());
    cache.close();
}
Also used : BuckEventBus(com.facebook.buck.event.BuckEventBus) BuckEventBus(com.facebook.buck.event.BuckEventBus) HttpURLConnection(java.net.HttpURLConnection) DataInputStream(java.io.DataInputStream) MoreExecutors(com.google.common.util.concurrent.MoreExecutors) ByteArrayOutputStream(java.io.ByteArrayOutputStream) BuckEvent(com.facebook.buck.event.BuckEvent) FakeProjectFilesystem(com.facebook.buck.testutil.FakeProjectFilesystem) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) ConsoleEvent(com.facebook.buck.event.ConsoleEvent) Assert.assertThat(org.junit.Assert.assertThat) Lists(com.google.common.collect.Lists) ByteArrayInputStream(java.io.ByteArrayInputStream) DataOutputStream(java.io.DataOutputStream) BuildId(com.facebook.buck.model.BuildId) RuleKey(com.facebook.buck.rules.RuleKey) Assert.assertArrayEquals(org.junit.Assert.assertArrayEquals) Response(okhttp3.Response) ByteSource(com.google.common.io.ByteSource) HttpResponse(com.facebook.buck.slb.HttpResponse) Path(java.nio.file.Path) MediaType(okhttp3.MediaType) ResponseBody(okhttp3.ResponseBody) HttpService(com.facebook.buck.slb.HttpService) Before(org.junit.Before) Charsets(com.google.common.base.Charsets) Request(okhttp3.Request) Buffer(okio.Buffer) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) Assert.assertTrue(org.junit.Assert.assertTrue) OkHttpResponseWrapper(com.facebook.buck.slb.OkHttpResponseWrapper) Matchers(org.hamcrest.Matchers) Set(java.util.Set) Test(org.junit.Test) IOException(java.io.IOException) Sets(com.google.common.collect.Sets) List(java.util.List) Paths(java.nio.file.Paths) LazyPath(com.facebook.buck.io.LazyPath) IncrementingFakeClock(com.facebook.buck.timing.IncrementingFakeClock) Optional(java.util.Optional) Protocol(okhttp3.Protocol) Assert.assertEquals(org.junit.Assert.assertEquals) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) Path(java.nio.file.Path) LazyPath(com.facebook.buck.io.LazyPath) RuleKey(com.facebook.buck.rules.RuleKey) FakeProjectFilesystem(com.facebook.buck.testutil.FakeProjectFilesystem) Request(okhttp3.Request) IncrementingFakeClock(com.facebook.buck.timing.IncrementingFakeClock) OkHttpResponseWrapper(com.facebook.buck.slb.OkHttpResponseWrapper) BuckEvent(com.facebook.buck.event.BuckEvent) Response(okhttp3.Response) HttpResponse(com.facebook.buck.slb.HttpResponse) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BuildId(com.facebook.buck.model.BuildId) ConsoleEvent(com.facebook.buck.event.ConsoleEvent) Test(org.junit.Test)

Example 3 with Response

use of oasis.names.tc.xacml._3_0.core.schema.wd_17.Response in project buck by facebook.

the class HttpArtifactCacheTest method testStore.

@Test
public void testStore() throws Exception {
    final RuleKey ruleKey = new RuleKey("00000000000000000000000000000000");
    final String data = "data";
    FakeProjectFilesystem filesystem = new FakeProjectFilesystem();
    Path output = Paths.get("output/file");
    filesystem.writeContentsToPath(data, output);
    final AtomicBoolean hasCalled = new AtomicBoolean(false);
    argsBuilder.setProjectFilesystem(filesystem);
    argsBuilder.setStoreClient(withMakeRequest(((path, requestBuilder) -> {
        Request request = requestBuilder.url(SERVER).build();
        hasCalled.set(true);
        Buffer buf = new Buffer();
        request.body().writeTo(buf);
        byte[] actualData = buf.readByteArray();
        byte[] expectedData;
        try (ByteArrayOutputStream out = new ByteArrayOutputStream();
            DataOutputStream dataOut = new DataOutputStream(out)) {
            dataOut.write(HttpArtifactCacheBinaryProtocol.createKeysHeader(ImmutableSet.of(ruleKey)));
            byte[] metadata = HttpArtifactCacheBinaryProtocol.createMetadataHeader(ImmutableSet.of(ruleKey), ImmutableMap.of(), ByteSource.wrap(data.getBytes(Charsets.UTF_8)));
            dataOut.writeInt(metadata.length);
            dataOut.write(metadata);
            dataOut.write(data.getBytes(Charsets.UTF_8));
            expectedData = out.toByteArray();
        }
        assertArrayEquals(expectedData, actualData);
        Response response = new Response.Builder().body(createDummyBody()).code(HttpURLConnection.HTTP_ACCEPTED).protocol(Protocol.HTTP_1_1).request(request).build();
        return new OkHttpResponseWrapper(response);
    })));
    HttpArtifactCache cache = new HttpArtifactCache(argsBuilder.build());
    cache.storeImpl(ArtifactInfo.builder().addRuleKeys(ruleKey).build(), output, createFinishedEventBuilder());
    assertTrue(hasCalled.get());
    cache.close();
}
Also used : Path(java.nio.file.Path) LazyPath(com.facebook.buck.io.LazyPath) Buffer(okio.Buffer) RuleKey(com.facebook.buck.rules.RuleKey) FakeProjectFilesystem(com.facebook.buck.testutil.FakeProjectFilesystem) DataOutputStream(java.io.DataOutputStream) Request(okhttp3.Request) ByteArrayOutputStream(java.io.ByteArrayOutputStream) OkHttpResponseWrapper(com.facebook.buck.slb.OkHttpResponseWrapper) Response(okhttp3.Response) HttpResponse(com.facebook.buck.slb.HttpResponse) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Test(org.junit.Test)

Example 4 with Response

use of oasis.names.tc.xacml._3_0.core.schema.wd_17.Response in project buck by facebook.

the class HttpArtifactCacheTest method testFetchWrongKey.

@Test
public void testFetchWrongKey() throws Exception {
    FakeProjectFilesystem filesystem = new FakeProjectFilesystem();
    final RuleKey ruleKey = new RuleKey("00000000000000000000000000000000");
    final RuleKey otherRuleKey = new RuleKey("11111111111111111111111111111111");
    final String data = "data";
    argsBuilder.setFetchClient(withMakeRequest(((path, requestBuilder) -> {
        Request request = requestBuilder.url(SERVER + path).build();
        Response response = new Response.Builder().request(request).protocol(Protocol.HTTP_1_1).code(HttpURLConnection.HTTP_OK).body(createResponseBody(ImmutableSet.of(otherRuleKey), ImmutableMap.of(), ByteSource.wrap(data.getBytes(Charsets.UTF_8)), data)).build();
        return new OkHttpResponseWrapper(response);
    })));
    HttpArtifactCache cache = new HttpArtifactCache(argsBuilder.build());
    Path output = Paths.get("output/file");
    CacheResult result = cache.fetch(ruleKey, LazyPath.ofInstance(output));
    assertEquals(CacheResultType.ERROR, result.getType());
    assertEquals(Optional.empty(), filesystem.readFileIfItExists(output));
    cache.close();
}
Also used : Response(okhttp3.Response) HttpResponse(com.facebook.buck.slb.HttpResponse) Path(java.nio.file.Path) LazyPath(com.facebook.buck.io.LazyPath) RuleKey(com.facebook.buck.rules.RuleKey) FakeProjectFilesystem(com.facebook.buck.testutil.FakeProjectFilesystem) Request(okhttp3.Request) OkHttpResponseWrapper(com.facebook.buck.slb.OkHttpResponseWrapper) Test(org.junit.Test)

Example 5 with Response

use of oasis.names.tc.xacml._3_0.core.schema.wd_17.Response in project buck by facebook.

the class ArtifactCaches method createHttpArtifactCache.

private static ArtifactCache createHttpArtifactCache(HttpCacheEntry cacheDescription, final String hostToReportToRemote, final BuckEventBus buckEventBus, ProjectFilesystem projectFilesystem, ListeningExecutorService httpWriteExecutorService, ArtifactCacheBuckConfig config, NetworkCacheFactory factory, boolean distributedBuildModeEnabled) {
    // Setup the default client to use.
    OkHttpClient.Builder storeClientBuilder = new OkHttpClient.Builder();
    storeClientBuilder.networkInterceptors().add(chain -> chain.proceed(chain.request().newBuilder().addHeader("X-BuckCache-User", stripNonAscii(System.getProperty("user.name", "<unknown>"))).addHeader("X-BuckCache-Host", stripNonAscii(hostToReportToRemote)).build()));
    int timeoutSeconds = cacheDescription.getTimeoutSeconds();
    setTimeouts(storeClientBuilder, timeoutSeconds);
    storeClientBuilder.connectionPool(new ConnectionPool(/* maxIdleConnections */
    (int) config.getThreadPoolSize(), /* keepAliveDurationMs */
    config.getThreadPoolKeepAliveDurationMillis(), TimeUnit.MILLISECONDS));
    // The artifact cache effectively only connects to a single host at a time. We should allow as
    // many concurrent connections to that host as we allow threads.
    Dispatcher dispatcher = new Dispatcher();
    dispatcher.setMaxRequestsPerHost((int) config.getThreadPoolSize());
    storeClientBuilder.dispatcher(dispatcher);
    final ImmutableMap<String, String> readHeaders = cacheDescription.getReadHeaders();
    final ImmutableMap<String, String> writeHeaders = cacheDescription.getWriteHeaders();
    // If write headers are specified, add them to every default client request.
    if (!writeHeaders.isEmpty()) {
        storeClientBuilder.networkInterceptors().add(chain -> chain.proceed(addHeadersToBuilder(chain.request().newBuilder(), writeHeaders).build()));
    }
    OkHttpClient storeClient = storeClientBuilder.build();
    // For fetches, use a client with a read timeout.
    OkHttpClient.Builder fetchClientBuilder = storeClient.newBuilder();
    setTimeouts(fetchClientBuilder, timeoutSeconds);
    // If read headers are specified, add them to every read client request.
    if (!readHeaders.isEmpty()) {
        fetchClientBuilder.networkInterceptors().add(chain -> chain.proceed(addHeadersToBuilder(chain.request().newBuilder(), readHeaders).build()));
    }
    fetchClientBuilder.networkInterceptors().add((chain -> {
        Response originalResponse = chain.proceed(chain.request());
        return originalResponse.newBuilder().body(new ProgressResponseBody(originalResponse.body(), buckEventBus)).build();
    }));
    OkHttpClient fetchClient = fetchClientBuilder.build();
    HttpService fetchService;
    HttpService storeService;
    switch(config.getLoadBalancingType()) {
        case CLIENT_SLB:
            HttpLoadBalancer clientSideSlb = config.getSlbConfig().createClientSideSlb(new DefaultClock(), buckEventBus, new CommandThreadFactory("ArtifactCaches.HttpLoadBalancer", SLB_THREAD_PRIORITY));
            fetchService = new RetryingHttpService(buckEventBus, new LoadBalancedService(clientSideSlb, fetchClient, buckEventBus), config.getMaxFetchRetries());
            storeService = new LoadBalancedService(clientSideSlb, storeClient, buckEventBus);
            break;
        case SINGLE_SERVER:
            URI url = cacheDescription.getUrl();
            fetchService = new SingleUriService(url, fetchClient);
            storeService = new SingleUriService(url, storeClient);
            break;
        default:
            throw new IllegalArgumentException("Unknown HttpLoadBalancer type: " + config.getLoadBalancingType());
    }
    String cacheName = cacheDescription.getName().map(input -> "http-" + input).orElse("http");
    boolean doStore = cacheDescription.getCacheReadMode().isDoStore();
    return factory.newInstance(NetworkCacheArgs.builder().setThriftEndpointPath(config.getHybridThriftEndpoint()).setCacheName(cacheName).setRepository(config.getRepository()).setScheduleType(config.getScheduleType()).setFetchClient(fetchService).setStoreClient(storeService).setDoStore(doStore).setProjectFilesystem(projectFilesystem).setBuckEventBus(buckEventBus).setHttpWriteExecutorService(httpWriteExecutorService).setErrorTextTemplate(cacheDescription.getErrorMessageFormat()).setDistributedBuildModeEnabled(distributedBuildModeEnabled).build());
}
Also used : ConnectionPool(okhttp3.ConnectionPool) BuckEventBus(com.facebook.buck.event.BuckEventBus) Okio(okio.Okio) Source(okio.Source) BytesReceivedEvent(com.facebook.buck.event.NetworkEvent.BytesReceivedEvent) RetryingHttpService(com.facebook.buck.slb.RetryingHttpService) Dispatcher(okhttp3.Dispatcher) ProjectFilesystem(com.facebook.buck.io.ProjectFilesystem) CommonGroups(com.facebook.buck.randomizedtrial.CommonGroups) BuckConfig(com.facebook.buck.cli.BuckConfig) ImmutableList(com.google.common.collect.ImmutableList) DefaultClock(com.facebook.buck.timing.DefaultClock) Map(java.util.Map) ForwardingSource(okio.ForwardingSource) Response(okhttp3.Response) HttpLoadBalancer(com.facebook.buck.slb.HttpLoadBalancer) URI(java.net.URI) LoadBalancedService(com.facebook.buck.slb.LoadBalancedService) Path(java.nio.file.Path) MediaType(okhttp3.MediaType) ResponseBody(okhttp3.ResponseBody) HttpService(com.facebook.buck.slb.HttpService) Logger(com.facebook.buck.log.Logger) Request(okhttp3.Request) Buffer(okio.Buffer) SingleUriService(com.facebook.buck.slb.SingleUriService) ImmutableSet(com.google.common.collect.ImmutableSet) ImmutableMap(com.google.common.collect.ImmutableMap) CharMatcher(com.google.common.base.CharMatcher) IOException(java.io.IOException) HumanReadableException(com.facebook.buck.util.HumanReadableException) CommandThreadFactory(com.facebook.buck.log.CommandThreadFactory) TimeUnit(java.util.concurrent.TimeUnit) BufferedSource(okio.BufferedSource) OkHttpClient(okhttp3.OkHttpClient) Optional(java.util.Optional) ConnectionPool(okhttp3.ConnectionPool) AsyncCloseable(com.facebook.buck.util.AsyncCloseable) RandomizedTrial(com.facebook.buck.randomizedtrial.RandomizedTrial) DirCacheExperimentEvent(com.facebook.buck.event.DirCacheExperimentEvent) ListeningExecutorService(com.google.common.util.concurrent.ListeningExecutorService) OkHttpClient(okhttp3.OkHttpClient) CommandThreadFactory(com.facebook.buck.log.CommandThreadFactory) Dispatcher(okhttp3.Dispatcher) RetryingHttpService(com.facebook.buck.slb.RetryingHttpService) URI(java.net.URI) Response(okhttp3.Response) RetryingHttpService(com.facebook.buck.slb.RetryingHttpService) HttpService(com.facebook.buck.slb.HttpService) DefaultClock(com.facebook.buck.timing.DefaultClock) LoadBalancedService(com.facebook.buck.slb.LoadBalancedService) SingleUriService(com.facebook.buck.slb.SingleUriService) HttpLoadBalancer(com.facebook.buck.slb.HttpLoadBalancer)

Aggregations

Response (okhttp3.Response)3634 Request (okhttp3.Request)2476 IOException (java.io.IOException)1531 OkHttpClient (okhttp3.OkHttpClient)855 Test (org.junit.Test)769 Call (okhttp3.Call)738 RequestBody (okhttp3.RequestBody)527 ResponseBody (okhttp3.ResponseBody)513 Response (retrofit2.Response)457 Callback (okhttp3.Callback)416 HttpUrl (okhttp3.HttpUrl)377 Test (org.junit.jupiter.api.Test)270 JSONObject (org.json.JSONObject)267 InputStream (java.io.InputStream)243 ArrayList (java.util.ArrayList)228 List (java.util.List)222 File (java.io.File)200 ServiceResponse (com.microsoft.rest.ServiceResponse)195 MockResponse (okhttp3.mockwebserver.MockResponse)192 Map (java.util.Map)189