Examples with TimerEventDispatcher com.google.cloud.tools.jib.builder.TimerEventDispatcher used on opensource projects

Example 26 with TimerEventDispatcher

use of com.google.cloud.tools.jib.builder.TimerEventDispatcher in project jib by GoogleContainerTools.

the class RegistryClient method pushBlob.

/**
 * Pushes the BLOB. If the {@code sourceRepository} is provided then the remote registry may skip
 * if the BLOB already exists on the registry.
 *
 * @param blobDigest the digest of the BLOB, used for existence-check
 * @param blob the BLOB to push
 * @param sourceRepository if pushing to the same registry then the source image, or {@code null}
 *     otherwise; used to optimize the BLOB push
 * @param writtenByteCountListener listens on byte count written to the registry during the push
 * @return {@code true} if the BLOB already exists on the registry and pushing was skipped; false
 *     if the BLOB was pushed
 * @throws IOException if communicating with the endpoint fails
 * @throws RegistryException if communicating with the endpoint fails
 */
public boolean pushBlob(DescriptorDigest blobDigest, Blob blob, @Nullable String sourceRepository, Consumer<Long> writtenByteCountListener) throws IOException, RegistryException {
    if (isBearerAuth(authorization.get()) && readOnlyBearerAuth) {
        throw new IllegalStateException("push may fail with pull-only bearer auth token");
    }
    if (sourceRepository != null && !(JibSystemProperties.useCrossRepositoryBlobMounts() && canAttemptBlobMount(authorization.get(), sourceRepository))) {
        // don't bother requesting a cross-repository blob-mount if we don't have access
        sourceRepository = null;
    }
    BlobPusher blobPusher = new BlobPusher(registryEndpointRequestProperties, blobDigest, blob, sourceRepository);
    try (TimerEventDispatcher timerEventDispatcher = new TimerEventDispatcher(eventHandlers, "pushBlob")) {
        try (TimerEventDispatcher timerEventDispatcher2 = timerEventDispatcher.subTimer("pushBlob POST " + blobDigest)) {
            // POST /v2/<name>/blobs/uploads/?mount={blob.digest}&from={sourceRepository}
            // POST /v2/<name>/blobs/uploads/
            Optional<URL> patchLocation = callRegistryEndpoint(blobPusher.initializer());
            if (!patchLocation.isPresent()) {
                // The BLOB exists already.
                return true;
            }
            timerEventDispatcher2.lap("pushBlob PATCH " + blobDigest);
            // PATCH <Location> with BLOB
            URL putLocation = callRegistryEndpoint(blobPusher.writer(patchLocation.get(), writtenByteCountListener));
            timerEventDispatcher2.lap("pushBlob PUT " + blobDigest);
            // PUT <Location>?digest={blob.digest}
            callRegistryEndpoint(blobPusher.committer(putLocation));
            return false;
        }
    }
}

Example 27 with TimerEventDispatcher

use of com.google.cloud.tools.jib.builder.TimerEventDispatcher in project jib by GoogleContainerTools.

the class CheckManifestStep method call.

@Override
public Optional<ManifestAndDigest<ManifestTemplate>> call() throws IOException, RegistryException {
    DescriptorDigest manifestDigest = Digests.computeJsonDigest(manifestTemplate);
    EventHandlers eventHandlers = buildContext.getEventHandlers();
    try (TimerEventDispatcher ignored = new TimerEventDispatcher(eventHandlers, DESCRIPTION);
        ProgressEventDispatcher ignored2 = progressEventDispatcherFactory.create("checking existence of manifest for " + manifestDigest, 1)) {
        eventHandlers.dispatch(LogEvent.info("Checking existence of manifest for " + manifestDigest + "..."));
        if (!JibSystemProperties.skipExistingImages()) {
            eventHandlers.dispatch(LogEvent.info("Skipping manifest existence check; system property set to false"));
            return Optional.empty();
        }
        return registryClient.checkManifest(manifestDigest.toString());
    }
}

Example 28 with TimerEventDispatcher

use of com.google.cloud.tools.jib.builder.TimerEventDispatcher in project jib by GoogleContainerTools.

the class LoadDockerStep method call.

@Override
public BuildResult call() throws InterruptedException, IOException {
    EventHandlers eventHandlers = buildContext.getEventHandlers();
    try (TimerEventDispatcher ignored = new TimerEventDispatcher(eventHandlers, "Loading to Docker daemon")) {
        eventHandlers.dispatch(LogEvent.progress("Loading to Docker daemon..."));
        ImageTarball imageTarball = new ImageTarball(builtImage, buildContext.getTargetImageConfiguration().getImage(), buildContext.getAllTargetImageTags());
        // See https://github.com/GoogleContainerTools/jib/pull/1960#discussion_r321898390
        try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create("loading to Docker daemon", imageTarball.getTotalLayerSize());
            ThrottledAccumulatingConsumer throttledProgressReporter = new ThrottledAccumulatingConsumer(progressEventDispatcher::dispatchProgress)) {
            // Load the image to docker daemon.
            eventHandlers.dispatch(LogEvent.debug(dockerClient.load(imageTarball, throttledProgressReporter)));
            return BuildResult.fromImage(builtImage, buildContext.getTargetFormat());
        }
    }
}

Example 29 with TimerEventDispatcher

use of com.google.cloud.tools.jib.builder.TimerEventDispatcher in project jib by GoogleContainerTools.

the class LocalBaseImageSteps method cacheDockerImageTar.

@VisibleForTesting
static LocalImage cacheDockerImageTar(BuildContext buildContext, Path tarPath, ProgressEventDispatcher.Factory progressEventDispatcherFactory, TempDirectoryProvider tempDirectoryProvider) throws IOException, LayerCountMismatchException {
    ExecutorService executorService = buildContext.getExecutorService();
    Path destination = tempDirectoryProvider.newDirectory();
    try (TimerEventDispatcher ignored = new TimerEventDispatcher(buildContext.getEventHandlers(), "Extracting tar " + tarPath + " into " + destination)) {
        TarExtractor.extract(tarPath, destination);
        InputStream manifestStream = Files.newInputStream(destination.resolve("manifest.json"));
        DockerManifestEntryTemplate loadManifest = new ObjectMapper().configure(MapperFeature.ACCEPT_CASE_INSENSITIVE_PROPERTIES, true).readValue(manifestStream, DockerManifestEntryTemplate[].class)[0];
        manifestStream.close();
        Path configPath = destination.resolve(loadManifest.getConfig());
        ContainerConfigurationTemplate configurationTemplate = JsonTemplateMapper.readJsonFromFile(configPath, ContainerConfigurationTemplate.class);
        // Don't compute the digest of the loaded Java JSON instance.
        BlobDescriptor originalConfigDescriptor = Blobs.from(configPath).writeTo(ByteStreams.nullOutputStream());
        List<String> layerFiles = loadManifest.getLayerFiles();
        if (configurationTemplate.getLayerCount() != layerFiles.size()) {
            throw new LayerCountMismatchException("Invalid base image format: manifest contains " + layerFiles.size() + " layers, but container configuration contains " + configurationTemplate.getLayerCount() + " layers");
        }
        buildContext.getBaseImageLayersCache().writeLocalConfig(originalConfigDescriptor.getDigest(), configurationTemplate);
        // Check the first layer to see if the layers are compressed already. 'docker save' output
        // is uncompressed, but a jib-built tar has compressed layers.
        boolean layersAreCompressed = !layerFiles.isEmpty() && isGzipped(destination.resolve(layerFiles.get(0)));
        // Process layer blobs
        try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create("processing base image layers", layerFiles.size())) {
            // Start compressing layers in parallel
            List<Future<PreparedLayer>> preparedLayers = new ArrayList<>();
            for (int index = 0; index < layerFiles.size(); index++) {
                Path layerFile = destination.resolve(layerFiles.get(index));
                DescriptorDigest diffId = configurationTemplate.getLayerDiffId(index);
                ProgressEventDispatcher.Factory layerProgressDispatcherFactory = progressEventDispatcher.newChildProducer();
                preparedLayers.add(executorService.submit(() -> compressAndCacheTarLayer(buildContext.getBaseImageLayersCache(), diffId, layerFile, layersAreCompressed, layerProgressDispatcherFactory)));
            }
            return new LocalImage(preparedLayers, configurationTemplate);
        }
    }
}

Example 30 with TimerEventDispatcher

use of com.google.cloud.tools.jib.builder.TimerEventDispatcher in project jib by GoogleContainerTools.

the class ObtainBaseImageLayerStep method call.

@Override
public PreparedLayer call() throws IOException, CacheCorruptedException, RegistryException {
    EventHandlers eventHandlers = buildContext.getEventHandlers();
    DescriptorDigest layerDigest = layer.getBlobDescriptor().getDigest();
    try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create("checking base image layer " + layerDigest, 1);
        TimerEventDispatcher ignored = new TimerEventDispatcher(eventHandlers, String.format(DESCRIPTION, layerDigest))) {
        StateInTarget stateInTarget = blobExistenceChecker.check(layerDigest);
        if (stateInTarget == StateInTarget.EXISTING) {
            eventHandlers.dispatch(LogEvent.info("Skipping pull; BLOB already exists on target registry : " + layer.getBlobDescriptor()));
            return new PreparedLayer.Builder(layer).setStateInTarget(stateInTarget).build();
        }
        Cache cache = buildContext.getBaseImageLayersCache();
        // Checks if the layer already exists in the cache.
        Optional<CachedLayer> optionalCachedLayer = cache.retrieve(layerDigest);
        if (optionalCachedLayer.isPresent()) {
            CachedLayer cachedLayer = optionalCachedLayer.get();
            return new PreparedLayer.Builder(cachedLayer).setStateInTarget(stateInTarget).build();
        } else if (buildContext.isOffline()) {
            throw new IOException("Cannot run Jib in offline mode; local Jib cache for base image is missing image layer " + layerDigest + ". Rerun Jib in online mode with \"-Djib.alwaysCacheBaseImage=true\" to " + "re-download the base image layers.");
        }
        try (ThrottledProgressEventDispatcherWrapper progressEventDispatcherWrapper = new ThrottledProgressEventDispatcherWrapper(progressEventDispatcher.newChildProducer(), "pulling base image layer " + layerDigest)) {
            CachedLayer cachedLayer = cache.writeCompressedLayer(Verify.verifyNotNull(registryClient).pullBlob(layerDigest, progressEventDispatcherWrapper::setProgressTarget, progressEventDispatcherWrapper::dispatchProgress));
            return new PreparedLayer.Builder(cachedLayer).setStateInTarget(stateInTarget).build();
        }
    }
}