Examples with ProgressEventDispatcher com.google.cloud.tools.jib.builder.ProgressEventDispatcher used on opensource projects

Example 1 with ProgressEventDispatcher

use of com.google.cloud.tools.jib.builder.ProgressEventDispatcher in project jib by GoogleContainerTools.

the class PullBaseImageStep method tryMirrors.

@VisibleForTesting
Optional<ImagesAndRegistryClient> tryMirrors(BuildContext buildContext, ProgressEventDispatcher.Factory progressDispatcherFactory) throws LayerCountMismatchException, BadContainerConfigurationFormatException {
    EventHandlers eventHandlers = buildContext.getEventHandlers();
    Collection<Map.Entry<String, String>> mirrorEntries = buildContext.getRegistryMirrors().entries();
    try (ProgressEventDispatcher progressDispatcher1 = progressDispatcherFactory.create("trying mirrors", mirrorEntries.size());
        TimerEventDispatcher ignored1 = new TimerEventDispatcher(eventHandlers, "trying mirrors")) {
        for (Map.Entry<String, String> entry : mirrorEntries) {
            String registry = entry.getKey();
            String mirror = entry.getValue();
            eventHandlers.dispatch(LogEvent.debug("mirror config: " + registry + " --> " + mirror));
            if (!buildContext.getBaseImageConfiguration().getImageRegistry().equals(registry)) {
                progressDispatcher1.dispatchProgress(1);
                continue;
            }
            eventHandlers.dispatch(LogEvent.info("trying mirror " + mirror + " for the base image"));
            try (ProgressEventDispatcher progressDispatcher2 = progressDispatcher1.newChildProducer().create("trying mirror " + mirror, 2)) {
                RegistryClient registryClient = buildContext.newBaseImageRegistryClientFactory(mirror).newRegistryClient();
                List<Image> images = pullPublicImages(registryClient, progressDispatcher2);
                eventHandlers.dispatch(LogEvent.info("pulled manifest from mirror " + mirror));
                return Optional.of(new ImagesAndRegistryClient(images, registryClient));
            } catch (IOException | RegistryException ex) {
                // Ignore errors from this mirror and continue.
                eventHandlers.dispatch(LogEvent.debug("failed to get manifest from mirror " + mirror + ": " + ex.getMessage()));
            }
        }
        return Optional.empty();
    }
}

Example 2 with ProgressEventDispatcher

use of com.google.cloud.tools.jib.builder.ProgressEventDispatcher in project jib by GoogleContainerTools.

the class PullBaseImageStep method pullBaseImages.

/**
 * Pulls the base images specified in the platforms list.
 *
 * @param registryClient to communicate with remote registry
 * @param progressDispatcherFactory the {@link ProgressEventDispatcher.Factory} for emitting
 *     {@link ProgressEvent}s
 * @return the list of pulled base images and a registry client
 * @throws IOException when an I/O exception occurs during the pulling
 * @throws RegistryException if communicating with the registry caused a known error
 * @throws LayerCountMismatchException if the manifest and configuration contain conflicting layer
 *     information
 * @throws LayerPropertyNotFoundException if adding image layers fails
 * @throws BadContainerConfigurationFormatException if the container configuration is in a bad
 *     format
 */
private List<Image> pullBaseImages(RegistryClient registryClient, ProgressEventDispatcher.Factory progressDispatcherFactory) throws IOException, RegistryException, LayerPropertyNotFoundException, LayerCountMismatchException, BadContainerConfigurationFormatException {
    Cache cache = buildContext.getBaseImageLayersCache();
    EventHandlers eventHandlers = buildContext.getEventHandlers();
    ImageConfiguration baseImageConfig = buildContext.getBaseImageConfiguration();
    try (ProgressEventDispatcher progressDispatcher1 = progressDispatcherFactory.create("pulling base image manifest and container config", 2)) {
        ManifestAndDigest<?> manifestAndDigest = registryClient.pullManifest(baseImageConfig.getImageQualifier());
        eventHandlers.dispatch(LogEvent.lifecycle("Using base image with digest: " + manifestAndDigest.getDigest()));
        progressDispatcher1.dispatchProgress(1);
        ProgressEventDispatcher.Factory childProgressDispatcherFactory = progressDispatcher1.newChildProducer();
        ManifestTemplate manifestTemplate = manifestAndDigest.getManifest();
        if (manifestTemplate instanceof V21ManifestTemplate) {
            V21ManifestTemplate v21Manifest = (V21ManifestTemplate) manifestTemplate;
            cache.writeMetadata(baseImageConfig.getImage(), v21Manifest);
            return Collections.singletonList(JsonToImageTranslator.toImage(v21Manifest));
        } else if (manifestTemplate instanceof BuildableManifestTemplate) {
            // V22ManifestTemplate or OciManifestTemplate
            BuildableManifestTemplate imageManifest = (BuildableManifestTemplate) manifestTemplate;
            ContainerConfigurationTemplate containerConfig = pullContainerConfigJson(manifestAndDigest, registryClient, childProgressDispatcherFactory);
            PlatformChecker.checkManifestPlatform(buildContext, containerConfig);
            cache.writeMetadata(baseImageConfig.getImage(), imageManifest, containerConfig);
            return Collections.singletonList(JsonToImageTranslator.toImage(imageManifest, containerConfig));
        }
        // TODO: support OciIndexTemplate once AbstractManifestPuller starts to accept it.
        Verify.verify(manifestTemplate instanceof V22ManifestListTemplate);
        List<ManifestAndConfigTemplate> manifestsAndConfigs = new ArrayList<>();
        ImmutableList.Builder<Image> images = ImmutableList.builder();
        Set<Platform> platforms = buildContext.getContainerConfiguration().getPlatforms();
        try (ProgressEventDispatcher progressDispatcher2 = childProgressDispatcherFactory.create("pulling platform-specific manifests and container configs", 2L * platforms.size())) {
            // If a manifest list, search for the manifests matching the given platforms.
            for (Platform platform : platforms) {
                String message = "Searching for architecture=%s, os=%s in the base image manifest list";
                eventHandlers.dispatch(LogEvent.info(String.format(message, platform.getArchitecture(), platform.getOs())));
                String manifestDigest = lookUpPlatformSpecificImageManifest((V22ManifestListTemplate) manifestTemplate, platform);
                // TODO: pull multiple manifests (+ container configs) in parallel.
                ManifestAndDigest<?> imageManifestAndDigest = registryClient.pullManifest(manifestDigest);
                progressDispatcher2.dispatchProgress(1);
                BuildableManifestTemplate imageManifest = (BuildableManifestTemplate) imageManifestAndDigest.getManifest();
                ContainerConfigurationTemplate containerConfig = pullContainerConfigJson(imageManifestAndDigest, registryClient, progressDispatcher2.newChildProducer());
                manifestsAndConfigs.add(new ManifestAndConfigTemplate(imageManifest, containerConfig, manifestDigest));
                images.add(JsonToImageTranslator.toImage(imageManifest, containerConfig));
            }
        }
        cache.writeMetadata(baseImageConfig.getImage(), new ImageMetadataTemplate(manifestTemplate, /* manifest list */
        manifestsAndConfigs));
        return images.build();
    }
}

Example 3 with ProgressEventDispatcher

use of com.google.cloud.tools.jib.builder.ProgressEventDispatcher in project jib by GoogleContainerTools.

the class PullBaseImageStep method call.

@Override
public ImagesAndRegistryClient call() throws IOException, RegistryException, LayerPropertyNotFoundException, LayerCountMismatchException, BadContainerConfigurationFormatException, CacheCorruptedException, CredentialRetrievalException {
    EventHandlers eventHandlers = buildContext.getEventHandlers();
    try (ProgressEventDispatcher progressDispatcher = progressDispatcherFactory.create("pulling base image manifest", 4);
        TimerEventDispatcher ignored1 = new TimerEventDispatcher(eventHandlers, DESCRIPTION)) {
        // Skip this step if this is a scratch image
        ImageReference imageReference = buildContext.getBaseImageConfiguration().getImage();
        if (imageReference.isScratch()) {
            Set<Platform> platforms = buildContext.getContainerConfiguration().getPlatforms();
            Verify.verify(!platforms.isEmpty());
            eventHandlers.dispatch(LogEvent.progress("Getting scratch base image..."));
            ImmutableList.Builder<Image> images = ImmutableList.builder();
            for (Platform platform : platforms) {
                Image.Builder imageBuilder = Image.builder(buildContext.getTargetFormat());
                imageBuilder.setArchitecture(platform.getArchitecture()).setOs(platform.getOs());
                images.add(imageBuilder.build());
            }
            return new ImagesAndRegistryClient(images.build(), null);
        }
        eventHandlers.dispatch(LogEvent.progress("Getting manifest for base image " + imageReference + "..."));
        if (buildContext.isOffline()) {
            List<Image> images = getCachedBaseImages();
            if (!images.isEmpty()) {
                return new ImagesAndRegistryClient(images, null);
            }
            throw new IOException("Cannot run Jib in offline mode; " + imageReference + " not found in local Jib cache");
        } else if (imageReference.getDigest().isPresent()) {
            List<Image> images = getCachedBaseImages();
            if (!images.isEmpty()) {
                RegistryClient noAuthRegistryClient = buildContext.newBaseImageRegistryClientFactory().newRegistryClient();
                // https://github.com/GoogleContainerTools/jib/issues/2220
                return new ImagesAndRegistryClient(images, noAuthRegistryClient);
            }
        }
        Optional<ImagesAndRegistryClient> mirrorPull = tryMirrors(buildContext, progressDispatcher.newChildProducer());
        if (mirrorPull.isPresent()) {
            return mirrorPull.get();
        }
        try {
            // First, try with no credentials. This works with public GCR images (but not Docker Hub).
            // TODO: investigate if we should just pass credentials up front. However, this involves
            // some risk. https://github.com/GoogleContainerTools/jib/pull/2200#discussion_r359069026
            // contains some related discussions.
            RegistryClient noAuthRegistryClient = buildContext.newBaseImageRegistryClientFactory().newRegistryClient();
            return new ImagesAndRegistryClient(pullBaseImages(noAuthRegistryClient, progressDispatcher.newChildProducer()), noAuthRegistryClient);
        } catch (RegistryUnauthorizedException ex) {
            eventHandlers.dispatch(LogEvent.lifecycle("The base image requires auth. Trying again for " + imageReference + "..."));
            Credential credential = RegistryCredentialRetriever.getBaseImageCredential(buildContext).orElse(null);
            RegistryClient registryClient = buildContext.newBaseImageRegistryClientFactory().setCredential(credential).newRegistryClient();
            String wwwAuthenticate = ex.getHttpResponseException().getHeaders().getAuthenticate();
            if (wwwAuthenticate != null) {
                eventHandlers.dispatch(LogEvent.debug("WWW-Authenticate for " + imageReference + ": " + wwwAuthenticate));
                registryClient.authPullByWwwAuthenticate(wwwAuthenticate);
                return new ImagesAndRegistryClient(pullBaseImages(registryClient, progressDispatcher.newChildProducer()), registryClient);
            } else {
                // TODO: consider removing this fallback branch.
                if (credential != null && !credential.isOAuth2RefreshToken()) {
                    eventHandlers.dispatch(LogEvent.debug("Trying basic auth as fallback for " + imageReference + "..."));
                    registryClient.configureBasicAuth();
                    try {
                        return new ImagesAndRegistryClient(pullBaseImages(registryClient, progressDispatcher.newChildProducer()), registryClient);
                    } catch (RegistryUnauthorizedException ignored) {
                    // Fall back to try bearer auth.
                    }
                }
                eventHandlers.dispatch(LogEvent.debug("Trying bearer auth as fallback for " + imageReference + "..."));
                registryClient.doPullBearerAuth();
                return new ImagesAndRegistryClient(pullBaseImages(registryClient, progressDispatcher.newChildProducer()), registryClient);
            }
        }
    }
}

Example 4 with ProgressEventDispatcher

use of com.google.cloud.tools.jib.builder.ProgressEventDispatcher in project jib by GoogleContainerTools.

the class PushBlobStep method call.

@Override
public BlobDescriptor call() throws IOException, RegistryException {
    EventHandlers eventHandlers = buildContext.getEventHandlers();
    DescriptorDigest blobDigest = blobDescriptor.getDigest();
    try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create("pushing blob " + blobDigest, blobDescriptor.getSize());
        TimerEventDispatcher ignored = new TimerEventDispatcher(eventHandlers, DESCRIPTION + blobDescriptor);
        ThrottledAccumulatingConsumer throttledProgressReporter = new ThrottledAccumulatingConsumer(progressEventDispatcher::dispatchProgress)) {
        // check if the BLOB is available
        if (!forcePush && registryClient.checkBlob(blobDigest).isPresent()) {
            eventHandlers.dispatch(LogEvent.info("Skipping push; BLOB already exists on target registry : " + blobDescriptor));
            return blobDescriptor;
        }
        // If base and target images are in the same registry, then use mount/from to try mounting the
        // BLOB from the base image repository to the target image repository and possibly avoid
        // having to push the BLOB. See
        // https://docs.docker.com/registry/spec/api/#cross-repository-blob-mount for details.
        String baseRegistry = buildContext.getBaseImageConfiguration().getImageRegistry();
        String baseRepository = buildContext.getBaseImageConfiguration().getImageRepository();
        String targetRegistry = buildContext.getTargetImageConfiguration().getImageRegistry();
        String sourceRepository = targetRegistry.equals(baseRegistry) ? baseRepository : null;
        registryClient.pushBlob(blobDigest, blob, sourceRepository, throttledProgressReporter);
        return blobDescriptor;
    }
}

Example 5 with ProgressEventDispatcher

use of com.google.cloud.tools.jib.builder.ProgressEventDispatcher in project jib by GoogleContainerTools.

the class PushImageStep method makeListForManifestList.

static ImmutableList<PushImageStep> makeListForManifestList(BuildContext buildContext, ProgressEventDispatcher.Factory progressEventDispatcherFactory, RegistryClient registryClient, ManifestTemplate manifestList, boolean manifestListAlreadyExists) throws IOException {
    Set<String> tags = buildContext.getAllTargetImageTags();
    EventHandlers eventHandlers = buildContext.getEventHandlers();
    try (TimerEventDispatcher ignored = new TimerEventDispatcher(eventHandlers, "Preparing manifest list pushers");
        ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create("launching manifest list pushers", tags.size())) {
        boolean singlePlatform = buildContext.getContainerConfiguration().getPlatforms().size() == 1;
        if (singlePlatform) {
            // single image; no need to push a manifest list
            return ImmutableList.of();
        }
        if (JibSystemProperties.skipExistingImages() && manifestListAlreadyExists) {
            eventHandlers.dispatch(LogEvent.info("Skipping pushing manifest list; already exists."));
            return ImmutableList.of();
        }
        DescriptorDigest manifestListDigest = Digests.computeJsonDigest(manifestList);
        return tags.stream().map(tag -> new PushImageStep(buildContext, progressEventDispatcher.newChildProducer(), registryClient, manifestList, tag, manifestListDigest, // return value and type.
        manifestListDigest)).collect(ImmutableList.toImmutableList());
    }
}