use of com.google.cloud.tools.jib.api.Containerizer in project jib by google.
the class War method call.
@Override
public Integer call() {
commonCliOptions.validate();
SingleThreadedExecutor executor = new SingleThreadedExecutor();
ConsoleLogger logger = CliLogger.newLogger(commonCliOptions.getVerbosity(), commonCliOptions.getHttpTrace(), commonCliOptions.getConsoleOutput(), spec.commandLine().getOut(), spec.commandLine().getErr(), executor);
Future<Optional<String>> updateCheckFuture = Futures.immediateFuture(Optional.empty());
try {
JibCli.configureHttpLogging(commonCliOptions.getHttpTrace().toJulLevel());
GlobalConfig globalConfig = GlobalConfig.readConfig();
updateCheckFuture = JibCli.newUpdateChecker(globalConfig, commonCliOptions.getVerbosity(), logEvent -> logger.log(logEvent.getLevel(), logEvent.getMessage()));
if (!Files.exists(warFile)) {
logger.log(LogEvent.Level.ERROR, "The file path provided does not exist: " + warFile);
return 1;
}
if (Files.isDirectory(warFile)) {
logger.log(LogEvent.Level.ERROR, "The file path provided is for a directory. Please provide a path to a WAR: " + warFile);
return 1;
}
Path warFileParentDir = Verify.verifyNotNull(warFile.toAbsolutePath().getParent());
CacheDirectories cacheDirectories = CacheDirectories.from(commonCliOptions, warFileParentDir);
ArtifactProcessor processor = ArtifactProcessors.fromWar(warFile, cacheDirectories, this, commonContainerConfigCliOptions);
JibContainerBuilder containerBuilder = WarFiles.toJibContainerBuilder(processor, commonCliOptions, commonContainerConfigCliOptions, logger);
Containerizer containerizer = Containerizers.from(commonCliOptions, logger, cacheDirectories);
// Enable registry mirrors
Multimaps.asMap(globalConfig.getRegistryMirrors()).forEach(containerizer::addRegistryMirrors);
JibContainer jibContainer = containerBuilder.containerize(containerizer);
JibCli.writeImageJson(commonCliOptions.getImageJsonPath(), jibContainer);
} catch (InterruptedException ex) {
JibCli.logTerminatingException(logger, ex, commonCliOptions.isStacktrace());
Thread.currentThread().interrupt();
return 1;
} catch (Exception ex) {
JibCli.logTerminatingException(logger, ex, commonCliOptions.isStacktrace());
return 1;
} finally {
JibCli.finishUpdateChecker(logger, updateCheckFuture);
executor.shutDownAndAwaitTermination(Duration.ofSeconds(3));
}
return 0;
}
use of com.google.cloud.tools.jib.api.Containerizer in project component-runtime by Talend.
the class RemoteEngineCustomizer method registerComponents.
// CHECKSTYLE:OFF
public void registerComponents(final String remoteEngineDirConf, final String workDirConf, final String cacheDirConf, final String baseImageConf, final String targetImageConf, final Collection<String> carPaths, final ImageType fromImageType, final ImageType targetImageType, final DockerConfiguration dockerConfiguration, final RegistryConfiguration registryConfiguration, final ConnectorLoader connectorLoader, final boolean updateOriginalFile) {
// CHECKSTYLE:ON
final Path remoteEngineDir = PathFactory.get(requireNonNull(remoteEngineDirConf, "Missing remote engine folder"));
final Path workDir = PathFactory.get(workDirConf);
final Path cacheDir = cacheDirConf.startsWith("${remote.engine.dir}/") ? remoteEngineDir.resolve(cacheDirConf.substring("${remote.engine.dir}/".length())) : PathFactory.get(cacheDirConf);
final Collection<Path> cars = carPaths.stream().map(PathFactory::get).collect(toList());
final List<Path> missingCars = cars.stream().filter(it -> !Files.exists(it)).collect(toList());
if (!missingCars.isEmpty()) {
throw new IllegalArgumentException("Missing component archives: " + missingCars);
}
try {
final Properties filtering = IO.loadProperties(remoteEngineDir.resolve(".env"));
final Path compose = remoteEngineDir.resolve("docker-compose.yml");
final List<String> lines = IO.readFile(compose);
final ImageAndLine connectorsImageRef = findImage(lines, "connectors");
final String fromConnectorsImage = ofNullable(baseImageConf).filter(it -> !"auto".equals(it)).orElseGet(() -> filterPlaceholders(filtering, connectorsImageRef.image));
final String toConnectorsImage = ofNullable(targetImageConf).filter(it -> !"auto".equals(it)).orElseGet(() -> timestampImage(fromConnectorsImage));
final Containerizer targetContainer = targetImageType == ImageType.DOCKER ? Containerizer.to(dockerConfiguration.toImage(toConnectorsImage)) : Containerizer.to(registryConfiguration.toImage(toConnectorsImage));
log.info("Building image '{}' from '{}' adding {}", toConnectorsImage, fromConnectorsImage, cars);
final ExecutorService executor = Executors.newFixedThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), 4));
try (final AutoCloseable ignored = IO.autoDir(workDir)) {
final Path registry = workDir.resolve("component-registry.properties");
final Path registryDigest = workDir.resolve("component-registry-digest.properties");
final AbsoluteUnixPath rootContainerPath = AbsoluteUnixPath.get("/opt/talend/connectors");
final Instant now = Instant.now();
final Collection<ConnectorLoader.ConnectorLayer> connectorsLayer = cars.stream().map(it -> connectorLoader.createConnectorLayer(rootContainerPath, workDir, it)).collect(toList());
final Path baseCache = cacheDir.resolve("base");
final Path appCache = cacheDir.resolve("application");
log.info("Looking for component-registry.properties configuration, this can be a bit long...");
final Image image;
try {
image = loadImage(fromConnectorsImage, toConnectorsImage, executor, baseCache, appCache, dockerConfiguration, fromImageType);
} catch (final ExecutionException ee) {
log.error("Please validate the connectors container image is an official one, " + "we don't support customizations on custom images or set the from image type");
throw ee;
}
final Map<String, Properties> propertiesContents = image.getLayers().reverse().stream().map(it -> extractProperties(it, Stream.of(registry, registryDigest).map(f -> "opt/talend/connectors/" + f.getFileName()).collect(toSet()))).filter(it -> !it.isEmpty()).findFirst().orElseThrow(() -> new IllegalStateException("No layer containing the component registry in '" + fromConnectorsImage + "'"));
final Properties componentProperties = requireNonNull(propertiesContents.get("/opt/talend/connectors/component-registry.properties"), "Missing component-registry.properties");
connectorsLayer.forEach(c -> componentProperties.put(c.getGav().split(":")[1], c.getGav()));
final Optional<Properties> digestProperties = ofNullable(propertiesContents.get("/opt/talend/connectors/component-registry-digest.properties"));
digestProperties.ifPresent(digests -> connectorsLayer.forEach(cl -> cl.getDependencies().forEach((key, path) -> {
try (final DigestOutputStream out = new DigestOutputStream(ByteStreams.nullOutputStream(), MessageDigest.getInstance("SHA-512"))) {
java.nio.file.Files.copy(path, out);
out.flush();
final byte[] digest = out.getMessageDigest().digest();
if (digests.put(key, Hex.hex(digest)) != null) {
log.info("'{}' digest will be overriding existing entry (entry='{}')", key, cl.getGav());
}
} catch (final NoSuchAlgorithmException | IOException e) {
throw new IllegalStateException(e);
}
})));
try (final Writer writer = Files.newBufferedWriter(registry)) {
componentProperties.store(writer, "Generated by " + getClass().getName());
}
if (digestProperties.isPresent()) {
try (final Writer writer = Files.newBufferedWriter(registryDigest)) {
digestProperties.orElseThrow(IllegalStateException::new).store(writer, "Generated by " + getClass().getName());
}
}
log.info("Building image '{}'", toConnectorsImage);
final JibContainerBuilder from = from(fromImageType, dockerConfiguration, fromConnectorsImage);
connectorsLayer.stream().map(ConnectorLoader.ConnectorLayer::getLayer).forEach(from::addLayer);
from.addLayer(LayerConfiguration.builder().addEntry(registry, rootContainerPath.resolve(registry.getFileName().toString()), FilePermissions.DEFAULT_FILE_PERMISSIONS, now).addEntry(registryDigest, rootContainerPath.resolve(registryDigest.getFileName().toString()), FilePermissions.DEFAULT_FILE_PERMISSIONS, now).build()).setCreationTime(now).containerize(targetContainer.setToolName("Talend Component Kit Remote Engine Customizer " + Versions.VERSION).setExecutorService(executor).setBaseImageLayersCache(baseCache).setApplicationLayersCache(appCache));
if (updateOriginalFile) {
rewriteCompose(remoteEngineDir, compose, lines, connectorsImageRef, toConnectorsImage);
log.info("Restart your remote engine to take into account the new connector image");
} else {
log.info("You can update '{}' connectors container with image '{}'", compose, toConnectorsImage);
}
} finally {
executor.shutdownNow();
if (!executor.awaitTermination(5, SECONDS)) {
log.warn("Executor is not terminated but exiting since it is not critical");
}
}
} catch (final InterruptedException e) {
Thread.currentThread().interrupt();
} catch (final Exception e) {
throw new IllegalStateException(e);
}
}
use of com.google.cloud.tools.jib.api.Containerizer in project jib by google.
the class Build method call.
@Override
public Integer call() {
commonCliOptions.validate();
Path buildFile = getBuildFile();
SingleThreadedExecutor executor = new SingleThreadedExecutor();
ConsoleLogger logger = CliLogger.newLogger(commonCliOptions.getVerbosity(), commonCliOptions.getHttpTrace(), commonCliOptions.getConsoleOutput(), spec.commandLine().getOut(), spec.commandLine().getErr(), executor);
Future<Optional<String>> updateCheckFuture = Futures.immediateFuture(Optional.empty());
try {
JibCli.configureHttpLogging(commonCliOptions.getHttpTrace().toJulLevel());
GlobalConfig globalConfig = GlobalConfig.readConfig();
updateCheckFuture = JibCli.newUpdateChecker(globalConfig, commonCliOptions.getVerbosity(), logEvent -> logger.log(logEvent.getLevel(), logEvent.getMessage()));
if (!Files.isReadable(buildFile)) {
logger.log(LogEvent.Level.ERROR, "The Build File YAML either does not exist or cannot be opened for reading: " + buildFile);
return 1;
}
if (!Files.isRegularFile(buildFile)) {
logger.log(LogEvent.Level.ERROR, "Build File YAML path is a not a file: " + buildFile);
return 1;
}
CacheDirectories cacheDirectories = CacheDirectories.from(commonCliOptions, contextRoot);
Containerizer containerizer = Containerizers.from(commonCliOptions, logger, cacheDirectories);
JibContainerBuilder containerBuilder = BuildFiles.toJibContainerBuilder(contextRoot, buildFile, this, commonCliOptions, logger);
// Enable registry mirrors
Multimaps.asMap(globalConfig.getRegistryMirrors()).forEach(containerizer::addRegistryMirrors);
JibContainer jibContainer = containerBuilder.containerize(containerizer);
JibCli.writeImageJson(commonCliOptions.getImageJsonPath(), jibContainer);
} catch (InterruptedException ex) {
JibCli.logTerminatingException(logger, ex, commonCliOptions.isStacktrace());
Thread.currentThread().interrupt();
return 1;
} catch (Exception ex) {
JibCli.logTerminatingException(logger, ex, commonCliOptions.isStacktrace());
return 1;
} finally {
JibCli.finishUpdateChecker(logger, updateCheckFuture);
executor.shutDownAndAwaitTermination(Duration.ofSeconds(3));
}
return 0;
}
use of com.google.cloud.tools.jib.api.Containerizer in project jib by google.
the class Containerizers method create.
private static Containerizer create(CommonCliOptions commonCliOptions, ConsoleLogger logger) throws InvalidImageReferenceException, FileNotFoundException {
String imageSpec = commonCliOptions.getTargetImage();
if (imageSpec.startsWith(DOCKER_DAEMON_IMAGE_PREFIX)) {
// TODO: allow setting docker env and docker executable (along with path/env)
return Containerizer.to(DockerDaemonImage.named(imageSpec.replaceFirst(DOCKER_DAEMON_IMAGE_PREFIX, "")));
}
if (imageSpec.startsWith(TAR_IMAGE_PREFIX)) {
return Containerizer.to(TarImage.at(Paths.get(imageSpec.replaceFirst(TAR_IMAGE_PREFIX, ""))).named(commonCliOptions.getName()));
}
ImageReference imageReference = ImageReference.parse(imageSpec.replaceFirst(REGISTRY_IMAGE_PREFIX, ""));
RegistryImage registryImage = RegistryImage.named(imageReference);
DefaultCredentialRetrievers defaultCredentialRetrievers = DefaultCredentialRetrievers.init(CredentialRetrieverFactory.forImage(imageReference, logEvent -> logger.log(logEvent.getLevel(), logEvent.getMessage())));
Credentials.getToCredentialRetrievers(commonCliOptions, defaultCredentialRetrievers).forEach(registryImage::addCredentialRetriever);
return Containerizer.to(registryImage);
}
use of com.google.cloud.tools.jib.api.Containerizer in project jib by google.
the class Jar method call.
@Override
public Integer call() {
commonCliOptions.validate();
SingleThreadedExecutor executor = new SingleThreadedExecutor();
ConsoleLogger logger = CliLogger.newLogger(commonCliOptions.getVerbosity(), commonCliOptions.getHttpTrace(), commonCliOptions.getConsoleOutput(), spec.commandLine().getOut(), spec.commandLine().getErr(), executor);
Future<Optional<String>> updateCheckFuture = Futures.immediateFuture(Optional.empty());
try {
JibCli.configureHttpLogging(commonCliOptions.getHttpTrace().toJulLevel());
GlobalConfig globalConfig = GlobalConfig.readConfig();
updateCheckFuture = JibCli.newUpdateChecker(globalConfig, commonCliOptions.getVerbosity(), logEvent -> logger.log(logEvent.getLevel(), logEvent.getMessage()));
if (!Files.exists(jarFile)) {
logger.log(LogEvent.Level.ERROR, "The file path provided does not exist: " + jarFile);
return 1;
}
if (Files.isDirectory(jarFile)) {
logger.log(LogEvent.Level.ERROR, "The file path provided is for a directory. Please provide a path to a JAR: " + jarFile);
return 1;
}
if (!commonContainerConfigCliOptions.getEntrypoint().isEmpty() && !jvmFlags.isEmpty()) {
logger.log(LogEvent.Level.WARN, "--jvm-flags is ignored when --entrypoint is specified");
}
Path jarFileParentDir = Verify.verifyNotNull(jarFile.toAbsolutePath().getParent());
CacheDirectories cacheDirectories = CacheDirectories.from(commonCliOptions, jarFileParentDir);
ArtifactProcessor processor = ArtifactProcessors.fromJar(jarFile, cacheDirectories, this, commonContainerConfigCliOptions);
JibContainerBuilder containerBuilder = JarFiles.toJibContainerBuilder(processor, this, commonCliOptions, commonContainerConfigCliOptions, logger);
Containerizer containerizer = Containerizers.from(commonCliOptions, logger, cacheDirectories);
// Enable registry mirrors
Multimaps.asMap(globalConfig.getRegistryMirrors()).forEach(containerizer::addRegistryMirrors);
JibContainer jibContainer = containerBuilder.containerize(containerizer);
JibCli.writeImageJson(commonCliOptions.getImageJsonPath(), jibContainer);
} catch (InterruptedException ex) {
JibCli.logTerminatingException(logger, ex, commonCliOptions.isStacktrace());
Thread.currentThread().interrupt();
return 1;
} catch (Exception ex) {
JibCli.logTerminatingException(logger, ex, commonCliOptions.isStacktrace());
return 1;
} finally {
JibCli.finishUpdateChecker(logger, updateCheckFuture);
executor.shutDownAndAwaitTermination(Duration.ofSeconds(3));
}
return 0;
}
Aggregations