Search in sources :

Example 1 with ConnectorLoader

use of org.talend.sdk.component.remoteengine.customizer.service.ConnectorLoader in project component-runtime by Talend.

the class Cli method run.

public static Object run(final String[] args) throws Exception {
    final Map<Class<?>, Object> services = new HashMap<>();
    services.put(ConnectorLoader.class, new ConnectorLoader());
    services.put(RemoteEngineCustomizer.class, new RemoteEngineCustomizer());
    Map<String, String> env = new HashMap<>();
    env.putAll(System.getenv());
    env.put("NOCOLOR", "true");
    env.put("NOLESS", "true");
    Environment.ENVIRONMENT_THREAD_LOCAL.set(new MyEnv(services, env));
    try {
        return new Main(Cli.class).exec(args);
    } finally {
        Environment.ENVIRONMENT_THREAD_LOCAL.remove();
    }
}
Also used : HashMap(java.util.HashMap) RemoteEngineCustomizer(org.talend.sdk.component.remoteengine.customizer.task.RemoteEngineCustomizer) Main(org.tomitribe.crest.Main) ConnectorLoader(org.talend.sdk.component.remoteengine.customizer.service.ConnectorLoader)

Example 2 with ConnectorLoader

use of org.talend.sdk.component.remoteengine.customizer.service.ConnectorLoader in project component-runtime by Talend.

the class RemoteEngineCustomizer method registerComponents.

// CHECKSTYLE:OFF
public void registerComponents(final String remoteEngineDirConf, final String workDirConf, final String cacheDirConf, final String baseImageConf, final String targetImageConf, final Collection<String> carPaths, final ImageType fromImageType, final ImageType targetImageType, final DockerConfiguration dockerConfiguration, final RegistryConfiguration registryConfiguration, final ConnectorLoader connectorLoader, final boolean updateOriginalFile) {
    // CHECKSTYLE:ON
    final Path remoteEngineDir = PathFactory.get(requireNonNull(remoteEngineDirConf, "Missing remote engine folder"));
    final Path workDir = PathFactory.get(workDirConf);
    final Path cacheDir = cacheDirConf.startsWith("${remote.engine.dir}/") ? remoteEngineDir.resolve(cacheDirConf.substring("${remote.engine.dir}/".length())) : PathFactory.get(cacheDirConf);
    final Collection<Path> cars = carPaths.stream().map(PathFactory::get).collect(toList());
    final List<Path> missingCars = cars.stream().filter(it -> !Files.exists(it)).collect(toList());
    if (!missingCars.isEmpty()) {
        throw new IllegalArgumentException("Missing component archives: " + missingCars);
    }
    try {
        final Properties filtering = IO.loadProperties(remoteEngineDir.resolve(".env"));
        final Path compose = remoteEngineDir.resolve("docker-compose.yml");
        final List<String> lines = IO.readFile(compose);
        final ImageAndLine connectorsImageRef = findImage(lines, "connectors");
        final String fromConnectorsImage = ofNullable(baseImageConf).filter(it -> !"auto".equals(it)).orElseGet(() -> filterPlaceholders(filtering, connectorsImageRef.image));
        final String toConnectorsImage = ofNullable(targetImageConf).filter(it -> !"auto".equals(it)).orElseGet(() -> timestampImage(fromConnectorsImage));
        final Containerizer targetContainer = targetImageType == ImageType.DOCKER ? Containerizer.to(dockerConfiguration.toImage(toConnectorsImage)) : Containerizer.to(registryConfiguration.toImage(toConnectorsImage));
        log.info("Building image '{}' from '{}' adding {}", toConnectorsImage, fromConnectorsImage, cars);
        final ExecutorService executor = Executors.newFixedThreadPool(Math.max(Runtime.getRuntime().availableProcessors(), 4));
        try (final AutoCloseable ignored = IO.autoDir(workDir)) {
            final Path registry = workDir.resolve("component-registry.properties");
            final Path registryDigest = workDir.resolve("component-registry-digest.properties");
            final AbsoluteUnixPath rootContainerPath = AbsoluteUnixPath.get("/opt/talend/connectors");
            final Instant now = Instant.now();
            final Collection<ConnectorLoader.ConnectorLayer> connectorsLayer = cars.stream().map(it -> connectorLoader.createConnectorLayer(rootContainerPath, workDir, it)).collect(toList());
            final Path baseCache = cacheDir.resolve("base");
            final Path appCache = cacheDir.resolve("application");
            log.info("Looking for component-registry.properties configuration, this can be a bit long...");
            final Image image;
            try {
                image = loadImage(fromConnectorsImage, toConnectorsImage, executor, baseCache, appCache, dockerConfiguration, fromImageType);
            } catch (final ExecutionException ee) {
                log.error("Please validate the connectors container image is an official one, " + "we don't support customizations on custom images or set the from image type");
                throw ee;
            }
            final Map<String, Properties> propertiesContents = image.getLayers().reverse().stream().map(it -> extractProperties(it, Stream.of(registry, registryDigest).map(f -> "opt/talend/connectors/" + f.getFileName()).collect(toSet()))).filter(it -> !it.isEmpty()).findFirst().orElseThrow(() -> new IllegalStateException("No layer containing the component registry in '" + fromConnectorsImage + "'"));
            final Properties componentProperties = requireNonNull(propertiesContents.get("/opt/talend/connectors/component-registry.properties"), "Missing component-registry.properties");
            connectorsLayer.forEach(c -> componentProperties.put(c.getGav().split(":")[1], c.getGav()));
            final Optional<Properties> digestProperties = ofNullable(propertiesContents.get("/opt/talend/connectors/component-registry-digest.properties"));
            digestProperties.ifPresent(digests -> connectorsLayer.forEach(cl -> cl.getDependencies().forEach((key, path) -> {
                try (final DigestOutputStream out = new DigestOutputStream(ByteStreams.nullOutputStream(), MessageDigest.getInstance("SHA-512"))) {
                    java.nio.file.Files.copy(path, out);
                    out.flush();
                    final byte[] digest = out.getMessageDigest().digest();
                    if (digests.put(key, Hex.hex(digest)) != null) {
                        log.info("'{}' digest will be overriding existing entry (entry='{}')", key, cl.getGav());
                    }
                } catch (final NoSuchAlgorithmException | IOException e) {
                    throw new IllegalStateException(e);
                }
            })));
            try (final Writer writer = Files.newBufferedWriter(registry)) {
                componentProperties.store(writer, "Generated by " + getClass().getName());
            }
            if (digestProperties.isPresent()) {
                try (final Writer writer = Files.newBufferedWriter(registryDigest)) {
                    digestProperties.orElseThrow(IllegalStateException::new).store(writer, "Generated by " + getClass().getName());
                }
            }
            log.info("Building image '{}'", toConnectorsImage);
            final JibContainerBuilder from = from(fromImageType, dockerConfiguration, fromConnectorsImage);
            connectorsLayer.stream().map(ConnectorLoader.ConnectorLayer::getLayer).forEach(from::addLayer);
            from.addLayer(LayerConfiguration.builder().addEntry(registry, rootContainerPath.resolve(registry.getFileName().toString()), FilePermissions.DEFAULT_FILE_PERMISSIONS, now).addEntry(registryDigest, rootContainerPath.resolve(registryDigest.getFileName().toString()), FilePermissions.DEFAULT_FILE_PERMISSIONS, now).build()).setCreationTime(now).containerize(targetContainer.setToolName("Talend Component Kit Remote Engine Customizer " + Versions.VERSION).setExecutorService(executor).setBaseImageLayersCache(baseCache).setApplicationLayersCache(appCache));
            if (updateOriginalFile) {
                rewriteCompose(remoteEngineDir, compose, lines, connectorsImageRef, toConnectorsImage);
                log.info("Restart your remote engine to take into account the new connector image");
            } else {
                log.info("You can update '{}' connectors container with image '{}'", compose, toConnectorsImage);
            }
        } finally {
            executor.shutdownNow();
            if (!executor.awaitTermination(5, SECONDS)) {
                log.warn("Executor is not terminated but exiting since it is not critical");
            }
        }
    } catch (final InterruptedException e) {
        Thread.currentThread().interrupt();
    } catch (final Exception e) {
        throw new IllegalStateException(e);
    }
}
Also used : DockerConfiguration(org.talend.sdk.component.remoteengine.customizer.model.DockerConfiguration) ImageReference(com.google.cloud.tools.jib.api.ImageReference) ZonedDateTime(java.time.ZonedDateTime) IO(org.talend.sdk.component.remoteengine.customizer.lang.IO) ImageType(org.talend.sdk.component.remoteengine.customizer.model.ImageType) RegistryConfiguration(org.talend.sdk.component.remoteengine.customizer.model.RegistryConfiguration) Future(java.util.concurrent.Future) ByteArrayInputStream(java.io.ByteArrayInputStream) Map(java.util.Map) Path(java.nio.file.Path) Collectors.toSet(java.util.stream.Collectors.toSet) Versions(org.talend.sdk.component.remoteengine.customizer.Versions) AbsoluteUnixPath(com.google.cloud.tools.jib.api.buildplan.AbsoluteUnixPath) Layer(com.google.cloud.tools.jib.image.Layer) Collection(java.util.Collection) StandardOpenOption(java.nio.file.StandardOpenOption) FilePermissions(com.google.cloud.tools.jib.api.buildplan.FilePermissions) Instant(java.time.Instant) InvalidImageReferenceException(com.google.cloud.tools.jib.api.InvalidImageReferenceException) StandardCharsets(java.nio.charset.StandardCharsets) Executors(java.util.concurrent.Executors) InvocationTargetException(java.lang.reflect.InvocationTargetException) DigestOutputStream(java.security.DigestOutputStream) List(java.util.List) Slf4j(lombok.extern.slf4j.Slf4j) Stream(java.util.stream.Stream) ImageFormat(com.google.cloud.tools.jib.api.buildplan.ImageFormat) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) Writer(java.io.Writer) ByteStreams(com.google.common.io.ByteStreams) Optional(java.util.Optional) EventHandlers(com.google.cloud.tools.jib.event.EventHandlers) Blobs(com.google.cloud.tools.jib.blob.Blobs) Image(com.google.cloud.tools.jib.image.Image) JibContainerBuilder(com.google.cloud.tools.jib.api.JibContainerBuilder) ImageConfiguration(com.google.cloud.tools.jib.configuration.ImageConfiguration) MessageDigest(java.security.MessageDigest) TarArchiveInputStream(org.apache.commons.compress.archivers.tar.TarArchiveInputStream) HashMap(java.util.HashMap) Hex(org.talend.sdk.component.remoteengine.customizer.lang.Hex) BuildContext(com.google.cloud.tools.jib.configuration.BuildContext) Jib(com.google.cloud.tools.jib.api.Jib) LayerConfiguration(com.google.cloud.tools.jib.api.LayerConfiguration) TarArchiveEntry(org.apache.commons.compress.archivers.tar.TarArchiveEntry) Objects.requireNonNull(java.util.Objects.requireNonNull) PathFactory(org.talend.sdk.component.remoteengine.customizer.lang.PathFactory) ExecutorService(java.util.concurrent.ExecutorService) GzipCompressorInputStream(org.apache.commons.compress.compressors.gzip.GzipCompressorInputStream) Properties(java.util.Properties) Files(java.nio.file.Files) Optional.ofNullable(java.util.Optional.ofNullable) IOException(java.io.IOException) Field(java.lang.reflect.Field) StepsRunner(com.google.cloud.tools.jib.builder.steps.StepsRunner) ExecutionException(java.util.concurrent.ExecutionException) LogEvent(com.google.cloud.tools.jib.api.LogEvent) Collectors.toList(java.util.stream.Collectors.toList) Reflects.asAccessible(org.talend.sdk.component.remoteengine.customizer.lang.Reflects.asAccessible) Containerizer(com.google.cloud.tools.jib.api.Containerizer) DateTimeFormatter(java.time.format.DateTimeFormatter) ProgressEventDispatcher(com.google.cloud.tools.jib.builder.ProgressEventDispatcher) ConnectorLoader(org.talend.sdk.component.remoteengine.customizer.service.ConnectorLoader) AllArgsConstructor(lombok.AllArgsConstructor) SECONDS(java.util.concurrent.TimeUnit.SECONDS) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) Properties(java.util.Properties) Image(com.google.cloud.tools.jib.image.Image) AbsoluteUnixPath(com.google.cloud.tools.jib.api.buildplan.AbsoluteUnixPath) DigestOutputStream(java.security.DigestOutputStream) Containerizer(com.google.cloud.tools.jib.api.Containerizer) JibContainerBuilder(com.google.cloud.tools.jib.api.JibContainerBuilder) ExecutionException(java.util.concurrent.ExecutionException) ConnectorLoader(org.talend.sdk.component.remoteengine.customizer.service.ConnectorLoader) Path(java.nio.file.Path) AbsoluteUnixPath(com.google.cloud.tools.jib.api.buildplan.AbsoluteUnixPath) Instant(java.time.Instant) IOException(java.io.IOException) InvalidImageReferenceException(com.google.cloud.tools.jib.api.InvalidImageReferenceException) InvocationTargetException(java.lang.reflect.InvocationTargetException) NoSuchAlgorithmException(java.security.NoSuchAlgorithmException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) ExecutorService(java.util.concurrent.ExecutorService) Writer(java.io.Writer)

Aggregations

HashMap (java.util.HashMap)2 Containerizer (com.google.cloud.tools.jib.api.Containerizer)1 ImageReference (com.google.cloud.tools.jib.api.ImageReference)1 InvalidImageReferenceException (com.google.cloud.tools.jib.api.InvalidImageReferenceException)1 Jib (com.google.cloud.tools.jib.api.Jib)1 JibContainerBuilder (com.google.cloud.tools.jib.api.JibContainerBuilder)1 LayerConfiguration (com.google.cloud.tools.jib.api.LayerConfiguration)1 LogEvent (com.google.cloud.tools.jib.api.LogEvent)1 AbsoluteUnixPath (com.google.cloud.tools.jib.api.buildplan.AbsoluteUnixPath)1 FilePermissions (com.google.cloud.tools.jib.api.buildplan.FilePermissions)1 ImageFormat (com.google.cloud.tools.jib.api.buildplan.ImageFormat)1 Blobs (com.google.cloud.tools.jib.blob.Blobs)1 ProgressEventDispatcher (com.google.cloud.tools.jib.builder.ProgressEventDispatcher)1 StepsRunner (com.google.cloud.tools.jib.builder.steps.StepsRunner)1 BuildContext (com.google.cloud.tools.jib.configuration.BuildContext)1 ImageConfiguration (com.google.cloud.tools.jib.configuration.ImageConfiguration)1 EventHandlers (com.google.cloud.tools.jib.event.EventHandlers)1 Image (com.google.cloud.tools.jib.image.Image)1 Layer (com.google.cloud.tools.jib.image.Layer)1 ByteStreams (com.google.common.io.ByteStreams)1