Search in sources :

Example 51 with InfrastructureException

use of org.eclipse.che.api.workspace.server.spi.InfrastructureException in project che-server by eclipse-che.

the class KubernetesInternalRuntime method checkServers.

/**
 * Returns a function, the result of which the completable stage that performs servers checks and
 * start of servers probes.
 */
private Function<Void, CompletionStage<Void>> checkServers(List<CompletableFuture<?>> toCancelFutures, KubernetesMachineImpl machine) {
    // Need to get active span here to allow use in returned function;
    final Span activeSpan = tracer.activeSpan();
    return ignored -> {
        // Span must be created within this lambda block, otherwise the span begins as soon as
        // this function is called (i.e. before the previous steps in the machine boot chain
        // are complete
        final Span tracingSpan = tracer.buildSpan(CHECK_SERVERS).asChildOf(activeSpan).start();
        TracingTags.WORKSPACE_ID.set(tracingSpan, getContext().getIdentity().getWorkspaceId());
        TracingTags.MACHINE_NAME.set(tracingSpan, machine.getName());
        // This completable future is used to unity the servers checks and start of probes
        final CompletableFuture<Void> serversAndProbesFuture = new CompletableFuture<>();
        final String machineName = machine.getName();
        final RuntimeIdentity runtimeId = getContext().getIdentity();
        final ServersChecker serverCheck = serverCheckerFactory.create(runtimeId, machineName, machine.getServers());
        final CompletableFuture<?> serversReadyFuture;
        LOG.debug("Performing servers check for machine '{}' in workspace '{}'", machineName, runtimeId.getWorkspaceId());
        try {
            serversReadyFuture = serverCheck.startAsync(new ServerReadinessHandler(machineName));
            toCancelFutures.add(serversReadyFuture);
            serversAndProbesFuture.whenComplete((ok, ex) -> {
                LOG.debug("Servers checks done for machine '{}' in workspace '{}'", machineName, runtimeId.getWorkspaceId());
                serversReadyFuture.cancel(true);
            });
        } catch (InfrastructureException ex) {
            serversAndProbesFuture.completeExceptionally(ex);
            TracingTags.setErrorStatus(tracingSpan, ex);
            tracingSpan.finish();
            return serversAndProbesFuture;
        }
        serversReadyFuture.whenComplete((BiConsumer<Object, Throwable>) (ok, ex) -> {
            if (ex != null) {
                serversAndProbesFuture.completeExceptionally(ex);
                TracingTags.setErrorStatus(tracingSpan, ex);
                tracingSpan.finish();
                return;
            }
            try {
                probeScheduler.schedule(probesFactory.getProbes(runtimeId, machineName, machine.getServers()), new ServerLivenessHandler());
            } catch (InfrastructureException iex) {
                serversAndProbesFuture.completeExceptionally(iex);
            }
            serversAndProbesFuture.complete(null);
            tracingSpan.finish();
        });
        return serversAndProbesFuture;
    };
}
Also used : ProbeResult(org.eclipse.che.api.workspace.server.hc.probe.ProbeResult) ServerStatus(org.eclipse.che.api.core.model.workspace.runtime.ServerStatus) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) KubernetesEnvironment(org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment) PodTemplateSpec(io.fabric8.kubernetes.api.model.PodTemplateSpec) Assisted(com.google.inject.assistedinject.Assisted) KubernetesObjectUtil.putLabels(org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.putLabels) KubernetesRuntimeState(org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesRuntimeState) Collectors.toMap(java.util.stream.Collectors.toMap) PodMerger(org.eclipse.che.workspace.infrastructure.kubernetes.environment.PodMerger) Map(java.util.Map) StateException(org.eclipse.che.api.workspace.server.spi.StateException) KubernetesObjectUtil.putAnnotations(org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.putAnnotations) CHECK_SERVERS(org.eclipse.che.workspace.infrastructure.kubernetes.util.TracingSpanConstants.CHECK_SERVERS) Command(org.eclipse.che.api.core.model.workspace.config.Command) WorkspaceProbes(org.eclipse.che.api.workspace.server.hc.probe.WorkspaceProbes) ImmutableMap(com.google.common.collect.ImmutableMap) KubernetesServerResolverFactory(org.eclipse.che.workspace.infrastructure.kubernetes.server.resolver.KubernetesServerResolverFactory) Collection(java.util.Collection) Set(java.util.Set) KubernetesMachineCache(org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesMachineCache) SidecarToolingProvisioner(org.eclipse.che.workspace.infrastructure.kubernetes.wsplugins.SidecarToolingProvisioner) NoOpURLRewriter(org.eclipse.che.api.workspace.server.URLRewriter.NoOpURLRewriter) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) RuntimeStartInterruptedException(org.eclipse.che.api.workspace.server.spi.RuntimeStartInterruptedException) KubernetesObjectUtil.shouldCreateInCheNamespace(org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesObjectUtil.shouldCreateInCheNamespace) Objects(java.util.Objects) InternalMachineConfig(org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig) KubernetesSharedPool(org.eclipse.che.workspace.infrastructure.kubernetes.util.KubernetesSharedPool) InfrastructureException(org.eclipse.che.api.workspace.server.spi.InfrastructureException) List(java.util.List) WorkspaceVolumesStrategy(org.eclipse.che.workspace.infrastructure.kubernetes.namespace.pvc.WorkspaceVolumesStrategy) CompletionStage(java.util.concurrent.CompletionStage) Ingress(io.fabric8.kubernetes.api.model.networking.v1.Ingress) TracingTags(org.eclipse.che.commons.tracing.TracingTags) UnrecoverablePodEventListenerFactory(org.eclipse.che.workspace.infrastructure.kubernetes.util.UnrecoverablePodEventListenerFactory) ObjectMeta(io.fabric8.kubernetes.api.model.ObjectMeta) RuntimeIdentity(org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity) Entry(java.util.Map.Entry) Secret(io.fabric8.kubernetes.api.model.Secret) Span(io.opentracing.Span) Optional(java.util.Optional) Scope(io.opentracing.Scope) WorkspaceProbesFactory(org.eclipse.che.api.workspace.server.hc.probe.WorkspaceProbesFactory) LogWatcher(org.eclipse.che.workspace.infrastructure.kubernetes.namespace.log.LogWatcher) WAIT_RUNNING_ASYNC(org.eclipse.che.workspace.infrastructure.kubernetes.util.TracingSpanConstants.WAIT_RUNNING_ASYNC) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) MachineStatus(org.eclipse.che.api.core.model.workspace.runtime.MachineStatus) InternalEnvironmentProvisioner(org.eclipse.che.api.workspace.server.spi.provision.InternalEnvironmentProvisioner) Container(io.fabric8.kubernetes.api.model.Container) CompletableFuture(java.util.concurrent.CompletableFuture) ServerResolver(org.eclipse.che.workspace.infrastructure.kubernetes.server.resolver.ServerResolver) KubernetesMachineImpl(org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesMachineImpl) Function(java.util.function.Function) Traced(org.eclipse.che.commons.annotation.Traced) ValidationException(org.eclipse.che.api.core.ValidationException) ArrayList(java.util.ArrayList) CheNamespace(org.eclipse.che.workspace.infrastructure.kubernetes.namespace.CheNamespace) LinkedHashMap(java.util.LinkedHashMap) Inject(javax.inject.Inject) ServersChecker(org.eclipse.che.api.workspace.server.hc.ServersChecker) KubernetesNamespace(org.eclipse.che.workspace.infrastructure.kubernetes.namespace.KubernetesNamespace) InternalInfrastructureException(org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException) EnvironmentContext(org.eclipse.che.commons.env.EnvironmentContext) BiConsumer(java.util.function.BiConsumer) WAIT_MACHINES_START(org.eclipse.che.workspace.infrastructure.kubernetes.util.TracingSpanConstants.WAIT_MACHINES_START) Service(io.fabric8.kubernetes.api.model.Service) Named(javax.inject.Named) RuntimeEventsPublisher(org.eclipse.che.workspace.infrastructure.kubernetes.util.RuntimeEventsPublisher) PodLogToEventPublisher(org.eclipse.che.workspace.infrastructure.kubernetes.namespace.log.PodLogToEventPublisher) ProbeScheduler(org.eclipse.che.api.workspace.server.hc.probe.ProbeScheduler) InternalRuntime(org.eclipse.che.api.workspace.server.spi.InternalRuntime) LogWatchTimeouts(org.eclipse.che.workspace.infrastructure.kubernetes.namespace.log.LogWatchTimeouts) Collections.emptyMap(java.util.Collections.emptyMap) Logger(org.slf4j.Logger) Tracer(io.opentracing.Tracer) Executor(java.util.concurrent.Executor) WorkspaceStatus(org.eclipse.che.api.core.model.workspace.WorkspaceStatus) Pod(io.fabric8.kubernetes.api.model.Pod) SecretAsContainerResourceProvisioner(org.eclipse.che.workspace.infrastructure.kubernetes.provision.secret.SecretAsContainerResourceProvisioner) ConfigMap(io.fabric8.kubernetes.api.model.ConfigMap) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) KubernetesRuntimeStateCache(org.eclipse.che.workspace.infrastructure.kubernetes.cache.KubernetesRuntimeStateCache) PodData(org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment.PodData) PodEvent(org.eclipse.che.workspace.infrastructure.kubernetes.namespace.event.PodEvent) PreviewUrlCommandProvisioner(org.eclipse.che.workspace.infrastructure.kubernetes.provision.PreviewUrlCommandProvisioner) ProbeStatus(org.eclipse.che.api.workspace.server.hc.probe.ProbeResult.ProbeStatus) Deployment(io.fabric8.kubernetes.api.model.apps.Deployment) ServersCheckerFactory(org.eclipse.che.api.workspace.server.hc.ServersCheckerFactory) RuntimeIdentity(org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity) CompletableFuture(java.util.concurrent.CompletableFuture) ServersChecker(org.eclipse.che.api.workspace.server.hc.ServersChecker) Span(io.opentracing.Span) InfrastructureException(org.eclipse.che.api.workspace.server.spi.InfrastructureException) InternalInfrastructureException(org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException) BiConsumer(java.util.function.BiConsumer)

Example 52 with InfrastructureException

use of org.eclipse.che.api.workspace.server.spi.InfrastructureException in project che-server by eclipse-che.

the class KubernetesInternalRuntime method storeStartingMachine.

/**
 * Puts createdPod in the {@code machines} map and sends the starting event for this machine
 */
private void storeStartingMachine(Pod createdPod, ObjectMeta toCreateMeta, Map<String, InternalMachineConfig> machineConfigs, ServerResolver serverResolver) throws InfrastructureException {
    final String workspaceId = getContext().getIdentity().getWorkspaceId();
    for (Container container : createdPod.getSpec().getContainers()) {
        String machineName = Names.machineName(toCreateMeta, container);
        LOG.debug("Creating machine '{}' in workspace '{}'", machineName, workspaceId);
        // Sometimes we facing NPE trying to retrieve machine config. Possible names mismatch. Need to
        // get more info on that cases.
        InternalMachineConfig machineConfig = Optional.ofNullable(machineConfigs.get(machineName)).orElseThrow(() -> {
            LOG.error("Workspace '{}' start failed. Machine with name '{}' requested but not found in configs map. Present machines are: {}.", workspaceId, machineName, String.join(",", machineConfigs.keySet()));
            return new InfrastructureException(format("Unable to start the workspace '%s' due to an internal inconsistency while composing the workspace runtime." + "Please report a bug. If possible, include the details from Che devfile and server log in bug report (your admin can help with that)", workspaceId));
        });
        machines.put(getContext().getIdentity(), new KubernetesMachineImpl(workspaceId, machineName, createdPod.getMetadata().getName(), container.getName(), MachineStatus.STARTING, machineConfig.getAttributes(), serverResolver.resolve(machineName)));
        eventPublisher.sendStartingEvent(machineName, getContext().getIdentity());
    }
}
Also used : InternalMachineConfig(org.eclipse.che.api.workspace.server.spi.environment.InternalMachineConfig) Container(io.fabric8.kubernetes.api.model.Container) KubernetesMachineImpl(org.eclipse.che.workspace.infrastructure.kubernetes.model.KubernetesMachineImpl) InfrastructureException(org.eclipse.che.api.workspace.server.spi.InfrastructureException) InternalInfrastructureException(org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException)

Example 53 with InfrastructureException

use of org.eclipse.che.api.workspace.server.spi.InfrastructureException in project che-server by eclipse-che.

the class KubernetesInternalRuntime method internalStop.

@Traced
@Override
protected void internalStop(Map<String, String> stopOptions) throws InfrastructureException {
    RuntimeIdentity identity = getContext().getIdentity();
    TracingTags.WORKSPACE_ID.set(identity.getWorkspaceId());
    runtimeHangingDetector.stopTracking(getContext().getIdentity());
    if (startSynchronizer.interrupt()) {
        // runtime is STARTING. Need to wait until start will be interrupted properly
        try {
            if (!startSynchronizer.awaitInterruption(workspaceStartTimeoutMin, TimeUnit.MINUTES)) {
                // Runtime is not interrupted yet. It may occur when start was performing by another
                // Che Server that is crashed so start is hung up in STOPPING phase.
                // Need to clean up runtime resources
                probeScheduler.cancel(identity.getWorkspaceId());
                runtimeCleaner.cleanUp(namespace, identity.getWorkspaceId());
            }
        } catch (InterruptedException e) {
            throw new InfrastructureException("Interrupted while waiting for start task cancellation", e);
        }
    } else {
        // runtime is RUNNING. Clean up used resources
        // Cancels workspace servers probes if any
        probeScheduler.cancel(identity.getWorkspaceId());
        runtimeCleaner.cleanUp(namespace, identity.getWorkspaceId());
    }
}
Also used : RuntimeIdentity(org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity) RuntimeStartInterruptedException(org.eclipse.che.api.workspace.server.spi.RuntimeStartInterruptedException) InfrastructureException(org.eclipse.che.api.workspace.server.spi.InfrastructureException) InternalInfrastructureException(org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException) Traced(org.eclipse.che.commons.annotation.Traced)

Example 54 with InfrastructureException

use of org.eclipse.che.api.workspace.server.spi.InfrastructureException in project che-server by eclipse-che.

the class DirectKubernetesAPIAccessHelper method call.

/**
 * This method just performs an HTTP request of given {@code httpMethod} on an URL composed of the
 * {@code masterUrl} and {@code relativeUri} using the provided {@code httpClient}, optionally
 * sending the provided {@code body}.
 *
 * @param masterUrl the base of the final URL
 * @param httpClient the HTTP client to perform the request with
 * @param httpMethod the HTTP method of the request
 * @param relativeUri the relative URI that should be appended ot the {@code masterUrl}
 * @param body the body to send with the request, if any
 * @return the HTTP response received
 * @throws InfrastructureException on failure to validate or perform the request
 */
public static Response call(String masterUrl, OkHttpClient httpClient, String httpMethod, URI relativeUri, @Nullable HttpHeaders headers, @Nullable InputStream body) throws InfrastructureException {
    if (relativeUri.isAbsolute() || relativeUri.isOpaque()) {
        throw new InfrastructureException("The direct infrastructure URL must be relative and not opaque.");
    }
    try {
        URL fullUrl = new URI(masterUrl).resolve(relativeUri).toURL();
        okhttp3.Response response = callApi(httpClient, fullUrl, httpMethod, headers, body);
        return convertResponse(response);
    } catch (URISyntaxException | MalformedURLException e) {
        throw new InfrastructureException("Could not compose the direct URI.", e);
    } catch (IOException e) {
        throw new InfrastructureException("Error sending the direct infrastructure request.", e);
    }
}
Also used : MalformedURLException(java.net.MalformedURLException) URISyntaxException(java.net.URISyntaxException) IOException(java.io.IOException) URI(java.net.URI) InfrastructureException(org.eclipse.che.api.workspace.server.spi.InfrastructureException) URL(java.net.URL)

Example 55 with InfrastructureException

use of org.eclipse.che.api.workspace.server.spi.InfrastructureException in project che-server by eclipse-che.

the class InconsistentRuntimesDetector method checkOne.

@VisibleForTesting
void checkOne(String workspaceId) throws InfrastructureException {
    LOG.debug("Checking consistency of runtime for workspace `{}`", workspaceId);
    KubernetesInternalRuntime k8sRuntime = getKubernetesInternalRuntime(workspaceId);
    RuntimeIdentity runtimeId = k8sRuntime.getContext().getIdentity();
    try {
        if (k8sRuntime.isConsistent()) {
            return;
        }
    } catch (InfrastructureException e) {
        throw new InfrastructureException(format("Error occurred during runtime '%s:%s' consistency checking. Cause: %s", runtimeId.getWorkspaceId(), runtimeId.getOwnerId(), e.getMessage()), e);
    }
    // not to initialize abnormal stop for a runtime that is not RUNNING anymore
    if (!isRunning(k8sRuntime)) {
        return;
    }
    LOG.warn("Found runtime `{}:{}` with inconsistent state. It's going to be stopped automatically", runtimeId.getWorkspaceId(), runtimeId.getOwnerId());
    stopAbnormally(k8sRuntime);
    LOG.debug("Checking consistency of runtime for workspace `{}` is finished", workspaceId);
}
Also used : RuntimeIdentity(org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity) InfrastructureException(org.eclipse.che.api.workspace.server.spi.InfrastructureException) VisibleForTesting(com.google.common.annotations.VisibleForTesting)

Aggregations

InfrastructureException (org.eclipse.che.api.workspace.server.spi.InfrastructureException)242 InternalInfrastructureException (org.eclipse.che.api.workspace.server.spi.InternalInfrastructureException)64 Test (org.testng.annotations.Test)56 KubernetesInfrastructureException (org.eclipse.che.workspace.infrastructure.kubernetes.KubernetesInfrastructureException)44 RuntimeIdentity (org.eclipse.che.api.core.model.workspace.runtime.RuntimeIdentity)42 KubernetesClientException (io.fabric8.kubernetes.client.KubernetesClientException)38 CompletableFuture (java.util.concurrent.CompletableFuture)36 ExecutionException (java.util.concurrent.ExecutionException)36 TimeoutException (java.util.concurrent.TimeoutException)32 ServerException (org.eclipse.che.api.core.ServerException)32 Pod (io.fabric8.kubernetes.api.model.Pod)30 Map (java.util.Map)26 ValidationException (org.eclipse.che.api.core.ValidationException)22 Traced (org.eclipse.che.commons.annotation.Traced)20 Container (io.fabric8.kubernetes.api.model.Container)18 List (java.util.List)18 Set (java.util.Set)18 Inject (javax.inject.Inject)18 RuntimeStartInterruptedException (org.eclipse.che.api.workspace.server.spi.RuntimeStartInterruptedException)18 KubernetesEnvironment (org.eclipse.che.workspace.infrastructure.kubernetes.environment.KubernetesEnvironment)18