Search in sources :

Example 26 with ContainerNodeSpec

use of com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec in project vespa by vespa-engine.

the class RealNodeRepositoryTest method testGetContainersToRunApi.

@Test
public void testGetContainersToRunApi() throws InterruptedException {
    waitForJdiscContainerToServe();
    NodeRepository nodeRepositoryApi = new RealNodeRepository(configServerApi);
    String dockerHostHostname = "dockerhost1.yahoo.com";
    final List<ContainerNodeSpec> containersToRun = nodeRepositoryApi.getContainersToRun(dockerHostHostname);
    assertThat(containersToRun.size(), is(1));
    final ContainerNodeSpec nodeSpec = containersToRun.get(0);
    assertThat(nodeSpec.hostname, is("host4.yahoo.com"));
    assertThat(nodeSpec.wantedDockerImage.get(), is(new DockerImage("docker-registry.domain.tld:8080/dist/vespa:6.42.0")));
    assertThat(nodeSpec.nodeState, is(Node.State.active));
    assertThat(nodeSpec.wantedRestartGeneration.get(), is(0L));
    assertThat(nodeSpec.currentRestartGeneration.get(), is(0L));
    assertThat(nodeSpec.minCpuCores, is(0.2));
    assertThat(nodeSpec.minMainMemoryAvailableGb, is(0.5));
    assertThat(nodeSpec.minDiskAvailableGb, is(100.0));
}
Also used : DockerImage(com.yahoo.vespa.hosted.dockerapi.DockerImage) ContainerNodeSpec(com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec) Test(org.junit.Test)

Example 27 with ContainerNodeSpec

use of com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec in project vespa by vespa-engine.

the class DockerFailTest method dockerFailTest.

@Test
public void dockerFailTest() throws Exception {
    try (DockerTester dockerTester = new DockerTester()) {
        ContainerNodeSpec containerNodeSpec = new ContainerNodeSpec.Builder().hostname("host1.test.yahoo.com").wantedDockerImage(new DockerImage("dockerImage")).nodeState(Node.State.active).nodeType("tenant").nodeFlavor("docker").wantedRestartGeneration(1L).currentRestartGeneration(1L).minCpuCores(1).minMainMemoryAvailableGb(1).minDiskAvailableGb(1).build();
        dockerTester.addContainerNodeSpec(containerNodeSpec);
        // Wait for node admin to be notified with node repo state and the docker container has been started
        while (dockerTester.nodeAdmin.getListOfHosts().size() == 0) {
            Thread.sleep(10);
        }
        dockerTester.callOrderVerifier.assertInOrder(1200, "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
        dockerTester.dockerMock.deleteContainer(new ContainerName("host1"));
        dockerTester.callOrderVerifier.assertInOrder("deleteContainer with ContainerName { name=host1 }", "createContainerCommand with DockerImage { imageId=dockerImage }, HostName: host1.test.yahoo.com, ContainerName { name=host1 }", "executeInContainerAsRoot with ContainerName { name=host1 }, args: [" + DockerTester.NODE_PROGRAM + ", resume]");
    }
}
Also used : ContainerName(com.yahoo.vespa.hosted.dockerapi.ContainerName) DockerImage(com.yahoo.vespa.hosted.dockerapi.DockerImage) ContainerNodeSpec(com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec) Test(org.junit.Test)

Example 28 with ContainerNodeSpec

use of com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec in project vespa by vespa-engine.

the class NodeAdminStateUpdaterImpl method updateHardwareDivergence.

private void updateHardwareDivergence(StorageMaintainer maintainer) {
    if (currentState != RESUMED)
        return;
    try {
        ContainerNodeSpec nodeSpec = nodeRepository.getContainerNodeSpec(dockerHostHostName).orElseThrow(() -> new RuntimeException("Failed to get host's node spec from node-repo"));
        String hardwareDivergence = maintainer.getHardwareDivergence(nodeSpec);
        // Only update hardware divergence if there is a change.
        if (!nodeSpec.hardwareDivergence.orElse("null").equals(hardwareDivergence)) {
            NodeAttributes nodeAttributes = new NodeAttributes().withHardwareDivergence(hardwareDivergence);
            nodeRepository.updateNodeAttributes(dockerHostHostName, nodeAttributes);
        }
    } catch (RuntimeException e) {
        log.log(Level.WARNING, "Failed to report hardware divergence", e);
    }
}
Also used : NodeAttributes(com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAttributes) ContainerNodeSpec(com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec)

Example 29 with ContainerNodeSpec

use of com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec in project vespa by vespa-engine.

the class NodeAgentImpl method converge.

// Public for testing
void converge() {
    final Optional<ContainerNodeSpec> nodeSpecOptional = nodeRepository.getContainerNodeSpec(hostname);
    // We just removed the node from node repo, so this is expected until NodeAdmin stop this NodeAgent
    if (!nodeSpecOptional.isPresent() && expectNodeNotInNodeRepo)
        return;
    final ContainerNodeSpec nodeSpec = nodeSpecOptional.orElseThrow(() -> new IllegalStateException(String.format("Node '%s' missing from node repository.", hostname)));
    expectNodeNotInNodeRepo = false;
    Optional<Container> container = getContainer();
    if (!nodeSpec.equals(lastNodeSpec)) {
        // will change and we will be reporting duplicate metrics.
        if (container.map(c -> c.state.isRunning()).orElse(false)) {
            storageMaintainer.writeMetricsConfig(containerName, nodeSpec);
        }
        addDebugMessage("Loading new node spec: " + nodeSpec.toString());
        lastNodeSpec = nodeSpec;
    }
    switch(nodeSpec.nodeState) {
        case ready:
        case reserved:
        case parked:
        case failed:
            removeContainerIfNeededUpdateContainerState(nodeSpec, container);
            updateNodeRepoWithCurrentAttributes(nodeSpec);
            break;
        case active:
            storageMaintainer.handleCoreDumpsForContainer(containerName, nodeSpec, false);
            storageMaintainer.getDiskUsageFor(containerName).map(diskUsage -> (double) diskUsage / BYTES_IN_GB / nodeSpec.minDiskAvailableGb).filter(diskUtil -> diskUtil >= 0.8).ifPresent(diskUtil -> storageMaintainer.removeOldFilesFromNode(containerName));
            scheduleDownLoadIfNeeded(nodeSpec);
            if (isDownloadingImage()) {
                addDebugMessage("Waiting for image to download " + imageBeingDownloaded.asString());
                return;
            }
            container = removeContainerIfNeededUpdateContainerState(nodeSpec, container);
            if (!container.isPresent()) {
                storageMaintainer.handleCoreDumpsForContainer(containerName, nodeSpec, false);
                containerState = STARTING;
                startContainer(nodeSpec);
                containerState = UNKNOWN;
            }
            runLocalResumeScriptIfNeeded(nodeSpec);
            // Because it's more important to stop a bad release from rolling out in prod,
            // we put the resume call last. So if we fail after updating the node repo attributes
            // but before resume, the app may go through the tenant pipeline but will halt in prod.
            // 
            // Note that this problem exists only because there are 2 different mechanisms
            // that should really be parts of a single mechanism:
            // - The content of node repo is used to determine whether a new Vespa+application
            // has been successfully rolled out.
            // - Slobrok and internal orchestrator state is used to determine whether
            // to allow upgrade (suspend).
            updateNodeRepoWithCurrentAttributes(nodeSpec);
            logger.info("Call resume against Orchestrator");
            orchestrator.resume(hostname);
            break;
        case inactive:
            removeContainerIfNeededUpdateContainerState(nodeSpec, container);
            updateNodeRepoWithCurrentAttributes(nodeSpec);
            break;
        case provisioned:
            nodeRepository.markAsDirty(hostname);
            break;
        case dirty:
            removeContainerIfNeededUpdateContainerState(nodeSpec, container);
            logger.info("State is " + nodeSpec.nodeState + ", will delete application storage and mark node as ready");
            storageMaintainer.cleanupNodeStorage(containerName, nodeSpec);
            updateNodeRepoWithCurrentAttributes(nodeSpec);
            nodeRepository.markNodeAvailableForNewAllocation(hostname);
            expectNodeNotInNodeRepo = true;
            break;
        default:
            throw new RuntimeException("UNKNOWN STATE " + nodeSpec.nodeState.name());
    }
}
Also used : DockerException(com.yahoo.vespa.hosted.dockerapi.DockerException) ContainerName(com.yahoo.vespa.hosted.dockerapi.ContainerName) Dimensions(com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions) StorageMaintainer(com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer) Environment(com.yahoo.vespa.hosted.node.admin.component.Environment) Date(java.util.Date) PromptContainerData(com.yahoo.vespa.hosted.node.admin.containerdata.PromptContainerData) ThreadFactoryFactory(com.yahoo.concurrent.ThreadFactoryFactory) ProcessResult(com.yahoo.vespa.hosted.dockerapi.ProcessResult) SimpleDateFormat(java.text.SimpleDateFormat) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Node(com.yahoo.vespa.hosted.provision.Node) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) MetricReceiverWrapper(com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper) Future(java.util.concurrent.Future) Duration(java.time.Duration) Map(java.util.Map) ContainerData(com.yahoo.vespa.hosted.node.admin.containerdata.ContainerData) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) LinkedList(java.util.LinkedList) PrefixLogger(com.yahoo.vespa.hosted.node.admin.util.PrefixLogger) ConfigServerContainerData(com.yahoo.vespa.hosted.node.admin.containerdata.ConfigServerContainerData) NodeType(com.yahoo.config.provision.NodeType) DockerOperations(com.yahoo.vespa.hosted.node.admin.docker.DockerOperations) Container(com.yahoo.vespa.hosted.dockerapi.Container) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) DimensionMetrics(com.yahoo.vespa.hosted.dockerapi.metrics.DimensionMetrics) NodeRepository(com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository) Instant(java.time.Instant) DockerExecTimeoutException(com.yahoo.vespa.hosted.dockerapi.DockerExecTimeoutException) Executors(java.util.concurrent.Executors) MotdContainerData(com.yahoo.vespa.hosted.node.admin.containerdata.MotdContainerData) UncheckedIOException(java.io.UncheckedIOException) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) List(java.util.List) ContainerNodeSpec(com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec) STARTING(com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.STARTING) DockerImage(com.yahoo.vespa.hosted.dockerapi.DockerImage) Docker(com.yahoo.vespa.hosted.dockerapi.Docker) ABSENT(com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.ABSENT) Orchestrator(com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator) UNKNOWN(com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.UNKNOWN) Clock(java.time.Clock) Optional(java.util.Optional) OrchestratorException(com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException) ContainerResources(com.yahoo.vespa.hosted.dockerapi.ContainerResources) Container(com.yahoo.vespa.hosted.dockerapi.Container) ContainerNodeSpec(com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec)

Example 30 with ContainerNodeSpec

use of com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec in project vespa by vespa-engine.

the class NodeAgentImpl method updateNodeRepoWithCurrentAttributes.

private void updateNodeRepoWithCurrentAttributes(final ContainerNodeSpec nodeSpec) {
    final NodeAttributes currentNodeAttributes = new NodeAttributes().withRestartGeneration(nodeSpec.currentRestartGeneration.orElse(null)).withRebootGeneration(nodeSpec.currentRebootGeneration.orElse(0L)).withDockerImage(nodeSpec.currentDockerImage.orElse(new DockerImage(""))).withVespaVersion(nodeSpec.vespaVersion.orElse(""));
    final NodeAttributes wantedNodeAttributes = new NodeAttributes().withRestartGeneration(nodeSpec.wantedRestartGeneration.orElse(null)).withRebootGeneration(nodeSpec.wantedRebootGeneration.orElse(0L)).withDockerImage(nodeSpec.wantedDockerImage.filter(node -> containerState == UNKNOWN).orElse(new DockerImage(""))).withVespaVersion(nodeSpec.wantedVespaVersion.filter(node -> containerState == UNKNOWN).orElse(""));
    publishStateToNodeRepoIfChanged(currentNodeAttributes, wantedNodeAttributes);
}
Also used : DockerException(com.yahoo.vespa.hosted.dockerapi.DockerException) ContainerName(com.yahoo.vespa.hosted.dockerapi.ContainerName) Dimensions(com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions) StorageMaintainer(com.yahoo.vespa.hosted.node.admin.maintenance.StorageMaintainer) Environment(com.yahoo.vespa.hosted.node.admin.component.Environment) Date(java.util.Date) PromptContainerData(com.yahoo.vespa.hosted.node.admin.containerdata.PromptContainerData) ThreadFactoryFactory(com.yahoo.concurrent.ThreadFactoryFactory) ProcessResult(com.yahoo.vespa.hosted.dockerapi.ProcessResult) SimpleDateFormat(java.text.SimpleDateFormat) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) Node(com.yahoo.vespa.hosted.provision.Node) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) MetricReceiverWrapper(com.yahoo.vespa.hosted.dockerapi.metrics.MetricReceiverWrapper) Future(java.util.concurrent.Future) Duration(java.time.Duration) Map(java.util.Map) ContainerData(com.yahoo.vespa.hosted.node.admin.containerdata.ContainerData) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) LinkedList(java.util.LinkedList) PrefixLogger(com.yahoo.vespa.hosted.node.admin.util.PrefixLogger) ConfigServerContainerData(com.yahoo.vespa.hosted.node.admin.containerdata.ConfigServerContainerData) NodeType(com.yahoo.config.provision.NodeType) DockerOperations(com.yahoo.vespa.hosted.node.admin.docker.DockerOperations) Container(com.yahoo.vespa.hosted.dockerapi.Container) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) DimensionMetrics(com.yahoo.vespa.hosted.dockerapi.metrics.DimensionMetrics) NodeRepository(com.yahoo.vespa.hosted.node.admin.configserver.noderepository.NodeRepository) Instant(java.time.Instant) DockerExecTimeoutException(com.yahoo.vespa.hosted.dockerapi.DockerExecTimeoutException) Executors(java.util.concurrent.Executors) MotdContainerData(com.yahoo.vespa.hosted.node.admin.containerdata.MotdContainerData) UncheckedIOException(java.io.UncheckedIOException) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) List(java.util.List) ContainerNodeSpec(com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec) STARTING(com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.STARTING) DockerImage(com.yahoo.vespa.hosted.dockerapi.DockerImage) Docker(com.yahoo.vespa.hosted.dockerapi.Docker) ABSENT(com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.ABSENT) Orchestrator(com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.Orchestrator) UNKNOWN(com.yahoo.vespa.hosted.node.admin.nodeagent.NodeAgentImpl.ContainerState.UNKNOWN) Clock(java.time.Clock) Optional(java.util.Optional) OrchestratorException(com.yahoo.vespa.hosted.node.admin.configserver.orchestrator.OrchestratorException) ContainerResources(com.yahoo.vespa.hosted.dockerapi.ContainerResources) DockerImage(com.yahoo.vespa.hosted.dockerapi.DockerImage)

Aggregations

ContainerNodeSpec (com.yahoo.vespa.hosted.node.admin.ContainerNodeSpec)30 Test (org.junit.Test)22 InOrder (org.mockito.InOrder)8 ContainerName (com.yahoo.vespa.hosted.dockerapi.ContainerName)7 DockerImage (com.yahoo.vespa.hosted.dockerapi.DockerImage)7 Docker (com.yahoo.vespa.hosted.dockerapi.Docker)4 DockerException (com.yahoo.vespa.hosted.dockerapi.DockerException)4 ArrayList (java.util.ArrayList)4 DimensionMetrics (com.yahoo.vespa.hosted.dockerapi.metrics.DimensionMetrics)3 Dimensions (com.yahoo.vespa.hosted.dockerapi.metrics.Dimensions)3 Node (com.yahoo.vespa.hosted.provision.Node)3 IOException (java.io.IOException)3 LinkedHashMap (java.util.LinkedHashMap)3 Map (java.util.Map)3 JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)2 ThreadFactoryFactory (com.yahoo.concurrent.ThreadFactoryFactory)2 NodeType (com.yahoo.config.provision.NodeType)2 Container (com.yahoo.vespa.hosted.dockerapi.Container)2 ContainerResources (com.yahoo.vespa.hosted.dockerapi.ContainerResources)2 DockerExecTimeoutException (com.yahoo.vespa.hosted.dockerapi.DockerExecTimeoutException)2