Examples with PrefixLogger com.yahoo.vespa.hosted.node.admin.util.PrefixLogger used on opensource projects

Example 1 with PrefixLogger

use of com.yahoo.vespa.hosted.node.admin.util.PrefixLogger in project vespa by vespa-engine.

the class DockerOperationsImpl method startContainer.

@Override
public void startContainer(ContainerName containerName, final ContainerNodeSpec nodeSpec) {
    PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName);
    logger.info("Starting container " + containerName);
    try {
        InetAddress nodeInetAddress = environment.getInetAddressForHost(nodeSpec.hostname);
        boolean isIPv6 = nodeInetAddress instanceof Inet6Address;
        if (isIPv6) {
            if (!docker.networkNPTed()) {
                docker.connectContainerToNetwork(containerName, "bridge");
            }
            docker.startContainer(containerName);
            setupContainerNetworkConnectivity(containerName);
        } else {
            docker.startContainer(containerName);
        }
        directoriesToMount.entrySet().stream().filter(Map.Entry::getValue).map(Map.Entry::getKey).forEach(path -> docker.executeInContainerAsRoot(containerName, "chmod", "-R", "a+w", path.toString()));
    } catch (IOException e) {
        throw new RuntimeException("Failed to start container " + containerName.asString(), e);
    }
}

Example 2 with PrefixLogger

use of com.yahoo.vespa.hosted.node.admin.util.PrefixLogger in project vespa by vespa-engine.

the class DockerOperationsImpl method createContainer.

@Override
public void createContainer(ContainerName containerName, final ContainerNodeSpec nodeSpec) {
    PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName);
    logger.info("Creating container " + containerName);
    try {
        InetAddress nodeInetAddress = environment.getInetAddressForHost(nodeSpec.hostname);
        String configServers = environment.getConfigServerUris().stream().map(URI::getHost).collect(Collectors.joining(","));
        Docker.CreateContainerCommand command = docker.createContainerCommand(nodeSpec.wantedDockerImage.get(), ContainerResources.from(nodeSpec.minCpuCores, nodeSpec.minMainMemoryAvailableGb), containerName, nodeSpec.hostname).withManagedBy(MANAGER_NAME).withEnvironment("CONFIG_SERVER_ADDRESS", // TODO: Remove when all images support CONTAINER_ENVIRONMENT_SETTINGS
        configServers).withEnvironment("CONTAINER_ENVIRONMENT_SETTINGS", environment.getContainerEnvironmentResolver().createSettings(environment, nodeSpec)).withUlimit("nofile", 262_144, 262_144).withUlimit("nproc", 32_768, 409_600).withUlimit("core", -1, -1).withAddCapability(// Needed for gcore, pstack etc.
        "SYS_PTRACE").withAddCapability(// Needed for perf
        "SYS_ADMIN");
        if (environment.getNodeType() == NodeType.confighost || environment.getNodeType() == NodeType.proxyhost) {
            command.withVolume("/var/lib/sia", "/var/lib/sia");
        }
        if (environment.getNodeType() == NodeType.proxyhost) {
            command.withVolume("/opt/yahoo/share/ssl/certs/", "/opt/yahoo/share/ssl/certs/");
        }
        if (!docker.networkNPTed()) {
            command.withIpAddress(nodeInetAddress);
            command.withNetworkMode(DockerImpl.DOCKER_CUSTOM_MACVLAN_NETWORK_NAME);
            // TODO This is probably not necessary - review later
            command.withVolume("/etc/hosts", "/etc/hosts");
        } else {
            // IPv6 - Assume always valid
            Inet6Address ipV6Address = this.retriever.getIPv6Address(nodeSpec.hostname).orElseThrow(() -> new RuntimeException("Unable to find a valid IPv6 address. Missing an AAAA DNS entry?"));
            InetAddress ipV6Prefix = InetAddress.getByName(IPV6_NPT_PREFIX);
            InetAddress ipV6Local = IPAddresses.prefixTranslate(ipV6Address, ipV6Prefix, 8);
            command.withIpAddress(ipV6Local);
            // IPv4 - Only present for some containers
            Optional<Inet4Address> ipV4Address = this.retriever.getIPv4Address(nodeSpec.hostname);
            if (ipV4Address.isPresent()) {
                InetAddress ipV4Prefix = InetAddress.getByName(IPV4_NPT_PREFIX);
                InetAddress ipV4Local = IPAddresses.prefixTranslate(ipV4Address.get(), ipV4Prefix, 2);
                command.withIpAddress(ipV4Local);
            }
            command.withNetworkMode(DOCKER_CUSTOM_BRIDGE_NETWORK_NAME);
        }
        for (Path pathInNode : directoriesToMount.keySet()) {
            String pathInHost = environment.pathInHostFromPathInNode(containerName, pathInNode).toString();
            command.withVolume(pathInHost, pathInNode.toString());
        }
        // TODO: Enforce disk constraints
        long minMainMemoryAvailableMb = (long) (nodeSpec.minMainMemoryAvailableGb * 1024);
        if (minMainMemoryAvailableMb > 0) {
            // VESPA_TOTAL_MEMORY_MB is used to make any jdisc container think the machine
            // only has this much physical memory (overrides total memory reported by `free -m`).
            command.withEnvironment("VESPA_TOTAL_MEMORY_MB", Long.toString(minMainMemoryAvailableMb));
        }
        logger.info("Creating new container with args: " + command);
        command.create();
        docker.createContainer(command);
    } catch (IOException e) {
        throw new RuntimeException("Failed to create container " + containerName.asString(), e);
    }
}

Example 3 with PrefixLogger

use of com.yahoo.vespa.hosted.node.admin.util.PrefixLogger in project vespa by vespa-engine.

the class StorageMaintainer method writeFilebeatConfig.

public void writeFilebeatConfig(ContainerName containerName, ContainerNodeSpec nodeSpec) {
    PrefixLogger logger = PrefixLogger.getNodeAgentLogger(StorageMaintainer.class, containerName);
    try {
        FilebeatConfigProvider filebeatConfigProvider = new FilebeatConfigProvider(environment);
        Optional<String> config = filebeatConfigProvider.getConfig(nodeSpec);
        if (!config.isPresent()) {
            logger.error("Was not able to generate a config for filebeat, ignoring filebeat file creation." + nodeSpec.toString());
            return;
        }
        Path filebeatPath = environment.pathInNodeAdminFromPathInNode(containerName, Paths.get("/etc/filebeat/filebeat.yml"));
        Files.write(filebeatPath, config.get().getBytes());
        logger.info("Wrote filebeat config.");
    } catch (Throwable t) {
        logger.error("Failed writing filebeat config; " + nodeSpec, t);
    }
}

Example 4 with PrefixLogger

use of com.yahoo.vespa.hosted.node.admin.util.PrefixLogger in project vespa by vespa-engine.

the class DockerOperationsImpl method removeContainer.

@Override
public void removeContainer(final Container existingContainer, ContainerNodeSpec nodeSpec) {
    final ContainerName containerName = existingContainer.name;
    PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName);
    if (existingContainer.state.isRunning()) {
        logger.info("Stopping container " + containerName.asString());
        docker.stopContainer(containerName);
    }
    logger.info("Deleting container " + containerName.asString());
    docker.deleteContainer(containerName);
}

Example 5 with PrefixLogger

use of com.yahoo.vespa.hosted.node.admin.util.PrefixLogger in project vespa by vespa-engine.

the class DockerOperationsImpl method executeCommandInNetworkNamespace.

@Override
public void executeCommandInNetworkNamespace(ContainerName containerName, String... command) {
    final PrefixLogger logger = PrefixLogger.getNodeAgentLogger(DockerOperationsImpl.class, containerName);
    final Integer containerPid = docker.getContainer(containerName).filter(container -> container.state.isRunning()).map(container -> container.pid).orElseThrow(() -> new RuntimeException("PID not found for container with name: " + containerName.asString()));
    Path procPath = environment.getPathResolver().getPathToRootOfHost().resolve("proc");
    final String[] wrappedCommand = Stream.concat(Stream.of("sudo", "nsenter", String.format("--net=%s/%d/ns/net", procPath, containerPid), "--"), Stream.of(command)).toArray(String[]::new);
    try {
        Pair<Integer, String> result = processExecuter.exec(wrappedCommand);
        if (result.getFirst() != 0) {
            String msg = String.format("Failed to execute %s in network namespace for %s (PID = %d), exit code: %d, output: %s", Arrays.toString(wrappedCommand), containerName.asString(), containerPid, result.getFirst(), result.getSecond());
            logger.error(msg);
            throw new RuntimeException(msg);
        }
    } catch (IOException e) {
        logger.warning(String.format("IOException while executing %s in network namespace for %s (PID = %d)", Arrays.toString(wrappedCommand), containerName.asString(), containerPid), e);
        throw new RuntimeException(e);
    }
}