Examples with ComputerJobSpec com.baidu.hugegraph.computer.k8s.crd.model.ComputerJobSpec used on opensource projects

Example 1 with ComputerJobSpec

use of com.baidu.hugegraph.computer.k8s.crd.model.ComputerJobSpec in project hugegraph-computer by hugegraph.

the class ComputerJobDeployer method desiredConfigMap.

private ConfigMap desiredConfigMap(HugeGraphComputerJob computerJob) {
    String crName = computerJob.getMetadata().getName();
    ComputerJobSpec spec = computerJob.getSpec();
    Map<String, String> computerConf = spec.getComputerConf();
    Map<String, String> data = new HashMap<>();
    data.put(Constants.COMPUTER_CONF_FILE, KubeUtil.asProperties(computerConf));
    String log4jXml = spec.getLog4jXml();
    if (StringUtils.isNotBlank(log4jXml)) {
        data.put(Constants.LOG_XML_FILE, log4jXml);
    }
    String name = KubeUtil.configMapName(crName);
    return new ConfigMapBuilder().withMetadata(this.getMetadata(computerJob, name)).withData(data).build();
}

Example 2 with ComputerJobSpec

use of com.baidu.hugegraph.computer.k8s.crd.model.ComputerJobSpec in project hugegraph-computer by hugegraph.

the class ComputerJobDeployer method desiredMasterJob.

public Job desiredMasterJob(HugeGraphComputerJob computerJob, Set<ContainerPort> ports) {
    String crName = computerJob.getMetadata().getName();
    String namespace = computerJob.getMetadata().getNamespace();
    ComputerJobSpec spec = computerJob.getSpec();
    List<String> command = spec.getMasterCommand();
    if (CollectionUtils.isEmpty(command)) {
        command = Constants.COMMAND;
    }
    List<String> args = spec.getMasterArgs();
    if (CollectionUtils.isEmpty(args)) {
        args = Constants.MASTER_ARGS;
    }
    String name = KubeUtil.masterJobName(crName);
    ObjectMeta metadata = this.getMetadata(computerJob, name);
    Container container = this.getContainer(name, namespace, spec, ports, command, args);
    List<Container> containers = Lists.newArrayList(container);
    final int instances = Constants.MASTER_INSTANCES;
    return this.getJob(crName, metadata, spec, instances, containers);
}

Example 3 with ComputerJobSpec

use of com.baidu.hugegraph.computer.k8s.crd.model.ComputerJobSpec in project hugegraph-computer by hugegraph.

the class ComputerJobDeployer method desiredWorkerJob.

public Job desiredWorkerJob(HugeGraphComputerJob computerJob, Set<ContainerPort> ports) {
    String crName = computerJob.getMetadata().getName();
    String namespace = computerJob.getMetadata().getNamespace();
    ComputerJobSpec spec = computerJob.getSpec();
    List<String> command = spec.getWorkerCommand();
    if (CollectionUtils.isEmpty(command)) {
        command = Constants.COMMAND;
    }
    List<String> args = spec.getWorkerArgs();
    if (CollectionUtils.isEmpty(args)) {
        args = Constants.WORKER_ARGS;
    }
    String name = KubeUtil.workerJobName(crName);
    ObjectMeta metadata = this.getMetadata(computerJob, name);
    Container container = this.getContainer(name, namespace, spec, ports, command, args);
    List<Container> containers = Lists.newArrayList(container);
    final int instances = spec.getWorkerInstances();
    return this.getJob(crName, metadata, spec, instances, containers);
}

Example 4 with ComputerJobSpec

use of com.baidu.hugegraph.computer.k8s.crd.model.ComputerJobSpec in project hugegraph-computer by hugegraph.

the class KubernetesDriverTest method testOnClose.

@Test
public void testOnClose() {
    Map<String, Pair<CompletableFuture<Void>, JobObserver>> waits = Whitebox.getInternalState(this.driver, "waits");
    waits.put("test-123", Pair.of(new CompletableFuture<>(), Mockito.mock(JobObserver.class)));
    AbstractWatchManager<HugeGraphComputerJob> watch = Whitebox.getInternalState(this.driver, "watch");
    Watcher<HugeGraphComputerJob> watcher = Whitebox.getInternalState(watch, "watcher");
    watcher.eventReceived(Watcher.Action.ADDED, null);
    watcher.eventReceived(Watcher.Action.ERROR, new HugeGraphComputerJob());
    HugeGraphComputerJob computerJob = new HugeGraphComputerJob();
    computerJob.setSpec(new ComputerJobSpec());
    watcher.eventReceived(Watcher.Action.MODIFIED, computerJob);
    WatcherException testClose = new WatcherException("test close");
    watcher.onClose(testClose);
    MutableBoolean watchActive = Whitebox.getInternalState(this.driver, "watchActive");
    Assert.assertFalse(watchActive.booleanValue());
}

Example 5 with ComputerJobSpec

use of com.baidu.hugegraph.computer.k8s.crd.model.ComputerJobSpec in project hugegraph-computer by hugegraph.

the class ComputerJobController method derivedCRStatus.

private ComputerJobStatus derivedCRStatus(ComputerJobComponent observed) {
    HugeGraphComputerJob computerJob = observed.computerJob();
    ComputerJobSpec spec = computerJob.getSpec();
    MutableInt failedComponents = new MutableInt(0);
    MutableInt succeededComponents = new MutableInt(0);
    MutableInt runningComponents = new MutableInt(0);
    ComputerJobStatus status = Serialization.clone(computerJob.getStatus());
    // ConfigMap
    ConfigMap configMap = observed.configMap();
    if (configMap != null) {
        ComponentState configMapState = new ComponentStateBuilder().withName(configMap.getMetadata().getName()).withState(CommonComponentState.READY.value()).build();
        status.getComponentStates().setConfigMap(configMapState);
    } else if (status.getComponentStates().getConfigMap() != null) {
        status.getComponentStates().getConfigMap().setState(CommonComponentState.DELETED.value());
    }
    // MasterJob
    Job masterJob = observed.masterJob();
    ComponentState masterJobState = this.deriveJobStatus(masterJob, observed.masterPods(), status.getComponentStates().getMasterJob(), Constants.MASTER_INSTANCES, failedComponents, succeededComponents, runningComponents);
    status.getComponentStates().setMasterJob(masterJobState);
    // WorkerJob
    Job workerJob = observed.workerJob();
    ComponentState workerJobState = this.deriveJobStatus(workerJob, observed.workerPods(), status.getComponentStates().getWorkerJob(), spec.getWorkerInstances(), failedComponents, succeededComponents, runningComponents);
    status.getComponentStates().setWorkerJob(workerJobState);
    if (failedComponents.intValue() > ALLOW_FAILED_COMPONENTS) {
        status.setJobStatus(JobStatus.FAILED.name());
        this.recordFailedEvent(computerJob, masterJobState, workerJobState);
        return status;
    } else if (succeededComponents.intValue() == TOTAL_COMPONENTS) {
        status.setJobStatus(JobStatus.SUCCEEDED.name());
        String crName = computerJob.getMetadata().getName();
        long cost = this.calculateJobCost(computerJob);
        this.recordEvent(computerJob, EventType.NORMAL, KubeUtil.succeedEventName(crName), "ComputerJobSucceed", String.format("Job %s run successfully, took %ss", crName, cost));
        return status;
    }
    int activeComponents = runningComponents.intValue() + succeededComponents.intValue();
    if (activeComponents == TOTAL_COMPONENTS) {
        status.setJobStatus(JobStatus.RUNNING.name());
    } else {
        status.setJobStatus(JobStatus.INITIALIZING.name());
    }
    return status;
}