use of com.spotify.helios.common.descriptors.Job in project helios by spotify.
the class DeploymentTest method testLotsOfConcurrentJobs.
@Test
public void testLotsOfConcurrentJobs() throws Exception {
startDefaultMaster();
final HeliosClient client = defaultClient();
startDefaultAgent(testHost());
awaitHostRegistered(client, testHost(), LONG_WAIT_SECONDS, SECONDS);
awaitHostStatus(client, testHost(), UP, LONG_WAIT_SECONDS, SECONDS);
final int numberOfJobs = 40;
final List<JobId> jobIds = Lists.newArrayListWithCapacity(numberOfJobs);
final String jobName = testJobName + "_" + toHexString(ThreadLocalRandom.current().nextInt());
// create and deploy a bunch of jobs
for (Integer i = 0; i < numberOfJobs; i++) {
final Job job = Job.newBuilder().setName(jobName).setVersion(i.toString()).setImage(BUSYBOX).setCommand(IDLE_COMMAND).setCreatingUser(TEST_USER).build();
final JobId jobId = job.getId();
final CreateJobResponse created = client.createJob(job).get();
assertEquals(CreateJobResponse.Status.OK, created.getStatus());
final Deployment deployment = Deployment.of(jobId, START, TEST_USER);
final JobDeployResponse deployed = client.deploy(deployment, testHost()).get();
assertEquals(JobDeployResponse.Status.OK, deployed.getStatus());
jobIds.add(jobId);
}
// get the container ID's for the jobs
final Set<String> containerIds = Sets.newHashSetWithExpectedSize(numberOfJobs);
for (final JobId jobId : jobIds) {
final TaskStatus taskStatus = awaitJobState(client, testHost(), jobId, RUNNING, LONG_WAIT_SECONDS, SECONDS);
containerIds.add(taskStatus.getContainerId());
}
try (final DockerClient dockerClient = getNewDockerClient()) {
// kill all the containers for the jobs
for (final String containerId : containerIds) {
dockerClient.killContainer(containerId);
}
// make sure all the containers come back up
final int restartedContainers = Polling.await(LONG_WAIT_SECONDS, SECONDS, new Callable<Integer>() {
@Override
public Integer call() throws Exception {
int matchingContainerCount = 0;
for (final Container c : dockerClient.listContainers()) {
for (final String name : c.names()) {
if (name.contains(jobName)) {
matchingContainerCount++;
}
}
}
if (matchingContainerCount < containerIds.size()) {
return null;
} else {
return matchingContainerCount;
}
}
});
assertEquals(numberOfJobs, restartedContainers);
}
}
use of com.spotify.helios.common.descriptors.Job in project helios by spotify.
the class DeregisterTest method testDeregisterJobDeployedWithoutStatus.
// Verify that we can deregister a host there are jobs deployed to it, for which there's no
// corresponding status information. For example, if a job was deployed to the host after is went
// down.
@Test
public void testDeregisterJobDeployedWithoutStatus() throws Exception {
startDefaultMaster();
final String host = testHost();
final HeliosClient client = defaultClient();
final DefaultZooKeeperClient zkClient = new DefaultZooKeeperClient(zk().curatorWithSuperAuth());
final String idPath = Paths.configHostId(host);
ZooKeeperRegistrarUtil.registerHost(zkClient, idPath, host, UUID.randomUUID().toString());
// Create a job
final Job job = Job.newBuilder().setName(testJobName).setVersion(testJobVersion).setImage(BUSYBOX).setCommand(IDLE_COMMAND).setPorts(ImmutableMap.of("foo", PortMapping.of(4711), "bar", PortMapping.of(4712, ports.localPort("bar")))).build();
final JobId jobId = job.getId();
final CreateJobResponse created = client.createJob(job).get();
assertEquals(CreateJobResponse.Status.OK, created.getStatus());
// Deploy the job on the agent
final Deployment deployment = Deployment.of(jobId, START);
final JobDeployResponse deployed = client.deploy(deployment, host).get();
assertEquals(JobDeployResponse.Status.OK, deployed.getStatus());
// Deregister agent
final HostDeregisterResponse deregisterResponse = client.deregisterHost(host).get();
assertEquals(HostDeregisterResponse.Status.OK, deregisterResponse.getStatus());
// Verify that it's possible to remove the job
final JobDeleteResponse deleteResponse = client.deleteJob(jobId).get();
assertEquals(JobDeleteResponse.Status.OK, deleteResponse.getStatus());
}
use of com.spotify.helios.common.descriptors.Job in project helios by spotify.
the class FlappingTest method test.
@Test
public void test() throws Exception {
// CircleCI boxes are too slow -- the job doesn't stop or restart fast enough to ever flap
assumeFalse(isCircleCi());
startDefaultMaster();
final String host = testHost();
startDefaultAgent(host);
final HeliosClient client = defaultClient();
awaitHostStatus(client, host, UP, LONG_WAIT_SECONDS, SECONDS);
final Job flapper = Job.newBuilder().setName(testJobName).setVersion(testJobVersion).setImage(BUSYBOX).setCommand(asList("nc", "-p", "4711", "-l")).addPort("poke", PortMapping.of(4711)).build();
final JobId jobId = createJob(flapper);
deployJob(jobId, host);
awaitTaskState(jobId, host, RUNNING);
// Poke the container to make it exit until it's classified as flapping
Polling.await(LONG_WAIT_SECONDS, SECONDS, new Callable<Object>() {
@Override
public Object call() throws Exception {
final JobStatus jobStatus = getOrNull(client.jobStatus(jobId));
final TaskStatus taskStatus = jobStatus.getTaskStatuses().get(host);
if (taskStatus.getThrottled() == FLAPPING) {
return true;
}
final PortMapping port = taskStatus.getPorts().get("poke");
assert port.getExternalPort() != null;
poke(port.getExternalPort());
return null;
}
});
// Verify that the job recovers after we stop poking
awaitJobThrottle(client, host, jobId, NO, LONG_WAIT_SECONDS, SECONDS);
}
use of com.spotify.helios.common.descriptors.Job in project helios by spotify.
the class HealthCheckTest method testContainerDiesDuringHealthcheck.
@Test
public void testContainerDiesDuringHealthcheck() throws Exception {
startDefaultMaster();
final HeliosClient client = defaultClient();
startDefaultAgent(testHost(), "--service-registry=" + registryAddress);
awaitHostStatus(client, testHost(), UP, LONG_WAIT_SECONDS, SECONDS);
final HealthCheck healthCheck = TcpHealthCheck.of("health");
final Job job = pokeJob(healthCheck);
final JobId jobId = createJob(job);
deployJob(jobId, testHost());
awaitTaskState(jobId, testHost(), HEALTHCHECKING);
// kill the underlying container
final JobStatus jobStatus = getOrNull(client.jobStatus(jobId));
final TaskStatus taskStatus = jobStatus.getTaskStatuses().get(testHost());
getNewDockerClient().killContainer(taskStatus.getContainerId());
// ensure the job is marked as failed
final int timeout = WAIT_TIMEOUT_SECONDS;
Polling.await(timeout, SECONDS, new Callable<Object>() {
@Override
public Object call() throws Exception {
final TaskStatusEvents jobHistory = getOrNull(client.jobHistory(jobId));
for (final TaskStatusEvent event : jobHistory.getEvents()) {
if (event.getStatus().getState() == FAILED) {
return true;
}
}
return null;
}
});
// wait for the job to come back up and start healthchecking again
awaitTaskState(jobId, testHost(), HEALTHCHECKING);
pokeAndVerifyRegistration(client, jobId, timeout);
}
use of com.spotify.helios.common.descriptors.Job in project helios by spotify.
the class CliDeploymentTest method testDeployAndUndeployJob.
@Test
public void testDeployAndUndeployJob() throws Exception {
startDefaultMaster();
// Wait for master to come up
Polling.await(LONG_WAIT_SECONDS, SECONDS, new Callable<String>() {
@Override
public String call() throws Exception {
final String output = cli("masters");
return output.contains(masterName()) ? output : null;
}
});
startDefaultAgent(testHost());
final String image = BUSYBOX;
final Map<String, PortMapping> ports = ImmutableMap.of("foo", PortMapping.of(4711), "bar", PortMapping.of(5000, externalPort));
final Map<ServiceEndpoint, ServicePorts> registration = ImmutableMap.of(ServiceEndpoint.of("foo-service", "tcp"), ServicePorts.of("foo"), ServiceEndpoint.of("bar-service", "http"), ServicePorts.of("bar"));
final Map<String, String> env = ImmutableMap.of("BAD", "f00d");
// Wait for agent to come up
awaitHostRegistered(testHost(), LONG_WAIT_SECONDS, SECONDS);
awaitHostStatus(testHost(), UP, LONG_WAIT_SECONDS, SECONDS);
// Create job
final JobId jobId = createJob(testJobName, testJobVersion, image, IDLE_COMMAND, env, ports, registration);
// Query for job
final Job expected = Job.newBuilder().setName(testJobName).setVersion(testJobVersion).setImage(image).setCommand(IDLE_COMMAND).setEnv(env).setPorts(ports).setRegistration(registration).setCreatingUser(TEST_USER).build();
final String inspectOutput = cli("inspect", "--json", jobId.toString());
final Job parsed = Json.read(inspectOutput, Job.class);
assertJobEquals(expected, parsed);
assertThat(cli("jobs", testJobName, "-q"), containsString(jobId.toString()));
assertThat(cli("jobs", testJobName + ":" + testJobVersion, "-q"), containsString(jobId.toString()));
assertEquals("job pattern foozbarz matched no jobs", cli("jobs", "foozbarz").trim());
assertTrue(cli("jobs", "foozbarz", "-q").isEmpty());
// Create a new job using the first job as a template
final Job expectedCloned = expected.toBuilder().setVersion(expected.getId().getVersion() + "-cloned").build();
final JobId clonedJobId = JobId.parse(WHITESPACE.trimFrom(cli("create", "-q", "-t", testJobName + ":" + testJobVersion, testJobName + ":" + testJobVersion + "-cloned")));
final String clonedInspectOutput = cli("inspect", "--json", clonedJobId.toString());
final Job clonedParsed = Json.read(clonedInspectOutput, Job.class);
assertJobEquals(expectedCloned, clonedParsed);
// Verify that port mapping and environment variables are correct
final String statusString = cli("status", "--job", jobId.toString(), "--json");
final Map<JobId, JobStatus> statuses = Json.read(statusString, STATUSES_TYPE);
final Job job = statuses.get(jobId).getJob();
assertEquals(ServicePorts.of("foo"), job.getRegistration().get(ServiceEndpoint.of("foo-service", "tcp")));
assertEquals(ServicePorts.of("bar"), job.getRegistration().get(ServiceEndpoint.of("bar-service", "http")));
assertEquals(4711, job.getPorts().get("foo").getInternalPort());
assertEquals(PortMapping.of(5000, externalPort), job.getPorts().get("bar"));
assertEquals("f00d", job.getEnv().get("BAD"));
final String duplicateJob = cli("create", testJobName + ":" + testJobVersion, image, "--", IDLE_COMMAND);
assertThat(duplicateJob, containsString("JOB_ALREADY_EXISTS"));
final String prestop = stopJob(jobId, testHost());
assertThat(prestop, containsString("JOB_NOT_DEPLOYED"));
// Deploy job
deployJob(jobId, testHost());
// Stop job
final String stop1 = stopJob(jobId, BOGUS_HOST);
assertThat(stop1, containsString("HOST_NOT_FOUND"));
final String stop2 = stopJob(BOGUS_JOB, testHost());
assertThat(stop2, containsString("Unknown job"));
final String stop3 = stopJob(jobId, testHost());
assertThat(stop3, containsString(testHost() + ": done"));
// Verify that undeploying the job from a nonexistent host fails
assertThat(cli("undeploy", jobId.toString(), BOGUS_HOST), containsString("HOST_NOT_FOUND"));
// Verify that undeploying a nonexistent job from the host fails
assertThat(cli("undeploy", BOGUS_JOB.toString(), testHost()), containsString("Unknown job"));
// Undeploy job
undeployJob(jobId, testHost());
}
Aggregations