Examples with ZooKeeperClient com.spotify.helios.servicescommon.coordination.ZooKeeperClient used on opensource projects

Example 26 with ZooKeeperClient

use of com.spotify.helios.servicescommon.coordination.ZooKeeperClient in project helios by spotify.

the class DeploymentGroupTest method testUpdateDeploymentGroupHosts.

// A test that ensures healthy deployment groups will perform a rolling update when their hosts
// change.
@Test
public void testUpdateDeploymentGroupHosts() throws Exception {
    final ZooKeeperClient client = spy(this.client);
    final ZooKeeperMasterModel masterModel = spy(newMasterModel(client));
    // Return a job so we can add a real deployment group.
    final Job job = Job.newBuilder().setCommand(ImmutableList.of("COMMAND")).setImage("IMAGE").setName("JOB_NAME").setVersion("VERSION").build();
    doReturn(job).when(masterModel).getJob(job.getId());
    // Add a real deployment group.
    final DeploymentGroup dg = DeploymentGroup.newBuilder().setName(GROUP_NAME).setHostSelectors(ImmutableList.of(HostSelector.parse("role=melmac"))).setJobId(job.getId()).setRolloutOptions(RolloutOptions.newBuilder().build()).setRollingUpdateReason(MANUAL).build();
    masterModel.addDeploymentGroup(dg);
    // Setup some hosts
    final String oldHost = "host1";
    final String newHost = "host2";
    client.ensurePath(Paths.configHost(oldHost));
    client.ensurePath(Paths.configHost(newHost));
    client.ensurePath(Paths.statusHostUp(oldHost));
    client.ensurePath(Paths.statusHostUp(newHost));
    // Give the deployment group a host.
    client.setData(Paths.statusDeploymentGroupHosts(dg.getName()), Json.asBytes(ImmutableList.of(oldHost)));
    // And a status...
    client.setData(Paths.statusDeploymentGroup(dg.getName()), DeploymentGroupStatus.newBuilder().setState(DONE).build().toJsonBytes());
    // Switch out our host!
    // TODO(negz): Use an unchanged host, make sure ordering remains the same.
    masterModel.updateDeploymentGroupHosts(dg.getName(), ImmutableList.of(newHost));
    verify(client, times(2)).transaction(opCaptor.capture());
    final DeploymentGroup changed = dg.toBuilder().setRollingUpdateReason(HOSTS_CHANGED).build();
    // Ensure we set the DG status to HOSTS_CHANGED.
    // This means we triggered a rolling update.
    final ZooKeeperOperation setDeploymentGroupHostChanged = set(Paths.configDeploymentGroup(dg.getName()), changed);
    // Ensure ZK tasks are written to:
    // - Perform a rolling undeploy for the removed (old) host
    // - Perform a rolling update for the added (new) host and the unchanged host
    final List<RolloutTask> tasks = ImmutableList.<RolloutTask>builder().addAll(RollingUndeployPlanner.of(changed).plan(singletonList(oldHost))).addAll(RollingUpdatePlanner.of(changed).plan(singletonList(newHost))).build();
    final ZooKeeperOperation setDeploymentGroupTasks = set(Paths.statusDeploymentGroupTasks(dg.getName()), DeploymentGroupTasks.newBuilder().setRolloutTasks(tasks).setTaskIndex(0).setDeploymentGroup(changed).build());
    assertThat(opCaptor.getValue(), hasItems(setDeploymentGroupHostChanged, setDeploymentGroupTasks));
}

Example 27 with ZooKeeperClient

use of com.spotify.helios.servicescommon.coordination.ZooKeeperClient in project helios by spotify.

the class ZooKeeperMasterModelIntegrationTest method setup.

@Before
public void setup() throws Exception {
    final RetryPolicy retryPolicy = new ExponentialBackoffRetry(1000, 3);
    final CuratorFramework curator = CuratorFrameworkFactory.newClient(zk.connectString(), retryPolicy);
    curator.start();
    final ZooKeeperClient client = new DefaultZooKeeperClient(curator);
    // TODO (dano): this bootstrapping is essentially duplicated from MasterService,
    // should be moved into ZooKeeperMasterModel?
    client.ensurePath(Paths.configHosts());
    client.ensurePath(Paths.configJobs());
    client.ensurePath(Paths.configJobRefs());
    client.ensurePath(Paths.statusHosts());
    client.ensurePath(Paths.statusMasters());
    client.ensurePath(Paths.historyJobs());
    final ZooKeeperClientProvider zkProvider = new ZooKeeperClientProvider(client, ZooKeeperModelReporter.noop());
    final List<EventSender> eventSenders = ImmutableList.of(eventSender);
    model = new ZooKeeperMasterModel(zkProvider, getClass().getName(), eventSenders, deploymentGroupEventTopic);
}

Example 28 with ZooKeeperClient

use of com.spotify.helios.servicescommon.coordination.ZooKeeperClient in project helios by spotify.

the class JobHistoryReaperTest method testJobHistoryReaper.

@Test
public void testJobHistoryReaper() throws Exception {
    final MasterModel masterModel = mock(MasterModel.class);
    final List<Datapoint> datapoints = Lists.newArrayList(// A job history with a corresponding job should NOT BE reaped.
    new Datapoint("job1", Job.newBuilder().setName("job1").build(), false), // A job history without a corresponding job should BE reaped.
    new Datapoint("job2", null, true));
    for (final Datapoint dp : datapoints) {
        when(masterModel.getJob(argThat(matchesName(dp.getJobName())))).thenReturn(dp.getJob());
    }
    final ZooKeeperClient client = mock(ZooKeeperClient.class);
    final List<String> jobHistories = ImmutableList.of("job1", "job2");
    when(client.getChildren(Paths.historyJobs())).thenReturn(jobHistories);
    final JobHistoryReaper reaper = new JobHistoryReaper(masterModel, client, 100, 0);
    reaper.startAsync().awaitRunning();
    for (final Datapoint datapoint : datapoints) {
        if (datapoint.expectReap) {
            verify(client, timeout(500)).deleteRecursive(Paths.historyJob(datapoint.getJobId()));
        } else {
            verify(client, never()).deleteRecursive(Paths.historyJob(datapoint.getJobId()));
        }
    }
}

Example 29 with ZooKeeperClient

use of com.spotify.helios.servicescommon.coordination.ZooKeeperClient in project helios by spotify.

the class AgentService method setupZookeeperClient.

/**
 * Create a Zookeeper client and create the control and state nodes if needed.
 *
 * @param config The service configuration.
 *
 * @return A zookeeper client.
 */
private ZooKeeperClient setupZookeeperClient(final AgentConfig config, final String id, final CountDownLatch zkRegistrationSignal) {
    ACLProvider aclProvider = null;
    List<AuthInfo> authorization = null;
    final String agentUser = config.getZookeeperAclAgentUser();
    final String agentPassword = config.getZooKeeperAclAgentPassword();
    final String masterUser = config.getZookeeperAclMasterUser();
    final String masterDigest = config.getZooKeeperAclMasterDigest();
    if (!isNullOrEmpty(agentPassword)) {
        if (isNullOrEmpty(agentUser)) {
            throw new HeliosRuntimeException("Agent username must be set if a password is set");
        }
        authorization = Lists.newArrayList(new AuthInfo("digest", String.format("%s:%s", agentUser, agentPassword).getBytes()));
    }
    if (config.isZooKeeperEnableAcls()) {
        if (isNullOrEmpty(agentUser) || isNullOrEmpty(agentPassword)) {
            throw new HeliosRuntimeException("ZooKeeper ACLs enabled but agent username and/or password not set");
        }
        if (isNullOrEmpty(masterUser) || isNullOrEmpty(masterDigest)) {
            throw new HeliosRuntimeException("ZooKeeper ACLs enabled but master username and/or digest not set");
        }
        aclProvider = heliosAclProvider(masterUser, masterDigest, agentUser, digest(agentUser, agentPassword));
    }
    final RetryPolicy zooKeeperRetryPolicy = new ExponentialBackoffRetry(1000, 3);
    final CuratorFramework curator = new CuratorClientFactoryImpl().newClient(config.getZooKeeperConnectionString(), config.getZooKeeperSessionTimeoutMillis(), config.getZooKeeperConnectionTimeoutMillis(), zooKeeperRetryPolicy, aclProvider, authorization);
    final ZooKeeperClient client = new DefaultZooKeeperClient(curator, config.getZooKeeperClusterId());
    client.start();
    // Register the agent
    final AgentZooKeeperRegistrar agentZooKeeperRegistrar = new AgentZooKeeperRegistrar(config.getName(), id, config.getZooKeeperRegistrationTtlMinutes(), new SystemClock());
    zkRegistrar = ZooKeeperRegistrarService.newBuilder().setZooKeeperClient(client).setZooKeeperRegistrar(agentZooKeeperRegistrar).setZkRegistrationSignal(zkRegistrationSignal).build();
    return client;
}

Example 30 with ZooKeeperClient

use of com.spotify.helios.servicescommon.coordination.ZooKeeperClient in project helios by spotify.

the class ZooKeeperMasterModel method removeDeploymentGroup.

/**
 * Remove a deployment group.
 *
 * <p>If successful, all ZK nodes associated with the DG will be deleted. Specifically these
 * nodes are guaranteed to be non-existent after a successful remove (not all of them might exist
 * before, though):
 * <ul>
 * <li>/config/deployment-groups/[group-name]</li>
 * <li>/status/deployment-groups/[group-name]</li>
 * <li>/status/deployment-groups/[group-name]/hosts</li>
 * <li>/status/deployment-groups/[group-name]/removed</li>
 * <li>/status/deployment-group-tasks/[group-name]</li>
 * </ul>
 * If the operation fails no ZK nodes will be removed.
 *
 * @throws DeploymentGroupDoesNotExistException If the DG does not exist.
 */
@Override
public void removeDeploymentGroup(final String name) throws DeploymentGroupDoesNotExistException {
    log.info("removing deployment-group: name={}", name);
    final ZooKeeperClient client = provider.get("removeDeploymentGroup");
    try {
        client.ensurePath(Paths.configDeploymentGroups());
        client.ensurePath(Paths.statusDeploymentGroups());
        client.ensurePath(Paths.statusDeploymentGroupTasks());
        final List<ZooKeeperOperation> operations = Lists.newArrayList();
        final List<String> paths = ImmutableList.of(Paths.configDeploymentGroup(name), Paths.statusDeploymentGroup(name), Paths.statusDeploymentGroupHosts(name), Paths.statusDeploymentGroupRemovedHosts(name), Paths.statusDeploymentGroupTasks(name));
        // DGs to become slower and spam logs with errors so we want to avoid it.
        for (final String path : paths) {
            if (client.exists(path) == null) {
                operations.add(create(path));
            }
        }
        for (final String path : Lists.reverse(paths)) {
            operations.add(delete(path));
        }
        client.transaction(operations);
    } catch (final NoNodeException e) {
        throw new DeploymentGroupDoesNotExistException(name);
    } catch (final KeeperException e) {
        throw new HeliosRuntimeException("removing deployment-group " + name + " failed", e);
    }
}