Examples with ZooKeeperClient com.spotify.helios.servicescommon.coordination.ZooKeeperClient used on opensource projects

Example 1 with ZooKeeperClient

use of com.spotify.helios.servicescommon.coordination.ZooKeeperClient in project helios by spotify.

the class TaskHistoryWriterTest method testZooKeeperErrorDoesntLoseItemsReally.

@Test
public void testZooKeeperErrorDoesntLoseItemsReally() throws Exception {
    final ZooKeeperClient mockClient = mock(ZooKeeperClient.class, delegatesTo(client));
    final String path = Paths.historyJobHostEventsTimestamp(JOB_ID, HOSTNAME, TIMESTAMP);
    // make save operations fail
    final AtomicBoolean throwExceptionOnCreateAndSet = new AtomicBoolean(true);
    final KeeperException exc = new ConnectionLossException();
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            if (throwExceptionOnCreateAndSet.get()) {
                throw exc;
            } else {
                client.createAndSetData((String) invocation.getArguments()[0], (byte[]) invocation.getArguments()[1]);
                return null;
            }
        }
    }).when(mockClient).createAndSetData(path, TASK_STATUS.toJsonBytes());
    makeWriter(mockClient);
    writer.saveHistoryItem(TASK_STATUS, TIMESTAMP);
    // wait up to 10s for it to fail twice -- and make sure I mocked it correctly.
    verify(mockClient, timeout(10000).atLeast(2)).createAndSetData(path, TASK_STATUS.toJsonBytes());
    // now make the client work
    throwExceptionOnCreateAndSet.set(false);
    awaitHistoryItems();
}

Example 2 with ZooKeeperClient

use of com.spotify.helios.servicescommon.coordination.ZooKeeperClient in project helios by spotify.

the class ZooKeeperAclInitializer method initializeAcl.

static void initializeAcl(final String zooKeeperConnectionString, final String zooKeeperClusterId, final String masterUser, final String masterPassword, final String agentUser, final String agentPassword) throws KeeperException {
    final ACLProvider aclProvider = heliosAclProvider(masterUser, digest(masterUser, masterPassword), agentUser, digest(agentUser, agentPassword));
    final List<AuthInfo> authorization = Lists.newArrayList(new AuthInfo("digest", String.format("%s:%s", masterUser, masterPassword).getBytes()));
    final RetryPolicy zooKeeperRetryPolicy = new ExponentialBackoffRetry(1000, 3);
    final CuratorFramework curator = new CuratorClientFactoryImpl().newClient(zooKeeperConnectionString, (int) TimeUnit.SECONDS.toMillis(60), (int) TimeUnit.SECONDS.toMillis(15), zooKeeperRetryPolicy, aclProvider, authorization);
    final ZooKeeperClient client = new DefaultZooKeeperClient(curator, zooKeeperClusterId);
    try {
        client.start();
        initializeAclRecursive(client, "/", aclProvider);
    } finally {
        client.close();
    }
}

Example 3 with ZooKeeperClient

use of com.spotify.helios.servicescommon.coordination.ZooKeeperClient in project helios by spotify.

the class ZooKeeperClusterIdTest method testZooKeeperClient.

@Test
public void testZooKeeperClient() throws Exception {
    // Create the cluster ID node
    zk().curatorWithSuperAuth().newNamespaceAwareEnsurePath(Paths.configId(zkClusterId)).ensure(zk().curatorWithSuperAuth().getZookeeperClient());
    // We need to create a new curator because ZooKeeperClient will try to start it,
    // and zk().curator() has already been started.
    final ExponentialBackoffRetry retryPolicy = new ExponentialBackoffRetry(1000, 3);
    final CuratorFramework curator = CuratorFrameworkFactory.builder().retryPolicy(retryPolicy).connectString(zk().connectString()).build();
    final ZooKeeperClient client = new DefaultZooKeeperClient(curator, zkClusterId);
    client.start();
    // This should work since the cluster ID exists
    client.create("/test");
    // Now let's remove the cluster ID
    client.delete(Paths.configId(zkClusterId));
    // Sleep so the watcher thread in ZooKeeperClient has a chance to update state
    Thread.sleep(500);
    // Try the same operation again, and it should fail this time
    try {
        client.ensurePath(Paths.configJobs());
        fail("ZooKeeper operation should have failed because cluster ID was removed");
    } catch (IllegalStateException ignore) {
    // ignored
    }
}

Example 4 with ZooKeeperClient

use of com.spotify.helios.servicescommon.coordination.ZooKeeperClient in project helios by spotify.

the class JobRemoveTest method testRemoveJobDeletesHistory.

@Test
public void testRemoveJobDeletesHistory() throws Exception {
    startDefaultAgent(testHost());
    awaitHostStatus(testHost(), UP, LONG_WAIT_SECONDS, SECONDS);
    final JobId jobId = createJob(testJobName, testJobVersion, BUSYBOX, IDLE_COMMAND);
    deployJob(jobId, testHost());
    awaitJobState(defaultClient(), testHost(), jobId, TaskStatus.State.RUNNING, LONG_WAIT_SECONDS, SECONDS);
    undeployJob(jobId, testHost());
    awaitJobUndeployed(defaultClient(), testHost(), jobId, LONG_WAIT_SECONDS, SECONDS);
    final ZooKeeperClient zkClient = new ZooKeeperClientProvider(new DefaultZooKeeperClient(zk().curatorWithSuperAuth()), ZooKeeperModelReporter.noop()).get("test-client");
    // Check that there's some history events
    assertNotNull(zkClient.stat(Paths.historyJob(jobId)));
    // Remove job
    final JobDeleteResponse response = defaultClient().deleteJob(jobId).get(WAIT_TIMEOUT_SECONDS, SECONDS);
    assertEquals(JobDeleteResponse.Status.OK, response.getStatus());
    // Verify that history is gone
    assertNull(zkClient.stat(Paths.historyJob(jobId)));
}

Example 5 with ZooKeeperClient

use of com.spotify.helios.servicescommon.coordination.ZooKeeperClient in project helios by spotify.

the class MasterService method setupZookeeperClient.

/**
 * Create a Zookeeper client and create the control and state nodes if needed.
 *
 * @param config The service configuration.
 *
 * @return A zookeeper client.
 */
private ZooKeeperClient setupZookeeperClient(final MasterConfig config) {
    ACLProvider aclProvider = null;
    List<AuthInfo> authorization = null;
    final String masterUser = config.getZookeeperAclMasterUser();
    final String masterPassword = config.getZooKeeperAclMasterPassword();
    final String agentUser = config.getZookeeperAclAgentUser();
    final String agentDigest = config.getZooKeeperAclAgentDigest();
    if (!isNullOrEmpty(masterPassword)) {
        if (isNullOrEmpty(masterUser)) {
            throw new HeliosRuntimeException("Master username must be set if a password is set");
        }
        authorization = Lists.newArrayList(new AuthInfo("digest", String.format("%s:%s", masterUser, masterPassword).getBytes()));
    }
    if (config.isZooKeeperEnableAcls()) {
        if (isNullOrEmpty(masterUser) || isNullOrEmpty(masterPassword)) {
            throw new HeliosRuntimeException("ZooKeeper ACLs enabled but master username and/or password not set");
        }
        if (isNullOrEmpty(agentUser) || isNullOrEmpty(agentDigest)) {
            throw new HeliosRuntimeException("ZooKeeper ACLs enabled but agent username and/or digest not set");
        }
        aclProvider = heliosAclProvider(masterUser, digest(masterUser, masterPassword), agentUser, agentDigest);
    }
    final RetryPolicy zooKeeperRetryPolicy = new ExponentialBackoffRetry(1000, 3);
    final CuratorFramework curator = curatorClientFactory.newClient(config.getZooKeeperConnectionString(), config.getZooKeeperSessionTimeoutMillis(), config.getZooKeeperConnectionTimeoutMillis(), zooKeeperRetryPolicy, aclProvider, authorization);
    final ZooKeeperClient client = new DefaultZooKeeperClient(curator, config.getZooKeeperClusterId());
    client.start();
    zkRegistrar = ZooKeeperRegistrarService.newBuilder().setZooKeeperClient(client).setZooKeeperRegistrar(new MasterZooKeeperRegistrar(config.getName())).build();
    // place where we have access to the ACL provider.
    if (aclProvider != null) {
        // effects are limited to a spurious log line.
        try {
            final List<ACL> curAcls = client.getAcl("/");
            final List<ACL> wantedAcls = aclProvider.getAclForPath("/");
            if (!Sets.newHashSet(curAcls).equals(Sets.newHashSet(wantedAcls))) {
                log.info("Current ACL's on the zookeeper root node differ from desired, updating: {} -> {}", curAcls, wantedAcls);
                client.getCuratorFramework().setACL().withACL(wantedAcls).forPath("/");
            }
        } catch (Exception e) {
            log.error("Failed to get/set ACLs on the zookeeper root node", e);
        }
    }
    return client;
}