Search in sources :

Example 36 with ACL

use of org.apache.zookeeper.data.ACL in project bookkeeper by apache.

the class ZKLogStreamMetadataStore method renameLogMetadata.

private CompletableFuture<Void> renameLogMetadata(URI uri, LogMetadataForWriter oldMetadata, String newStreamName) {
    final LinkedList<Op> createOps = Lists.newLinkedList();
    final LinkedList<Op> deleteOps = Lists.newLinkedList();
    List<ACL> acls = zooKeeperClient.getDefaultACL();
    // get the root path
    String oldRootPath = oldMetadata.getLogRootPath();
    String newRootPath = LogMetadata.getLogRootPath(uri, newStreamName, conf.getUnpartitionedStreamName());
    // 0. the log path
    deleteOps.addFirst(Op.delete(LogMetadata.getLogStreamPath(uri, oldMetadata.getLogName()), -1));
    // 1. the root path
    createOps.addLast(Op.create(newRootPath, EMPTY_BYTES, acls, CreateMode.PERSISTENT));
    deleteOps.addFirst(Op.delete(oldRootPath, -1));
    // 2. max id
    Versioned<byte[]> maxTxIdData = oldMetadata.getMaxTxIdData();
    deleteOldPathAndCreateNewPath(oldRootPath, MAX_TXID_PATH, maxTxIdData, newRootPath, DLUtils.serializeTransactionId(0L), acls, createOps, deleteOps);
    // 3. version
    createOps.addLast(Op.create(newRootPath + VERSION_PATH, intToBytes(LAYOUT_VERSION), acls, CreateMode.PERSISTENT));
    deleteOps.addFirst(Op.delete(oldRootPath + VERSION_PATH, -1));
    // 4. lock path (NOTE: if the stream is locked by a writer, then the delete will fail as you can not
    // delete the lock path if children is not empty.
    createOps.addLast(Op.create(newRootPath + LOCK_PATH, EMPTY_BYTES, acls, CreateMode.PERSISTENT));
    deleteOps.addFirst(Op.delete(oldRootPath + LOCK_PATH, -1));
    // 5. read lock path (NOTE: same reason as the write lock)
    createOps.addLast(Op.create(newRootPath + READ_LOCK_PATH, EMPTY_BYTES, acls, CreateMode.PERSISTENT));
    deleteOps.addFirst(Op.delete(oldRootPath + READ_LOCK_PATH, -1));
    // 6. allocation path
    Versioned<byte[]> allocationData = oldMetadata.getAllocationData();
    deleteOldPathAndCreateNewPath(oldRootPath, ALLOCATION_PATH, allocationData, newRootPath, EMPTY_BYTES, acls, createOps, deleteOps);
    // 7. log segments
    Versioned<byte[]> maxLSSNData = oldMetadata.getMaxLSSNData();
    deleteOldPathAndCreateNewPath(oldRootPath, LOGSEGMENTS_PATH, maxLSSNData, newRootPath, DLUtils.serializeLogSegmentSequenceNumber(UNASSIGNED_LOGSEGMENT_SEQNO), acls, createOps, deleteOps);
    // 8. copy the log segments
    CompletableFuture<List<LogSegmentMetadata>> segmentsFuture;
    if (pathExists(maxLSSNData)) {
        segmentsFuture = getLogSegments(zooKeeperClient, oldRootPath + LOGSEGMENTS_PATH);
    } else {
        segmentsFuture = FutureUtils.value(Collections.emptyList());
    }
    return segmentsFuture.thenApply(segments -> {
        for (LogSegmentMetadata segment : segments) {
            deleteOldSegmentAndCreateNewSegment(segment, newRootPath + LOGSEGMENTS_PATH, acls, createOps, deleteOps);
        }
        return null;
    }).thenCompose(ignored -> getMissingPaths(zooKeeperClient, uri, newStreamName)).thenCompose(paths -> {
        for (String path : paths) {
            createOps.addFirst(Op.create(path, EMPTY_BYTES, acls, CreateMode.PERSISTENT));
        }
        return executeRenameTxn(oldRootPath, newRootPath, createOps, deleteOps);
    });
}
Also used : CreateMode(org.apache.zookeeper.CreateMode) LogExistsException(org.apache.distributedlog.exceptions.LogExistsException) ZKDistributedLock(org.apache.distributedlog.lock.ZKDistributedLock) LogSegmentMetadataStore(org.apache.distributedlog.logsegment.LogSegmentMetadataStore) LoggerFactory(org.slf4j.LoggerFactory) PermitManager(org.apache.distributedlog.common.util.PermitManager) LogMetadataForReader(org.apache.distributedlog.metadata.LogMetadataForReader) Stat(org.apache.zookeeper.data.Stat) LogNotFoundException(org.apache.distributedlog.exceptions.LogNotFoundException) LOGSEGMENTS_PATH(org.apache.distributedlog.metadata.LogMetadata.LOGSEGMENTS_PATH) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) UnexpectedException(org.apache.distributedlog.exceptions.UnexpectedException) Optional(com.google.common.base.Optional) READ_LOCK_PATH(org.apache.distributedlog.metadata.LogMetadata.READ_LOCK_PATH) SchedulerUtils(org.apache.distributedlog.common.util.SchedulerUtils) Transaction(org.apache.distributedlog.util.Transaction) URI(java.net.URI) DistributedLogConstants(org.apache.distributedlog.DistributedLogConstants) ZKUtil(org.apache.zookeeper.ZKUtil) ZKException(org.apache.distributedlog.exceptions.ZKException) ZooKeeper(org.apache.zookeeper.ZooKeeper) Op(org.apache.zookeeper.Op) OrderedScheduler(org.apache.bookkeeper.common.util.OrderedScheduler) CancellationException(java.util.concurrent.CancellationException) FutureUtils(org.apache.bookkeeper.common.concurrent.FutureUtils) Create(org.apache.zookeeper.Op.Create) List(java.util.List) LockCancelledException(org.apache.distributedlog.exceptions.LockCancelledException) StatsLogger(org.apache.bookkeeper.stats.StatsLogger) ZKLogSegmentMetadataStore(org.apache.distributedlog.impl.ZKLogSegmentMetadataStore) EMPTY_BYTES(org.apache.distributedlog.DistributedLogConstants.EMPTY_BYTES) Code(org.apache.zookeeper.KeeperException.Code) LongVersion(org.apache.bookkeeper.versioning.LongVersion) DLInterruptedException(org.apache.distributedlog.exceptions.DLInterruptedException) LockingException(org.apache.distributedlog.exceptions.LockingException) CompletableFuture(java.util.concurrent.CompletableFuture) ACL(org.apache.zookeeper.data.ACL) UTF_8(com.google.common.base.Charsets.UTF_8) LOCK_PATH(org.apache.distributedlog.metadata.LogMetadata.LOCK_PATH) ZooKeeperConnectionException(org.apache.distributedlog.ZooKeeperClient.ZooKeeperConnectionException) Lists(com.google.common.collect.Lists) LogSegmentMetadata(org.apache.distributedlog.LogSegmentMetadata) DistributedLogConfiguration(org.apache.distributedlog.DistributedLogConfiguration) Versioned(org.apache.bookkeeper.versioning.Versioned) Utils(org.apache.distributedlog.util.Utils) LimitedPermitManager(org.apache.distributedlog.zk.LimitedPermitManager) ZKTransaction(org.apache.distributedlog.zk.ZKTransaction) OpResult(org.apache.zookeeper.OpResult) LinkedList(java.util.LinkedList) LogStreamMetadataStore(org.apache.distributedlog.metadata.LogStreamMetadataStore) Delete(org.apache.zookeeper.Op.Delete) Logger(org.slf4j.Logger) LAYOUT_VERSION(org.apache.distributedlog.metadata.LogMetadata.LAYOUT_VERSION) FutureEventListener(org.apache.bookkeeper.common.concurrent.FutureEventListener) KeeperException(org.apache.zookeeper.KeeperException) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) ALLOCATION_PATH(org.apache.distributedlog.metadata.LogMetadata.ALLOCATION_PATH) IOException(java.io.IOException) MAX_TXID_PATH(org.apache.distributedlog.metadata.LogMetadata.MAX_TXID_PATH) LogMetadata(org.apache.distributedlog.metadata.LogMetadata) TimeUnit(java.util.concurrent.TimeUnit) DLUtils(org.apache.distributedlog.util.DLUtils) DistributedLock(org.apache.distributedlog.lock.DistributedLock) PathUtils(org.apache.zookeeper.common.PathUtils) LogMetadataForWriter(org.apache.distributedlog.metadata.LogMetadataForWriter) AsyncCallback(org.apache.zookeeper.AsyncCallback) VisibleForTesting(com.google.common.annotations.VisibleForTesting) ZKSessionLockFactory(org.apache.distributedlog.lock.ZKSessionLockFactory) UNASSIGNED_LOGSEGMENT_SEQNO(org.apache.distributedlog.DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO) Collections(java.util.Collections) InvalidStreamNameException(org.apache.distributedlog.exceptions.InvalidStreamNameException) VERSION_PATH(org.apache.distributedlog.metadata.LogMetadata.VERSION_PATH) ZooKeeperClient(org.apache.distributedlog.ZooKeeperClient) SessionLockFactory(org.apache.distributedlog.lock.SessionLockFactory) Op(org.apache.zookeeper.Op) LogSegmentMetadata(org.apache.distributedlog.LogSegmentMetadata) ACL(org.apache.zookeeper.data.ACL) List(java.util.List) LinkedList(java.util.LinkedList)

Example 37 with ACL

use of org.apache.zookeeper.data.ACL in project bookkeeper by apache.

the class EnableZkSecurityBasicTest method checkACls.

private void checkACls(ZooKeeper zk, String path) throws KeeperException, InterruptedException {
    List<String> children = zk.getChildren(path, null);
    for (String child : children) {
        if (child.equals(READONLY)) {
            continue;
        }
        String fullPath = path.equals("/") ? path + child : path + "/" + child;
        List<ACL> acls = zk.getACL(fullPath, new Stat());
        checkACls(zk, fullPath);
        if (// skip zookeeper internal nodes
        !fullPath.startsWith("/zookeeper") && // node created by test setup
        !fullPath.equals("/ledgers") && // node created by test setup
        !fullPath.equals("/ledgers/" + BookKeeperConstants.AVAILABLE_NODE)) {
            assertEquals(1, acls.size());
            assertEquals(31, acls.get(0).getPerms());
            assertEquals(31, acls.get(0).getPerms());
            assertEquals("unexpected ACLS on " + fullPath + ": " + acls.get(0), "foo", acls.get(0).getId().getId());
            assertEquals("unexpected ACLS on " + fullPath + ": " + acls.get(0), "sasl", acls.get(0).getId().getScheme());
        }
    }
}
Also used : Stat(org.apache.zookeeper.data.Stat) ACL(org.apache.zookeeper.data.ACL)

Example 38 with ACL

use of org.apache.zookeeper.data.ACL in project oozie by apache.

the class ZKUtils method checkAndSetACLs.

private void checkAndSetACLs() throws Exception {
    if (Services.get().getConf().getBoolean(ZK_SECURE, false)) {
        // If znodes were previously created without security enabled, and now it is, we need to go through all existing znodes
        // and set the ACLs for them
        // We can't get the namespace znode through curator; have to go through zk client
        String namespace = "/" + client.getNamespace();
        if (client.getZookeeperClient().getZooKeeper().exists(namespace, null) != null) {
            List<ACL> acls = client.getZookeeperClient().getZooKeeper().getACL(namespace, new Stat());
            if (!acls.get(0).getId().getScheme().equals("sasl")) {
                log.info("'sasl' ACLs not set; setting...");
                List<String> children = client.getZookeeperClient().getZooKeeper().getChildren(namespace, null);
                for (String child : children) {
                    checkAndSetACLs("/" + child);
                }
                client.getZookeeperClient().getZooKeeper().setACL(namespace, saslACL, -1);
            }
        }
    }
}
Also used : Stat(org.apache.zookeeper.data.Stat) ACL(org.apache.zookeeper.data.ACL)

Example 39 with ACL

use of org.apache.zookeeper.data.ACL in project cdap by caskdata.

the class SharedResourceCacheTest method testCache.

@Test
public void testCache() throws Exception {
    String parentZNode = ZK_NAMESPACE + "/testCache";
    List<ACL> acls = Lists.newArrayList(ZooDefs.Ids.OPEN_ACL_UNSAFE);
    // create 2 cache instances
    ZKClientService zkClient1 = injector1.getInstance(ZKClientService.class);
    zkClient1.startAndWait();
    SharedResourceCache<String> cache1 = new SharedResourceCache<>(zkClient1, new StringCodec(), parentZNode, acls);
    cache1.init();
    // add items to one and wait for them to show up in the second
    String key1 = "key1";
    String value1 = "value1";
    cache1.put(key1, value1);
    ZKClientService zkClient2 = injector2.getInstance(ZKClientService.class);
    zkClient2.startAndWait();
    SharedResourceCache<String> cache2 = new SharedResourceCache<>(zkClient2, new StringCodec(), parentZNode, acls);
    cache2.init();
    waitForEntry(cache2, key1, value1, 10000);
    assertEquals(cache1.get(key1), cache2.get(key1));
    final String key2 = "key2";
    String value2 = "value2";
    cache1.put(key2, value2);
    waitForEntry(cache2, key2, value2, 10000);
    assertEquals(cache1.get(key2), cache2.get(key2));
    final String key3 = "key3";
    String value3 = "value3";
    cache2.put(key3, value3);
    waitForEntry(cache1, key3, value3, 10000);
    assertEquals(cache2.get(key3), cache1.get(key3));
    // replace an existing key
    final String value2new = "value2.2";
    final SettableFuture<String> value2future = SettableFuture.create();
    ResourceListener<String> value2listener = new BaseResourceListener<String>() {

        @Override
        public void onResourceUpdate(String name, String instance) {
            LOG.info("Resource updated: {}={}", name, instance);
            if (key2.equals(name) && value2new.equals(instance)) {
                value2future.set(instance);
            }
        }
    };
    cache2.addListener(value2listener);
    cache1.put(key2, value2new);
    assertEquals(value2new, value2future.get(10, TimeUnit.SECONDS));
    assertEquals(value2new, cache2.get(key2));
    cache2.removeListener(value2listener);
    // remove items from the second and wait for them to disappear from the first
    // Use a latch to make sure both cache see the changes
    final CountDownLatch key3RemoveLatch = new CountDownLatch(2);
    cache1.addListener(new BaseResourceListener<String>() {

        @Override
        public void onResourceDelete(String name) {
            LOG.info("Resource deleted on cache 1 {}", name);
            if (name.equals(key3)) {
                key3RemoveLatch.countDown();
            }
        }
    });
    final SettableFuture<String> key3RemoveFuture = SettableFuture.create();
    ResourceListener<String> key3Listener = new BaseResourceListener<String>() {

        @Override
        public void onResourceDelete(String name) {
            LOG.info("Resource deleted on cache 2 {}", name);
            if (name.equals(key3)) {
                key3RemoveFuture.set(name);
                key3RemoveLatch.countDown();
            }
        }
    };
    cache2.addListener(key3Listener);
    cache1.remove(key3);
    String removedKey = key3RemoveFuture.get();
    assertEquals(key3, removedKey);
    assertNull(cache2.get(key3));
    key3RemoveLatch.await(5, TimeUnit.SECONDS);
    // verify that cache contents are equal
    assertEquals(cache1, cache2);
}
Also used : ACL(org.apache.zookeeper.data.ACL) CountDownLatch(java.util.concurrent.CountDownLatch) ZKClientService(org.apache.twill.zookeeper.ZKClientService) Test(org.junit.Test)

Example 40 with ACL

use of org.apache.zookeeper.data.ACL in project xian by happyyangyuan.

the class TestFramework method testCreateACLWithReset.

@Test
public void testCreateACLWithReset() throws Exception {
    Timing timing = new Timing();
    CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder();
    CuratorFramework client = builder.connectString(server.getConnectString()).sessionTimeoutMs(timing.session()).connectionTimeoutMs(timing.connection()).authorization("digest", "me:pass".getBytes()).retryPolicy(new RetryOneTime(1)).build();
    client.start();
    try {
        final CountDownLatch lostLatch = new CountDownLatch(1);
        ConnectionStateListener listener = new ConnectionStateListener() {

            @Override
            public void stateChanged(CuratorFramework client, ConnectionState newState) {
                if (newState == ConnectionState.LOST) {
                    lostLatch.countDown();
                }
            }
        };
        client.getConnectionStateListenable().addListener(listener);
        ACL acl = new ACL(ZooDefs.Perms.WRITE, ZooDefs.Ids.AUTH_IDS);
        List<ACL> aclList = Lists.newArrayList(acl);
        client.create().withACL(aclList).forPath("/test", "test".getBytes());
        server.stop();
        Assert.assertTrue(timing.awaitLatch(lostLatch));
        try {
            client.checkExists().forPath("/");
            Assert.fail("Connection should be down");
        } catch (KeeperException.ConnectionLossException e) {
        // expected
        }
        server.restart();
        try {
            client.setData().forPath("/test", "test".getBytes());
        } catch (KeeperException.NoAuthException e) {
            Assert.fail("Auth failed");
        }
    } finally {
        CloseableUtils.closeQuietly(client);
    }
}
Also used : RetryOneTime(org.apache.curator.retry.RetryOneTime) CuratorFrameworkFactory(org.apache.curator.framework.CuratorFrameworkFactory) ACL(org.apache.zookeeper.data.ACL) CountDownLatch(java.util.concurrent.CountDownLatch) CuratorFramework(org.apache.curator.framework.CuratorFramework) Timing(org.apache.curator.test.Timing) ConnectionState(org.apache.curator.framework.state.ConnectionState) ConnectionStateListener(org.apache.curator.framework.state.ConnectionStateListener) KeeperException(org.apache.zookeeper.KeeperException) Test(org.testng.annotations.Test)

Aggregations

ACL (org.apache.zookeeper.data.ACL)214 Id (org.apache.zookeeper.data.Id)83 ArrayList (java.util.ArrayList)58 Test (org.junit.Test)58 Stat (org.apache.zookeeper.data.Stat)53 KeeperException (org.apache.zookeeper.KeeperException)35 Test (org.testng.annotations.Test)32 CuratorFramework (org.apache.curator.framework.CuratorFramework)19 Test (org.junit.jupiter.api.Test)18 Configuration (org.apache.hadoop.conf.Configuration)17 ZooKeeper (org.apache.zookeeper.ZooKeeper)16 ACLProvider (org.apache.curator.framework.api.ACLProvider)15 List (java.util.List)11 IOException (java.io.IOException)10 CountDownLatch (java.util.concurrent.CountDownLatch)9 TestableZooKeeper (org.apache.zookeeper.TestableZooKeeper)7 ByteArrayOutputStream (java.io.ByteArrayOutputStream)6 ExponentialBackoffRetry (org.apache.curator.retry.ExponentialBackoffRetry)6 RetryOneTime (org.apache.curator.retry.RetryOneTime)6 CreateMode (org.apache.zookeeper.CreateMode)6