Examples with Uuid org.apache.kafka.common.Uuid used on opensource projects

Example 31 with Uuid

use of org.apache.kafka.common.Uuid in project kafka by apache.

the class FetchResponseBenchmark method setup.

@Setup(Level.Trial)
public void setup() {
    MemoryRecords records = MemoryRecords.withRecords(CompressionType.NONE, new SimpleRecord(1000, "key1".getBytes(StandardCharsets.UTF_8), "value1".getBytes(StandardCharsets.UTF_8)), new SimpleRecord(1001, "key2".getBytes(StandardCharsets.UTF_8), "value2".getBytes(StandardCharsets.UTF_8)), new SimpleRecord(1002, "key3".getBytes(StandardCharsets.UTF_8), "value3".getBytes(StandardCharsets.UTF_8)));
    this.responseData = new LinkedHashMap<>();
    this.topicIds = new HashMap<>();
    this.topicNames = new HashMap<>();
    for (int topicIdx = 0; topicIdx < topicCount; topicIdx++) {
        String topic = UUID.randomUUID().toString();
        Uuid id = Uuid.randomUuid();
        topicIds.put(topic, id);
        topicNames.put(id, topic);
        for (int partitionId = 0; partitionId < partitionCount; partitionId++) {
            FetchResponseData.PartitionData partitionData = new FetchResponseData.PartitionData().setPartitionIndex(partitionId).setLastStableOffset(0).setLogStartOffset(0).setRecords(records);
            responseData.put(new TopicIdPartition(id, new TopicPartition(topic, partitionId)), partitionData);
        }
    }
    this.header = new ResponseHeader(100, ApiKeys.FETCH.responseHeaderVersion(ApiKeys.FETCH.latestVersion()));
    this.fetchResponse = FetchResponse.of(Errors.NONE, 0, 0, responseData);
    this.fetchResponseData = this.fetchResponse.data();
}

Example 32 with Uuid

use of org.apache.kafka.common.Uuid in project kafka by apache.

the class AclControlManager method deleteAclsForFilter.

AclDeleteResult deleteAclsForFilter(AclBindingFilter filter, List<ApiMessageAndVersion> records) {
    List<AclBindingDeleteResult> deleted = new ArrayList<>();
    for (Entry<Uuid, StandardAcl> entry : idToAcl.entrySet()) {
        Uuid id = entry.getKey();
        StandardAcl acl = entry.getValue();
        AclBinding binding = acl.toBinding();
        if (filter.matches(binding)) {
            deleted.add(new AclBindingDeleteResult(binding));
            records.add(new ApiMessageAndVersion(new RemoveAccessControlEntryRecord().setId(id), (short) 0));
        }
    }
    return new AclDeleteResult(deleted);
}

Example 33 with Uuid

use of org.apache.kafka.common.Uuid in project kafka by apache.

the class ReplicaFetcherThreadBenchmark method setup.

@Setup(Level.Trial)
public void setup() throws IOException {
    if (!logDir.mkdir())
        throw new IOException("error creating test directory");
    scheduler.startup();
    Properties props = new Properties();
    props.put("zookeeper.connect", "127.0.0.1:9999");
    KafkaConfig config = new KafkaConfig(props);
    LogConfig logConfig = createLogConfig();
    BrokerTopicStats brokerTopicStats = new BrokerTopicStats();
    LogDirFailureChannel logDirFailureChannel = Mockito.mock(LogDirFailureChannel.class);
    List<File> logDirs = Collections.singletonList(logDir);
    logManager = new LogManagerBuilder().setLogDirs(logDirs).setInitialOfflineDirs(Collections.emptyList()).setConfigRepository(new MockConfigRepository()).setInitialDefaultConfig(logConfig).setCleanerConfig(new CleanerConfig(0, 0, 0, 0, 0, 0.0, 0, false, "MD5")).setRecoveryThreadsPerDataDir(1).setFlushCheckMs(1000L).setFlushRecoveryOffsetCheckpointMs(10000L).setFlushStartOffsetCheckpointMs(10000L).setRetentionCheckMs(1000L).setMaxPidExpirationMs(60000).setInterBrokerProtocolVersion(ApiVersion.latestVersion()).setScheduler(scheduler).setBrokerTopicStats(brokerTopicStats).setLogDirFailureChannel(logDirFailureChannel).setTime(Time.SYSTEM).setKeepPartitionMetadataFile(true).build();
    LinkedHashMap<TopicIdPartition, FetchResponseData.PartitionData> initialFetched = new LinkedHashMap<>();
    HashMap<String, Uuid> topicIds = new HashMap<>();
    scala.collection.mutable.Map<TopicPartition, InitialFetchState> initialFetchStates = new scala.collection.mutable.HashMap<>();
    List<UpdateMetadataRequestData.UpdateMetadataPartitionState> updatePartitionState = new ArrayList<>();
    for (int i = 0; i < partitionCount; i++) {
        TopicPartition tp = new TopicPartition("topic", i);
        List<Integer> replicas = Arrays.asList(0, 1, 2);
        LeaderAndIsrRequestData.LeaderAndIsrPartitionState partitionState = new LeaderAndIsrRequestData.LeaderAndIsrPartitionState().setControllerEpoch(0).setLeader(0).setLeaderEpoch(0).setIsr(replicas).setZkVersion(1).setReplicas(replicas).setIsNew(true);
        IsrChangeListener isrChangeListener = Mockito.mock(IsrChangeListener.class);
        OffsetCheckpoints offsetCheckpoints = Mockito.mock(OffsetCheckpoints.class);
        Mockito.when(offsetCheckpoints.fetch(logDir.getAbsolutePath(), tp)).thenReturn(Option.apply(0L));
        AlterIsrManager isrChannelManager = Mockito.mock(AlterIsrManager.class);
        Partition partition = new Partition(tp, 100, ApiVersion$.MODULE$.latestVersion(), 0, Time.SYSTEM, isrChangeListener, new DelayedOperationsMock(tp), Mockito.mock(MetadataCache.class), logManager, isrChannelManager);
        partition.makeFollower(partitionState, offsetCheckpoints, topicId);
        pool.put(tp, partition);
        initialFetchStates.put(tp, new InitialFetchState(topicId, new BrokerEndPoint(3, "host", 3000), 0, 0));
        BaseRecords fetched = new BaseRecords() {

            @Override
            public int sizeInBytes() {
                return 0;
            }

            @Override
            public RecordsSend<? extends BaseRecords> toSend() {
                return null;
            }
        };
        initialFetched.put(new TopicIdPartition(topicId.get(), tp), new FetchResponseData.PartitionData().setPartitionIndex(tp.partition()).setLastStableOffset(0).setLogStartOffset(0).setRecords(fetched));
        updatePartitionState.add(new UpdateMetadataRequestData.UpdateMetadataPartitionState().setTopicName("topic").setPartitionIndex(i).setControllerEpoch(0).setLeader(0).setLeaderEpoch(0).setIsr(replicas).setZkVersion(1).setReplicas(replicas));
    }
    UpdateMetadataRequest updateMetadataRequest = new UpdateMetadataRequest.Builder(ApiKeys.UPDATE_METADATA.latestVersion(), 0, 0, 0, updatePartitionState, Collections.emptyList(), topicIds).build();
    // TODO: fix to support raft
    ZkMetadataCache metadataCache = new ZkMetadataCache(0);
    metadataCache.updateMetadata(0, updateMetadataRequest);
    replicaManager = new ReplicaManagerBuilder().setConfig(config).setMetrics(metrics).setTime(new MockTime()).setZkClient(Mockito.mock(KafkaZkClient.class)).setScheduler(scheduler).setLogManager(logManager).setQuotaManagers(Mockito.mock(QuotaFactory.QuotaManagers.class)).setBrokerTopicStats(brokerTopicStats).setMetadataCache(metadataCache).setLogDirFailureChannel(new LogDirFailureChannel(logDirs.size())).setAlterIsrManager(TestUtils.createAlterIsrManager()).build();
    fetcher = new ReplicaFetcherBenchThread(config, replicaManager, pool);
    fetcher.addPartitions(initialFetchStates);
    // force a pass to move partitions to fetching state. We do this in the setup phase
    // so that we do not measure this time as part of the steady state work
    fetcher.doWork();
    // handle response to engage the incremental fetch session handler
    fetcher.fetchSessionHandler().handleResponse(FetchResponse.of(Errors.NONE, 0, 999, initialFetched), ApiKeys.FETCH.latestVersion());
}

Example 34 with Uuid

use of org.apache.kafka.common.Uuid in project kafka by apache.

the class QuorumControllerTest method testTimeouts.

/**
 * Test that certain controller operations time out if they stay on the controller
 * queue for too long.
 */
@Test
public void testTimeouts() throws Throwable {
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(1, Optional.empty())) {
        try (QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
            QuorumController controller = controlEnv.activeController();
            CountDownLatch countDownLatch = controller.pause();
            CompletableFuture<CreateTopicsResponseData> createFuture = controller.createTopics(new CreateTopicsRequestData().setTimeoutMs(0).setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName("foo")).iterator())));
            long now = controller.time().nanoseconds();
            CompletableFuture<Map<Uuid, ApiError>> deleteFuture = controller.deleteTopics(now, Collections.singletonList(Uuid.ZERO_UUID));
            CompletableFuture<Map<String, ResultOrError<Uuid>>> findTopicIdsFuture = controller.findTopicIds(now, Collections.singletonList("foo"));
            CompletableFuture<Map<Uuid, ResultOrError<String>>> findTopicNamesFuture = controller.findTopicNames(now, Collections.singletonList(Uuid.ZERO_UUID));
            CompletableFuture<List<CreatePartitionsTopicResult>> createPartitionsFuture = controller.createPartitions(now, Collections.singletonList(new CreatePartitionsTopic()));
            CompletableFuture<ElectLeadersResponseData> electLeadersFuture = controller.electLeaders(new ElectLeadersRequestData().setTimeoutMs(0).setTopicPartitions(null));
            CompletableFuture<AlterPartitionReassignmentsResponseData> alterReassignmentsFuture = controller.alterPartitionReassignments(new AlterPartitionReassignmentsRequestData().setTimeoutMs(0).setTopics(Collections.singletonList(new ReassignableTopic())));
            CompletableFuture<ListPartitionReassignmentsResponseData> listReassignmentsFuture = controller.listPartitionReassignments(new ListPartitionReassignmentsRequestData().setTopics(null).setTimeoutMs(0));
            while (controller.time().nanoseconds() == now) {
                Thread.sleep(0, 10);
            }
            countDownLatch.countDown();
            assertYieldsTimeout(createFuture);
            assertYieldsTimeout(deleteFuture);
            assertYieldsTimeout(findTopicIdsFuture);
            assertYieldsTimeout(findTopicNamesFuture);
            assertYieldsTimeout(createPartitionsFuture);
            assertYieldsTimeout(electLeadersFuture);
            assertYieldsTimeout(alterReassignmentsFuture);
            assertYieldsTimeout(listReassignmentsFuture);
        }
    }
}

Example 35 with Uuid

use of org.apache.kafka.common.Uuid in project kafka by apache.

the class QuorumControllerTest method testMissingInMemorySnapshot.

@Test
public void testMissingInMemorySnapshot() throws Exception {
    int numBrokers = 3;
    int numPartitions = 3;
    String topicName = "topic-name";
    try (LocalLogManagerTestEnv logEnv = new LocalLogManagerTestEnv(1, Optional.empty());
        QuorumControllerTestEnv controlEnv = new QuorumControllerTestEnv(logEnv, b -> b.setConfigDefs(CONFIGS))) {
        QuorumController controller = controlEnv.activeController();
        Map<Integer, Long> brokerEpochs = registerBrokers(controller, numBrokers);
        // Create a lot of partitions
        List<CreatableReplicaAssignment> partitions = IntStream.range(0, numPartitions).mapToObj(partitionIndex -> new CreatableReplicaAssignment().setPartitionIndex(partitionIndex).setBrokerIds(Arrays.asList(0, 1, 2))).collect(Collectors.toList());
        Uuid topicId = controller.createTopics(new CreateTopicsRequestData().setTopics(new CreatableTopicCollection(Collections.singleton(new CreatableTopic().setName(topicName).setNumPartitions(-1).setReplicationFactor((short) -1).setAssignments(new CreatableReplicaAssignmentCollection(partitions.iterator()))).iterator()))).get().topics().find(topicName).topicId();
        // Create a lot of alter isr
        List<AlterIsrRequestData.PartitionData> alterIsrs = IntStream.range(0, numPartitions).mapToObj(partitionIndex -> {
            PartitionRegistration partitionRegistration = controller.replicationControl().getPartition(topicId, partitionIndex);
            return new AlterIsrRequestData.PartitionData().setPartitionIndex(partitionIndex).setLeaderEpoch(partitionRegistration.leaderEpoch).setCurrentIsrVersion(partitionRegistration.partitionEpoch).setNewIsr(Arrays.asList(0, 1));
        }).collect(Collectors.toList());
        AlterIsrRequestData.TopicData topicData = new AlterIsrRequestData.TopicData().setName(topicName);
        topicData.partitions().addAll(alterIsrs);
        int leaderId = 0;
        AlterIsrRequestData alterIsrRequest = new AlterIsrRequestData().setBrokerId(leaderId).setBrokerEpoch(brokerEpochs.get(leaderId));
        alterIsrRequest.topics().add(topicData);
        logEnv.logManagers().get(0).resignAfterNonAtomicCommit();
        int oldClaimEpoch = controller.curClaimEpoch();
        assertThrows(ExecutionException.class, () -> controller.alterIsr(alterIsrRequest).get());
        // Wait for the controller to become active again
        assertSame(controller, controlEnv.activeController());
        assertTrue(oldClaimEpoch < controller.curClaimEpoch(), String.format("oldClaimEpoch = %s, newClaimEpoch = %s", oldClaimEpoch, controller.curClaimEpoch()));
        // Since the alterIsr partially failed we expect to see
        // some partitions to still have 2 in the ISR.
        int partitionsWithReplica2 = Utils.toList(controller.replicationControl().brokersToIsrs().partitionsWithBrokerInIsr(2)).size();
        int partitionsWithReplica0 = Utils.toList(controller.replicationControl().brokersToIsrs().partitionsWithBrokerInIsr(0)).size();
        assertEquals(numPartitions, partitionsWithReplica0);
        assertNotEquals(0, partitionsWithReplica2);
        assertTrue(partitionsWithReplica0 > partitionsWithReplica2, String.format("partitionsWithReplica0 = %s, partitionsWithReplica2 = %s", partitionsWithReplica0, partitionsWithReplica2));
    }
}