Examples with SessionKey org.apache.kafka.connect.runtime.SessionKey used on opensource projects

Example 1 with SessionKey

use of org.apache.kafka.connect.runtime.SessionKey in project kafka by apache.

the class DistributedHerder method tick.

// public for testing
public void tick() {
    try {
        // Joining and immediately leaving for failure to read configs is exceedingly impolite
        if (!canReadConfigs) {
            if (readConfigToEnd(workerSyncTimeoutMs)) {
                canReadConfigs = true;
            } else {
                // Safe to return and tick immediately because readConfigToEnd will do the backoff for us
                return;
            }
        }
        log.debug("Ensuring group membership is still active");
        member.ensureActive();
        // Ensure we're in a good state in our group. If not restart and everything should be setup to rejoin
        if (!handleRebalanceCompleted())
            return;
    } catch (WakeupException e) {
        // May be due to a request from another thread, or might be stopping. If the latter, we need to check the
        // flag immediately. If the former, we need to re-run the ensureActive call since we can't handle requests
        // unless we're in the group.
        log.trace("Woken up while ensure group membership is still active");
        return;
    }
    long now = time.milliseconds();
    if (checkForKeyRotation(now)) {
        log.debug("Distributing new session key");
        keyExpiration = Long.MAX_VALUE;
        try {
            configBackingStore.putSessionKey(new SessionKey(keyGenerator.generateKey(), now));
        } catch (Exception e) {
            log.info("Failed to write new session key to config topic; forcing a read to the end of the config topic before possibly retrying");
            canReadConfigs = false;
            return;
        }
    }
    // Process any external requests
    // TODO: Some of these can be performed concurrently or even optimized away entirely.
    // For example, if three different connectors are slated to be restarted, it's fine to
    // restart all three at the same time instead.
    // Another example: if multiple configurations are submitted for the same connector,
    // the only one that actually has to be written to the config topic is the
    // most-recently one.
    long nextRequestTimeoutMs = Long.MAX_VALUE;
    while (true) {
        final DistributedHerderRequest next = peekWithoutException();
        if (next == null) {
            break;
        } else if (now >= next.at) {
            requests.pollFirst();
        } else {
            nextRequestTimeoutMs = next.at - now;
            break;
        }
        try {
            next.action().call();
            next.callback().onCompletion(null, null);
        } catch (Throwable t) {
            next.callback().onCompletion(t, null);
        }
    }
    // Process all pending connector restart requests
    processRestartRequests();
    if (scheduledRebalance < Long.MAX_VALUE) {
        nextRequestTimeoutMs = Math.min(nextRequestTimeoutMs, Math.max(scheduledRebalance - now, 0));
        rebalanceResolved = false;
        log.debug("Scheduled rebalance at: {} (now: {} nextRequestTimeoutMs: {}) ", scheduledRebalance, now, nextRequestTimeoutMs);
    }
    if (isLeader() && internalRequestValidationEnabled() && keyExpiration < Long.MAX_VALUE) {
        nextRequestTimeoutMs = Math.min(nextRequestTimeoutMs, Math.max(keyExpiration - now, 0));
        log.debug("Scheduled next key rotation at: {} (now: {} nextRequestTimeoutMs: {}) ", keyExpiration, now, nextRequestTimeoutMs);
    }
    // Process any configuration updates
    AtomicReference<Set<String>> connectorConfigUpdatesCopy = new AtomicReference<>();
    AtomicReference<Set<String>> connectorTargetStateChangesCopy = new AtomicReference<>();
    AtomicReference<Set<ConnectorTaskId>> taskConfigUpdatesCopy = new AtomicReference<>();
    boolean shouldReturn;
    if (member.currentProtocolVersion() == CONNECT_PROTOCOL_V0) {
        shouldReturn = updateConfigsWithEager(connectorConfigUpdatesCopy, connectorTargetStateChangesCopy);
        // been set to retain the old workflow
        if (shouldReturn) {
            return;
        }
        if (connectorConfigUpdatesCopy.get() != null) {
            processConnectorConfigUpdates(connectorConfigUpdatesCopy.get());
        }
        if (connectorTargetStateChangesCopy.get() != null) {
            processTargetStateChanges(connectorTargetStateChangesCopy.get());
        }
    } else {
        shouldReturn = updateConfigsWithIncrementalCooperative(connectorConfigUpdatesCopy, connectorTargetStateChangesCopy, taskConfigUpdatesCopy);
        if (connectorConfigUpdatesCopy.get() != null) {
            processConnectorConfigUpdates(connectorConfigUpdatesCopy.get());
        }
        if (connectorTargetStateChangesCopy.get() != null) {
            processTargetStateChanges(connectorTargetStateChangesCopy.get());
        }
        if (taskConfigUpdatesCopy.get() != null) {
            processTaskConfigUpdatesWithIncrementalCooperative(taskConfigUpdatesCopy.get());
        }
        if (shouldReturn) {
            return;
        }
    }
    // Let the group take any actions it needs to
    try {
        log.trace("Polling for group activity; will wait for {}ms or until poll is interrupted by " + "either config backing store updates or a new external request", nextRequestTimeoutMs);
        member.poll(nextRequestTimeoutMs);
        // Ensure we're in a good state in our group. If not restart and everything should be setup to rejoin
        handleRebalanceCompleted();
    } catch (WakeupException e) {
        // FIXME should not be WakeupException
        log.trace("Woken up while polling for group activity");
    // Ignore. Just indicates we need to check the exit flag, for requested actions, etc.
    }
}

Example 2 with SessionKey

use of org.apache.kafka.connect.runtime.SessionKey in project kafka by apache.

the class DistributedHerderTest method testKeyRotationWhenWorkerBecomesLeader.

@Test
public void testKeyRotationWhenWorkerBecomesLeader() throws Exception {
    EasyMock.expect(member.memberId()).andStubReturn("member");
    EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V2);
    expectRebalance(1, Collections.emptyList(), Collections.emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    // First rebalance: poll indefinitely as no key has been read yet, so expiration doesn't come into play
    member.poll(Long.MAX_VALUE);
    EasyMock.expectLastCall();
    expectRebalance(2, Collections.emptyList(), Collections.emptyList());
    SessionKey initialKey = new SessionKey(EasyMock.mock(SecretKey.class), 0);
    ClusterConfigState snapshotWithKey = new ClusterConfigState(2, initialKey, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED), TASK_CONFIGS_MAP, Collections.emptySet());
    expectPostRebalanceCatchup(snapshotWithKey);
    // Second rebalance: poll indefinitely as worker is follower, so expiration still doesn't come into play
    member.poll(Long.MAX_VALUE);
    EasyMock.expectLastCall();
    expectRebalance(2, Collections.emptyList(), Collections.emptyList(), "member", MEMBER_URL);
    Capture<SessionKey> updatedKey = EasyMock.newCapture();
    configBackingStore.putSessionKey(EasyMock.capture(updatedKey));
    EasyMock.expectLastCall().andAnswer(() -> {
        configUpdateListener.onSessionKeyUpdate(updatedKey.getValue());
        return null;
    });
    // Third rebalance: poll for a limited time as worker has become leader and must wake up for key expiration
    Capture<Long> pollTimeout = EasyMock.newCapture();
    member.poll(EasyMock.captureLong(pollTimeout));
    EasyMock.expectLastCall();
    PowerMock.replayAll();
    herder.tick();
    configUpdateListener.onSessionKeyUpdate(initialKey);
    herder.tick();
    herder.tick();
    assertTrue(pollTimeout.getValue() <= DistributedConfig.INTER_WORKER_KEY_TTL_MS_MS_DEFAULT);
    PowerMock.verifyAll();
}

Example 3 with SessionKey

use of org.apache.kafka.connect.runtime.SessionKey in project kafka by apache.

the class DistributedHerderTest method testKeyRotationDisabledWhenWorkerBecomesFollower.

@Test
public void testKeyRotationDisabledWhenWorkerBecomesFollower() throws Exception {
    EasyMock.expect(member.memberId()).andStubReturn("member");
    EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V2);
    expectRebalance(1, Collections.emptyList(), Collections.emptyList(), "member", MEMBER_URL);
    SecretKey initialSecretKey = EasyMock.mock(SecretKey.class);
    EasyMock.expect(initialSecretKey.getAlgorithm()).andReturn(DistributedConfig.INTER_WORKER_KEY_GENERATION_ALGORITHM_DEFAULT).anyTimes();
    EasyMock.expect(initialSecretKey.getEncoded()).andReturn(new byte[32]).anyTimes();
    SessionKey initialKey = new SessionKey(initialSecretKey, time.milliseconds());
    ClusterConfigState snapshotWithKey = new ClusterConfigState(1, initialKey, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED), TASK_CONFIGS_MAP, Collections.emptySet());
    expectPostRebalanceCatchup(snapshotWithKey);
    // First rebalance: poll for a limited time as worker is leader and must wake up for key expiration
    Capture<Long> firstPollTimeout = EasyMock.newCapture();
    member.poll(EasyMock.captureLong(firstPollTimeout));
    EasyMock.expectLastCall();
    expectRebalance(1, Collections.emptyList(), Collections.emptyList());
    // Second rebalance: poll indefinitely as worker is no longer leader, so key expiration doesn't come into play
    member.poll(Long.MAX_VALUE);
    EasyMock.expectLastCall();
    PowerMock.replayAll(initialSecretKey);
    configUpdateListener.onSessionKeyUpdate(initialKey);
    herder.tick();
    assertTrue(firstPollTimeout.getValue() <= DistributedConfig.INTER_WORKER_KEY_TTL_MS_MS_DEFAULT);
    herder.tick();
    PowerMock.verifyAll();
}

Example 4 with SessionKey

use of org.apache.kafka.connect.runtime.SessionKey in project kafka by apache.

the class DistributedHerderTest method testFailedToReadBackNewlyWrittenSessionKey.

@Test
public void testFailedToReadBackNewlyWrittenSessionKey() throws Exception {
    SecretKey secretKey = EasyMock.niceMock(SecretKey.class);
    EasyMock.expect(secretKey.getAlgorithm()).andReturn(INTER_WORKER_KEY_GENERATION_ALGORITHM_DEFAULT);
    EasyMock.expect(secretKey.getEncoded()).andReturn(new byte[32]);
    SessionKey sessionKey = new SessionKey(secretKey, time.milliseconds());
    ClusterConfigState snapshotWithSessionKey = new ClusterConfigState(1, sessionKey, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED), TASK_CONFIGS_MAP, Collections.emptySet());
    // First tick -- after joining the group, we try to write a new session key to
    // the config topic, and fail (in this case, we're trying to simulate that we've
    // actually written the key successfully, but haven't been able to read it back
    // from the config topic, so to the herder it looks the same as if it'd just failed
    // to write the key)
    EasyMock.expect(member.memberId()).andStubReturn("leader");
    EasyMock.expect(member.currentProtocolVersion()).andStubReturn(CONNECT_PROTOCOL_V2);
    expectRebalance(1, Collections.emptyList(), Collections.emptyList());
    expectPostRebalanceCatchup(SNAPSHOT);
    configBackingStore.putSessionKey(anyObject(SessionKey.class));
    EasyMock.expectLastCall().andThrow(new ConnectException("Oh no!"));
    // Second tick -- we read to the end of the config topic first, and pick up
    // the session key that we were able to write the last time,
    // then ensure we're still active in the group
    // then finally begin polling for group activity
    // Importantly, we do not try to write a new session key this time around
    configBackingStore.refresh(EasyMock.anyLong(), EasyMock.anyObject(TimeUnit.class));
    EasyMock.expectLastCall().andAnswer(() -> {
        configUpdateListener.onSessionKeyUpdate(sessionKey);
        return null;
    });
    EasyMock.expect(configBackingStore.snapshot()).andReturn(snapshotWithSessionKey);
    member.ensureActive();
    PowerMock.expectLastCall();
    member.poll(EasyMock.anyInt());
    PowerMock.expectLastCall();
    PowerMock.replayAll(secretKey);
    herder.tick();
    herder.tick();
    PowerMock.verifyAll();
}