Examples with GlobalStreamThread org.apache.kafka.streams.processor.internals.GlobalStreamThread used on opensource projects

Example 1 with GlobalStreamThread

use of org.apache.kafka.streams.processor.internals.GlobalStreamThread in project kafka by apache.

the class KafkaStreams method start.

/**
     * Start the {@code KafkaStreams} instance by starting all its threads.
     * <p>
     * Note, for brokers with version {@code 0.9.x} or lower, the broker version cannot be checked.
     * There will be no error and the client will hang and retry to verify the broker version until it
     * {@link StreamsConfig#REQUEST_TIMEOUT_MS_CONFIG times out}.

     * @throws IllegalStateException if process was already started
     * @throws StreamsException if the Kafka brokers have version 0.10.0.x
     */
public synchronized void start() throws IllegalStateException, StreamsException {
    log.debug("{} Starting Kafka Stream process.", logPrefix);
    if (state == State.CREATED) {
        checkBrokerVersionCompatibility();
        setState(State.RUNNING);
        if (globalStreamThread != null) {
            globalStreamThread.start();
        }
        for (final StreamThread thread : threads) {
            thread.start();
        }
        log.info("{} Started Kafka Stream process", logPrefix);
    } else {
        throw new IllegalStateException("Cannot start again.");
    }
}

Example 2 with GlobalStreamThread

use of org.apache.kafka.streams.processor.internals.GlobalStreamThread in project apache-kafka-on-k8s by banzaicloud.

the class KafkaStreamsTest method testStateThreadClose.

@Test
public void testStateThreadClose() throws Exception {
    final StreamsBuilder builder = new StreamsBuilder();
    // make sure we have the global state thread running too
    builder.globalTable("anyTopic");
    final KafkaStreams streams = new KafkaStreams(builder.build(), props);
    final java.lang.reflect.Field threadsField = streams.getClass().getDeclaredField("threads");
    threadsField.setAccessible(true);
    final StreamThread[] threads = (StreamThread[]) threadsField.get(streams);
    assertEquals(NUM_THREADS, threads.length);
    assertEquals(streams.state(), KafkaStreams.State.CREATED);
    streams.start();
    TestUtils.waitForCondition(new TestCondition() {

        @Override
        public boolean conditionMet() {
            return streams.state() == KafkaStreams.State.RUNNING;
        }
    }, 10 * 1000, "Streams never started.");
    for (int i = 0; i < NUM_THREADS; i++) {
        final StreamThread tmpThread = threads[i];
        tmpThread.shutdown();
        TestUtils.waitForCondition(new TestCondition() {

            @Override
            public boolean conditionMet() {
                return tmpThread.state() == StreamThread.State.DEAD;
            }
        }, 10 * 1000, "Thread never stopped.");
        threads[i].join();
    }
    TestUtils.waitForCondition(new TestCondition() {

        @Override
        public boolean conditionMet() {
            return streams.state() == KafkaStreams.State.ERROR;
        }
    }, 10 * 1000, "Streams never stopped.");
    streams.close();
    TestUtils.waitForCondition(new TestCondition() {

        @Override
        public boolean conditionMet() {
            return streams.state() == KafkaStreams.State.NOT_RUNNING;
        }
    }, 10 * 1000, "Streams never stopped.");
    final java.lang.reflect.Field globalThreadField = streams.getClass().getDeclaredField("globalStreamThread");
    globalThreadField.setAccessible(true);
    final GlobalStreamThread globalStreamThread = (GlobalStreamThread) globalThreadField.get(streams);
    assertEquals(globalStreamThread, null);
}

Example 3 with GlobalStreamThread

use of org.apache.kafka.streams.processor.internals.GlobalStreamThread in project apache-kafka-on-k8s by banzaicloud.

the class KafkaStreamsTest method testStateGlobalThreadClose.

@Test
public void testStateGlobalThreadClose() throws Exception {
    final StreamsBuilder builder = new StreamsBuilder();
    // make sure we have the global state thread running too
    builder.globalTable("anyTopic");
    final KafkaStreams streams = new KafkaStreams(builder.build(), props);
    streams.start();
    TestUtils.waitForCondition(new TestCondition() {

        @Override
        public boolean conditionMet() {
            return streams.state() == KafkaStreams.State.RUNNING;
        }
    }, 10 * 1000, "Streams never started.");
    final java.lang.reflect.Field globalThreadField = streams.getClass().getDeclaredField("globalStreamThread");
    globalThreadField.setAccessible(true);
    final GlobalStreamThread globalStreamThread = (GlobalStreamThread) globalThreadField.get(streams);
    globalStreamThread.shutdown();
    TestUtils.waitForCondition(new TestCondition() {

        @Override
        public boolean conditionMet() {
            return globalStreamThread.state() == GlobalStreamThread.State.DEAD;
        }
    }, 10 * 1000, "Thread never stopped.");
    globalStreamThread.join();
    assertEquals(streams.state(), KafkaStreams.State.ERROR);
    streams.close();
    assertEquals(streams.state(), KafkaStreams.State.NOT_RUNNING);
}

Example 4 with GlobalStreamThread

use of org.apache.kafka.streams.processor.internals.GlobalStreamThread in project apache-kafka-on-k8s by banzaicloud.

the class KafkaStreams method start.

/**
 * Start the {@code KafkaStreams} instance by starting all its threads.
 * This function is expected to be called only once during the life cycle of the client.
 * <p>
 * Because threads are started in the background, this method does not block.
 * As a consequence, any fatal exception that happens during processing is by default only logged.
 * If you want to be notified about dying threads, you can
 * {@link #setUncaughtExceptionHandler(Thread.UncaughtExceptionHandler) register an uncaught exception handler}
 * before starting the {@code KafkaStreams} instance.
 * <p>
 * Note, for brokers with version {@code 0.9.x} or lower, the broker version cannot be checked.
 * There will be no error and the client will hang and retry to verify the broker version until it
 * {@link StreamsConfig#REQUEST_TIMEOUT_MS_CONFIG times out}.
 *
 * @throws IllegalStateException if process was already started
 * @throws StreamsException if the Kafka brokers have version 0.10.0.x or
 *                          if {@link StreamsConfig#PROCESSING_GUARANTEE_CONFIG exactly-once} is enabled for pre 0.11.0.x brokers
 */
public synchronized void start() throws IllegalStateException, StreamsException {
    log.debug("Starting Streams client");
    // making sure the state will always transit to RUNNING before REBALANCING
    if (setRunningFromCreated()) {
        if (globalStreamThread != null) {
            globalStreamThread.start();
        }
        for (final StreamThread thread : threads) {
            thread.start();
        }
        final Long cleanupDelay = config.getLong(StreamsConfig.STATE_CLEANUP_DELAY_MS_CONFIG);
        stateDirCleaner.scheduleAtFixedRate(new Runnable() {

            @Override
            public void run() {
                if (state == State.RUNNING) {
                    stateDirectory.cleanRemovedTasks(cleanupDelay);
                }
            }
        }, cleanupDelay, cleanupDelay, TimeUnit.MILLISECONDS);
        log.info("Started Streams client");
    } else {
        // if transition failed but no exception is thrown; currently it is not possible
        // since we do not allow calling start multiple times whether or not it is already shutdown.
        // TODO: In the future if we lift this restriction this code path could then be triggered and be updated
        log.error("Already stopped, cannot re-start");
    }
}

Example 5 with GlobalStreamThread

use of org.apache.kafka.streams.processor.internals.GlobalStreamThread in project apache-kafka-on-k8s by banzaicloud.

the class KafkaStreams method close.

/**
 * Shutdown this {@code KafkaStreams} by signaling all the threads to stop, and then wait up to the timeout for the
 * threads to join.
 * A {@code timeout} of 0 means to wait forever.
 *
 * @param timeout  how long to wait for the threads to shutdown
 * @param timeUnit unit of time used for timeout
 * @return {@code true} if all threads were successfully stopped—{@code false} if the timeout was reached
 * before all threads stopped
 * Note that this method must not be called in the {@code onChange} callback of {@link StateListener}.
 */
public synchronized boolean close(final long timeout, final TimeUnit timeUnit) {
    log.debug("Stopping Streams client with timeoutMillis = {} ms.", timeUnit.toMillis(timeout));
    if (!setState(State.PENDING_SHUTDOWN)) {
        // if transition failed, it means it was either in PENDING_SHUTDOWN
        // or NOT_RUNNING already; just check that all threads have been stopped
        log.info("Already in the pending shutdown state, wait to complete shutdown");
    } else {
        stateDirCleaner.shutdownNow();
        // wait for all threads to join in a separate thread;
        // save the current thread so that if it is a stream thread
        // we don't attempt to join it and cause a deadlock
        final Thread shutdownThread = new Thread(new Runnable() {

            @Override
            public void run() {
                // further state reports from the thread since we're shutting down
                for (final StreamThread thread : threads) {
                    thread.setStateListener(null);
                    thread.shutdown();
                }
                if (globalStreamThread != null) {
                    globalStreamThread.setStateListener(null);
                    globalStreamThread.shutdown();
                }
                for (final StreamThread thread : threads) {
                    try {
                        if (!thread.isRunning()) {
                            thread.join();
                        }
                    } catch (final InterruptedException ex) {
                        Thread.currentThread().interrupt();
                    }
                }
                if (globalStreamThread != null && !globalStreamThread.stillRunning()) {
                    try {
                        globalStreamThread.join();
                    } catch (final InterruptedException e) {
                        Thread.currentThread().interrupt();
                    }
                    globalStreamThread = null;
                }
                adminClient.close();
                metrics.close();
                setState(State.NOT_RUNNING);
            }
        }, "kafka-streams-close-thread");
        shutdownThread.setDaemon(true);
        shutdownThread.start();
    }
    if (waitOnState(State.NOT_RUNNING, timeUnit.toMillis(timeout))) {
        log.info("Streams client stopped completely");
        return true;
    } else {
        log.info("Streams client cannot stop completely within the timeout");
        return false;
    }
}