Examples with Time org.apache.kafka.common.utils.Time used on opensource projects

Example 76 with Time

use of org.apache.kafka.common.utils.Time in project apache-kafka-on-k8s by banzaicloud.

the class KafkaConsumerTest method testAutoCommitSentBeforePositionUpdate.

@Test
public void testAutoCommitSentBeforePositionUpdate() {
    Time time = new MockTime();
    Cluster cluster = TestUtils.singletonCluster(topic, 1);
    Node node = cluster.nodes().get(0);
    Metadata metadata = createMetadata();
    metadata.update(cluster, Collections.<String>emptySet(), time.milliseconds());
    MockClient client = new MockClient(time, metadata);
    client.setNode(node);
    PartitionAssignor assignor = new RoundRobinAssignor();
    KafkaConsumer<String, String> consumer = newConsumer(time, client, metadata, assignor, true);
    consumer.subscribe(singleton(topic), getConsumerRebalanceListener(consumer));
    Node coordinator = prepareRebalance(client, node, assignor, singletonList(tp0), null);
    consumer.poll(0);
    // respond to the outstanding fetch so that we have data available on the next poll
    client.respondFrom(fetchResponse(tp0, 0, 5), node);
    client.poll(0, time.milliseconds());
    time.sleep(autoCommitIntervalMs);
    client.prepareResponseFrom(fetchResponse(tp0, 5, 0), node);
    // no data has been returned to the user yet, so the committed offset should be 0
    AtomicBoolean commitReceived = prepareOffsetCommitResponse(client, coordinator, tp0, 0);
    consumer.poll(0);
    assertTrue(commitReceived.get());
    consumer.close(0, TimeUnit.MILLISECONDS);
}

Example 77 with Time

use of org.apache.kafka.common.utils.Time in project apache-kafka-on-k8s by banzaicloud.

the class KafkaConsumerTest method testResetUsingAutoResetPolicy.

@Test
public void testResetUsingAutoResetPolicy() {
    Time time = new MockTime();
    Cluster cluster = TestUtils.singletonCluster(topic, 1);
    Node node = cluster.nodes().get(0);
    Metadata metadata = createMetadata();
    metadata.update(cluster, Collections.<String>emptySet(), time.milliseconds());
    MockClient client = new MockClient(time, metadata);
    client.setNode(node);
    PartitionAssignor assignor = new RoundRobinAssignor();
    KafkaConsumer<String, String> consumer = newConsumer(time, client, metadata, assignor, OffsetResetStrategy.LATEST, true);
    consumer.assign(singletonList(tp0));
    client.prepareResponseFrom(new FindCoordinatorResponse(Errors.NONE, node), node);
    Node coordinator = new Node(Integer.MAX_VALUE - node.id(), node.host(), node.port());
    client.prepareResponseFrom(offsetResponse(Collections.singletonMap(tp0, -1L), Errors.NONE), coordinator);
    client.prepareResponse(listOffsetsResponse(Collections.singletonMap(tp0, 50L)));
    consumer.poll(0);
    assertEquals(50L, consumer.position(tp0));
}

Example 78 with Time

use of org.apache.kafka.common.utils.Time in project apache-kafka-on-k8s by banzaicloud.

the class ConnectDistributed method main.

public static void main(String[] args) throws Exception {
    if (args.length < 1) {
        log.info("Usage: ConnectDistributed worker.properties");
        Exit.exit(1);
    }
    try {
        Time time = Time.SYSTEM;
        log.info("Kafka Connect distributed worker initializing ...");
        long initStart = time.hiResClockMs();
        WorkerInfo initInfo = new WorkerInfo();
        initInfo.logAll();
        String workerPropsFile = args[0];
        Map<String, String> workerProps = !workerPropsFile.isEmpty() ? Utils.propsToStringMap(Utils.loadProps(workerPropsFile)) : Collections.<String, String>emptyMap();
        log.info("Scanning for plugin classes. This might take a moment ...");
        Plugins plugins = new Plugins(workerProps);
        plugins.compareAndSwapWithDelegatingLoader();
        DistributedConfig config = new DistributedConfig(workerProps);
        String kafkaClusterId = ConnectUtils.lookupKafkaClusterId(config);
        log.debug("Kafka cluster ID: {}", kafkaClusterId);
        RestServer rest = new RestServer(config);
        URI advertisedUrl = rest.advertisedUrl();
        String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort();
        KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore();
        offsetBackingStore.configure(config);
        Worker worker = new Worker(workerId, time, plugins, config, offsetBackingStore);
        Converter internalValueConverter = worker.getInternalValueConverter();
        StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, internalValueConverter);
        statusBackingStore.configure(config);
        ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(internalValueConverter, config);
        DistributedHerder herder = new DistributedHerder(config, time, worker, kafkaClusterId, statusBackingStore, configBackingStore, advertisedUrl.toString());
        final Connect connect = new Connect(herder, rest);
        log.info("Kafka Connect distributed worker initialization took {}ms", time.hiResClockMs() - initStart);
        try {
            connect.start();
        } catch (Exception e) {
            log.error("Failed to start Connect", e);
            connect.stop();
            Exit.exit(3);
        }
        // Shutdown will be triggered by Ctrl-C or via HTTP shutdown request
        connect.awaitStop();
    } catch (Throwable t) {
        log.error("Stopping due to error", t);
        Exit.exit(2);
    }
}

Example 79 with Time

use of org.apache.kafka.common.utils.Time in project apache-kafka-on-k8s by banzaicloud.

the class KafkaAdminClient method createInternal.

static KafkaAdminClient createInternal(AdminClientConfig config, TimeoutProcessorFactory timeoutProcessorFactory) {
    Metrics metrics = null;
    NetworkClient networkClient = null;
    Time time = Time.SYSTEM;
    String clientId = generateClientId(config);
    ChannelBuilder channelBuilder = null;
    Selector selector = null;
    ApiVersions apiVersions = new ApiVersions();
    LogContext logContext = createLogContext(clientId);
    try {
        // Since we only request node information, it's safe to pass true for allowAutoTopicCreation (and it
        // simplifies communication with older brokers)
        Metadata metadata = new Metadata(config.getLong(AdminClientConfig.RETRY_BACKOFF_MS_CONFIG), config.getLong(AdminClientConfig.METADATA_MAX_AGE_CONFIG), true);
        List<MetricsReporter> reporters = config.getConfiguredInstances(AdminClientConfig.METRIC_REPORTER_CLASSES_CONFIG, MetricsReporter.class);
        Map<String, String> metricTags = Collections.singletonMap("client-id", clientId);
        MetricConfig metricConfig = new MetricConfig().samples(config.getInt(AdminClientConfig.METRICS_NUM_SAMPLES_CONFIG)).timeWindow(config.getLong(AdminClientConfig.METRICS_SAMPLE_WINDOW_MS_CONFIG), TimeUnit.MILLISECONDS).recordLevel(Sensor.RecordingLevel.forName(config.getString(AdminClientConfig.METRICS_RECORDING_LEVEL_CONFIG))).tags(metricTags);
        reporters.add(new JmxReporter(JMX_PREFIX));
        metrics = new Metrics(metricConfig, reporters, time);
        String metricGrpPrefix = "admin-client";
        channelBuilder = ClientUtils.createChannelBuilder(config);
        selector = new Selector(config.getLong(AdminClientConfig.CONNECTIONS_MAX_IDLE_MS_CONFIG), metrics, time, metricGrpPrefix, channelBuilder, logContext);
        networkClient = new NetworkClient(selector, metadata, clientId, 1, config.getLong(AdminClientConfig.RECONNECT_BACKOFF_MS_CONFIG), config.getLong(AdminClientConfig.RECONNECT_BACKOFF_MAX_MS_CONFIG), config.getInt(AdminClientConfig.SEND_BUFFER_CONFIG), config.getInt(AdminClientConfig.RECEIVE_BUFFER_CONFIG), (int) TimeUnit.HOURS.toMillis(1), time, true, apiVersions, logContext);
        return new KafkaAdminClient(config, clientId, time, metadata, metrics, networkClient, timeoutProcessorFactory, logContext);
    } catch (Throwable exc) {
        closeQuietly(metrics, "Metrics");
        closeQuietly(networkClient, "NetworkClient");
        closeQuietly(selector, "Selector");
        closeQuietly(channelBuilder, "ChannelBuilder");
        throw new KafkaException("Failed create new KafkaAdminClient", exc);
    }
}

Example 80 with Time

use of org.apache.kafka.common.utils.Time in project kafka by apache.

the class SensorTest method testStrictQuotaEnforcementWithTokenBucket.

@Test
public void testStrictQuotaEnforcementWithTokenBucket() {
    final Time time = new MockTime(0, System.currentTimeMillis(), 0);
    final Metrics metrics = new Metrics(time);
    final Sensor sensor = metrics.sensor("sensor", new MetricConfig().quota(Quota.upperBound(2)).timeWindow(1, TimeUnit.SECONDS).samples(10));
    final MetricName metricName = metrics.metricName("credits", "test-group");
    assertTrue(sensor.add(metricName, new TokenBucket()));
    final KafkaMetric tkMetric = metrics.metric(metricName);
    // Recording a first value at T+0 to bring the remaining credits below zero
    strictRecord(sensor, 30, time.milliseconds());
    assertEquals(-10, tkMetric.measurableValue(time.milliseconds()), 0.1);
    // Theoretically, we should wait 5s to bring back the avg rate to the define quota:
    // 10 / 2 = 5s
    time.sleep(5000);
    // Unlike the default rate based on a windowed sum, it works as expected.
    assertEquals(0, tkMetric.measurableValue(time.milliseconds()), 0.1);
    strictRecord(sensor, 30, time.milliseconds());
    assertEquals(-30, tkMetric.measurableValue(time.milliseconds()), 0.1);
    metrics.close();
}