Examples with SystemTime org.apache.kafka.common.utils.SystemTime used on opensource projects

Example 11 with SystemTime

use of org.apache.kafka.common.utils.SystemTime in project kafka by apache.

the class TopologyTestDriverTest method shouldReturnAllStores.

@Test
public void shouldReturnAllStores() {
    final Topology topology = setupSourceSinkTopology();
    topology.addProcessor("processor", new MockProcessorSupplier(), "source");
    topology.addStateStore(new KeyValueStoreBuilder<>(Stores.inMemoryKeyValueStore("store"), Serdes.ByteArray(), Serdes.ByteArray(), new SystemTime()), "processor");
    topology.addGlobalStore(new KeyValueStoreBuilder<>(Stores.inMemoryKeyValueStore("globalStore"), Serdes.ByteArray(), Serdes.ByteArray(), new SystemTime()).withLoggingDisabled(), "sourceProcessorName", Serdes.ByteArray().deserializer(), Serdes.ByteArray().deserializer(), "globalTopicName", "globalProcessorName", voidProcessorSupplier);
    testDriver = new TopologyTestDriver(topology, config);
    final Set<String> expectedStoreNames = new HashSet<>();
    expectedStoreNames.add("store");
    expectedStoreNames.add("globalStore");
    final Map<String, StateStore> allStores = testDriver.getAllStateStores();
    assertThat(allStores.keySet(), equalTo(expectedStoreNames));
    for (final StateStore store : allStores.values()) {
        assertNotNull(store);
    }
}

Example 12 with SystemTime

use of org.apache.kafka.common.utils.SystemTime in project ignite by apache.

the class IgniteSinkConnectorTest method beforeTest.

/**
 * {@inheritDoc}
 */
@SuppressWarnings("unchecked")
@Override
protected void beforeTest() throws Exception {
    kafkaBroker = new TestKafkaBroker();
    for (String topic : TOPICS) kafkaBroker.createTopic(topic, PARTITIONS, REPLICATION_FACTOR);
    WorkerConfig workerCfg = new StandaloneConfig(makeWorkerProps());
    OffsetBackingStore offBackingStore = mock(OffsetBackingStore.class);
    offBackingStore.configure(workerCfg);
    worker = new Worker(WORKER_ID, new SystemTime(), workerCfg, offBackingStore);
    worker.start();
    herder = new StandaloneHerder(worker);
    herder.start();
}

Example 13 with SystemTime

use of org.apache.kafka.common.utils.SystemTime in project cruise-control by linkedin.

the class OfflineProposalGenerator method main.

public static void main(String[] argv) throws Exception {
    // TODO: probably need to save this in the original model file
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    KafkaCruiseControlConfig config = new KafkaCruiseControlConfig(props);
    ModelUtils.init(config);
    ModelParameters.init(config);
    BalancingConstraint balancingConstraint = new BalancingConstraint(config);
    long start = System.currentTimeMillis();
    ClusterModel clusterModel = clusterModelFromFile(argv[0]);
    long end = System.currentTimeMillis();
    double duration = (end - start) / 1000.0;
    System.out.println("Model loaded in " + duration + "s.");
    ClusterModelStats origStats = clusterModel.getClusterStats(balancingConstraint);
    String loadBeforeOptimization = clusterModel.brokerStats().toString();
    // Instantiate the components.
    GoalOptimizer goalOptimizer = new GoalOptimizer(config, null, new SystemTime(), new MetricRegistry());
    start = System.currentTimeMillis();
    GoalOptimizer.OptimizerResult optimizerResult = goalOptimizer.optimizations(clusterModel, new OperationProgress());
    end = System.currentTimeMillis();
    duration = (end - start) / 1000.0;
    String loadAfterOptimization = clusterModel.brokerStats().toString();
    System.out.println("Optimize goals in " + duration + "s.");
    System.out.println(optimizerResult.goalProposals().size());
    System.out.println(loadBeforeOptimization);
    System.out.println(loadAfterOptimization);
    ClusterModelStats optimizedStats = clusterModel.getClusterStats(balancingConstraint);
    double[] testStatistics = AnalyzerUtils.testDifference(origStats.utilizationMatrix(), optimizedStats.utilizationMatrix());
    System.out.println(Arrays.stream(RawAndDerivedResource.values()).map(x -> x.toString()).collect(Collectors.joining(", ")));
    System.out.println(Arrays.stream(testStatistics).boxed().map(pValue -> Double.toString(pValue)).collect(Collectors.joining(", ")));
}

Example 14 with SystemTime

use of org.apache.kafka.common.utils.SystemTime in project cruise-control by linkedin.

the class OptimizationVerifier method executeGoalsFor.

/**
 * Execute given goals in the given cluster enforcing the given constraint. Return pass / fail status of a test.
 * A test fails if:
 * 1) Rebalance: During the optimization process, optimization of a goal leads to a worse cluster state (in terms of
 * the requirements of the same goal) than the cluster state just before starting the optimization.
 * 2) Self Healing: There are replicas on dead brokers after self healing.
 * 3) Adding a new broker causes the replicas to move among old brokers.
 *
 * @param constraint         Balancing constraint for the given cluster.
 * @param clusterModel       The state of the cluster.
 * @param goalNameByPriority Name of goals by the order of execution priority.
 * @param excludedTopics     The excluded topics.
 * @param verifications      The verifications to make after the optimization.
 * @return Pass / fail status of a test.
 */
static boolean executeGoalsFor(BalancingConstraint constraint, ClusterModel clusterModel, Map<Integer, String> goalNameByPriority, Collection<String> excludedTopics, List<Verification> verifications) throws Exception {
    // Get the initial stats from the cluster.
    ClusterModelStats preOptimizedStats = clusterModel.getClusterStats(constraint);
    // Set goals by their priority.
    SortedMap<Integer, Goal> goalByPriority = new TreeMap<>();
    for (Map.Entry<Integer, String> goalEntry : goalNameByPriority.entrySet()) {
        Integer priority = goalEntry.getKey();
        String goalClassName = goalEntry.getValue();
        Class<? extends Goal> goalClass = (Class<? extends Goal>) Class.forName(goalClassName);
        try {
            Constructor<? extends Goal> constructor = goalClass.getDeclaredConstructor(BalancingConstraint.class);
            constructor.setAccessible(true);
            goalByPriority.put(priority, constructor.newInstance(constraint));
        } catch (NoSuchMethodException badConstructor) {
            // Try default constructor
            goalByPriority.put(priority, goalClass.newInstance());
        }
    }
    // Generate the goalOptimizer and optimize given goals.
    long startTime = System.currentTimeMillis();
    Properties props = KafkaCruiseControlUnitTestUtils.getKafkaCruiseControlProperties();
    StringJoiner stringJoiner = new StringJoiner(",");
    excludedTopics.forEach(stringJoiner::add);
    props.setProperty(KafkaCruiseControlConfig.TOPICS_EXCLUDED_FROM_PARTITION_MOVEMENT_CONFIG, stringJoiner.toString());
    GoalOptimizer goalOptimizer = new GoalOptimizer(new KafkaCruiseControlConfig(constraint.setProps(props)), null, new SystemTime(), new MetricRegistry());
    GoalOptimizer.OptimizerResult optimizerResult = goalOptimizer.optimizations(clusterModel, goalByPriority, new OperationProgress());
    LOG.trace("Took {} ms to execute {} to generate {} proposals.", System.currentTimeMillis() - startTime, goalByPriority, optimizerResult.goalProposals().size());
    for (Verification verification : verifications) {
        switch(verification) {
            case GOAL_VIOLATION:
                if (!verifyGoalViolations(optimizerResult)) {
                    return false;
                }
                break;
            case NEW_BROKERS:
                if (!clusterModel.newBrokers().isEmpty() && !verifyNewBrokers(clusterModel, constraint)) {
                    return false;
                }
                break;
            case DEAD_BROKERS:
                if (!clusterModel.deadBrokers().isEmpty() && !verifyDeadBrokers(clusterModel)) {
                    return false;
                }
                break;
            case REGRESSION:
                if (clusterModel.selfHealingEligibleReplicas().isEmpty() && !verifyRegression(optimizerResult, preOptimizedStats)) {
                    return false;
                }
                break;
            default:
                throw new IllegalStateException("Invalid verification " + verification);
        }
    }
    return true;
}

Example 15 with SystemTime

use of org.apache.kafka.common.utils.SystemTime in project cruise-control by linkedin.

the class ExecutorTest method executeAndVerifyProposals.

private void executeAndVerifyProposals(ZkUtils zkUtils, Collection<ExecutionProposal> proposalsToExecute, Collection<ExecutionProposal> proposalsToCheck) {
    KafkaCruiseControlConfig configs = new KafkaCruiseControlConfig(getExecutorProperties());
    Executor executor = new Executor(configs, new SystemTime(), new MetricRegistry());
    executor.executeProposals(proposalsToExecute, Collections.emptySet(), EasyMock.mock(LoadMonitor.class));
    Map<TopicPartition, Integer> replicationFactors = new HashMap<>();
    for (ExecutionProposal proposal : proposalsToCheck) {
        int replicationFactor = zkUtils.getReplicasForPartition(proposal.topic(), proposal.partitionId()).size();
        replicationFactors.put(new TopicPartition(proposal.topic(), proposal.partitionId()), replicationFactor);
    }
    long now = System.currentTimeMillis();
    while (executor.state().state() != ExecutorState.State.NO_TASK_IN_PROGRESS && System.currentTimeMillis() < now + 30000) {
        try {
            Thread.sleep(100);
        } catch (InterruptedException e) {
            e.printStackTrace();
        }
    }
    if (executor.state().state() != ExecutorState.State.NO_TASK_IN_PROGRESS) {
        fail("The execution did not finish in 30 seconds.");
    }
    for (ExecutionProposal proposal : proposalsToCheck) {
        TopicPartition tp = new TopicPartition(proposal.topic(), proposal.partitionId());
        int expectedReplicationFactor = replicationFactors.get(tp);
        assertEquals("Replication factor for partition " + tp + " should be " + expectedReplicationFactor, expectedReplicationFactor, zkUtils.getReplicasForPartition(tp.topic(), tp.partition()).size());
        if (proposal.hasReplicaAction()) {
            for (int brokerId : proposal.newReplicas()) {
                assertTrue("The partition should have moved for " + tp, zkUtils.getReplicasForPartition(tp.topic(), tp.partition()).contains(brokerId));
            }
        }
        assertEquals("The leader should have moved for " + tp, proposal.newLeader(), zkUtils.getLeaderForPartition(tp.topic(), tp.partition()).get());
    }
}