Search in sources :

Example 6 with BaseConfiguration

use of org.apache.commons.configuration.BaseConfiguration in project archaius by Netflix.

the class InitWithNullConfigTest method testCreateProperty.

@Test
public void testCreateProperty() {
    try {
        DynamicPropertyFactory.initWithConfigurationSource((AbstractConfiguration) null);
        fail("NPE expected");
    } catch (NullPointerException e) {
        assertNotNull(e);
    }
    DynamicBooleanProperty prop = DynamicPropertyFactory.getInstance().getBooleanProperty("abc", false);
    BaseConfiguration baseConfig = new BaseConfiguration();
    DynamicPropertyFactory.initWithConfigurationSource(baseConfig);
    baseConfig.setProperty("abc", "true");
    assertTrue(prop.get());
}
Also used : BaseConfiguration(org.apache.commons.configuration.BaseConfiguration) Test(org.junit.Test)

Example 7 with BaseConfiguration

use of org.apache.commons.configuration.BaseConfiguration in project archaius by Netflix.

the class MultiThreadedInit method test.

@Test
public void test() {
    final BaseConfiguration baseConfig = new BaseConfiguration();
    baseConfig.setProperty("abc", 1);
    (new Thread() {

        public void run() {
            Random r = new Random();
            while (DynamicPropertyFactory.getBackingConfigurationSource() != baseConfig) {
                try {
                    Thread.sleep(r.nextInt(100) + 1);
                } catch (InterruptedException e) {
                }
                System.setProperty(DynamicPropertyFactory.DISABLE_DEFAULT_CONFIG, "true");
                ConfigurationManager.install(baseConfig);
            }
        }
    }).start();
    Object config = null;
    DynamicIntProperty prop = DynamicPropertyFactory.getInstance().getIntProperty("abc", 0);
    while ((config = DynamicPropertyFactory.getBackingConfigurationSource()) != baseConfig && prop.get() != 1) {
        // prop = DynamicPropertyFactory.getInstance().getIntProperty("abc", 0);
        try {
            Thread.sleep(10);
        } catch (InterruptedException e) {
        }
    }
}
Also used : BaseConfiguration(org.apache.commons.configuration.BaseConfiguration) Random(java.util.Random) Test(org.junit.Test)

Example 8 with BaseConfiguration

use of org.apache.commons.configuration.BaseConfiguration in project archaius by Netflix.

the class PollingSourceTest method testNoneDeletingPollingSource.

@Test
public void testNoneDeletingPollingSource() throws Exception {
    BaseConfiguration config = new BaseConfiguration();
    config.addProperty("prop1", "original");
    DummyPollingSource source = new DummyPollingSource(false);
    source.setFull("");
    FixedDelayPollingScheduler scheduler = new FixedDelayPollingScheduler(0, 10, true);
    ConfigurationWithPollingSource pollingConfig = new ConfigurationWithPollingSource(config, source, scheduler);
    Thread.sleep(200);
    assertEquals("original", pollingConfig.getProperty("prop1"));
    source.setFull("prop1=changed");
    Thread.sleep(200);
    assertEquals("changed", pollingConfig.getProperty("prop1"));
    source.setFull("prop1=changedagain,prop2=new");
    Thread.sleep(200);
    assertEquals("changedagain", pollingConfig.getProperty("prop1"));
    assertEquals("new", pollingConfig.getProperty("prop2"));
    source.setFull("prop3=new");
    Thread.sleep(200);
    assertEquals("changedagain", pollingConfig.getProperty("prop1"));
    assertEquals("new", pollingConfig.getProperty("prop2"));
    assertEquals("new", pollingConfig.getProperty("prop3"));
}
Also used : BaseConfiguration(org.apache.commons.configuration.BaseConfiguration) Test(org.junit.Test)

Example 9 with BaseConfiguration

use of org.apache.commons.configuration.BaseConfiguration in project pinot by linkedin.

the class RoutingTableTest method testHelixExternalViewBasedRoutingTable.

@Test
public void testHelixExternalViewBasedRoutingTable() throws Exception {
    RoutingTableBuilder routingStrategy = new RandomRoutingTableBuilder(100);
    HelixExternalViewBasedRouting routingTable = new HelixExternalViewBasedRouting(null, NO_LLC_ROUTING, null, new BaseConfiguration());
    routingTable.setSmallClusterRoutingTableBuilder(routingStrategy);
    ExternalView externalView = new ExternalView("testResource0_OFFLINE");
    externalView.setState("segment0", "dataServer_instance_0", "ONLINE");
    externalView.setState("segment0", "dataServer_instance_1", "ONLINE");
    externalView.setState("segment1", "dataServer_instance_1", "ONLINE");
    externalView.setState("segment1", "dataServer_instance_2", "ONLINE");
    externalView.setState("segment2", "dataServer_instance_2", "ONLINE");
    externalView.setState("segment2", "dataServer_instance_0", "ONLINE");
    List<InstanceConfig> instanceConfigs = generateInstanceConfigs("dataServer_instance", 0, 2);
    routingTable.markDataResourceOnline("testResource0_OFFLINE", externalView, instanceConfigs);
    ExternalView externalView1 = new ExternalView("testResource1_OFFLINE");
    externalView1.setState("segment10", "dataServer_instance_0", "ONLINE");
    externalView1.setState("segment11", "dataServer_instance_1", "ONLINE");
    externalView1.setState("segment12", "dataServer_instance_2", "ONLINE");
    routingTable.markDataResourceOnline("testResource1_OFFLINE", externalView1, instanceConfigs);
    ExternalView externalView2 = new ExternalView("testResource2_OFFLINE");
    externalView2.setState("segment20", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_0", "ONLINE");
    externalView2.setState("segment20", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_1", "ONLINE");
    externalView2.setState("segment20", "dataServer_instance_2", "ONLINE");
    externalView2.setState("segment21", "dataServer_instance_2", "ONLINE");
    externalView2.setState("segment22", "dataServer_instance_2", "ONLINE");
    routingTable.markDataResourceOnline("testResource2_OFFLINE", externalView2, instanceConfigs);
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource0_OFFLINE", "[segment0, segment1, segment2]", 3);
    }
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource1_OFFLINE", "[segment10, segment11, segment12]", 3);
    }
    for (int numRun = 0; numRun < 100; ++numRun) {
        assertResourceRequest(routingTable, "testResource2_OFFLINE", "[segment20, segment21, segment22]", 3);
    }
}
Also used : RandomRoutingTableBuilder(com.linkedin.pinot.routing.builder.RandomRoutingTableBuilder) KafkaHighLevelConsumerBasedRoutingTableBuilder(com.linkedin.pinot.routing.builder.KafkaHighLevelConsumerBasedRoutingTableBuilder) RoutingTableBuilder(com.linkedin.pinot.routing.builder.RoutingTableBuilder) ExternalView(org.apache.helix.model.ExternalView) BaseConfiguration(org.apache.commons.configuration.BaseConfiguration) InstanceConfig(org.apache.helix.model.InstanceConfig) RandomRoutingTableBuilder(com.linkedin.pinot.routing.builder.RandomRoutingTableBuilder) Test(org.testng.annotations.Test)

Example 10 with BaseConfiguration

use of org.apache.commons.configuration.BaseConfiguration in project pinot by linkedin.

the class KafkaLowLevelConsumerRoutingTableBuilderTest method testAllOnlineRoutingTable.

@Test
public void testAllOnlineRoutingTable() {
    final int ITERATIONS = 1000;
    Random random = new Random();
    KafkaLowLevelConsumerRoutingTableBuilder routingTableBuilder = new KafkaLowLevelConsumerRoutingTableBuilder();
    routingTableBuilder.init(new BaseConfiguration());
    long totalNanos = 0L;
    for (int i = 0; i < ITERATIONS; i++) {
        // 3 to 14 instances
        int instanceCount = random.nextInt(12) + 3;
        // 4 to 11 partitions
        int partitionCount = random.nextInt(8) + 4;
        // 3 to 5 replicas
        int replicationFactor = random.nextInt(3) + 3;
        // Generate instances
        String[] instanceNames = new String[instanceCount];
        for (int serverInstanceId = 0; serverInstanceId < instanceCount; serverInstanceId++) {
            instanceNames[serverInstanceId] = "Server_localhost_" + serverInstanceId;
        }
        // Generate partitions
        String[][] segmentNames = new String[partitionCount][];
        int totalSegmentCount = 0;
        for (int partitionId = 0; partitionId < partitionCount; partitionId++) {
            // 0 to 31 segments in partition
            int segmentCount = random.nextInt(32);
            segmentNames[partitionId] = new String[segmentCount];
            for (int sequenceNumber = 0; sequenceNumber < segmentCount; sequenceNumber++) {
                segmentNames[partitionId][sequenceNumber] = new LLCSegmentName("table", partitionId, sequenceNumber, System.currentTimeMillis()).getSegmentName();
            }
            totalSegmentCount += segmentCount;
        }
        // Generate instance configurations
        List<InstanceConfig> instanceConfigs = new ArrayList<InstanceConfig>();
        for (String instanceName : instanceNames) {
            InstanceConfig instanceConfig = new InstanceConfig(instanceName);
            instanceConfigs.add(instanceConfig);
            instanceConfig.getRecord().setSimpleField(CommonConstants.Helix.IS_SHUTDOWN_IN_PROGRESS, "false");
        }
        // Generate a random external view
        ExternalView externalView = new ExternalView("table_REALTIME");
        int[] segmentCountForInstance = new int[instanceCount];
        int maxSegmentCountOnInstance = 0;
        for (int partitionId = 0; partitionId < segmentNames.length; partitionId++) {
            String[] segments = segmentNames[partitionId];
            // Assign each segment for this partition
            for (int replicaId = 0; replicaId < replicationFactor; ++replicaId) {
                for (int segmentIndex = 0; segmentIndex < segments.length; segmentIndex++) {
                    int instanceIndex = -1;
                    int randomOffset = random.nextInt(instanceCount);
                    // Pick the first random instance that has fewer than maxSegmentCountOnInstance segments assigned to it
                    for (int j = 0; j < instanceCount; j++) {
                        int potentialInstanceIndex = (j + randomOffset) % instanceCount;
                        if (segmentCountForInstance[potentialInstanceIndex] < maxSegmentCountOnInstance) {
                            instanceIndex = potentialInstanceIndex;
                            break;
                        }
                    }
                    // All replicas have exactly maxSegmentCountOnInstance, pick a replica and increment the max
                    if (instanceIndex == -1) {
                        maxSegmentCountOnInstance++;
                        instanceIndex = randomOffset;
                    }
                    // Increment the segment count for the instance
                    segmentCountForInstance[instanceIndex]++;
                    // Add the segment to the external view
                    externalView.setState(segmentNames[partitionId][segmentIndex], instanceNames[instanceIndex], "ONLINE");
                }
            }
        }
        // Create routing tables
        long startTime = System.nanoTime();
        List<ServerToSegmentSetMap> routingTables = routingTableBuilder.computeRoutingTableFromExternalView("table_REALTIME", externalView, instanceConfigs);
        long endTime = System.nanoTime();
        totalNanos += endTime - startTime;
        // Check that all routing tables generated match all segments, with no duplicates
        for (ServerToSegmentSetMap routingTable : routingTables) {
            Set<String> assignedSegments = new HashSet<String>();
            for (String server : routingTable.getServerSet()) {
                for (String segment : routingTable.getSegmentSet(server)) {
                    assertFalse(assignedSegments.contains(segment));
                    assignedSegments.add(segment);
                }
            }
            assertEquals(assignedSegments.size(), totalSegmentCount);
        }
    }
    LOGGER.warn("Routing table building avg ms: " + totalNanos / (ITERATIONS * 1000000.0));
}
Also used : ExternalView(org.apache.helix.model.ExternalView) ArrayList(java.util.ArrayList) ServerToSegmentSetMap(com.linkedin.pinot.routing.ServerToSegmentSetMap) LLCSegmentName(com.linkedin.pinot.common.utils.LLCSegmentName) BaseConfiguration(org.apache.commons.configuration.BaseConfiguration) Random(java.util.Random) InstanceConfig(org.apache.helix.model.InstanceConfig) HashSet(java.util.HashSet) Test(org.testng.annotations.Test)

Aggregations

BaseConfiguration (org.apache.commons.configuration.BaseConfiguration)32 Test (org.junit.Test)17 CommonsConfiguration (com.thinkaurelius.titan.diskstorage.configuration.backend.CommonsConfiguration)8 ExternalView (org.apache.helix.model.ExternalView)7 Test (org.testng.annotations.Test)7 ModifiableConfiguration (com.thinkaurelius.titan.diskstorage.configuration.ModifiableConfiguration)5 GraphDatabaseConfiguration (com.thinkaurelius.titan.graphdb.configuration.GraphDatabaseConfiguration)5 InstanceConfig (org.apache.helix.model.InstanceConfig)5 LLCSegmentName (com.linkedin.pinot.common.utils.LLCSegmentName)4 BasicConfiguration (com.thinkaurelius.titan.diskstorage.configuration.BasicConfiguration)4 Configuration (com.thinkaurelius.titan.diskstorage.configuration.Configuration)4 ElasticSearchIndex (com.thinkaurelius.titan.diskstorage.es.ElasticSearchIndex)4 ArrayList (java.util.ArrayList)4 AbstractConfiguration (org.apache.commons.configuration.AbstractConfiguration)4 RoutingTableBuilder (com.linkedin.pinot.routing.builder.RoutingTableBuilder)3 File (java.io.File)3 Configuration (org.apache.commons.configuration.Configuration)3 ServerInstance (com.linkedin.pinot.common.response.ServerInstance)2 HLCSegmentName (com.linkedin.pinot.common.utils.HLCSegmentName)2 ServerToSegmentSetMap (com.linkedin.pinot.routing.ServerToSegmentSetMap)2