Search in sources :

Example 1 with LlapFixedRegistryImpl

use of org.apache.hadoop.hive.llap.registry.impl.LlapFixedRegistryImpl in project hive by apache.

the class TestUtils method testGetSplitLocationProvider.

@Test
public void testGetSplitLocationProvider() throws IOException, URISyntaxException {
    // Create test LlapServiceInstances to make sure that we can handle all of the instance types
    List<LlapServiceInstance> instances = new ArrayList<>(3);
    // Set 1 inactive instance to make sure that this does not cause problem for us
    LlapServiceInstance inactive = new InactiveServiceInstance(INACTIVE);
    instances.add(inactive);
    HiveConf conf = new HiveConf();
    conf.set(HiveConf.ConfVars.HIVE_ZOOKEEPER_QUORUM.varname, "localhost");
    LlapZookeeperRegistryImpl dynRegistry = new LlapZookeeperRegistryImpl("dyn", conf);
    Endpoint rpcEndpoint = RegistryTypeUtils.ipcEndpoint("llap", new InetSocketAddress(ACTIVE, 4000));
    Endpoint shuffle = RegistryTypeUtils.ipcEndpoint("shuffle", new InetSocketAddress(ACTIVE, 4000));
    Endpoint mng = RegistryTypeUtils.ipcEndpoint("llapmng", new InetSocketAddress(ACTIVE, 4000));
    Endpoint outputFormat = RegistryTypeUtils.ipcEndpoint("llapoutputformat", new InetSocketAddress(ACTIVE, 4000));
    Endpoint services = RegistryTypeUtils.webEndpoint("services", new URI(ACTIVE + ":4000"));
    // Set 1 active instance
    ServiceRecord enabledSrv = new ServiceRecord();
    enabledSrv.addInternalEndpoint(rpcEndpoint);
    enabledSrv.addInternalEndpoint(shuffle);
    enabledSrv.addInternalEndpoint(mng);
    enabledSrv.addInternalEndpoint(outputFormat);
    enabledSrv.addExternalEndpoint(services);
    enabledSrv.set(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS, 10);
    enabledSrv.set(HiveConf.ConfVars.LLAP_DAEMON_MEMORY_PER_INSTANCE_MB.varname, 100);
    LlapZookeeperRegistryImpl.DynamicServiceInstance dynamic = dynRegistry.new DynamicServiceInstance(enabledSrv);
    instances.add(dynamic);
    // Set 1 instance with 0 executors
    ServiceRecord disabledSrv = new ServiceRecord(enabledSrv);
    disabledSrv.set(LlapRegistryService.LLAP_DAEMON_NUM_ENABLED_EXECUTORS, 0);
    LlapZookeeperRegistryImpl.DynamicServiceInstance disabled = dynRegistry.new DynamicServiceInstance(disabledSrv);
    disabled.setHost(DISABLED);
    instances.add(disabled);
    when(mockRegistry.getInstances()).thenReturn(mockInstanceSet);
    when(mockInstanceSet.getAllInstancesOrdered(anyBoolean())).thenReturn(instances);
    SplitLocationProvider provider = Utils.getCustomSplitLocationProvider(mockRegistry, LOG);
    assertLocations((HostAffinitySplitLocationProvider) provider, new String[] { ACTIVE });
    // Check if fixed stuff is working as well
    LlapFixedRegistryImpl fixRegistry = new LlapFixedRegistryImpl("llap", new HiveConf());
    // Instance for testing fixed registry instances
    LlapServiceInstance fixed = fixRegistry.new FixedServiceInstance(FIXED);
    instances.remove(dynamic);
    instances.add(fixed);
    provider = Utils.getCustomSplitLocationProvider(mockRegistry, LOG);
    assertLocations((HostAffinitySplitLocationProvider) provider, new String[] { FIXED });
}
Also used : InetSocketAddress(java.net.InetSocketAddress) ArrayList(java.util.ArrayList) LlapServiceInstance(org.apache.hadoop.hive.llap.registry.LlapServiceInstance) LlapFixedRegistryImpl(org.apache.hadoop.hive.llap.registry.impl.LlapFixedRegistryImpl) URI(java.net.URI) ServiceRecord(org.apache.hadoop.registry.client.types.ServiceRecord) InactiveServiceInstance(org.apache.hadoop.hive.llap.registry.impl.InactiveServiceInstance) Endpoint(org.apache.hadoop.registry.client.types.Endpoint) LlapZookeeperRegistryImpl(org.apache.hadoop.hive.llap.registry.impl.LlapZookeeperRegistryImpl) HiveConf(org.apache.hadoop.hive.conf.HiveConf) SplitLocationProvider(org.apache.hadoop.mapred.split.SplitLocationProvider) Test(org.junit.Test)

Aggregations

InetSocketAddress (java.net.InetSocketAddress)1 URI (java.net.URI)1 ArrayList (java.util.ArrayList)1 HiveConf (org.apache.hadoop.hive.conf.HiveConf)1 LlapServiceInstance (org.apache.hadoop.hive.llap.registry.LlapServiceInstance)1 InactiveServiceInstance (org.apache.hadoop.hive.llap.registry.impl.InactiveServiceInstance)1 LlapFixedRegistryImpl (org.apache.hadoop.hive.llap.registry.impl.LlapFixedRegistryImpl)1 LlapZookeeperRegistryImpl (org.apache.hadoop.hive.llap.registry.impl.LlapZookeeperRegistryImpl)1 SplitLocationProvider (org.apache.hadoop.mapred.split.SplitLocationProvider)1 Endpoint (org.apache.hadoop.registry.client.types.Endpoint)1 ServiceRecord (org.apache.hadoop.registry.client.types.ServiceRecord)1 Test (org.junit.Test)1