Examples with HubNodeAddress alluxio.hub.proto.HubNodeAddress used on opensource projects

Example 6 with HubNodeAddress

use of alluxio.hub.proto.HubNodeAddress in project alluxio by Alluxio.

the class ManagerProcessContext method updateConfigurationFor.

/**
 * Updates the configuration for a set of Alluxio nodes.
 *
 * @param type the type of node to write the configuration to
 * @param conf the configuration to write
 * @return a set of the nodes which had their configuration updated
 */
public Set<HubNodeAddress> updateConfigurationFor(AlluxioNodeType type, AlluxioConfigurationSet conf) {
    Set<HubNodeAddress> nodes = mHubCluster.nodesFromAlluxio(mAlluxioCluster, type);
    Map<HubNodeAddress, AgentWriteConfigurationSetResponse> confSet = mHubCluster.exec(nodes, mConf, (client) -> client.writeConfigurationSet(AgentWriteConfigurationSetRequest.newBuilder().setConfSet(conf).build()), mSvc);
    // update the file system client to reflect the new conf
    if (type.equals(AlluxioNodeType.MASTER)) {
        updateFileSystemClient(conf);
    }
    // if running in K8s, update configmap
    if (mK8sConfig != null && (type.equals(AlluxioNodeType.MASTER) || type.equals(AlluxioNodeType.ALL))) {
        // We maintain a single configMap for all pods, so the config set is symmetric
        LOG.info("Persist configuration to K8s configMap {}", mK8sConfigMapName);
        try (KubernetesClient client = new DefaultKubernetesClient(mK8sConfig)) {
            String namespace = client.getNamespace();
            if (namespace == null) {
                LOG.info("Use K8s default namespace ");
                namespace = "default";
            }
            // Update configMap
            ConfigMap existingConfig = client.configMaps().inNamespace(namespace).withName(mK8sConfigMapName).get();
            if (existingConfig != null) {
                ConfigMapBuilder builder = new ConfigMapBuilder().withMetadata(existingConfig.getMetadata()).withData(existingConfig.getData());
                // Replace log4 properties
                builder.addToData(K8S_CONFIG_MAP_ENV_LOG4J, conf.getLog4JProperties());
                // Replace site properties
                builder.addToData(K8S_CONFIG_MAP_ENV_SITE, conf.getSiteProperties());
                // Replace environment variables
                for (Map.Entry<String, String> entry : HubUtil.getEnvVarsFromFile(conf.getAlluxioEnv(), LOG).entrySet()) {
                    builder.addToData(entry.getKey(), entry.getValue());
                }
                // Write config map
                client.configMaps().inNamespace(namespace).withName(mK8sConfigMapName).replace(builder.build());
                LOG.info("Updated K8s configMap {}", mK8sConfigMapName);
            }
        } catch (Exception e) {
            LOG.error("Unable to persist configuration to K8s configMap {}", mK8sConfigMapName, e);
        }
    }
    return confSet.keySet();
}

Example 7 with HubNodeAddress

use of alluxio.hub.proto.HubNodeAddress in project alluxio by Alluxio.

the class ManagerProcessContext method configurationSetFor.

/**
 * Gets the configuration for a given alluxio node type.
 *
 * @param type the type of node to get the configuration from
 * @return a mapping of responses from each hub agent the action was executed on to the
 *         agent's address
 */
public AlluxioConfigurationSet configurationSetFor(AlluxioNodeType type) {
    Set<HubNodeAddress> nodes = mHubCluster.nodesFromAlluxio(mAlluxioCluster, type);
    Iterator<HubNodeAddress> iter = nodes.iterator();
    if (!iter.hasNext()) {
        throw new IllegalStateException("No nodes of type " + type + " to get configuration in the " + "cluster.");
    }
    HubNodeAddress addr = iter.next();
    Map<HubNodeAddress, AlluxioConfigurationSet> confSet = mHubCluster.exec(Collections.singleton(addr), mConf, (client) -> client.getConfigurationSet(AgentGetConfigurationSetRequest.newBuilder().build()).getConfSet(), mSvc);
    return confSet.get(addr);
}

Example 8 with HubNodeAddress

use of alluxio.hub.proto.HubNodeAddress in project alluxio by Alluxio.

the class ManagerProcessContextTest method testAddFile.

@Test
public void testAddFile() {
    AgentFileUploadResponse r = AgentFileUploadResponse.newBuilder().setSuccess(true).setVersion(2).setFileName("uptime").build();
    HubNodeAddress addr = HubTestUtils.generateNodeAddress();
    Map<HubNodeAddress, AgentFileUploadResponse> resp = new HashMap<>();
    resp.put(addr, r);
    UploadFile req = UploadFile.newBuilder().setDestination(AlluxioNodeType.WORKER).setProcessType(UploadProcessType.PRESTO).setContent("#!/usr/bin/env bash\nuptime").setName("uptime").setPermission("0777").build();
    doReturn(resp).when(mContext).execOnHub(any(), any(), any());
    boolean addFileResp = mContext.uploadFile(Collections.singletonList(req));
    assertTrue(addFileResp);
    resp.put(HubTestUtils.generateNodeAddress(), r.toBuilder().setSuccess(false).build());
    assertFalse(mContext.uploadFile(Collections.singletonList(req)));
}

Example 9 with HubNodeAddress

use of alluxio.hub.proto.HubNodeAddress in project alluxio by Alluxio.

the class ManagerProcessContextTest method testListCatalogs.

@Test
public void testListCatalogs() throws IOException {
    Set<String> nodes = Collections.singleton(UUID.randomUUID().toString());
    doReturn(nodes).when(mMockHub).nodesFromAlluxio(mMockAlluxio, AlluxioNodeType.MASTER);
    Path prestoCatalogDir = Paths.get(mTemp.getRoot().getAbsolutePath(), "etc/presto/conf/catalog");
    Files.createDirectories(prestoCatalogDir);
    PrestoCatalogListing r = PrestoCatalogListing.newBuilder().setCatalogName("hive-test").setHiveMetastoreUri("thrift://blahblahblah:9083").build();
    Map<HubNodeAddress, AgentListCatalogResponse> map = new HashMap<>();
    map.put(HubNodeAddress.newBuilder().build(), AgentListCatalogResponse.newBuilder().addCatalog(r).build());
    doReturn(map).when(mMockHub).exec(any(), any(), any(), any());
    assertEquals(PrestoCatalogListingResult.newBuilder().addCatalog(r).build(), mContext.listCatalogs(prestoCatalogDir.toString()));
}

Example 10 with HubNodeAddress

use of alluxio.hub.proto.HubNodeAddress in project alluxio by Alluxio.

the class ManagerProcessContextTest method testExecOnHubEmpty.

@Test
public void testExecOnHubEmpty() {
    Set<HubNodeAddress> nodes = Collections.singleton(HubNodeAddress.newBuilder().setHostname(UUID.randomUUID().toString()).build());
    doReturn(nodes).when(mMockHub).nodesFromAlluxio(mMockAlluxio, AlluxioNodeType.MASTER);
    AtomicInteger i = new AtomicInteger(0);
    Function<AgentManagerServiceGrpc.AgentManagerServiceBlockingStub, Integer> action = client -> i.get();
    mContext.execOnHub(action, AlluxioNodeType.MASTER);
    verify(mMockHub).nodesFromAlluxio(mMockAlluxio, AlluxioNodeType.MASTER);
    verify(mMockHub).exec(nodes, mConf, action, mMockExec);
}