Examples with NetworkAddress org.apache.hyracks.api.comm.NetworkAddress used on opensource projects

Example 11 with NetworkAddress

use of org.apache.hyracks.api.comm.NetworkAddress in project asterixdb by apache.

the class MessagingNetworkManager method establishNewConnection.

private IChannelControlBlock establishNewConnection(String nodeId) throws Exception {
    Map<String, NodeControllerInfo> nodeControllers = ncs.getNodeControllersInfo();
    // Get the node messaging address from its info
    NodeControllerInfo nodeControllerInfo = nodeControllers.get(nodeId);
    if (nodeControllerInfo == null) {
        throw new NetException("Could not find node: " + nodeId);
    }
    NetworkAddress nodeMessagingNeAddress = nodeControllerInfo.getMessagingNetworkAddress();
    SocketAddress nodeAddress = new InetSocketAddress(InetAddress.getByName(nodeMessagingNeAddress.getAddress()), nodeMessagingNeAddress.getPort());
    // Open the channel
    IChannelControlBlock ccb = connect(nodeAddress);
    try {
        // Prepare the initial message buffer
        ByteBuffer initialBuffer = ccb.getReadInterface().getBufferFactory().createBuffer();
        prepareMessagingInitialMessage(ncs.getId(), initialBuffer);
        // Send the initial messaging channel handshake message to register the opened channel on both nodes
        ccb.getWriteInterface().getFullBufferAcceptor().accept(initialBuffer);
        return ccb;
    } catch (NetException e) {
        closeChannel(ccb);
        throw e;
    }
}

Example 12 with NetworkAddress

use of org.apache.hyracks.api.comm.NetworkAddress in project asterixdb by apache.

the class MessagingNetworkManager method start.

public void start() throws IOException {
    md.start();
    InetSocketAddress sockAddr = md.getLocalAddress();
    localNetworkAddress = new NetworkAddress(sockAddr.getHostString(), sockAddr.getPort());
    // make it a copy of localNetworkAddress
    if (publicNetworkAddress.getAddress() == null) {
        publicNetworkAddress = localNetworkAddress;
    } else {
        // Likewise for public port
        if (publicNetworkAddress.getPort() == 0) {
            publicNetworkAddress = new NetworkAddress(publicNetworkAddress.getAddress(), sockAddr.getPort());
        }
    }
}

Example 13 with NetworkAddress

use of org.apache.hyracks.api.comm.NetworkAddress in project asterixdb by apache.

the class PartitionUtils method reportPartitionMatch.

public static void reportPartitionMatch(ClusterControllerService ccs, final PartitionId pid, Pair<PartitionDescriptor, PartitionRequest> match) throws Exception {
    PartitionDescriptor desc = match.getLeft();
    PartitionRequest req = match.getRight();
    INodeManager nodeManager = ccs.getNodeManager();
    NodeControllerState producerNCS = nodeManager.getNodeControllerState(desc.getNodeId());
    NodeControllerState requestorNCS = nodeManager.getNodeControllerState(req.getNodeId());
    final NetworkAddress dataport = producerNCS.getDataPort();
    final INodeController requestorNC = requestorNCS.getNodeController();
    requestorNC.reportPartitionAvailability(pid, dataport);
}

Example 14 with NetworkAddress

use of org.apache.hyracks.api.comm.NetworkAddress in project asterixdb by apache.

the class JobExecutor method assignTaskLocations.

private void assignTaskLocations(TaskCluster tc, Map<String, List<TaskAttemptDescriptor>> taskAttemptMap) throws HyracksException {
    ActivityClusterGraph acg = jobRun.getActivityClusterGraph();
    Task[] tasks = tc.getTasks();
    List<TaskClusterAttempt> tcAttempts = tc.getAttempts();
    int attempts = tcAttempts.size();
    TaskClusterAttempt tcAttempt = new TaskClusterAttempt(tc, attempts);
    Map<TaskId, TaskAttempt> taskAttempts = new HashMap<>();
    Map<TaskId, LValueConstraintExpression> locationMap = new HashMap<>();
    for (int i = 0; i < tasks.length; ++i) {
        Task ts = tasks[i];
        TaskId tid = ts.getTaskId();
        TaskAttempt taskAttempt = new TaskAttempt(tcAttempt, new TaskAttemptId(new TaskId(tid.getActivityId(), tid.getPartition()), attempts), ts);
        taskAttempt.setStatus(TaskAttempt.TaskStatus.INITIALIZED, null);
        locationMap.put(tid, new PartitionLocationExpression(tid.getActivityId().getOperatorDescriptorId(), tid.getPartition()));
        taskAttempts.put(tid, taskAttempt);
    }
    tcAttempt.setTaskAttempts(taskAttempts);
    solver.solve(locationMap.values());
    for (int i = 0; i < tasks.length; ++i) {
        Task ts = tasks[i];
        TaskId tid = ts.getTaskId();
        TaskAttempt taskAttempt = taskAttempts.get(tid);
        String nodeId = assignLocation(acg, locationMap, tid, taskAttempt);
        taskAttempt.setNodeId(nodeId);
        taskAttempt.setStatus(TaskAttempt.TaskStatus.RUNNING, null);
        taskAttempt.setStartTime(System.currentTimeMillis());
        List<TaskAttemptDescriptor> tads = taskAttemptMap.get(nodeId);
        if (tads == null) {
            tads = new ArrayList<>();
            taskAttemptMap.put(nodeId, tads);
        }
        OperatorDescriptorId opId = tid.getActivityId().getOperatorDescriptorId();
        jobRun.registerOperatorLocation(opId, tid.getPartition(), nodeId);
        ActivityPartitionDetails apd = ts.getActivityPlan().getActivityPartitionDetails();
        TaskAttemptDescriptor tad = new TaskAttemptDescriptor(taskAttempt.getTaskAttemptId(), apd.getPartitionCount(), apd.getInputPartitionCounts(), apd.getOutputPartitionCounts());
        tads.add(tad);
    }
    tcAttempt.initializePendingTaskCounter();
    tcAttempts.add(tcAttempt);
    /**
         * Improvement for reducing master/slave message communications, for each TaskAttemptDescriptor,
         * we set the NetworkAddress[][] partitionLocations, in which each row is for an incoming connector descriptor
         * and each column is for an input channel of the connector.
         */
    INodeManager nodeManager = ccs.getNodeManager();
    for (Map.Entry<String, List<TaskAttemptDescriptor>> e : taskAttemptMap.entrySet()) {
        List<TaskAttemptDescriptor> tads = e.getValue();
        for (TaskAttemptDescriptor tad : tads) {
            TaskAttemptId taid = tad.getTaskAttemptId();
            int attempt = taid.getAttempt();
            TaskId tid = taid.getTaskId();
            ActivityId aid = tid.getActivityId();
            List<IConnectorDescriptor> inConnectors = acg.getActivityInputs(aid);
            int[] inPartitionCounts = tad.getInputPartitionCounts();
            if (inPartitionCounts == null) {
                continue;
            }
            NetworkAddress[][] partitionLocations = new NetworkAddress[inPartitionCounts.length][];
            for (int i = 0; i < inPartitionCounts.length; ++i) {
                ConnectorDescriptorId cdId = inConnectors.get(i).getConnectorId();
                IConnectorPolicy policy = jobRun.getConnectorPolicyMap().get(cdId);
                /**
                     * carry sender location information into a task
                     * when it is not the case that it is an re-attempt and the send-side
                     * is materialized blocking.
                     */
                if (attempt > 0 && policy.materializeOnSendSide() && policy.consumerWaitsForProducerToFinish()) {
                    continue;
                }
                ActivityId producerAid = acg.getProducerActivity(cdId);
                partitionLocations[i] = new NetworkAddress[inPartitionCounts[i]];
                for (int j = 0; j < inPartitionCounts[i]; ++j) {
                    TaskId producerTaskId = new TaskId(producerAid, j);
                    String nodeId = findTaskLocation(producerTaskId);
                    partitionLocations[i][j] = nodeManager.getNodeControllerState(nodeId).getDataPort();
                }
            }
            tad.setInputPartitionLocations(partitionLocations);
        }
    }
    tcAttempt.setStatus(TaskClusterAttempt.TaskClusterStatus.RUNNING);
    tcAttempt.setStartTime(System.currentTimeMillis());
    inProgressTaskClusters.add(tc);
}

Example 15 with NetworkAddress

use of org.apache.hyracks.api.comm.NetworkAddress in project asterixdb by apache.

the class GetDatasetDirectoryServiceInfoWork method doRun.

@Override
public void doRun() {
    try {
        NetworkAddress addr = ccs.getDatasetDirectoryServiceInfo();
        callback.setValue(addr);
    } catch (Exception e) {
        callback.setException(e);
    }
}