Search in sources :

Example 76 with Address

use of com.hazelcast.cluster.Address in project hazelcast by hazelcast.

the class ExecutionPlan method initialize.

/**
 * A method called on the members as part of the InitExecutionOperation.
 * Creates tasklets, inboxes/outboxes and connects these to make them ready
 * for a later StartExecutionOperation.
 */
public void initialize(NodeEngineImpl nodeEngine, long jobId, long executionId, @Nonnull SnapshotContext snapshotContext, ConcurrentHashMap<String, File> tempDirectories, InternalSerializationService jobSerializationService) {
    this.nodeEngine = nodeEngine;
    this.jobClassLoaderService = ((JetServiceBackend) nodeEngine.getService(JetServiceBackend.SERVICE_NAME)).getJobClassLoaderService();
    this.executionId = executionId;
    initProcSuppliers(jobId, tempDirectories, jobSerializationService);
    initDag(jobSerializationService);
    this.ptionArrgmt = new PartitionArrangement(partitionAssignment, nodeEngine.getThisAddress());
    Set<Integer> higherPriorityVertices = VertexDef.getHigherPriorityVertices(vertices);
    for (Address destAddr : remoteMembers.get()) {
        Connection conn = getMemberConnection(nodeEngine, destAddr);
        if (conn == null) {
            throw new TopologyChangedException("no connection to job participant: " + destAddr);
        }
        memberConnections.put(destAddr, conn);
    }
    for (VertexDef vertex : vertices) {
        ClassLoader processorClassLoader = isLightJob ? null : jobClassLoaderService.getProcessorClassLoader(jobId, vertex.name());
        Collection<? extends Processor> processors = doWithClassLoader(processorClassLoader, () -> createProcessors(vertex, vertex.localParallelism()));
        String jobPrefix = prefix(jobConfig.getName(), jobId, vertex.name());
        // create StoreSnapshotTasklet and the queues to it
        ConcurrentConveyor<Object> ssConveyor = null;
        if (!isLightJob) {
            // Note that we create the snapshot queues for all non-light jobs, even if they don't have
            // processing guarantee enabled, because in EE one can request a snapshot also for
            // non-snapshotted jobs.
            @SuppressWarnings("unchecked") QueuedPipe<Object>[] snapshotQueues = new QueuedPipe[vertex.localParallelism()];
            Arrays.setAll(snapshotQueues, i -> new OneToOneConcurrentArrayQueue<>(SNAPSHOT_QUEUE_SIZE));
            ssConveyor = ConcurrentConveyor.concurrentConveyor(null, snapshotQueues);
            ILogger storeSnapshotLogger = prefixedLogger(nodeEngine.getLogger(StoreSnapshotTasklet.class), jobPrefix);
            StoreSnapshotTasklet ssTasklet = new StoreSnapshotTasklet(snapshotContext, ConcurrentInboundEdgeStream.create(ssConveyor, 0, 0, true, jobPrefix + "/ssFrom", null), new AsyncSnapshotWriterImpl(nodeEngine, snapshotContext, vertex.name(), memberIndex, memberCount, jobSerializationService), storeSnapshotLogger, vertex.name(), higherPriorityVertices.contains(vertex.vertexId()));
            tasklets.add(ssTasklet);
        }
        int localProcessorIdx = 0;
        for (Processor processor : processors) {
            int globalProcessorIndex = memberIndex * vertex.localParallelism() + localProcessorIdx;
            String processorPrefix = prefix(jobConfig.getName(), jobId, vertex.name(), globalProcessorIndex);
            ILogger logger = prefixedLogger(nodeEngine.getLogger(processor.getClass()), processorPrefix);
            ProcCtx context = new ProcCtx(nodeEngine, jobId, executionId, getJobConfig(), logger, vertex.name(), localProcessorIdx, globalProcessorIndex, isLightJob, partitionAssignment, vertex.localParallelism(), memberIndex, memberCount, tempDirectories, jobSerializationService, subject, processorClassLoader);
            // createOutboundEdgeStreams() populates localConveyorMap and edgeSenderConveyorMap.
            // Also populates instance fields: senderMap, receiverMap, tasklets.
            List<OutboundEdgeStream> outboundStreams = createOutboundEdgeStreams(vertex, localProcessorIdx, jobPrefix, jobSerializationService);
            List<InboundEdgeStream> inboundStreams = createInboundEdgeStreams(vertex, localProcessorIdx, jobPrefix, globalProcessorIndex);
            OutboundCollector snapshotCollector = ssConveyor == null ? null : new ConveyorCollector(ssConveyor, localProcessorIdx, null);
            // vertices which are only used for snapshot restore will not be marked as "source=true" in metrics
            // also do not consider snapshot restore edges for determining source tag
            boolean isSource = vertex.inboundEdges().stream().allMatch(EdgeDef::isSnapshotRestoreEdge) && !vertex.isSnapshotVertex();
            ProcessorTasklet processorTasklet = new ProcessorTasklet(context, nodeEngine.getExecutionService().getExecutor(TASKLET_INIT_CLOSE_EXECUTOR_NAME), jobSerializationService, processor, inboundStreams, outboundStreams, snapshotContext, snapshotCollector, isSource);
            tasklets.add(processorTasklet);
            this.processors.add(processor);
            localProcessorIdx++;
        }
    }
    List<ReceiverTasklet> allReceivers = receiverMap.values().stream().flatMap(o -> o.values().stream()).flatMap(a -> a.values().stream()).collect(toList());
    tasklets.addAll(allReceivers);
}
Also used : Address(com.hazelcast.cluster.Address) ImdgUtil.getMemberConnection(com.hazelcast.jet.impl.util.ImdgUtil.getMemberConnection) Arrays(java.util.Arrays) SnapshotContext(com.hazelcast.jet.impl.execution.SnapshotContext) Collections.unmodifiableList(java.util.Collections.unmodifiableList) ConcurrentConveyor.concurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor.concurrentConveyor) Processor(com.hazelcast.jet.core.Processor) OutboundCollector.compositeCollector(com.hazelcast.jet.impl.execution.OutboundCollector.compositeCollector) ObjectWithPartitionId(com.hazelcast.jet.impl.util.ObjectWithPartitionId) ProcessorTasklet(com.hazelcast.jet.impl.execution.ProcessorTasklet) ImdgUtil(com.hazelcast.jet.impl.util.ImdgUtil) Collectors.toMap(java.util.stream.Collectors.toMap) ConcurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor) Map(java.util.Map) Util.memoize(com.hazelcast.jet.impl.util.Util.memoize) SerializationServiceAware(com.hazelcast.internal.serialization.SerializationServiceAware) DISTRIBUTE_TO_ALL(com.hazelcast.jet.core.Edge.DISTRIBUTE_TO_ALL) ObjectDataInput(com.hazelcast.nio.ObjectDataInput) TASKLET_INIT_CLOSE_EXECUTOR_NAME(com.hazelcast.jet.impl.execution.TaskletExecutionService.TASKLET_INIT_CLOSE_EXECUTOR_NAME) InboundEdgeStream(com.hazelcast.jet.impl.execution.InboundEdgeStream) PrefixedLogger.prefix(com.hazelcast.jet.impl.util.PrefixedLogger.prefix) Collection(java.util.Collection) IPartitionService(com.hazelcast.internal.partition.IPartitionService) JobConfig(com.hazelcast.jet.config.JobConfig) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) Stream(java.util.stream.Stream) DEFAULT_QUEUE_SIZE(com.hazelcast.jet.config.EdgeConfig.DEFAULT_QUEUE_SIZE) StoreSnapshotTasklet(com.hazelcast.jet.impl.execution.StoreSnapshotTasklet) ObjectDataOutput(com.hazelcast.nio.ObjectDataOutput) TopologyChangedException(com.hazelcast.jet.core.TopologyChangedException) IntStream(java.util.stream.IntStream) ComparatorEx(com.hazelcast.function.ComparatorEx) IdentifiedDataSerializable(com.hazelcast.nio.serialization.IdentifiedDataSerializable) ImdgUtil.writeList(com.hazelcast.jet.impl.util.ImdgUtil.writeList) OutboundEdgeStream(com.hazelcast.jet.impl.execution.OutboundEdgeStream) RoutingPolicy(com.hazelcast.jet.core.Edge.RoutingPolicy) Util.doWithClassLoader(com.hazelcast.jet.impl.util.Util.doWithClassLoader) SenderTasklet(com.hazelcast.jet.impl.execution.SenderTasklet) HashMap(java.util.HashMap) Supplier(java.util.function.Supplier) ProcSupplierCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcSupplierCtx) ArrayList(java.util.ArrayList) PrefixedLogger.prefixedLogger(com.hazelcast.jet.impl.util.PrefixedLogger.prefixedLogger) JetException(com.hazelcast.jet.JetException) ConveyorCollector(com.hazelcast.jet.impl.execution.ConveyorCollector) ReceiverTasklet(com.hazelcast.jet.impl.execution.ReceiverTasklet) ILogger(com.hazelcast.logging.ILogger) InternalSerializationService(com.hazelcast.internal.serialization.InternalSerializationService) Nonnull(javax.annotation.Nonnull) ProcessorSupplier(com.hazelcast.jet.core.ProcessorSupplier) QueuedPipe(com.hazelcast.internal.util.concurrent.QueuedPipe) IntFunction(java.util.function.IntFunction) JetConfig(com.hazelcast.jet.config.JetConfig) NodeEngineImpl(com.hazelcast.spi.impl.NodeEngineImpl) OneToOneConcurrentArrayQueue(com.hazelcast.internal.util.concurrent.OneToOneConcurrentArrayQueue) Connection(com.hazelcast.internal.nio.Connection) Tasklet(com.hazelcast.jet.impl.execution.Tasklet) ProcCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx) AsyncSnapshotWriterImpl(com.hazelcast.jet.impl.util.AsyncSnapshotWriterImpl) IOException(java.io.IOException) ConveyorCollectorWithPartition(com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition) Subject(javax.security.auth.Subject) File(java.io.File) ImdgUtil.readList(com.hazelcast.jet.impl.util.ImdgUtil.readList) Collectors.toList(java.util.stream.Collectors.toList) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector) JobClassLoaderService(com.hazelcast.jet.impl.JobClassLoaderService) ProcessingGuarantee(com.hazelcast.jet.config.ProcessingGuarantee) JetServiceBackend(com.hazelcast.jet.impl.JetServiceBackend) Processor(com.hazelcast.jet.core.Processor) Address(com.hazelcast.cluster.Address) AsyncSnapshotWriterImpl(com.hazelcast.jet.impl.util.AsyncSnapshotWriterImpl) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector) ConveyorCollector(com.hazelcast.jet.impl.execution.ConveyorCollector) ProcCtx(com.hazelcast.jet.impl.execution.init.Contexts.ProcCtx) InboundEdgeStream(com.hazelcast.jet.impl.execution.InboundEdgeStream) ConcurrentInboundEdgeStream(com.hazelcast.jet.impl.execution.ConcurrentInboundEdgeStream) Util.doWithClassLoader(com.hazelcast.jet.impl.util.Util.doWithClassLoader) ILogger(com.hazelcast.logging.ILogger) StoreSnapshotTasklet(com.hazelcast.jet.impl.execution.StoreSnapshotTasklet) ProcessorTasklet(com.hazelcast.jet.impl.execution.ProcessorTasklet) ImdgUtil.getMemberConnection(com.hazelcast.jet.impl.util.ImdgUtil.getMemberConnection) Connection(com.hazelcast.internal.nio.Connection) OutboundEdgeStream(com.hazelcast.jet.impl.execution.OutboundEdgeStream) ReceiverTasklet(com.hazelcast.jet.impl.execution.ReceiverTasklet) TopologyChangedException(com.hazelcast.jet.core.TopologyChangedException) QueuedPipe(com.hazelcast.internal.util.concurrent.QueuedPipe)

Example 77 with Address

use of com.hazelcast.cluster.Address in project hazelcast by hazelcast.

the class ExecutionPlan method createRemoteOutboundCollectors.

private OutboundCollector[] createRemoteOutboundCollectors(EdgeDef edge, String jobPrefix, int processorIndex, int totalPartitionCount, int[][] partitionsPerProcessor, InternalSerializationService jobSerializationService) {
    // the distributed-to-one edge must be partitioned and the target member must be present
    if (!edge.getDistributedTo().equals(DISTRIBUTE_TO_ALL)) {
        if (edge.routingPolicy() != RoutingPolicy.PARTITIONED) {
            throw new JetException("An edge distributing to a specific member must be partitioned: " + edge);
        }
        if (!ptionArrgmt.getRemotePartitionAssignment().containsKey(edge.getDistributedTo()) && !edge.getDistributedTo().equals(nodeEngine.getThisAddress())) {
            throw new JetException("The target member of an edge is not present in the cluster or is a lite member: " + edge);
        }
    }
    Map<Address, ConcurrentConveyor<Object>> senderConveyorMap = memberToSenderConveyorMap(edgeSenderConveyorMap, edge, jobPrefix, jobSerializationService);
    createIfAbsentReceiverTasklet(edge, jobPrefix, partitionsPerProcessor, totalPartitionCount, jobSerializationService);
    // assign remote partitions to outbound data collectors
    Address distributeTo = edge.getDistributedTo();
    Map<Address, int[]> memberToPartitions = distributeTo.equals(DISTRIBUTE_TO_ALL) ? ptionArrgmt.getRemotePartitionAssignment() : ptionArrgmt.remotePartitionAssignmentToOne(distributeTo);
    OutboundCollector[] remoteCollectors = new OutboundCollector[memberToPartitions.size()];
    int index = 0;
    for (Map.Entry<Address, int[]> entry : memberToPartitions.entrySet()) {
        Address memberAddress = entry.getKey();
        int[] memberPartitions = entry.getValue();
        ConcurrentConveyor<Object> conveyor = senderConveyorMap.get(memberAddress);
        remoteCollectors[index++] = new ConveyorCollectorWithPartition(conveyor, processorIndex, memberPartitions);
    }
    return remoteCollectors;
}
Also used : ConcurrentConveyor(com.hazelcast.internal.util.concurrent.ConcurrentConveyor) Address(com.hazelcast.cluster.Address) OutboundCollector(com.hazelcast.jet.impl.execution.OutboundCollector) JetException(com.hazelcast.jet.JetException) ConveyorCollectorWithPartition(com.hazelcast.jet.impl.execution.ConveyorCollectorWithPartition) Collectors.toMap(java.util.stream.Collectors.toMap) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap)

Example 78 with Address

use of com.hazelcast.cluster.Address in project hazelcast by hazelcast.

the class TerminateExecutionOperation method run.

@Override
public void run() {
    JetServiceBackend service = getJetServiceBackend();
    JobExecutionService executionService = service.getJobExecutionService();
    Address callerAddress = getCallerAddress();
    executionService.terminateExecution(jobId(), executionId, callerAddress, mode);
}
Also used : JobExecutionService(com.hazelcast.jet.impl.JobExecutionService) Address(com.hazelcast.cluster.Address) JetServiceBackend(com.hazelcast.jet.impl.JetServiceBackend)

Example 79 with Address

use of com.hazelcast.cluster.Address in project hazelcast by hazelcast.

the class LocalMapStatsProvider method getReplicaAddress.

/**
 * Gets replica address. Waits if necessary.
 *
 * @see #waitForReplicaAddress
 */
private Address getReplicaAddress(int partitionId, int replicaNumber, int backupCount) {
    IPartition partition = partitionService.getPartition(partitionId);
    Address replicaAddress = partition.getReplicaAddress(replicaNumber);
    if (replicaAddress == null) {
        replicaAddress = waitForReplicaAddress(replicaNumber, partition, backupCount);
    }
    return replicaAddress;
}
Also used : Address(com.hazelcast.cluster.Address) IPartition(com.hazelcast.internal.partition.IPartition)

Example 80 with Address

use of com.hazelcast.cluster.Address in project hazelcast by hazelcast.

the class LocalMapStatsProvider method waitForReplicaAddress.

/**
 * Waits partition table update to get replica address if current replica address is null.
 */
private Address waitForReplicaAddress(int replica, IPartition partition, int backupCount) {
    int tryCount = RETRY_COUNT;
    Address replicaAddress = null;
    while (replicaAddress == null && partitionService.getMaxAllowedBackupCount() >= backupCount && tryCount-- > 0) {
        sleep();
        replicaAddress = partition.getReplicaAddress(replica);
    }
    return replicaAddress;
}
Also used : Address(com.hazelcast.cluster.Address)

Aggregations

Address (com.hazelcast.cluster.Address)540 Test (org.junit.Test)211 QuickTest (com.hazelcast.test.annotation.QuickTest)191 ParallelJVMTest (com.hazelcast.test.annotation.ParallelJVMTest)178 HazelcastInstance (com.hazelcast.core.HazelcastInstance)92 InetAddress (java.net.InetAddress)75 ArrayList (java.util.ArrayList)66 Member (com.hazelcast.cluster.Member)63 Accessors.getAddress (com.hazelcast.test.Accessors.getAddress)54 MemberImpl (com.hazelcast.cluster.impl.MemberImpl)48 Config (com.hazelcast.config.Config)43 PartitionReplica (com.hazelcast.internal.partition.PartitionReplica)43 UUID (java.util.UUID)43 ILogger (com.hazelcast.logging.ILogger)37 HashMap (java.util.HashMap)36 Operation (com.hazelcast.spi.impl.operationservice.Operation)35 List (java.util.List)35 OperationService (com.hazelcast.spi.impl.operationservice.OperationService)34 Map (java.util.Map)33 InetSocketAddress (java.net.InetSocketAddress)32