Examples with VcrClusterParticipantListener com.github.ambry.clustermap.VcrClusterParticipantListener used on opensource projects

Example 1 with VcrClusterParticipantListener

use of com.github.ambry.clustermap.VcrClusterParticipantListener in project ambry by linkedin.

the class HelixVcrClusterParticipant method removePartition.

/**
 * Remove {@link PartitionId} from assignedPartitionIds set, if {@param partitionIdStr} valid.
 * Used in one of the vcr state model classes {@link OnlineOfflineHelixVcrStateModel} or
 * {@link LeaderStandbyHelixVcrStateModel} if current VCR becomes offline for a partition.
 * @param partitionIdStr The partitionIdStr notified by Helix.
 */
public void removePartition(String partitionIdStr) {
    PartitionId partitionId = partitionIdMap.remove(partitionIdStr);
    if (partitionId == null) {
        metrics.partitionIdNotInClusterMapOnRemove.inc();
        logger.warn("Partition {} not exists on current VCR: {}", partitionIdStr, vcrInstanceName);
    } else {
        for (VcrClusterParticipantListener listener : listeners) {
            listener.onPartitionRemoved(partitionId);
        }
        assignedPartitionIds.remove(partitionId);
        logger.info("Partition {} is removed from current VCR: {}. Number of assigned partitions: {}", partitionIdStr, vcrInstanceName, assignedPartitionIds.size());
        logger.debug("Current assigned partitions: {}", partitionIdMap.keySet());
    }
}

Example 2 with VcrClusterParticipantListener

use of com.github.ambry.clustermap.VcrClusterParticipantListener in project ambry by linkedin.

the class HelixVcrClusterParticipant method doAddPartition.

private void doAddPartition(String partitionIdStr, Callback<Object> callback) {
    PartitionId partitionId = clusterMap.getPartitionIdByName(partitionIdStr);
    if (partitionId == null) {
        metrics.partitionIdNotInClusterMapOnAdd.inc();
        callback.onCompletion(null, new IllegalStateException("Partition not in clusterMap on add: Partition Id: " + partitionIdStr));
    } else {
        if (partitionIdMap.putIfAbsent(partitionIdStr, partitionId) == null) {
            // TODO: get rid of assignedPartitionIds
            assignedPartitionIds.add(partitionId);
            for (VcrClusterParticipantListener listener : listeners) {
                listener.onPartitionAdded(partitionId);
            }
            logger.debug("Assigned Partitions: {}", assignedPartitionIds);
        } else {
            logger.info("Partition {} exists on current VCR: {}", partitionIdStr, vcrInstanceName);
        }
        callback.onCompletion(null, null);
    }
}

Example 3 with VcrClusterParticipantListener

use of com.github.ambry.clustermap.VcrClusterParticipantListener in project ambry by linkedin.

the class VcrReplicationManager method start.

@Override
public void start() throws ReplicationException {
    // Add listener for new coming assigned partition
    vcrClusterParticipant.addListener(new VcrClusterParticipantListener() {

        @Override
        public void onPartitionAdded(PartitionId partitionId) {
            if (partitionId.isEqual(cloudConfig.vcrHelixUpdaterPartitionId)) {
                vcrHelixUpdateLock.lock();
                try {
                    if (!isAmbryListenerToUpdateVcrHelixRegistered) {
                        // Prepare the vcrUpdateDistributedLock. Only one instance can update vcr helix cluster at one time.
                        // It's possible isVcrHelixUpdater to be true on two nodes.
                        // For example, at time "t" node A is the owner of partition 1. Due to some partition reassignment
                        // (lets say new node addition), partition 1 get assigned to node B at time "t+1". In this case it's possible
                        // for Node B to get notification of addPartition of partition 1 at "t+2" before Node A gets removePartition
                        // notification (at t+4). If a main cluster update happens between "t+2" and "t+4", then two nodes might try
                        // to update vcr cluster at the same time. Therefore, we need this distributed lock.
                        LockScope distributedLockScope = new HelixLockScope(HelixLockScope.LockScopeProperty.CLUSTER, Arrays.asList(cloudConfig.vcrClusterName, cloudConfig.vcrClusterName));
                        vcrUpdateDistributedLock = new ZKDistributedNonblockingLock(distributedLockScope, cloudConfig.vcrClusterZkConnectString, cloudConfig.vcrHelixLockTimeoutInMs, "Updating VCR Cluster", clusterMapConfig.clusterMapHostName);
                        // Only register the listener once. Unfortunately, we can't unregister a listener, so we use
                        // isAmbryListenerToUpdateVcrHelixRegistered as the flag.
                        clusterMap.registerClusterMapListener(new AmbryListenerToUpdateVcrHelix());
                        isAmbryListenerToUpdateVcrHelixRegistered = true;
                        // Schedule a fixed rate task to check if ambry helix and vcr helix on sync.
                        ambryVcrHelixSyncCheckTaskFuture = scheduler.scheduleAtFixedRate(() -> checkAmbryHelixAndVcrHelixOnSync(), cloudConfig.vcrHelixSyncCheckIntervalInSeconds, cloudConfig.vcrHelixSyncCheckIntervalInSeconds, TimeUnit.SECONDS);
                        logger.info("VCR updater registered.");
                    }
                    isVcrHelixUpdater = true;
                    scheduleVcrHelix("VCR starts");
                } finally {
                    vcrHelixUpdateLock.unlock();
                }
            }
            try {
                addReplica(partitionId);
                logger.info("Partition {} added to {}", partitionId, dataNodeId);
            } catch (ReplicationException e) {
                vcrMetrics.addPartitionErrorCount.inc();
                logger.error("Exception on adding Partition {} to {}: ", partitionId, dataNodeId, e);
            } catch (Exception e) {
                // Helix will run into error state if exception throws in Helix context.
                vcrMetrics.addPartitionErrorCount.inc();
                logger.error("Unknown Exception on adding Partition {} to {}: ", partitionId, dataNodeId, e);
            }
        }

        @Override
        public void onPartitionRemoved(PartitionId partitionId) {
            if (partitionId.isEqual(cloudConfig.vcrHelixUpdaterPartitionId)) {
                vcrHelixUpdateLock.lock();
                try {
                    isVcrHelixUpdater = false;
                    if (vcrHelixUpdateFuture != null) {
                        vcrHelixUpdateFuture.cancel(false);
                    }
                    if (ambryVcrHelixSyncCheckTaskFuture != null) {
                        ambryVcrHelixSyncCheckTaskFuture.cancel(false);
                    }
                } finally {
                    vcrHelixUpdateLock.unlock();
                }
            }
            try {
                removeReplica(partitionId);
            } catch (Exception e) {
                // Helix will run into error state if exception throws in Helix context.
                vcrMetrics.removePartitionErrorCount.inc();
                logger.error("Exception on removing Partition {} from {}: ", partitionId, dataNodeId, e);
            }
        }
    });
    try {
        vcrClusterParticipant.participate();
    } catch (Exception e) {
        throw new ReplicationException("Cluster participate failed.", e);
    }
    // start background persistent thread
    // start scheduler thread to persist index in the background
    scheduleTask(persistor, true, replicationConfig.replicationTokenFlushDelaySeconds, replicationConfig.replicationTokenFlushIntervalSeconds, "replica token persistor");
    // Schedule thread to purge dead blobs for this VCR's partitions
    // after delay to allow startup to finish.
    scheduleTask(cloudStorageCompactor, cloudConfig.cloudBlobCompactionEnabled, cloudConfig.cloudBlobCompactionStartupDelaySecs, TimeUnit.HOURS.toSeconds(cloudConfig.cloudBlobCompactionIntervalHours), "cloud blob compaction");
    // Schedule thread to purge blobs belonging to deprecated containers for this VCR's partitions
    // after delay to allow startup to finish.
    scheduleTask(() -> cloudContainerCompactor.compactAssignedDeprecatedContainers(vcrClusterParticipant.getAssignedPartitionIds()), cloudConfig.cloudContainerCompactionEnabled, cloudConfig.cloudContainerCompactionStartupDelaySecs, TimeUnit.HOURS.toSeconds(cloudConfig.cloudContainerCompactionIntervalHours), "cloud container compaction");
    started = true;
    startupLatch.countDown();
}