Examples with SubnetRoute org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute used on opensource projects

Example 6 with SubnetRoute

use of org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute in project netvirt by opendaylight.

the class ArpNotificationHandler method addMipAdjacency.

private void addMipAdjacency(String vpnName, String vpnInterface, IpAddress srcPrefix, String mipMacAddress, IpAddress dstPrefix) {
    LOG.trace("Adding {} adjacency to VPN Interface {} ", srcPrefix, vpnInterface);
    InstanceIdentifier<VpnInterface> vpnIfId = VpnUtil.getVpnInterfaceIdentifier(vpnInterface);
    InstanceIdentifier<Adjacencies> path = vpnIfId.augmentation(Adjacencies.class);
    synchronized (vpnInterface.intern()) {
        Optional<Adjacencies> adjacencies = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, path);
        String nextHopIpAddr = null;
        String nextHopMacAddress = null;
        String ip = srcPrefix.getIpv4Address().getValue();
        if (interfaceManager.isExternalInterface(vpnInterface)) {
            String subnetId = getSubnetId(vpnName, dstPrefix.getIpv4Address().getValue());
            if (subnetId == null) {
                LOG.trace("Can't find corresponding subnet for src IP {}, src MAC {}, dst IP {},  in VPN {}", srcPrefix, mipMacAddress, dstPrefix, vpnName);
                return;
            }
            ip = VpnUtil.getIpPrefix(ip);
            AdjacencyBuilder newAdjBuilder = new AdjacencyBuilder().setIpAddress(ip).setKey(new AdjacencyKey(ip)).setAdjacencyType(AdjacencyType.PrimaryAdjacency).setMacAddress(mipMacAddress).setSubnetId(new Uuid(subnetId)).setPhysNetworkFunc(true);
            List<Adjacency> adjacencyList = adjacencies.isPresent() ? adjacencies.get().getAdjacency() : new ArrayList<>();
            adjacencyList.add(newAdjBuilder.build());
            Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencyList);
            Optional<VpnInterface> optionalVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vpnIfId);
            VpnInterface newVpnIntf;
            if (optionalVpnInterface.isPresent()) {
                newVpnIntf = new VpnInterfaceBuilder(optionalVpnInterface.get()).addAugmentation(Adjacencies.class, aug).build();
                VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, vpnIfId, newVpnIntf);
            }
            LOG.debug(" Successfully stored subnetroute Adjacency into VpnInterface {}", vpnInterface);
            return;
        }
        if (adjacencies.isPresent()) {
            List<Adjacency> adjacencyList = adjacencies.get().getAdjacency();
            ip = VpnUtil.getIpPrefix(ip);
            for (Adjacency adjacs : adjacencyList) {
                if (adjacs.getAdjacencyType() == AdjacencyType.PrimaryAdjacency) {
                    if (adjacs.getIpAddress().equals(ip)) {
                        LOG.error("The MIP {} is already present as a primary adjacency for interface {} vpn {}." + "Skipping adjacency addition.", ip, vpnInterface, vpnName);
                        return;
                    }
                    nextHopIpAddr = adjacs.getIpAddress();
                    nextHopMacAddress = adjacs.getMacAddress();
                    break;
                }
            }
            if (nextHopIpAddr != null) {
                String rd = VpnUtil.getVpnRd(dataBroker, vpnName);
                long label = VpnUtil.getUniqueId(idManager, VpnConstants.VPN_IDPOOL_NAME, VpnUtil.getNextHopLabelKey(rd != null ? rd : vpnName, ip));
                if (label == 0) {
                    LOG.error("Unable to fetch label from Id Manager. Bailing out of adding MIP adjacency {} " + "to vpn interface {} for vpn {}", ip, vpnInterface, vpnName);
                    return;
                }
                String nextHopIp = nextHopIpAddr.split("/")[0];
                AdjacencyBuilder newAdjBuilder = new AdjacencyBuilder().setIpAddress(ip).setKey(new AdjacencyKey(ip)).setNextHopIpList(Collections.singletonList(nextHopIp)).setAdjacencyType(AdjacencyType.LearntIp);
                if (mipMacAddress != null && !mipMacAddress.equalsIgnoreCase(nextHopMacAddress)) {
                    newAdjBuilder.setMacAddress(mipMacAddress);
                }
                adjacencyList.add(newAdjBuilder.build());
                Adjacencies aug = VpnUtil.getVpnInterfaceAugmentation(adjacencyList);
                Optional<VpnInterface> optionalVpnInterface = VpnUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, vpnIfId);
                VpnInterface newVpnIntf;
                if (optionalVpnInterface.isPresent()) {
                    newVpnIntf = new VpnInterfaceBuilder(optionalVpnInterface.get()).addAugmentation(Adjacencies.class, aug).build();
                    VpnUtil.syncWrite(dataBroker, LogicalDatastoreType.CONFIGURATION, vpnIfId, newVpnIntf);
                }
                LOG.debug(" Successfully stored subnetroute Adjacency into VpnInterface {}", vpnInterface);
            }
        }
    }
}

Example 7 with SubnetRoute

use of org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute in project netvirt by opendaylight.

the class VrfEntryListener method installSubnetBroadcastAddrDropRule.

private void installSubnetBroadcastAddrDropRule(final BigInteger dpnId, final String rd, final long vpnId, final VrfEntry vrfEntry, int addOrRemove, WriteTransaction tx) {
    List<MatchInfo> matches = new ArrayList<>();
    LOG.debug("SUBNETROUTE: installSubnetBroadcastAddrDropRule: destPrefix {} rd {} vpnId {} dpnId {}", vrfEntry.getDestPrefix(), rd, vpnId, dpnId);
    String[] ipAddress = vrfEntry.getDestPrefix().split("/");
    String subnetBroadcastAddr = FibUtil.getBroadcastAddressFromCidr(vrfEntry.getDestPrefix());
    final int prefixLength = ipAddress.length == 1 ? 0 : Integer.parseInt(ipAddress[1]);
    InetAddress destPrefix;
    try {
        destPrefix = InetAddress.getByName(subnetBroadcastAddr);
    } catch (UnknownHostException e) {
        LOG.error("Failed to get destPrefix for prefix {} rd {} VpnId {} DPN {}", vrfEntry.getDestPrefix(), rd, vpnId, dpnId, e);
        return;
    }
    // Match on VpnId and SubnetBroadCast IP address
    matches.add(new MatchMetadata(MetaDataUtil.getVpnIdMetadata(vpnId), MetaDataUtil.METADATA_MASK_VRFID));
    matches.add(MatchEthernetType.IPV4);
    if (prefixLength != 0) {
        matches.add(new MatchIpv4Destination(subnetBroadcastAddr, Integer.toString(IPV4_ADDR_PREFIX_LENGTH)));
    }
    // Action is to drop the packet
    List<InstructionInfo> dropInstructions = new ArrayList<>();
    List<ActionInfo> actionsInfos = new ArrayList<>();
    actionsInfos.add(new ActionDrop());
    dropInstructions.add(new InstructionApplyActions(actionsInfos));
    int priority = DEFAULT_FIB_FLOW_PRIORITY + IPV4_ADDR_PREFIX_LENGTH;
    String flowRef = FibUtil.getFlowRef(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, rd, priority, destPrefix);
    FlowEntity flowEntity = MDSALUtil.buildFlowEntity(dpnId, NwConstants.L3_SUBNET_ROUTE_TABLE, flowRef, priority, flowRef, 0, 0, COOKIE_TABLE_MISS, matches, dropInstructions);
    Flow flow = flowEntity.getFlowBuilder().build();
    String flowId = flowEntity.getFlowId();
    FlowKey flowKey = new FlowKey(new FlowId(flowId));
    Node nodeDpn = FibUtil.buildDpnNode(dpnId);
    InstanceIdentifier<Flow> flowInstanceId = InstanceIdentifier.builder(Nodes.class).child(Node.class, nodeDpn.getKey()).augmentation(FlowCapableNode.class).child(Table.class, new TableKey(flow.getTableId())).child(Flow.class, flowKey).build();
    if (addOrRemove == NwConstants.ADD_FLOW) {
        tx.put(LogicalDatastoreType.CONFIGURATION, flowInstanceId, flow, true);
    } else {
        tx.delete(LogicalDatastoreType.CONFIGURATION, flowInstanceId);
    }
}

Example 8 with SubnetRoute

use of org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute in project netvirt by opendaylight.

the class VrfEntryListener method cleanUpDpnForVpn.

public void cleanUpDpnForVpn(final BigInteger dpnId, final long vpnId, final String rd, final FutureCallback<List<Void>> callback) {
    LOG.trace("cleanUpDpnForVpn: Remove dpn {} for vpn {} : cleanUpDpnForVpn", dpnId, rd);
    InstanceIdentifier<VrfTables> id = buildVrfId(rd);
    final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(rd);
    List<SubTransaction> txnObjects = new ArrayList<>();
    final Optional<VrfTables> vrfTable = MDSALUtil.read(dataBroker, LogicalDatastoreType.CONFIGURATION, id);
    jobCoordinator.enqueueJob(FibUtil.getJobKeyForVpnIdDpnId(vpnId, dpnId), () -> {
        List<ListenableFuture<Void>> futures = new ArrayList<>();
        if (vrfTable.isPresent()) {
            synchronized (vpnInstance.getVpnInstanceName().intern()) {
                futures.add(retryingTxRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
                    for (final VrfEntry vrfEntry : vrfTable.get().getVrfEntry()) {
                        /* Handle subnet routes here */
                        SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
                        if (subnetRoute != null) {
                            LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Cleaning subnetroute {} on dpn {}" + " for vpn {}", vrfEntry.getDestPrefix(), dpnId, rd);
                            baseVrfEntryHandler.makeConnectedRoute(dpnId, vpnId, vrfEntry, rd, null, NwConstants.DEL_FLOW, tx, null);
                            List<RoutePaths> routePaths = vrfEntry.getRoutePaths();
                            if (routePaths != null) {
                                for (RoutePaths routePath : routePaths) {
                                    makeLFibTableEntry(dpnId, routePath.getLabel(), null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx);
                                    LOG.trace("SUBNETROUTE: cleanUpDpnForVpn: Released subnetroute label {}" + " for rd {} prefix {}", routePath.getLabel(), rd, vrfEntry.getDestPrefix());
                                }
                            }
                            installSubnetBroadcastAddrDropRule(dpnId, rd, vpnId, vrfEntry, NwConstants.DEL_FLOW, tx);
                            continue;
                        }
                        // ping responder for router interfaces
                        RouterInterface routerInt = vrfEntry.getAugmentation(RouterInterface.class);
                        if (routerInt != null) {
                            LOG.trace("Router augmented vrfentry found for rd:{}, uuid:{}, ip:{}, mac:{}", rd, routerInt.getUuid(), routerInt.getIpAddress(), routerInt.getMacAddress());
                            routerInterfaceVrfEntryHandler.installRouterFibEntry(vrfEntry, dpnId, vpnId, routerInt.getIpAddress(), new MacAddress(routerInt.getMacAddress()), NwConstants.DEL_FLOW);
                            continue;
                        }
                        // Handle local flow deletion for imports
                        if (RouteOrigin.value(vrfEntry.getOrigin()) == RouteOrigin.SELF_IMPORTED) {
                            java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
                            if (optionalLabel.isPresent()) {
                                List<String> nextHopList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
                                LabelRouteInfo lri = getLabelRouteInfo(optionalLabel.get());
                                if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopList, lri) && lri.getDpnId().equals(dpnId)) {
                                    deleteLocalFibEntry(vpnId, rd, vrfEntry);
                                }
                            }
                        }
                        // Passing null as we don't know the dpn
                        // to which prefix is attached at this point
                        List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
                        String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
                        Optional<Routes> extraRouteOptional;
                        // an adjacency in the vpn
                        if (usedRds != null && !usedRds.isEmpty()) {
                            if (usedRds.size() > 1) {
                                LOG.error("The extra route prefix is still present in some DPNs");
                                return;
                            } else {
                                extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
                            }
                        } else {
                            extraRouteOptional = Optional.absent();
                        }
                        if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
                            bgpRouteVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, extraRouteOptional, tx, txnObjects);
                        } else {
                            baseVrfEntryHandler.deleteRemoteRoute(null, dpnId, vpnId, vrfTable.get().getKey(), vrfEntry, extraRouteOptional, tx);
                        }
                    }
                }));
            }
            if (callback != null) {
                ListenableFuture<List<Void>> listenableFuture = Futures.allAsList(futures);
                Futures.addCallback(listenableFuture, callback, MoreExecutors.directExecutor());
            }
        }
        return futures;
    });
}

Example 9 with SubnetRoute

use of org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute in project netvirt by opendaylight.

the class VrfEntryListener method createFibEntries.

private void createFibEntries(final InstanceIdentifier<VrfEntry> vrfEntryIid, final VrfEntry vrfEntry) {
    final VrfTablesKey vrfTableKey = vrfEntryIid.firstKeyOf(VrfTables.class);
    List<SubTransaction> txnObjects = new ArrayList<>();
    final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
    Preconditions.checkNotNull(vpnInstance, "Vpn Instance not available " + vrfTableKey.getRouteDistinguisher());
    Preconditions.checkNotNull(vpnInstance.getVpnId(), "Vpn Instance with rd " + vpnInstance.getVrfId() + " has null vpnId!");
    final Collection<VpnToDpnList> vpnToDpnList;
    if (vrfEntry.getParentVpnRd() != null && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
        // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
        VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
        vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() : vpnInstance.getVpnToDpnList();
        LOG.info("createFibEntries: Processing creation of PNF FIB entry with rd {} prefix {}", vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
    } else {
        vpnToDpnList = vpnInstance.getVpnToDpnList();
    }
    final Long vpnId = vpnInstance.getVpnId();
    final String rd = vrfTableKey.getRouteDistinguisher();
    SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
    if (subnetRoute != null) {
        final long elanTag = subnetRoute.getElantag();
        LOG.trace("SUBNETROUTE: createFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}" + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
        if (vpnToDpnList != null) {
            jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()), () -> {
                WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
                for (final VpnToDpnList curDpn : vpnToDpnList) {
                    if (curDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
                        installSubnetRouteInFib(curDpn.getDpnId(), elanTag, rd, vpnId, vrfEntry, tx);
                        installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnId.longValue(), vrfEntry, NwConstants.ADD_FLOW, tx);
                    }
                }
                List<ListenableFuture<Void>> futures = new ArrayList<>();
                futures.add(tx.submit());
                return futures;
            });
        }
        return;
    }
    final List<BigInteger> localDpnIdList = createLocalFibEntry(vpnInstance.getVpnId(), rd, vrfEntry);
    if (!localDpnIdList.isEmpty() && vpnToDpnList != null) {
        jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()), () -> {
            WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
            synchronized (vpnInstance.getVpnInstanceName().intern()) {
                for (VpnToDpnList vpnDpn : vpnToDpnList) {
                    if (!localDpnIdList.contains(vpnDpn.getDpnId())) {
                        if (vpnDpn.getDpnState() == VpnToDpnList.DpnState.Active) {
                            try {
                                if (RouteOrigin.BGP.getValue().equals(vrfEntry.getOrigin())) {
                                    bgpRouteVrfEntryHandler.createRemoteFibEntry(vpnDpn.getDpnId(), vpnId, vrfTableKey.getRouteDistinguisher(), vrfEntry, tx, txnObjects);
                                } else {
                                    createRemoteFibEntry(vpnDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry, tx);
                                }
                            } catch (NullPointerException e) {
                                LOG.error("Failed to get create remote fib flows for prefix {} ", vrfEntry.getDestPrefix(), e);
                            }
                        }
                    }
                }
            }
            List<ListenableFuture<Void>> futures = new ArrayList<>();
            futures.add(tx.submit());
            return futures;
        }, MAX_RETRIES);
    }
    Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
    if (optVpnUuid.isPresent()) {
        String vpnUuid = optVpnUuid.get();
        InterVpnLinkDataComposite interVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid).orNull();
        if (interVpnLink != null) {
            LOG.debug("InterVpnLink {} found in Cache linking Vpn {}", interVpnLink.getInterVpnLinkName(), vpnUuid);
            FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
                if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
                    // This is an static route that points to the other endpoint of an InterVpnLink
                    // In that case, we should add another entry in FIB table pointing to LPortDispatcher table.
                    installIVpnLinkSwitchingFlows(interVpnLink, vpnUuid, vrfEntry, vpnId);
                    installInterVpnRouteInLFib(interVpnLink, vpnUuid, vrfEntry);
                }
            });
        }
    }
}

Example 10 with SubnetRoute

use of org.opendaylight.yang.gen.v1.urn.opendaylight.netvirt.fibmanager.rev150330.SubnetRoute in project netvirt by opendaylight.

the class VrfEntryListener method deleteFibEntries.

private void deleteFibEntries(final InstanceIdentifier<VrfEntry> identifier, final VrfEntry vrfEntry) {
    final VrfTablesKey vrfTableKey = identifier.firstKeyOf(VrfTables.class);
    final String rd = vrfTableKey.getRouteDistinguisher();
    final VpnInstanceOpDataEntry vpnInstance = fibUtil.getVpnInstance(vrfTableKey.getRouteDistinguisher());
    if (vpnInstance == null) {
        LOG.error("VPN Instance for rd {} is not available from VPN Op Instance Datastore", rd);
        return;
    }
    final Collection<VpnToDpnList> vpnToDpnList;
    if (vrfEntry.getParentVpnRd() != null && FibHelper.isControllerManagedNonSelfImportedRoute(RouteOrigin.value(vrfEntry.getOrigin()))) {
        // This block MUST BE HIT only for PNF (Physical Network Function) FIB Entries.
        VpnInstanceOpDataEntry parentVpnInstance = fibUtil.getVpnInstance(vrfEntry.getParentVpnRd());
        vpnToDpnList = parentVpnInstance != null ? parentVpnInstance.getVpnToDpnList() : vpnInstance.getVpnToDpnList();
        LOG.info("deleteFibEntries: Processing deletion of PNF FIB entry with rd {} prefix {}", vrfEntry.getParentVpnRd(), vrfEntry.getDestPrefix());
    } else {
        vpnToDpnList = vpnInstance.getVpnToDpnList();
    }
    SubnetRoute subnetRoute = vrfEntry.getAugmentation(SubnetRoute.class);
    final java.util.Optional<Long> optionalLabel = FibUtil.getLabelFromRoutePaths(vrfEntry);
    List<String> nextHopAddressList = FibHelper.getNextHopListFromRoutePaths(vrfEntry);
    String vpnName = fibUtil.getVpnNameFromId(vpnInstance.getVpnId());
    if (subnetRoute != null) {
        long elanTag = subnetRoute.getElantag();
        LOG.trace("SUBNETROUTE: deleteFibEntries: SubnetRoute augmented vrfentry found for rd {} prefix {}" + " with elantag {}", rd, vrfEntry.getDestPrefix(), elanTag);
        if (vpnToDpnList != null) {
            jobCoordinator.enqueueJob(FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix()), () -> Collections.singletonList(txRunner.callWithNewWriteOnlyTransactionAndSubmit(tx -> {
                for (final VpnToDpnList curDpn : vpnToDpnList) {
                    baseVrfEntryHandler.makeConnectedRoute(curDpn.getDpnId(), vpnInstance.getVpnId(), vrfEntry, vrfTableKey.getRouteDistinguisher(), null, NwConstants.DEL_FLOW, tx, null);
                    if (RouteOrigin.value(vrfEntry.getOrigin()) != RouteOrigin.SELF_IMPORTED) {
                        optionalLabel.ifPresent(label -> makeLFibTableEntry(curDpn.getDpnId(), label, null, DEFAULT_FIB_FLOW_PRIORITY, NwConstants.DEL_FLOW, tx));
                    }
                    installSubnetBroadcastAddrDropRule(curDpn.getDpnId(), rd, vpnInstance.getVpnId(), vrfEntry, NwConstants.DEL_FLOW, tx);
                }
            })));
        }
        optionalLabel.ifPresent(label -> {
            synchronized (label.toString().intern()) {
                LabelRouteInfo lri = getLabelRouteInfo(label);
                if (isPrefixAndNextHopPresentInLri(vrfEntry.getDestPrefix(), nextHopAddressList, lri)) {
                    Optional<VpnInstanceOpDataEntry> vpnInstanceOpDataEntryOptional = fibUtil.getVpnInstanceOpData(rd);
                    String vpnInstanceName = "";
                    if (vpnInstanceOpDataEntryOptional.isPresent()) {
                        vpnInstanceName = vpnInstanceOpDataEntryOptional.get().getVpnInstanceName();
                    }
                    boolean lriRemoved = this.deleteLabelRouteInfo(lri, vpnInstanceName, null);
                    if (lriRemoved) {
                        String parentRd = lri.getParentVpnRd();
                        fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(parentRd, vrfEntry.getDestPrefix()));
                        LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}" + " as labelRouteInfo cleared", label, rd, vrfEntry.getDestPrefix());
                    }
                } else {
                    fibUtil.releaseId(FibConstants.VPN_IDPOOL_NAME, FibUtil.getNextHopLabelKey(rd, vrfEntry.getDestPrefix()));
                    LOG.trace("SUBNETROUTE: deleteFibEntries: Released subnetroute label {} for rd {} prefix {}", label, rd, vrfEntry.getDestPrefix());
                }
            }
        });
        return;
    }
    final List<BigInteger> localDpnIdList = deleteLocalFibEntry(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
    if (vpnToDpnList != null) {
        List<String> usedRds = VpnExtraRouteHelper.getUsedRds(dataBroker, vpnInstance.getVpnId(), vrfEntry.getDestPrefix());
        String jobKey;
        Optional<Routes> extraRouteOptional;
        // Is this fib route an extra route? If yes, get the nexthop which would be an adjacency in the vpn
        if (usedRds != null && !usedRds.isEmpty()) {
            jobKey = FibUtil.getJobKeyForRdPrefix(usedRds.get(0), vrfEntry.getDestPrefix());
            if (usedRds.size() > 1) {
                LOG.error("The extra route prefix is still present in some DPNs");
                return;
            } else {
                // The first rd is retrieved from usedrds as Only 1 rd would be present as extra route prefix
                // is not present in any other DPN
                extraRouteOptional = VpnExtraRouteHelper.getVpnExtraroutes(dataBroker, vpnName, usedRds.get(0), vrfEntry.getDestPrefix());
            }
        } else {
            jobKey = FibUtil.getJobKeyForRdPrefix(rd, vrfEntry.getDestPrefix());
            extraRouteOptional = Optional.absent();
        }
        jobCoordinator.enqueueJob(jobKey, () -> {
            WriteTransaction tx = dataBroker.newWriteOnlyTransaction();
            if (localDpnIdList.size() <= 0) {
                for (VpnToDpnList curDpn : vpnToDpnList) {
                    baseVrfEntryHandler.deleteRemoteRoute(BigInteger.ZERO, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
                }
            } else {
                for (BigInteger localDpnId : localDpnIdList) {
                    for (VpnToDpnList curDpn : vpnToDpnList) {
                        if (!curDpn.getDpnId().equals(localDpnId)) {
                            baseVrfEntryHandler.deleteRemoteRoute(localDpnId, curDpn.getDpnId(), vpnInstance.getVpnId(), vrfTableKey, vrfEntry, extraRouteOptional, tx);
                        }
                    }
                }
            }
            List<ListenableFuture<Void>> futures = new ArrayList<>();
            futures.add(tx.submit());
            return futures;
        }, MAX_RETRIES);
    }
    // The flow/group entry has been deleted from config DS; need to clean up associated operational
    // DS entries in VPN Op DS, VpnInstanceOpData and PrefixToInterface to complete deletion
    cleanUpOpDataForFib(vpnInstance.getVpnId(), vrfTableKey.getRouteDistinguisher(), vrfEntry);
    // Remove all fib entries configured due to interVpnLink, when nexthop is the opposite endPoint
    // of the interVpnLink.
    Optional<String> optVpnUuid = fibUtil.getVpnNameFromRd(rd);
    if (optVpnUuid.isPresent()) {
        String vpnUuid = optVpnUuid.get();
        FibUtil.getFirstNextHopAddress(vrfEntry).ifPresent(routeNexthop -> {
            Optional<InterVpnLinkDataComposite> optInterVpnLink = interVpnLinkCache.getInterVpnLinkByVpnId(vpnUuid);
            if (optInterVpnLink.isPresent()) {
                InterVpnLinkDataComposite interVpnLink = optInterVpnLink.get();
                if (interVpnLink.isIpAddrTheOtherVpnEndpoint(routeNexthop, vpnUuid)) {
                    // This is route that points to the other endpoint of an InterVpnLink
                    // In that case, we should look for the FIB table pointing to
                    // LPortDispatcher table and remove it.
                    removeInterVPNLinkRouteFlows(interVpnLink, vpnUuid, vrfEntry);
                }
            }
        });
    }
}