Examples with HelixDataAccessor org.apache.helix.HelixDataAccessor used on opensource projects

Example 6 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project pinot by linkedin.

the class HelixSetupUtils method createHelixClusterIfNeeded.

public static void createHelixClusterIfNeeded(String helixClusterName, String zkPath, boolean isUpdateStateModel) {
    final HelixAdmin admin = new ZKHelixAdmin(zkPath);
    final String segmentStateModelName = PinotHelixSegmentOnlineOfflineStateModelGenerator.PINOT_SEGMENT_ONLINE_OFFLINE_STATE_MODEL;
    if (admin.getClusters().contains(helixClusterName)) {
        LOGGER.info("cluster already exists ********************************************* ");
        if (isUpdateStateModel) {
            final StateModelDefinition curStateModelDef = admin.getStateModelDef(helixClusterName, segmentStateModelName);
            List<String> states = curStateModelDef.getStatesPriorityList();
            if (states.contains(PinotHelixSegmentOnlineOfflineStateModelGenerator.CONSUMING_STATE)) {
                LOGGER.info("State model {} already updated to contain CONSUMING state", segmentStateModelName);
                return;
            } else {
                LOGGER.info("Updating {} to add states for low level kafka consumers", segmentStateModelName);
                StateModelDefinition newStateModelDef = PinotHelixSegmentOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition();
                ZkClient zkClient = new ZkClient(zkPath);
                zkClient.waitUntilConnected(20, TimeUnit.SECONDS);
                zkClient.setZkSerializer(new ZNRecordSerializer());
                HelixDataAccessor accessor = new ZKHelixDataAccessor(helixClusterName, new ZkBaseDataAccessor<ZNRecord>(zkClient));
                PropertyKey.Builder keyBuilder = accessor.keyBuilder();
                accessor.setProperty(keyBuilder.stateModelDef(segmentStateModelName), newStateModelDef);
                LOGGER.info("Completed updating statemodel {}", segmentStateModelName);
                zkClient.close();
            }
        }
        return;
    }
    LOGGER.info("Creating a new cluster, as the helix cluster : " + helixClusterName + " was not found ********************************************* ");
    admin.addCluster(helixClusterName, false);
    LOGGER.info("Enable auto join.");
    final HelixConfigScope scope = new HelixConfigScopeBuilder(ConfigScopeProperty.CLUSTER).forCluster(helixClusterName).build();
    final Map<String, String> props = new HashMap<String, String>();
    props.put(ZKHelixManager.ALLOW_PARTICIPANT_AUTO_JOIN, String.valueOf(true));
    //we need only one segment to be loaded at a time
    props.put(MessageType.STATE_TRANSITION + "." + HelixTaskExecutor.MAX_THREADS, String.valueOf(1));
    admin.setConfig(scope, props);
    LOGGER.info("Adding state model {} (with CONSUMED state) generated using {} **********************************************", segmentStateModelName, PinotHelixSegmentOnlineOfflineStateModelGenerator.class.toString());
    // If this is a fresh cluster we are creating, then the cluster will see the CONSUMING state in the
    // state model. But then the servers will never be asked to go to that STATE (whether they have the code
    // to handle it or not) unil we complete the feature using low-level kafka consumers and turn the feature on.
    admin.addStateModelDef(helixClusterName, segmentStateModelName, PinotHelixSegmentOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition());
    LOGGER.info("Adding state model definition named : " + PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.PINOT_BROKER_RESOURCE_ONLINE_OFFLINE_STATE_MODEL + " generated using : " + PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.class.toString() + " ********************************************** ");
    admin.addStateModelDef(helixClusterName, PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.PINOT_BROKER_RESOURCE_ONLINE_OFFLINE_STATE_MODEL, PinotHelixBrokerResourceOnlineOfflineStateModelGenerator.generatePinotStateModelDefinition());
    LOGGER.info("Adding empty ideal state for Broker!");
    HelixHelper.updateResourceConfigsFor(new HashMap<String, String>(), CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, helixClusterName, admin);
    IdealState idealState = PinotTableIdealStateBuilder.buildEmptyIdealStateForBrokerResource(admin, helixClusterName);
    admin.setResourceIdealState(helixClusterName, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, idealState);
    initPropertyStorePath(helixClusterName, zkPath);
    LOGGER.info("New Cluster setup completed... ********************************************** ");
}

Example 7 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project pinot by linkedin.

the class HelixExternalViewBasedRouting method processInstanceConfigChange.

public void processInstanceConfigChange() {
    long startTime = System.currentTimeMillis();
    // Get stats for all relevant instance configs
    HelixDataAccessor helixDataAccessor = _helixManager.getHelixDataAccessor();
    PropertyKey.Builder propertyKeyBuilder = helixDataAccessor.keyBuilder();
    List<String> instancesUsed = new ArrayList<>(_tablesForInstance.keySet());
    List<String> instancePaths = new ArrayList<>(instancesUsed.size());
    for (String instanceName : instancesUsed) {
        PropertyKey propertyKey = propertyKeyBuilder.instanceConfig(instanceName);
        instancePaths.add(propertyKey.getPath());
    }
    if (instancePaths.isEmpty()) {
        return;
    }
    long statFetchStart = System.currentTimeMillis();
    Stat[] instanceConfigStats = helixDataAccessor.getBaseDataAccessor().getStats(instancePaths, AccessOption.PERSISTENT);
    long statFetchEnd = System.currentTimeMillis();
    // Make a list of instance configs that changed
    long icConfigCheckStart = System.currentTimeMillis();
    List<String> instancesThatChanged = new ArrayList<>();
    for (int i = 0; i < instanceConfigStats.length; i++) {
        Stat instanceConfigStat = instanceConfigStats[i];
        if (instanceConfigStat != null) {
            String instanceName = instancesUsed.get(i);
            int currentInstanceConfigVersion = instanceConfigStat.getVersion();
            int lastKnownInstanceConfigVersion = _lastKnownInstanceConfigs.get(instanceName).getRecord().getVersion();
            if (currentInstanceConfigVersion != lastKnownInstanceConfigVersion) {
                instancesThatChanged.add(instanceName);
            }
        }
    }
    // Make a list of all tables affected by the instance config changes
    Set<String> affectedTables = new HashSet<>();
    for (String instanceName : instancesThatChanged) {
        affectedTables.addAll(_tablesForInstance.get(instanceName));
    }
    long icConfigCheckEnd = System.currentTimeMillis();
    // Update the routing tables
    long icFetchTime = 0;
    long evFetchTime = 0;
    long rebuildCheckTime = 0;
    long buildTime = 0;
    int routingTablesRebuiltCount = 0;
    if (!affectedTables.isEmpty()) {
        long icFetchStart = System.currentTimeMillis();
        List<InstanceConfig> instanceConfigs = helixDataAccessor.getChildValues(propertyKeyBuilder.instanceConfigs());
        long icFetchEnd = System.currentTimeMillis();
        icFetchTime = icFetchEnd - icFetchStart;
        for (String tableName : affectedTables) {
            long evFetchStart = System.currentTimeMillis();
            ExternalView externalView = helixDataAccessor.getProperty(propertyKeyBuilder.externalView(tableName));
            long evFetchEnd = System.currentTimeMillis();
            evFetchTime += evFetchEnd - evFetchStart;
            long rebuildCheckStart = System.currentTimeMillis();
            final boolean routingTableRebuildRequired = isRoutingTableRebuildRequired(tableName, externalView, instanceConfigs);
            long rebuildCheckEnd = System.currentTimeMillis();
            rebuildCheckTime += rebuildCheckEnd - rebuildCheckStart;
            if (routingTableRebuildRequired) {
                long rebuildStart = System.currentTimeMillis();
                buildRoutingTable(tableName, externalView, instanceConfigs);
                long rebuildEnd = System.currentTimeMillis();
                buildTime += rebuildEnd - rebuildStart;
                routingTablesRebuiltCount++;
            }
        }
    }
    long endTime = System.currentTimeMillis();
    LOGGER.info("Processed instance config change in {} ms (stat {} ms, IC check {} ms, IC fetch {} ms, EV fetch {} ms, rebuild check {} ms, rebuild {} ms), {} / {} routing tables rebuilt", (endTime - startTime), (statFetchEnd - statFetchStart), (icConfigCheckEnd - icConfigCheckStart), icFetchTime, evFetchTime, rebuildCheckTime, buildTime, routingTablesRebuiltCount, _lastKnownExternalViewVersionMap.size());
}

Example 8 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project pinot by linkedin.

the class DeleteOverlappingSegmentsInPinot method deleteOverlappingSegments.

public static boolean deleteOverlappingSegments(String zkUrl, String zkCluster, String tableName) {
    boolean updateSuccessful = false;
    if (!tableName.endsWith("_OFFLINE")) {
        tableName = tableName + "_OFFLINE";
    }
    ZkClient zkClient = new ZkClient(zkUrl);
    ZNRecordSerializer zkSerializer = new ZNRecordSerializer();
    zkClient.setZkSerializer(zkSerializer);
    BaseDataAccessor<ZNRecord> baseDataAccessor = new ZkBaseDataAccessor<>(zkClient);
    HelixDataAccessor helixDataAccessor = new ZKHelixDataAccessor(zkCluster, baseDataAccessor);
    Builder keyBuilder = helixDataAccessor.keyBuilder();
    PropertyKey idealStateKey = keyBuilder.idealStates(tableName);
    PropertyKey externalViewKey = keyBuilder.externalView(tableName);
    IdealState currentIdealState = helixDataAccessor.getProperty(idealStateKey);
    byte[] serializeIS = zkSerializer.serialize(currentIdealState.getRecord());
    String name = tableName + ".idealstate." + System.currentTimeMillis();
    File outputFile = new File("/tmp", name);
    try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
        IOUtils.write(serializeIS, fileOutputStream);
    } catch (IOException e) {
        LOG.error("Exception in delete overlapping segments", e);
        return updateSuccessful;
    }
    LOG.info("Saved current idealstate to {}", outputFile);
    IdealState newIdealState;
    do {
        newIdealState = computeNewIdealStateAfterDeletingOverlappingSegments(helixDataAccessor, idealStateKey);
        LOG.info("Updating IdealState");
        updateSuccessful = helixDataAccessor.getBaseDataAccessor().set(idealStateKey.getPath(), newIdealState.getRecord(), newIdealState.getRecord().getVersion(), AccessOption.PERSISTENT);
        if (updateSuccessful) {
            int numSegmentsDeleted = currentIdealState.getPartitionSet().size() - newIdealState.getPartitionSet().size();
            LOG.info("Successfully updated IdealState: Removed segments: {}", (numSegmentsDeleted));
        }
    } while (!updateSuccessful);
    try {
        while (true) {
            Thread.sleep(10000);
            ExternalView externalView = helixDataAccessor.getProperty(externalViewKey);
            IdealState idealState = helixDataAccessor.getProperty(idealStateKey);
            Set<String> evPartitionSet = externalView.getPartitionSet();
            Set<String> isPartitionSet = idealState.getPartitionSet();
            if (evPartitionSet.equals(isPartitionSet)) {
                LOG.info("Table {} has reached stable state. i.e segments in external view match idealstates", tableName);
                break;
            }
        }
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    return updateSuccessful;
}

Example 9 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project incubator-gobblin by apache.

the class GobblinHelixMessagingService method generateMessage.

@Override
public Map<InstanceType, List<Message>> generateMessage(final Criteria recipientCriteria, final Message message) {
    Map<InstanceType, List<Message>> messagesToSendMap = new HashMap<InstanceType, List<Message>>();
    InstanceType instanceType = recipientCriteria.getRecipientInstanceType();
    if (instanceType == InstanceType.CONTROLLER) {
        List<Message> messages = generateMessagesForController(message);
        messagesToSendMap.put(InstanceType.CONTROLLER, messages);
    // _dataAccessor.setControllerProperty(PropertyType.MESSAGES,
    // newMessage.getRecord(), CreateMode.PERSISTENT);
    } else if (instanceType == InstanceType.PARTICIPANT) {
        List<Message> messages = new ArrayList<Message>();
        List<Map<String, String>> matchedList = _gobblinHelixCriteriaEvaluator.evaluateCriteria(recipientCriteria, _manager);
        if (!matchedList.isEmpty()) {
            Map<String, String> sessionIdMap = new HashMap<String, String>();
            if (recipientCriteria.isSessionSpecific()) {
                HelixDataAccessor accessor = _manager.getHelixDataAccessor();
                PropertyKey.Builder keyBuilder = accessor.keyBuilder();
                List<LiveInstance> liveInstances = accessor.getChildValues(keyBuilder.liveInstances());
                for (LiveInstance liveInstance : liveInstances) {
                    sessionIdMap.put(liveInstance.getInstanceName(), liveInstance.getSessionId());
                }
            }
            for (Map<String, String> map : matchedList) {
                String id = UUID.randomUUID().toString();
                Message newMessage = new Message(message.getRecord(), id);
                String srcInstanceName = _manager.getInstanceName();
                String tgtInstanceName = map.get("instanceName");
                // Don't send message to self
                if (recipientCriteria.isSelfExcluded() && srcInstanceName.equalsIgnoreCase(tgtInstanceName)) {
                    continue;
                }
                newMessage.setSrcName(srcInstanceName);
                newMessage.setTgtName(tgtInstanceName);
                newMessage.setResourceName(map.get("resourceName"));
                newMessage.setPartitionName(map.get("partitionName"));
                if (recipientCriteria.isSessionSpecific()) {
                    newMessage.setTgtSessionId(sessionIdMap.get(tgtInstanceName));
                }
                messages.add(newMessage);
            }
            messagesToSendMap.put(InstanceType.PARTICIPANT, messages);
        }
    }
    return messagesToSendMap;
}

Example 10 with HelixDataAccessor

use of org.apache.helix.HelixDataAccessor in project helix by apache.

the class ReadClusterDataStage method process.

@Override
public void process(ClusterEvent event) throws Exception {
    HelixManager manager = event.getAttribute(AttributeName.helixmanager.name());
    if (manager == null) {
        throw new StageException("HelixManager attribute value is null");
    }
    ClusterDataCache cache = event.getAttribute(AttributeName.ClusterDataCache.name());
    if (cache == null && _cache == null) {
        cache = new ClusterDataCache(event.getClusterName());
    }
    _cache = cache;
    HelixDataAccessor dataAccessor = manager.getHelixDataAccessor();
    _cache.refresh(dataAccessor);
    final ClusterConfig clusterConfig = cache.getClusterConfig();
    if (!_cache.isTaskCache()) {
        final ClusterStatusMonitor clusterStatusMonitor = event.getAttribute(AttributeName.clusterStatusMonitor.name());
        asyncExecute(_cache.getAsyncTasksThreadPool(), new Callable<Object>() {

            @Override
            public Object call() {
                // Update the cluster status gauges
                if (clusterStatusMonitor != null) {
                    logger.debug("Update cluster status monitors");
                    Set<String> instanceSet = Sets.newHashSet();
                    Set<String> liveInstanceSet = Sets.newHashSet();
                    Set<String> disabledInstanceSet = Sets.newHashSet();
                    Map<String, Map<String, List<String>>> disabledPartitions = Maps.newHashMap();
                    Map<String, List<String>> oldDisabledPartitions = Maps.newHashMap();
                    Map<String, Set<String>> tags = Maps.newHashMap();
                    Map<String, LiveInstance> liveInstanceMap = _cache.getLiveInstances();
                    for (Map.Entry<String, InstanceConfig> e : _cache.getInstanceConfigMap().entrySet()) {
                        String instanceName = e.getKey();
                        InstanceConfig config = e.getValue();
                        instanceSet.add(instanceName);
                        if (liveInstanceMap.containsKey(instanceName)) {
                            liveInstanceSet.add(instanceName);
                        }
                        if (!config.getInstanceEnabled() || (clusterConfig.getDisabledInstances() != null && clusterConfig.getDisabledInstances().containsKey(instanceName))) {
                            disabledInstanceSet.add(instanceName);
                        }
                        // TODO : Get rid of this data structure once the API is removed.
                        oldDisabledPartitions.put(instanceName, config.getDisabledPartitions());
                        disabledPartitions.put(instanceName, config.getDisabledPartitionsMap());
                        Set<String> instanceTags = Sets.newHashSet(config.getTags());
                        tags.put(instanceName, instanceTags);
                    }
                    clusterStatusMonitor.setClusterInstanceStatus(liveInstanceSet, instanceSet, disabledInstanceSet, disabledPartitions, oldDisabledPartitions, tags);
                    logger.debug("Complete cluster status monitors update.");
                }
                return null;
            }
        });
    }
    event.addAttribute(AttributeName.ClusterDataCache.name(), _cache);
}