Examples with PropertyKey org.apache.helix.PropertyKey used on opensource projects

Example 1 with PropertyKey

use of org.apache.helix.PropertyKey in project pinot by linkedin.

the class HelixHelper method updateIdealState.

/**
   * Updates the ideal state, retrying if necessary in case of concurrent updates to the ideal state.
   *
   * @param helixManager The HelixManager used to interact with the Helix cluster
   * @param resourceName The resource for which to update the ideal state
   * @param updater A function that returns an updated ideal state given an input ideal state
   */
public static void updateIdealState(final HelixManager helixManager, final String resourceName, final Function<IdealState, IdealState> updater, RetryPolicy policy) {
    boolean successful = policy.attempt(new Callable<Boolean>() {

        @Override
        public Boolean call() {
            HelixDataAccessor dataAccessor = helixManager.getHelixDataAccessor();
            PropertyKey propertyKey = dataAccessor.keyBuilder().idealStates(resourceName);
            // Create an updated version of the ideal state
            IdealState idealState = dataAccessor.getProperty(propertyKey);
            PropertyKey key = dataAccessor.keyBuilder().idealStates(resourceName);
            String path = key.getPath();
            // Make a copy of the the idealState above to pass it to the updater, instead of querying again,
            // as the state my change between the queries.
            ZNRecordSerializer znRecordSerializer = new ZNRecordSerializer();
            IdealState idealStateCopy = new IdealState((ZNRecord) znRecordSerializer.deserialize(znRecordSerializer.serialize(idealState.getRecord())));
            IdealState updatedIdealState;
            try {
                updatedIdealState = updater.apply(idealStateCopy);
            } catch (Exception e) {
                LOGGER.error("Caught exception while updating ideal state", e);
                return false;
            }
            // If there are changes to apply, apply them
            if (!EqualityUtils.isEqual(idealState, updatedIdealState) && updatedIdealState != null) {
                BaseDataAccessor<ZNRecord> baseDataAccessor = dataAccessor.getBaseDataAccessor();
                boolean success;
                // If the ideal state is large enough, enable compression
                if (MAX_PARTITION_COUNT_IN_UNCOMPRESSED_IDEAL_STATE < updatedIdealState.getPartitionSet().size()) {
                    updatedIdealState.getRecord().setBooleanField("enableCompression", true);
                }
                try {
                    success = baseDataAccessor.set(path, updatedIdealState.getRecord(), idealState.getRecord().getVersion(), AccessOption.PERSISTENT);
                } catch (Exception e) {
                    boolean idealStateIsCompressed = updatedIdealState.getRecord().getBooleanField("enableCompression", false);
                    LOGGER.warn("Caught exception while updating ideal state for resource {} (compressed={}), retrying.", resourceName, idealStateIsCompressed, e);
                    return false;
                }
                if (success) {
                    return true;
                } else {
                    LOGGER.warn("Failed to update ideal state for resource {}, retrying.", resourceName);
                    return false;
                }
            } else {
                LOGGER.warn("Idempotent or null ideal state update for resource {}, skipping update.", resourceName);
                return true;
            }
        }
    });
    if (!successful) {
        throw new RuntimeException("Failed to update ideal state for resource " + resourceName);
    }
}

Example 2 with PropertyKey

use of org.apache.helix.PropertyKey in project pinot by linkedin.

the class PinotHelixResourceManager method updateExistedSegment.

private boolean updateExistedSegment(SegmentZKMetadata segmentZKMetadata) {
    final String tableName;
    if (segmentZKMetadata instanceof RealtimeSegmentZKMetadata) {
        tableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(segmentZKMetadata.getTableName());
    } else {
        tableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(segmentZKMetadata.getTableName());
    }
    final String segmentName = segmentZKMetadata.getSegmentName();
    HelixDataAccessor helixDataAccessor = _helixZkManager.getHelixDataAccessor();
    PropertyKey idealStatePropertyKey = _keyBuilder.idealStates(tableName);
    // Set all partitions to offline to unload them from the servers
    boolean updateSuccessful;
    do {
        final IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
        final Set<String> instanceSet = idealState.getInstanceSet(segmentName);
        if (instanceSet == null || instanceSet.size() == 0) {
            // We are trying to refresh a segment, but there are no instances currently assigned for fielding this segment.
            // When those instances do come up, the segment will be uploaded correctly, so return success but log a warning.
            LOGGER.warn("No instances as yet for segment {}, table {}", segmentName, tableName);
            return true;
        }
        for (final String instance : instanceSet) {
            idealState.setPartitionState(segmentName, instance, "OFFLINE");
        }
        updateSuccessful = helixDataAccessor.updateProperty(idealStatePropertyKey, idealState);
    } while (!updateSuccessful);
    // Check that the ideal state has been written to ZK
    IdealState updatedIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
    Map<String, String> instanceStateMap = updatedIdealState.getInstanceStateMap(segmentName);
    for (String state : instanceStateMap.values()) {
        if (!"OFFLINE".equals(state)) {
            LOGGER.error("Failed to write OFFLINE ideal state!");
            return false;
        }
    }
    // Wait until the partitions are offline in the external view
    LOGGER.info("Wait until segment - " + segmentName + " to be OFFLINE in ExternalView");
    if (!ifExternalViewChangeReflectedForState(tableName, segmentName, "OFFLINE", _externalViewOnlineToOfflineTimeoutMillis, false)) {
        LOGGER.error("External view for segment {} did not reflect the ideal state of OFFLINE within the {} ms time limit", segmentName, _externalViewOnlineToOfflineTimeoutMillis);
        return false;
    }
    // Set all partitions to online so that they load the new segment data
    do {
        final IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
        final Set<String> instanceSet = idealState.getInstanceSet(segmentName);
        LOGGER.info("Found {} instances for segment '{}', in ideal state", instanceSet.size(), segmentName);
        for (final String instance : instanceSet) {
            idealState.setPartitionState(segmentName, instance, "ONLINE");
            LOGGER.info("Setting Ideal State for segment '{}' to ONLINE for instance '{}'", segmentName, instance);
        }
        updateSuccessful = helixDataAccessor.updateProperty(idealStatePropertyKey, idealState);
    } while (!updateSuccessful);
    // Check that the ideal state has been written to ZK
    updatedIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
    instanceStateMap = updatedIdealState.getInstanceStateMap(segmentName);
    LOGGER.info("Found {} instances for segment '{}', after updating ideal state", instanceStateMap.size(), segmentName);
    for (String state : instanceStateMap.values()) {
        if (!"ONLINE".equals(state)) {
            LOGGER.error("Failed to write ONLINE ideal state!");
            return false;
        }
    }
    LOGGER.info("Refresh is done for segment - " + segmentName);
    return true;
}

Example 3 with PropertyKey

use of org.apache.helix.PropertyKey in project pinot by linkedin.

the class PinotHelixResourceManager method dropInstance.

/**
   * Drop the instance from helix cluster. Instance will not be dropped if:
   * - It is a live instance.
   * - Has at least one ONLINE segment.
   *
   * @param instanceName: Name of the instance to be dropped.
   * @return
   */
public PinotResourceManagerResponse dropInstance(String instanceName) {
    if (!instanceExists(instanceName)) {
        return new PinotResourceManagerResponse("Instance " + instanceName + " does not exist.", false);
    }
    HelixDataAccessor helixDataAccessor = _helixZkManager.getHelixDataAccessor();
    LiveInstance liveInstance = helixDataAccessor.getProperty(_keyBuilder.liveInstance(instanceName));
    if (liveInstance != null) {
        PropertyKey currentStatesKey = _keyBuilder.currentStates(instanceName, liveInstance.getSessionId());
        List<CurrentState> currentStates = _helixDataAccessor.getChildValues(currentStatesKey);
        if (currentStates != null) {
            for (CurrentState currentState : currentStates) {
                for (String state : currentState.getPartitionStateMap().values()) {
                    if (state.equalsIgnoreCase(SegmentOnlineOfflineStateModel.ONLINE)) {
                        return new PinotResourceManagerResponse(("Instance " + instanceName + " has online partitions"), false);
                    }
                }
            }
        } else {
            return new PinotResourceManagerResponse("Cannot drop live instance " + instanceName + " please stop the instance first.", false);
        }
    }
    // Disable the instance first.
    toogleInstance(instanceName, false, 10);
    _helixAdmin.dropInstance(_helixClusterName, getHelixInstanceConfig(instanceName));
    return new PinotResourceManagerResponse("Instance " + instanceName + " dropped.", true);
}

Example 4 with PropertyKey

use of org.apache.helix.PropertyKey in project pinot by linkedin.

the class HelixExternalViewBasedRouting method processInstanceConfigChange.

public void processInstanceConfigChange() {
    long startTime = System.currentTimeMillis();
    // Get stats for all relevant instance configs
    HelixDataAccessor helixDataAccessor = _helixManager.getHelixDataAccessor();
    PropertyKey.Builder propertyKeyBuilder = helixDataAccessor.keyBuilder();
    List<String> instancesUsed = new ArrayList<>(_tablesForInstance.keySet());
    List<String> instancePaths = new ArrayList<>(instancesUsed.size());
    for (String instanceName : instancesUsed) {
        PropertyKey propertyKey = propertyKeyBuilder.instanceConfig(instanceName);
        instancePaths.add(propertyKey.getPath());
    }
    if (instancePaths.isEmpty()) {
        return;
    }
    long statFetchStart = System.currentTimeMillis();
    Stat[] instanceConfigStats = helixDataAccessor.getBaseDataAccessor().getStats(instancePaths, AccessOption.PERSISTENT);
    long statFetchEnd = System.currentTimeMillis();
    // Make a list of instance configs that changed
    long icConfigCheckStart = System.currentTimeMillis();
    List<String> instancesThatChanged = new ArrayList<>();
    for (int i = 0; i < instanceConfigStats.length; i++) {
        Stat instanceConfigStat = instanceConfigStats[i];
        if (instanceConfigStat != null) {
            String instanceName = instancesUsed.get(i);
            int currentInstanceConfigVersion = instanceConfigStat.getVersion();
            int lastKnownInstanceConfigVersion = _lastKnownInstanceConfigs.get(instanceName).getRecord().getVersion();
            if (currentInstanceConfigVersion != lastKnownInstanceConfigVersion) {
                instancesThatChanged.add(instanceName);
            }
        }
    }
    // Make a list of all tables affected by the instance config changes
    Set<String> affectedTables = new HashSet<>();
    for (String instanceName : instancesThatChanged) {
        affectedTables.addAll(_tablesForInstance.get(instanceName));
    }
    long icConfigCheckEnd = System.currentTimeMillis();
    // Update the routing tables
    long icFetchTime = 0;
    long evFetchTime = 0;
    long rebuildCheckTime = 0;
    long buildTime = 0;
    int routingTablesRebuiltCount = 0;
    if (!affectedTables.isEmpty()) {
        long icFetchStart = System.currentTimeMillis();
        List<InstanceConfig> instanceConfigs = helixDataAccessor.getChildValues(propertyKeyBuilder.instanceConfigs());
        long icFetchEnd = System.currentTimeMillis();
        icFetchTime = icFetchEnd - icFetchStart;
        for (String tableName : affectedTables) {
            long evFetchStart = System.currentTimeMillis();
            ExternalView externalView = helixDataAccessor.getProperty(propertyKeyBuilder.externalView(tableName));
            long evFetchEnd = System.currentTimeMillis();
            evFetchTime += evFetchEnd - evFetchStart;
            long rebuildCheckStart = System.currentTimeMillis();
            final boolean routingTableRebuildRequired = isRoutingTableRebuildRequired(tableName, externalView, instanceConfigs);
            long rebuildCheckEnd = System.currentTimeMillis();
            rebuildCheckTime += rebuildCheckEnd - rebuildCheckStart;
            if (routingTableRebuildRequired) {
                long rebuildStart = System.currentTimeMillis();
                buildRoutingTable(tableName, externalView, instanceConfigs);
                long rebuildEnd = System.currentTimeMillis();
                buildTime += rebuildEnd - rebuildStart;
                routingTablesRebuiltCount++;
            }
        }
    }
    long endTime = System.currentTimeMillis();
    LOGGER.info("Processed instance config change in {} ms (stat {} ms, IC check {} ms, IC fetch {} ms, EV fetch {} ms, rebuild check {} ms, rebuild {} ms), {} / {} routing tables rebuilt", (endTime - startTime), (statFetchEnd - statFetchStart), (icConfigCheckEnd - icConfigCheckStart), icFetchTime, evFetchTime, rebuildCheckTime, buildTime, routingTablesRebuiltCount, _lastKnownExternalViewVersionMap.size());
}

Example 5 with PropertyKey

use of org.apache.helix.PropertyKey in project pinot by linkedin.

the class DeleteOverlappingSegmentsInPinot method deleteOverlappingSegments.

public static boolean deleteOverlappingSegments(String zkUrl, String zkCluster, String tableName) {
    boolean updateSuccessful = false;
    if (!tableName.endsWith("_OFFLINE")) {
        tableName = tableName + "_OFFLINE";
    }
    ZkClient zkClient = new ZkClient(zkUrl);
    ZNRecordSerializer zkSerializer = new ZNRecordSerializer();
    zkClient.setZkSerializer(zkSerializer);
    BaseDataAccessor<ZNRecord> baseDataAccessor = new ZkBaseDataAccessor<>(zkClient);
    HelixDataAccessor helixDataAccessor = new ZKHelixDataAccessor(zkCluster, baseDataAccessor);
    Builder keyBuilder = helixDataAccessor.keyBuilder();
    PropertyKey idealStateKey = keyBuilder.idealStates(tableName);
    PropertyKey externalViewKey = keyBuilder.externalView(tableName);
    IdealState currentIdealState = helixDataAccessor.getProperty(idealStateKey);
    byte[] serializeIS = zkSerializer.serialize(currentIdealState.getRecord());
    String name = tableName + ".idealstate." + System.currentTimeMillis();
    File outputFile = new File("/tmp", name);
    try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
        IOUtils.write(serializeIS, fileOutputStream);
    } catch (IOException e) {
        LOG.error("Exception in delete overlapping segments", e);
        return updateSuccessful;
    }
    LOG.info("Saved current idealstate to {}", outputFile);
    IdealState newIdealState;
    do {
        newIdealState = computeNewIdealStateAfterDeletingOverlappingSegments(helixDataAccessor, idealStateKey);
        LOG.info("Updating IdealState");
        updateSuccessful = helixDataAccessor.getBaseDataAccessor().set(idealStateKey.getPath(), newIdealState.getRecord(), newIdealState.getRecord().getVersion(), AccessOption.PERSISTENT);
        if (updateSuccessful) {
            int numSegmentsDeleted = currentIdealState.getPartitionSet().size() - newIdealState.getPartitionSet().size();
            LOG.info("Successfully updated IdealState: Removed segments: {}", (numSegmentsDeleted));
        }
    } while (!updateSuccessful);
    try {
        while (true) {
            Thread.sleep(10000);
            ExternalView externalView = helixDataAccessor.getProperty(externalViewKey);
            IdealState idealState = helixDataAccessor.getProperty(idealStateKey);
            Set<String> evPartitionSet = externalView.getPartitionSet();
            Set<String> isPartitionSet = idealState.getPartitionSet();
            if (evPartitionSet.equals(isPartitionSet)) {
                LOG.info("Table {} has reached stable state. i.e segments in external view match idealstates", tableName);
                break;
            }
        }
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    return updateSuccessful;
}