Examples with PropertyKey org.apache.helix.PropertyKey used on opensource projects

Example 36 with PropertyKey

use of org.apache.helix.PropertyKey in project helix by apache.

the class ParticipantManager method carryOverPreviousCurrentState.

/**
 * carry over current-states from last sessions
 * set to initial state for current session only when state doesn't exist in current session
 */
public static synchronized void carryOverPreviousCurrentState(HelixDataAccessor dataAccessor, String instanceName, String sessionId, StateMachineEngine stateMachineEngine, boolean setToInitState) {
    PropertyKey.Builder keyBuilder = dataAccessor.keyBuilder();
    List<String> sessions = dataAccessor.getChildNames(keyBuilder.sessions(instanceName));
    for (String session : sessions) {
        if (session.equals(sessionId)) {
            continue;
        }
        // Ignore if any current states in the previous folder cannot be read.
        List<CurrentState> lastCurStates = dataAccessor.getChildValues(keyBuilder.currentStates(instanceName, session), false);
        for (CurrentState lastCurState : lastCurStates) {
            LOG.info("Carrying over old session: " + session + ", resource: " + lastCurState.getId() + " to current session: " + sessionId + ", setToInitState: " + setToInitState);
            String stateModelDefRef = lastCurState.getStateModelDefRef();
            if (stateModelDefRef == null) {
                LOG.error("skip carry-over because previous current state doesn't have a state model definition. previous current-state: " + lastCurState);
                continue;
            }
            // Note: this check is not necessary due to TaskCurrentStates, but keep it for backwards compatibility
            if (stateModelDefRef.equals(TaskConstants.STATE_MODEL_NAME)) {
                continue;
            }
            StateModelDefinition stateModelDef = dataAccessor.getProperty(keyBuilder.stateModelDef(stateModelDefRef));
            String initState = stateModelDef.getInitialState();
            Map<String, String> partitionExpectedStateMap = new HashMap<>();
            if (setToInitState) {
                lastCurState.getPartitionStateMap().keySet().forEach(partition -> partitionExpectedStateMap.put(partition, initState));
            } else {
                String factoryName = lastCurState.getStateModelFactoryName();
                StateModelFactory<? extends StateModel> stateModelFactory = stateMachineEngine.getStateModelFactory(stateModelDefRef, factoryName);
                lastCurState.getPartitionStateMap().keySet().forEach(partition -> {
                    StateModel stateModel = stateModelFactory.getStateModel(lastCurState.getResourceName(), partition);
                    if (stateModel != null) {
                        partitionExpectedStateMap.put(partition, stateModel.getCurrentState());
                    }
                });
            }
            BaseDataAccessor<ZNRecord> baseAccessor = dataAccessor.getBaseDataAccessor();
            String curStatePath = keyBuilder.currentState(instanceName, sessionId, lastCurState.getResourceName()).getPath();
            if (lastCurState.getBucketSize() > 0) {
                // update parent node
                ZNRecord metaRecord = new ZNRecord(lastCurState.getId());
                metaRecord.setSimpleFields(lastCurState.getRecord().getSimpleFields());
                DataUpdater<ZNRecord> metaRecordUpdater = new CurStateCarryOverUpdater(sessionId, partitionExpectedStateMap, new CurrentState(metaRecord));
                boolean success = baseAccessor.update(curStatePath, metaRecordUpdater, AccessOption.PERSISTENT);
                if (success) {
                    // update current state buckets
                    ZNRecordBucketizer bucketizer = new ZNRecordBucketizer(lastCurState.getBucketSize());
                    Map<String, ZNRecord> map = bucketizer.bucketize(lastCurState.getRecord());
                    List<String> paths = new ArrayList<String>();
                    List<DataUpdater<ZNRecord>> updaters = new ArrayList<DataUpdater<ZNRecord>>();
                    for (String bucketName : map.keySet()) {
                        paths.add(curStatePath + "/" + bucketName);
                        updaters.add(new CurStateCarryOverUpdater(sessionId, partitionExpectedStateMap, new CurrentState(map.get(bucketName))));
                    }
                    baseAccessor.updateChildren(paths, updaters, AccessOption.PERSISTENT);
                }
            } else {
                dataAccessor.getBaseDataAccessor().update(curStatePath, new CurStateCarryOverUpdater(sessionId, partitionExpectedStateMap, lastCurState), AccessOption.PERSISTENT);
            }
        }
    }
    /**
     * remove previous current state parent nodes
     */
    for (String session : sessions) {
        if (session.equals(sessionId)) {
            continue;
        }
        PropertyKey currentStatesProperty = keyBuilder.currentStates(instanceName, session);
        String path = currentStatesProperty.getPath();
        LOG.info("Removing current states from previous sessions. path: {}", path);
        if (!dataAccessor.removeProperty(currentStatesProperty)) {
            throw new ZkClientException("Failed to delete " + path);
        }
    }
}

Example 37 with PropertyKey

use of org.apache.helix.PropertyKey in project helix by apache.

the class ParticipantManager method persistHistory.

private void persistHistory(ParticipantHistory history, boolean skipOnEmptyPath) {
    PropertyKey propertyKey = _keyBuilder.participantHistory(_instanceName);
    boolean result = skipOnEmptyPath ? _dataAccessor.updateProperty(propertyKey, currentData -> (currentData == null) ? null : history.getRecord(), history) : _dataAccessor.setProperty(propertyKey, history);
    if (!result) {
        LOG.error("Failed to persist participant history to zk!");
    }
}

Example 38 with PropertyKey

use of org.apache.helix.PropertyKey in project helix by apache.

the class TestCurrentStateSnapshot method testGetNewCurrentStateEndTimes.

// This test makes sure that currentStateEndTimes calculation would record correct partition replica.
// Specifically, if a replicate has not endTime field set, we should not put an entry into currentStateEndTime
// calculation. Otherwise, we see huge statePropagation latency of 1.4Tms.
@Test(description = "test getNewCurrentStateEndTimes")
public void testGetNewCurrentStateEndTimes() {
    String instance1 = "instance1";
    String session1 = "session1";
    String resource1 = "resource1";
    String partition1 = "partition1";
    String partition2 = "partition2";
    PropertyKey key = new PropertyKey.Builder("cluster").currentState(instance1, session1, resource1);
    CurrentState nxtState = new CurrentState(resource1);
    // partition 1, expect to record in endTimesMap
    nxtState.setState(partition1, "SLAVE");
    nxtState.setEndTime(partition1, 200);
    // partition 2, expect to not record in endTimeMap. This is fixing current 1.4T observed timestamp issue
    nxtState.setState(partition2, "MASTER");
    Map<PropertyKey, CurrentState> currentStateMap = new HashMap<>();
    Map<PropertyKey, CurrentState> nextStateMap = new HashMap<>();
    nextStateMap.put(key, nxtState);
    Set<PropertyKey> updateKeys = new HashSet<>();
    updateKeys.add(key);
    CurrentStateSnapshot snapshot = new CurrentStateSnapshot(nextStateMap, currentStateMap, updateKeys);
    Map<PropertyKey, Map<String, Long>> endTimesMap = snapshot.getNewCurrentStateEndTimes();
    Assert.assertEquals(endTimesMap.size(), 1);
    Assert.assertTrue(endTimesMap.get(key).get(partition1) == 200);
}

Example 39 with PropertyKey

use of org.apache.helix.PropertyKey in project helix by apache.

the class TestCurrentStateSnapshot method testRefreshCurrentStateCache.

// This test makes sure that all the changed current State is reflected in newCurrentStateEndTimes calculation.
// Previously, we have bugs that all newly created current state would be reflected in newCurrentStateEndTimes
// calculation.
@Test(description = "testRefreshCurrentStateCache")
public void testRefreshCurrentStateCache() {
    String instanceName = "instance1";
    long instanceSession = 12345;
    String resourceName = "resource";
    String partitionName = "resource_partition1";
    MockAccessor accessor = new MockAccessor();
    PropertyKey.Builder keyBuilder = accessor.keyBuilder();
    // construct liveInstance
    ZNRecord record = new ZNRecord(instanceName);
    record.setEphemeralOwner(instanceSession);
    LiveInstance instance = new LiveInstance(record);
    boolean retVal = accessor.setProperty(keyBuilder.liveInstance(instanceName), instance);
    Assert.assertTrue(retVal);
    // construct currentstate
    CurrentState originState = new CurrentState(resourceName);
    originState.setEndTime(partitionName, 100);
    CurrentState currentState = new CurrentState(resourceName);
    currentState.setEndTime(partitionName, 300);
    retVal = accessor.setProperty(keyBuilder.currentState(instanceName, instance.getEphemeralOwner(), resourceName), originState);
    Assert.assertTrue(retVal);
    CurrentStateCache cache = new CurrentStateCache("cluster");
    Map<String, LiveInstance> liveInstanceMap = new HashMap<>();
    liveInstanceMap.put(instanceName, instance);
    retVal = cache.refresh(accessor, liveInstanceMap);
    Assert.assertTrue(retVal);
    retVal = accessor.setProperty(keyBuilder.currentState(instanceName, instance.getEphemeralOwner(), resourceName), currentState);
    Assert.assertTrue(retVal);
    retVal = cache.refresh(accessor, liveInstanceMap);
    Assert.assertTrue(retVal);
    CurrentStateSnapshot snapshot = cache.getSnapshot();
    Map<PropertyKey, Map<String, Long>> endTimesMap = snapshot.getNewCurrentStateEndTimes();
    Assert.assertEquals(endTimesMap.size(), 1);
    // note, without this fix, the endTimesMap would be size zero.
    Assert.assertTrue(endTimesMap.get(keyBuilder.currentState(instanceName, instance.getEphemeralOwner(), resourceName)).get(partitionName) == 300);
}

Example 40 with PropertyKey

use of org.apache.helix.PropertyKey in project helix by apache.

the class StatusUpdateUtil method publishStatusUpdateRecord.

/**
 * Write a status update record to zookeeper to the zookeeper store.
 * @param record
 *          the status update record
 * @param message
 *          the message to be logged
 * @param level
 *          the error level of the message update
 * @param accessor
 *          the zookeeper data accessor that writes the status update to zookeeper
 * @param isController
 *          if the update is for a controller instance or not
 */
void publishStatusUpdateRecord(ZNRecord record, Message message, Level level, HelixDataAccessor accessor, boolean isController) {
    String instanceName = message.getTgtName();
    String statusUpdateSubPath = getStatusUpdateSubPath(message);
    String statusUpdateKey = getStatusUpdateKey(message);
    String sessionId = message.getExecutionSessionId();
    if (sessionId == null) {
        sessionId = message.getTgtSessionId();
    }
    if (sessionId == null) {
        sessionId = "*";
    }
    Builder keyBuilder = accessor.keyBuilder();
    if (!_recordedMessages.containsKey(message.getMsgId())) {
        ZNRecord statusUpdateRecord = createMessageLogRecord(message);
        PropertyKey propertyKey;
        if (isController) {
            propertyKey = keyBuilder.controllerTaskStatus(statusUpdateSubPath, statusUpdateKey);
        } else {
            propertyKey = keyBuilder.stateTransitionStatus(instanceName, sessionId, statusUpdateSubPath, statusUpdateKey);
        }
        accessor.updateProperty(propertyKey, new StatusUpdate(statusUpdateRecord));
        if (_logger.isTraceEnabled()) {
            _logger.trace("StatusUpdate path:" + propertyKey.getPath() + ", updates:" + statusUpdateRecord);
        }
        _recordedMessages.put(message.getMsgId(), message.getMsgId());
    }
    PropertyKey propertyKey;
    if (isController) {
        propertyKey = keyBuilder.controllerTaskStatus(statusUpdateSubPath, statusUpdateKey);
    } else {
        propertyKey = keyBuilder.stateTransitionStatus(instanceName, sessionId, statusUpdateSubPath, statusUpdateKey);
    }
    accessor.updateProperty(propertyKey, new StatusUpdate(record));
    if (_logger.isTraceEnabled()) {
        _logger.trace("StatusUpdate path:" + propertyKey.getPath() + ", updates:" + record);
    }
    // If the error level is ERROR, also write the record to "ERROR" ZNode
    if (Level.HELIX_ERROR == level) {
        publishErrorRecord(record, instanceName, statusUpdateSubPath, statusUpdateKey, sessionId, accessor, isController);
    }
}