Examples with OfflineSegmentZKMetadata com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata used on opensource projects

Example 6 with OfflineSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata in project pinot by linkedin.

the class SegmentStatusChecker method runSegmentMetrics.

/**
   * Runs a segment status pass over the currently loaded tables.
   */
public void runSegmentMetrics() {
    if (!_pinotHelixResourceManager.isLeader()) {
        LOGGER.info("Skipping Segment Status check, not leader!");
        setStatusToDefault();
        stop();
        return;
    }
    long startTime = System.nanoTime();
    LOGGER.info("Starting Segment Status check for metrics");
    // Fetch the list of tables
    List<String> allTableNames = _pinotHelixResourceManager.getAllPinotTableNames();
    String helixClusterName = _pinotHelixResourceManager.getHelixClusterName();
    HelixAdmin helixAdmin = _pinotHelixResourceManager.getHelixAdmin();
    int realTimeTableCount = 0;
    int offlineTableCount = 0;
    ZkHelixPropertyStore<ZNRecord> propertyStore = _pinotHelixResourceManager.getPropertyStore();
    for (String tableName : allTableNames) {
        if (TableNameBuilder.getTableTypeFromTableName(tableName).equals(CommonConstants.Helix.TableType.OFFLINE)) {
            offlineTableCount++;
        } else {
            realTimeTableCount++;
        }
        IdealState idealState = helixAdmin.getResourceIdealState(helixClusterName, tableName);
        if ((idealState == null) || (idealState.getPartitionSet().isEmpty())) {
            _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS, 1);
            _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS, 100);
            _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE, 100);
            continue;
        }
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.IDEALSTATE_ZNODE_SIZE, idealState.toString().length());
        ExternalView externalView = helixAdmin.getResourceExternalView(helixClusterName, tableName);
        // Keeps track of maximum number of replicas in ideal state
        int nReplicasIdealMax = 0;
        // Keeps track of minimum number of replicas in external view
        int nReplicasExternal = -1;
        // Keeps track of number of segments in error state
        int nErrors = 0;
        // Keeeps track of number segments with no online replicas
        int nOffline = 0;
        // Counts number of segments
        int nSegments = 0;
        for (String partitionName : idealState.getPartitionSet()) {
            int nReplicas = 0;
            int nIdeal = 0;
            nSegments++;
            // Skip segments not online in ideal state
            for (Map.Entry<String, String> serverAndState : idealState.getInstanceStateMap(partitionName).entrySet()) {
                if (serverAndState == null) {
                    break;
                }
                if (serverAndState.getValue().equals(ONLINE)) {
                    nIdeal++;
                    break;
                }
            }
            if (nIdeal == 0) {
                // No online segments in ideal state
                continue;
            }
            nReplicasIdealMax = (idealState.getInstanceStateMap(partitionName).size() > nReplicasIdealMax) ? idealState.getInstanceStateMap(partitionName).size() : nReplicasIdealMax;
            if ((externalView == null) || (externalView.getStateMap(partitionName) == null)) {
                // No replicas for this segment
                TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName);
                if ((tableType != null) && (tableType.equals(TableType.OFFLINE))) {
                    OfflineSegmentZKMetadata segmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(propertyStore, tableName, partitionName);
                    if (segmentZKMetadata != null && segmentZKMetadata.getPushTime() > System.currentTimeMillis() - _waitForPushTimeSeconds * 1000) {
                        // push not yet finished, skip
                        continue;
                    }
                }
                nOffline++;
                if (nOffline < MaxOfflineSegmentsToLog) {
                    LOGGER.warn("Segment {} of table {} has no replicas", partitionName, tableName);
                }
                nReplicasExternal = 0;
                continue;
            }
            for (Map.Entry<String, String> serverAndState : externalView.getStateMap(partitionName).entrySet()) {
                // Count number of online replicas
                if (serverAndState.getValue().equals(ONLINE)) {
                    nReplicas++;
                }
                if (serverAndState.getValue().equals(ERROR)) {
                    nErrors++;
                }
            }
            if (nReplicas == 0) {
                if (nOffline < MaxOfflineSegmentsToLog) {
                    LOGGER.warn("Segment {} of table {} has no online replicas", partitionName, tableName);
                }
                nOffline++;
            }
            nReplicasExternal = ((nReplicasExternal > nReplicas) || (nReplicasExternal == -1)) ? nReplicas : nReplicasExternal;
        }
        if (nReplicasExternal == -1) {
            nReplicasExternal = (nReplicasIdealMax == 0) ? 1 : 0;
        }
        // Synchronization provided by Controller Gauge to make sure that only one thread updates the gauge
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS, nReplicasExternal);
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS, (nReplicasIdealMax > 0) ? (nReplicasExternal * 100 / nReplicasIdealMax) : 100);
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE, nErrors);
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE, (nSegments > 0) ? (100 - (nOffline * 100 / nSegments)) : 100);
        if (nOffline > 0) {
            LOGGER.warn("Table {} has {} segments with no online replicas", tableName, nOffline);
        }
        if (nReplicasExternal < nReplicasIdealMax) {
            LOGGER.warn("Table {} has {} replicas, below replication threshold :{}", tableName, nReplicasExternal, nReplicasIdealMax);
        }
    }
    _metricsRegistry.setValueOfGlobalGauge(ControllerGauge.REALTIME_TABLE_COUNT, realTimeTableCount);
    _metricsRegistry.setValueOfGlobalGauge(ControllerGauge.OFFLINE_TABLE_COUNT, offlineTableCount);
    long totalNanos = System.nanoTime() - startTime;
    LOGGER.info("Segment status metrics completed in {}ms", TimeUnit.MILLISECONDS.convert(totalNanos, TimeUnit.NANOSECONDS));
}

Example 7 with OfflineSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata in project pinot by linkedin.

the class PinotHelixResourceManager method ifRefreshAnExistedSegment.

private boolean ifRefreshAnExistedSegment(SegmentMetadata segmentMetadata, String segmentName, String tableName) {
    OfflineSegmentZKMetadata offlineSegmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(_propertyStore, segmentMetadata.getTableName(), segmentMetadata.getName());
    if (offlineSegmentZKMetadata == null) {
        LOGGER.info("Rejecting because Zk metadata is null for segment {} of table {}", segmentName, tableName);
        return false;
    }
    final SegmentMetadata existedSegmentMetadata = new SegmentMetadataImpl(offlineSegmentZKMetadata);
    if (segmentMetadata.getIndexCreationTime() <= existedSegmentMetadata.getIndexCreationTime()) {
        LOGGER.info("Rejecting because of older or same creation time {} (we have {}) for segment {} of table {}", segmentMetadata.getIndexCreationTime(), existedSegmentMetadata.getIndexCreationTime(), segmentName, tableName);
        return false;
    }
    if (segmentMetadata.getCrc().equals(existedSegmentMetadata.getCrc())) {
        LOGGER.info("Rejecting because of matching CRC exists (incoming={}, existing={}) for {} of table {}", segmentMetadata.getCrc(), existedSegmentMetadata.getCrc(), segmentName, tableName);
        return false;
    }
    return true;
}

Example 8 with OfflineSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata in project pinot by linkedin.

the class PinotHelixResourceManager method addSegment.

public PinotResourceManagerResponse addSegment(final SegmentMetadata segmentMetadata, String downloadUrl) {
    final PinotResourceManagerResponse res = new PinotResourceManagerResponse();
    String segmentName = "Unknown";
    String tableName = "Unknown";
    try {
        if (!matchTableName(segmentMetadata)) {
            throw new RuntimeException("Reject segment: table name is not registered." + " table name: " + segmentMetadata.getTableName() + "\n");
        }
        segmentName = segmentMetadata.getName();
        tableName = segmentMetadata.getTableName();
        if (ifSegmentExisted(segmentMetadata)) {
            if (ifRefreshAnExistedSegment(segmentMetadata, segmentName, tableName)) {
                OfflineSegmentZKMetadata offlineSegmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(_propertyStore, segmentMetadata.getTableName(), segmentMetadata.getName());
                offlineSegmentZKMetadata = ZKMetadataUtils.updateSegmentMetadata(offlineSegmentZKMetadata, segmentMetadata);
                offlineSegmentZKMetadata.setDownloadUrl(downloadUrl);
                offlineSegmentZKMetadata.setRefreshTime(System.currentTimeMillis());
                ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore, offlineSegmentZKMetadata);
                LOGGER.info("Refresh segment {} of table {} to propertystore ", segmentName, tableName);
                boolean success = true;
                if (shouldSendMessage(offlineSegmentZKMetadata)) {
                    // Send a message to the servers to update the segment.
                    // We return success even if we are not able to send messages (which can happen if no servers are alive).
                    // For segment validation errors we would have returned earlier.
                    sendSegmentRefreshMessage(offlineSegmentZKMetadata);
                } else {
                    // Go through the ONLINE->OFFLINE->ONLINE state transition to update the segment
                    success = updateExistedSegment(offlineSegmentZKMetadata);
                }
                if (success) {
                    res.status = ResponseStatus.success;
                } else {
                    LOGGER.error("Failed to refresh segment {} of table {}, marking crc and creation time as invalid", segmentName, tableName);
                    offlineSegmentZKMetadata.setCrc(-1L);
                    offlineSegmentZKMetadata.setCreationTime(-1L);
                    ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore, offlineSegmentZKMetadata);
                }
            } else {
                String msg = "Not refreshing identical segment " + segmentName + "of table " + tableName + " with creation time " + segmentMetadata.getIndexCreationTime() + " and crc " + segmentMetadata.getCrc();
                LOGGER.info(msg);
                res.status = ResponseStatus.success;
                res.message = msg;
            }
        } else {
            OfflineSegmentZKMetadata offlineSegmentZKMetadata = new OfflineSegmentZKMetadata();
            offlineSegmentZKMetadata = ZKMetadataUtils.updateSegmentMetadata(offlineSegmentZKMetadata, segmentMetadata);
            offlineSegmentZKMetadata.setDownloadUrl(downloadUrl);
            offlineSegmentZKMetadata.setPushTime(System.currentTimeMillis());
            ZKMetadataProvider.setOfflineSegmentZKMetadata(_propertyStore, offlineSegmentZKMetadata);
            LOGGER.info("Added segment {} of table {} to propertystore", segmentName, tableName);
            addNewOfflineSegment(segmentMetadata);
            res.status = ResponseStatus.success;
        }
    } catch (final Exception e) {
        LOGGER.error("Caught exception while adding segment {} of table {}", segmentName, tableName, e);
        res.status = ResponseStatus.failure;
        res.message = e.getMessage();
    }
    return res;
}

Example 9 with OfflineSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata in project pinot by linkedin.

the class PinotSegmentRestletResource method getSegmentMetaData.

/**
   * Get meta-data for segment of table. Table name is the suffixed (offline/realtime)
   * name.
   * @param tableName: Suffixed (realtime/offline) table Name
   * @param segmentName: Segment for which to get the meta-data.
   * @return
   * @throws JSONException
   */
private StringRepresentation getSegmentMetaData(String tableName, String segmentName, TableType tableType) throws JSONException {
    if (!ZKMetadataProvider.isSegmentExisted(_pinotHelixResourceManager.getPropertyStore(), tableName, segmentName)) {
        String error = new String("Error: segment " + segmentName + " not found.");
        LOGGER.error(error);
        setStatus(Status.CLIENT_ERROR_BAD_REQUEST);
        return new StringRepresentation(error);
    }
    JSONArray ret = new JSONArray();
    JSONObject jsonObj = new JSONObject();
    jsonObj.put(TABLE_NAME, tableName);
    ZkHelixPropertyStore<ZNRecord> propertyStore = _pinotHelixResourceManager.getPropertyStore();
    if (tableType == tableType.OFFLINE) {
        OfflineSegmentZKMetadata offlineSegmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(propertyStore, tableName, segmentName);
        jsonObj.put(STATE, offlineSegmentZKMetadata.toMap());
    }
    if (tableType == TableType.REALTIME) {
        RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = ZKMetadataProvider.getRealtimeSegmentZKMetadata(propertyStore, tableName, segmentName);
        jsonObj.put(STATE, realtimeSegmentZKMetadata.toMap());
    }
    ret.put(jsonObj);
    return new StringRepresentation(ret.toString());
}

Example 10 with OfflineSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.OfflineSegmentZKMetadata in project pinot by linkedin.

the class TableRetentionValidator method run.

public void run() throws Exception {
    // Get all resources in cluster
    List<String> resourcesInCluster = _helixAdmin.getResourcesInCluster(_clusterName);
    for (String tableName : resourcesInCluster) {
        // Skip non-table resources
        if (!tableName.endsWith("_OFFLINE") && !tableName.endsWith("_REALTIME")) {
            continue;
        }
        // Skip tables that do not match the defined name pattern
        if (_tableNamePattern != null && !tableName.matches(_tableNamePattern)) {
            continue;
        }
        // Get the retention config
        SegmentsValidationAndRetentionConfig retentionConfig = getTableConfig(tableName).getValidationConfig();
        if (retentionConfig == null) {
            LOGGER.error("Table: {}, \"segmentsConfig\" field is missing in table config", tableName);
            continue;
        }
        String segmentPushType = retentionConfig.getSegmentPushType();
        if (segmentPushType == null) {
            LOGGER.error("Table: {}, null push type", tableName);
            continue;
        } else if (segmentPushType.equalsIgnoreCase("REFRESH")) {
            continue;
        } else if (!segmentPushType.equalsIgnoreCase("APPEND")) {
            LOGGER.error("Table: {}, invalid push type: {}", tableName, segmentPushType);
            continue;
        }
        // APPEND use case
        // Get time unit
        String timeUnitString = retentionConfig.getRetentionTimeUnit();
        TimeUnit timeUnit;
        try {
            timeUnit = TimeUnit.valueOf(timeUnitString.toUpperCase());
        } catch (Exception e) {
            LOGGER.error("Table: {}, invalid time unit: {}", tableName, timeUnitString);
            continue;
        }
        // Get time duration in days
        String timeValueString = retentionConfig.getRetentionTimeValue();
        long durationInDays;
        try {
            durationInDays = timeUnit.toDays(Long.valueOf(timeValueString));
        } catch (Exception e) {
            LOGGER.error("Table: {}, invalid time value: {}", tableName, timeValueString);
            continue;
        }
        if (durationInDays <= 0) {
            LOGGER.error("Table: {}, invalid retention duration in days: {}", tableName, durationInDays);
            continue;
        }
        if (durationInDays > _durationInDaysThreshold) {
            LOGGER.warn("Table: {}, retention duration in days is too large: {}", tableName, durationInDays);
        }
        // Skip segments metadata check for realtime tables
        if (tableName.endsWith("REALTIME")) {
            continue;
        }
        // Check segments metadata (only for offline tables)
        List<String> segmentNames = getSegmentNames(tableName);
        if (segmentNames == null || segmentNames.isEmpty()) {
            LOGGER.warn("Table: {}, no segment metadata in property store", tableName);
            continue;
        }
        List<String> errorMessages = new ArrayList<>();
        for (String segmentName : segmentNames) {
            OfflineSegmentZKMetadata offlineSegmentMetadata = getOfflineSegmentMetadata(tableName, segmentName);
            TimeUnit segmentTimeUnit = offlineSegmentMetadata.getTimeUnit();
            if (segmentTimeUnit == null) {
                errorMessages.add("Segment: " + segmentName + " has null time unit");
                continue;
            }
            long startTimeInMillis = segmentTimeUnit.toMillis(offlineSegmentMetadata.getStartTime());
            if (!TimeUtils.timeValueInValidRange(startTimeInMillis)) {
                errorMessages.add("Segment: " + segmentName + " has invalid start time in millis: " + startTimeInMillis);
            }
            long endTimeInMillis = segmentTimeUnit.toMillis(offlineSegmentMetadata.getEndTime());
            if (!TimeUtils.timeValueInValidRange(endTimeInMillis)) {
                errorMessages.add("Segment: " + segmentName + " has invalid end time in millis: " + endTimeInMillis);
            }
        }
        if (!errorMessages.isEmpty()) {
            LOGGER.error("Table: {}, invalid segments: {}", tableName, errorMessages);
        }
    }
}