Examples with RealtimeSegmentZKMetadata com.linkedin.pinot.common.metadata.segment.RealtimeSegmentZKMetadata used on opensource projects

Example 1 with RealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.RealtimeSegmentZKMetadata in project pinot by linkedin.

the class ZKMetadataProvider method getRealtimeSegmentZKMetadata.

@Nullable
public static RealtimeSegmentZKMetadata getRealtimeSegmentZKMetadata(ZkHelixPropertyStore<ZNRecord> propertyStore, String tableName, String segmentName) {
    String realtimeTableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(tableName);
    ZNRecord znRecord = propertyStore.get(constructPropertyStorePathForSegment(realtimeTableName, segmentName), null, AccessOption.PERSISTENT);
    // It is possible that the segment metadata has just been deleted due to retention.
    if (znRecord == null) {
        return null;
    }
    if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) {
        return new RealtimeSegmentZKMetadata(znRecord);
    } else {
        return new LLCRealtimeSegmentZKMetadata(znRecord);
    }
}

Example 2 with RealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.RealtimeSegmentZKMetadata in project pinot by linkedin.

the class ZKMetadataProvider method getRealtimeSegmentZKMetadataListForTable.

public static List<RealtimeSegmentZKMetadata> getRealtimeSegmentZKMetadataListForTable(ZkHelixPropertyStore<ZNRecord> propertyStore, String resourceName) {
    List<RealtimeSegmentZKMetadata> resultList = new ArrayList<RealtimeSegmentZKMetadata>();
    if (propertyStore == null) {
        return resultList;
    }
    String realtimeTableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(resourceName);
    if (propertyStore.exists(constructPropertyStorePathForResource(realtimeTableName), AccessOption.PERSISTENT)) {
        List<ZNRecord> znRecordList = propertyStore.getChildren(constructPropertyStorePathForResource(realtimeTableName), null, AccessOption.PERSISTENT);
        if (znRecordList != null) {
            for (ZNRecord record : znRecordList) {
                resultList.add(new RealtimeSegmentZKMetadata(record));
            }
        }
    }
    return resultList;
}

Example 3 with RealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.RealtimeSegmentZKMetadata in project pinot by linkedin.

the class PinotHelixResourceManager method updateExistedSegment.

private boolean updateExistedSegment(SegmentZKMetadata segmentZKMetadata) {
    final String tableName;
    if (segmentZKMetadata instanceof RealtimeSegmentZKMetadata) {
        tableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(segmentZKMetadata.getTableName());
    } else {
        tableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(segmentZKMetadata.getTableName());
    }
    final String segmentName = segmentZKMetadata.getSegmentName();
    HelixDataAccessor helixDataAccessor = _helixZkManager.getHelixDataAccessor();
    PropertyKey idealStatePropertyKey = _keyBuilder.idealStates(tableName);
    // Set all partitions to offline to unload them from the servers
    boolean updateSuccessful;
    do {
        final IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
        final Set<String> instanceSet = idealState.getInstanceSet(segmentName);
        if (instanceSet == null || instanceSet.size() == 0) {
            // We are trying to refresh a segment, but there are no instances currently assigned for fielding this segment.
            // When those instances do come up, the segment will be uploaded correctly, so return success but log a warning.
            LOGGER.warn("No instances as yet for segment {}, table {}", segmentName, tableName);
            return true;
        }
        for (final String instance : instanceSet) {
            idealState.setPartitionState(segmentName, instance, "OFFLINE");
        }
        updateSuccessful = helixDataAccessor.updateProperty(idealStatePropertyKey, idealState);
    } while (!updateSuccessful);
    // Check that the ideal state has been written to ZK
    IdealState updatedIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
    Map<String, String> instanceStateMap = updatedIdealState.getInstanceStateMap(segmentName);
    for (String state : instanceStateMap.values()) {
        if (!"OFFLINE".equals(state)) {
            LOGGER.error("Failed to write OFFLINE ideal state!");
            return false;
        }
    }
    // Wait until the partitions are offline in the external view
    LOGGER.info("Wait until segment - " + segmentName + " to be OFFLINE in ExternalView");
    if (!ifExternalViewChangeReflectedForState(tableName, segmentName, "OFFLINE", _externalViewOnlineToOfflineTimeoutMillis, false)) {
        LOGGER.error("External view for segment {} did not reflect the ideal state of OFFLINE within the {} ms time limit", segmentName, _externalViewOnlineToOfflineTimeoutMillis);
        return false;
    }
    // Set all partitions to online so that they load the new segment data
    do {
        final IdealState idealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
        final Set<String> instanceSet = idealState.getInstanceSet(segmentName);
        LOGGER.info("Found {} instances for segment '{}', in ideal state", instanceSet.size(), segmentName);
        for (final String instance : instanceSet) {
            idealState.setPartitionState(segmentName, instance, "ONLINE");
            LOGGER.info("Setting Ideal State for segment '{}' to ONLINE for instance '{}'", segmentName, instance);
        }
        updateSuccessful = helixDataAccessor.updateProperty(idealStatePropertyKey, idealState);
    } while (!updateSuccessful);
    // Check that the ideal state has been written to ZK
    updatedIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, tableName);
    instanceStateMap = updatedIdealState.getInstanceStateMap(segmentName);
    LOGGER.info("Found {} instances for segment '{}', after updating ideal state", instanceStateMap.size(), segmentName);
    for (String state : instanceStateMap.values()) {
        if (!"ONLINE".equals(state)) {
            LOGGER.error("Failed to write ONLINE ideal state!");
            return false;
        }
    }
    LOGGER.info("Refresh is done for segment - " + segmentName);
    return true;
}

Example 4 with RealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.RealtimeSegmentZKMetadata in project pinot by linkedin.

the class PinotSegmentRestletResource method getSegmentMetaData.

/**
   * Get meta-data for segment of table. Table name is the suffixed (offline/realtime)
   * name.
   * @param tableName: Suffixed (realtime/offline) table Name
   * @param segmentName: Segment for which to get the meta-data.
   * @return
   * @throws JSONException
   */
private StringRepresentation getSegmentMetaData(String tableName, String segmentName, TableType tableType) throws JSONException {
    if (!ZKMetadataProvider.isSegmentExisted(_pinotHelixResourceManager.getPropertyStore(), tableName, segmentName)) {
        String error = new String("Error: segment " + segmentName + " not found.");
        LOGGER.error(error);
        setStatus(Status.CLIENT_ERROR_BAD_REQUEST);
        return new StringRepresentation(error);
    }
    JSONArray ret = new JSONArray();
    JSONObject jsonObj = new JSONObject();
    jsonObj.put(TABLE_NAME, tableName);
    ZkHelixPropertyStore<ZNRecord> propertyStore = _pinotHelixResourceManager.getPropertyStore();
    if (tableType == tableType.OFFLINE) {
        OfflineSegmentZKMetadata offlineSegmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(propertyStore, tableName, segmentName);
        jsonObj.put(STATE, offlineSegmentZKMetadata.toMap());
    }
    if (tableType == TableType.REALTIME) {
        RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = ZKMetadataProvider.getRealtimeSegmentZKMetadata(propertyStore, tableName, segmentName);
        jsonObj.put(STATE, realtimeSegmentZKMetadata.toMap());
    }
    ret.put(jsonObj);
    return new StringRepresentation(ret.toString());
}

Example 5 with RealtimeSegmentZKMetadata

use of com.linkedin.pinot.common.metadata.segment.RealtimeSegmentZKMetadata in project pinot by linkedin.

the class ValidationManager method runValidation.

/**
   * Runs a validation pass over the currently loaded tables.
   */
public void runValidation() {
    if (!_pinotHelixResourceManager.isLeader()) {
        LOGGER.info("Skipping validation, not leader!");
        return;
    }
    LOGGER.info("Starting validation");
    // Fetch the list of tables
    List<String> allTableNames = _pinotHelixResourceManager.getAllPinotTableNames();
    ZkHelixPropertyStore<ZNRecord> propertyStore = _pinotHelixResourceManager.getPropertyStore();
    for (String tableName : allTableNames) {
        List<SegmentMetadata> segmentMetadataList = new ArrayList<SegmentMetadata>();
        TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName);
        AbstractTableConfig tableConfig = null;
        _pinotHelixResourceManager.rebuildBrokerResourceFromHelixTags(tableName);
        // For each table, fetch the metadata for all its segments
        if (tableType.equals(TableType.OFFLINE)) {
            validateOfflineSegmentPush(propertyStore, tableName, segmentMetadataList);
        } else if (tableType.equals(TableType.REALTIME)) {
            LOGGER.info("Starting to validate table {}", tableName);
            List<RealtimeSegmentZKMetadata> realtimeSegmentZKMetadatas = ZKMetadataProvider.getRealtimeSegmentZKMetadataListForTable(propertyStore, tableName);
            // false if this table has ONLY LLC segments (i.e. fully migrated)
            boolean countHLCSegments = true;
            KafkaStreamMetadata streamMetadata = null;
            try {
                tableConfig = _pinotHelixResourceManager.getRealtimeTableConfig(tableName);
                streamMetadata = new KafkaStreamMetadata(tableConfig.getIndexingConfig().getStreamConfigs());
                if (streamMetadata.hasSimpleKafkaConsumerType() && !streamMetadata.hasHighLevelKafkaConsumerType()) {
                    countHLCSegments = false;
                }
                for (RealtimeSegmentZKMetadata realtimeSegmentZKMetadata : realtimeSegmentZKMetadatas) {
                    SegmentMetadata segmentMetadata = new SegmentMetadataImpl(realtimeSegmentZKMetadata);
                    segmentMetadataList.add(segmentMetadata);
                }
                // Update the gauge to contain the total document count in the segments
                _validationMetrics.updateTotalDocumentsGauge(tableName, computeRealtimeTotalDocumentInSegments(segmentMetadataList, countHLCSegments));
                if (streamMetadata.hasSimpleKafkaConsumerType()) {
                    validateLLCSegments(tableName, tableConfig);
                }
            } catch (Exception e) {
                if (tableConfig == null) {
                    LOGGER.warn("Cannot get realtime tableconfig for {}", tableName);
                } else if (streamMetadata == null) {
                    LOGGER.warn("Cannot get streamconfig for {}", tableName);
                } else {
                    LOGGER.error("Exception while validating table {}", tableName, e);
                }
            }
        } else {
            LOGGER.warn("Ignoring table type {} for table {}", tableType, tableName);
        }
    }
    LOGGER.info("Validation completed");
}