Examples with TableType com.linkedin.pinot.common.utils.CommonConstants.Helix.TableType used on opensource projects

Example 1 with TableType

use of com.linkedin.pinot.common.utils.CommonConstants.Helix.TableType in project pinot by linkedin.

the class SegmentStatusChecker method runSegmentMetrics.

/**
   * Runs a segment status pass over the currently loaded tables.
   */
public void runSegmentMetrics() {
    if (!_pinotHelixResourceManager.isLeader()) {
        LOGGER.info("Skipping Segment Status check, not leader!");
        setStatusToDefault();
        stop();
        return;
    }
    long startTime = System.nanoTime();
    LOGGER.info("Starting Segment Status check for metrics");
    // Fetch the list of tables
    List<String> allTableNames = _pinotHelixResourceManager.getAllPinotTableNames();
    String helixClusterName = _pinotHelixResourceManager.getHelixClusterName();
    HelixAdmin helixAdmin = _pinotHelixResourceManager.getHelixAdmin();
    int realTimeTableCount = 0;
    int offlineTableCount = 0;
    ZkHelixPropertyStore<ZNRecord> propertyStore = _pinotHelixResourceManager.getPropertyStore();
    for (String tableName : allTableNames) {
        if (TableNameBuilder.getTableTypeFromTableName(tableName).equals(CommonConstants.Helix.TableType.OFFLINE)) {
            offlineTableCount++;
        } else {
            realTimeTableCount++;
        }
        IdealState idealState = helixAdmin.getResourceIdealState(helixClusterName, tableName);
        if ((idealState == null) || (idealState.getPartitionSet().isEmpty())) {
            _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS, 1);
            _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS, 100);
            _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE, 100);
            continue;
        }
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.IDEALSTATE_ZNODE_SIZE, idealState.toString().length());
        ExternalView externalView = helixAdmin.getResourceExternalView(helixClusterName, tableName);
        // Keeps track of maximum number of replicas in ideal state
        int nReplicasIdealMax = 0;
        // Keeps track of minimum number of replicas in external view
        int nReplicasExternal = -1;
        // Keeps track of number of segments in error state
        int nErrors = 0;
        // Keeeps track of number segments with no online replicas
        int nOffline = 0;
        // Counts number of segments
        int nSegments = 0;
        for (String partitionName : idealState.getPartitionSet()) {
            int nReplicas = 0;
            int nIdeal = 0;
            nSegments++;
            // Skip segments not online in ideal state
            for (Map.Entry<String, String> serverAndState : idealState.getInstanceStateMap(partitionName).entrySet()) {
                if (serverAndState == null) {
                    break;
                }
                if (serverAndState.getValue().equals(ONLINE)) {
                    nIdeal++;
                    break;
                }
            }
            if (nIdeal == 0) {
                // No online segments in ideal state
                continue;
            }
            nReplicasIdealMax = (idealState.getInstanceStateMap(partitionName).size() > nReplicasIdealMax) ? idealState.getInstanceStateMap(partitionName).size() : nReplicasIdealMax;
            if ((externalView == null) || (externalView.getStateMap(partitionName) == null)) {
                // No replicas for this segment
                TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName);
                if ((tableType != null) && (tableType.equals(TableType.OFFLINE))) {
                    OfflineSegmentZKMetadata segmentZKMetadata = ZKMetadataProvider.getOfflineSegmentZKMetadata(propertyStore, tableName, partitionName);
                    if (segmentZKMetadata != null && segmentZKMetadata.getPushTime() > System.currentTimeMillis() - _waitForPushTimeSeconds * 1000) {
                        // push not yet finished, skip
                        continue;
                    }
                }
                nOffline++;
                if (nOffline < MaxOfflineSegmentsToLog) {
                    LOGGER.warn("Segment {} of table {} has no replicas", partitionName, tableName);
                }
                nReplicasExternal = 0;
                continue;
            }
            for (Map.Entry<String, String> serverAndState : externalView.getStateMap(partitionName).entrySet()) {
                // Count number of online replicas
                if (serverAndState.getValue().equals(ONLINE)) {
                    nReplicas++;
                }
                if (serverAndState.getValue().equals(ERROR)) {
                    nErrors++;
                }
            }
            if (nReplicas == 0) {
                if (nOffline < MaxOfflineSegmentsToLog) {
                    LOGGER.warn("Segment {} of table {} has no online replicas", partitionName, tableName);
                }
                nOffline++;
            }
            nReplicasExternal = ((nReplicasExternal > nReplicas) || (nReplicasExternal == -1)) ? nReplicas : nReplicasExternal;
        }
        if (nReplicasExternal == -1) {
            nReplicasExternal = (nReplicasIdealMax == 0) ? 1 : 0;
        }
        // Synchronization provided by Controller Gauge to make sure that only one thread updates the gauge
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.NUMBER_OF_REPLICAS, nReplicasExternal);
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.PERCENT_OF_REPLICAS, (nReplicasIdealMax > 0) ? (nReplicasExternal * 100 / nReplicasIdealMax) : 100);
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.SEGMENTS_IN_ERROR_STATE, nErrors);
        _metricsRegistry.setValueOfTableGauge(tableName, ControllerGauge.PERCENT_SEGMENTS_AVAILABLE, (nSegments > 0) ? (100 - (nOffline * 100 / nSegments)) : 100);
        if (nOffline > 0) {
            LOGGER.warn("Table {} has {} segments with no online replicas", tableName, nOffline);
        }
        if (nReplicasExternal < nReplicasIdealMax) {
            LOGGER.warn("Table {} has {} replicas, below replication threshold :{}", tableName, nReplicasExternal, nReplicasIdealMax);
        }
    }
    _metricsRegistry.setValueOfGlobalGauge(ControllerGauge.REALTIME_TABLE_COUNT, realTimeTableCount);
    _metricsRegistry.setValueOfGlobalGauge(ControllerGauge.OFFLINE_TABLE_COUNT, offlineTableCount);
    long totalNanos = System.nanoTime() - startTime;
    LOGGER.info("Segment status metrics completed in {}ms", TimeUnit.MILLISECONDS.convert(totalNanos, TimeUnit.NANOSECONDS));
}

Example 2 with TableType

use of com.linkedin.pinot.common.utils.CommonConstants.Helix.TableType in project pinot by linkedin.

the class PinotHelixResourceManager method rebuildBrokerResourceFromHelixTags.

public PinotResourceManagerResponse rebuildBrokerResourceFromHelixTags(final String tableName) {
    // Get the broker tag for this table
    String brokerTag = null;
    TenantConfig tenantConfig = null;
    try {
        final TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName);
        AbstractTableConfig tableConfig;
        if (tableType == TableType.OFFLINE) {
            tableConfig = ZKMetadataProvider.getOfflineTableConfig(getPropertyStore(), tableName);
        } else if (tableType == TableType.REALTIME) {
            tableConfig = ZKMetadataProvider.getRealtimeTableConfig(getPropertyStore(), tableName);
        } else {
            return new PinotResourceManagerResponse("Table " + tableName + " does not have a table type", false);
        }
        if (tableConfig == null) {
            return new PinotResourceManagerResponse("Table " + tableName + " does not exist", false);
        }
        tenantConfig = tableConfig.getTenantConfig();
    } catch (Exception e) {
        LOGGER.warn("Caught exception while getting tenant config for table {}", tableName, e);
        return new PinotResourceManagerResponse("Failed to fetch broker tag for table " + tableName + " due to exception: " + e.getMessage(), false);
    }
    brokerTag = tenantConfig.getBroker();
    // Look for all instances tagged with this broker tag
    final Set<String> brokerInstances = getAllInstancesForBrokerTenant(brokerTag);
    // If we add a new broker, we want to rebuild the broker resource.
    HelixAdmin helixAdmin = getHelixAdmin();
    String clusterName = getHelixClusterName();
    IdealState brokerIdealState = HelixHelper.getBrokerIdealStates(helixAdmin, clusterName);
    Set<String> idealStateBrokerInstances = brokerIdealState.getInstanceSet(tableName);
    if (idealStateBrokerInstances.equals(brokerInstances)) {
        return new PinotResourceManagerResponse("Broker resource is not rebuilt because ideal state is the same for table {} " + tableName, false);
    }
    // Reset ideal state with the instance list
    try {
        HelixHelper.updateIdealState(getHelixZkManager(), CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, new Function<IdealState, IdealState>() {

            @Nullable
            @Override
            public IdealState apply(@Nullable IdealState idealState) {
                Map<String, String> instanceStateMap = idealState.getInstanceStateMap(tableName);
                if (instanceStateMap != null) {
                    instanceStateMap.clear();
                }
                for (String brokerInstance : brokerInstances) {
                    idealState.setPartitionState(tableName, brokerInstance, BrokerOnlineOfflineStateModel.ONLINE);
                }
                return idealState;
            }
        }, DEFAULT_RETRY_POLICY);
        LOGGER.info("Successfully rebuilt brokerResource for table {}", tableName);
        return new PinotResourceManagerResponse("Rebuilt brokerResource for table " + tableName, true);
    } catch (Exception e) {
        LOGGER.warn("Caught exception while rebuilding broker resource from Helix tags for table {}", e, tableName);
        return new PinotResourceManagerResponse("Failed to rebuild brokerResource for table " + tableName + " due to exception: " + e.getMessage(), false);
    }
}

Example 3 with TableType

use of com.linkedin.pinot.common.utils.CommonConstants.Helix.TableType in project pinot by linkedin.

the class PinotTableRestletResource method updateTableConfig.

/*
   * NOTE: There is inconsistency in these APIs. GET returns OFFLINE + REALTIME configuration
   * in a single response but POST and this PUT request only operate on either offline or realtime
   * table configuration. If we make this API take both realtime and offline table configuration
   * then the update is not guaranteed to be transactional for both table types. This is more of a PATCH request
   * than PUT.
   */
@HttpVerb("put")
@Summary("Update table configuration. Request body is offline or realtime table configuration")
@Tags({ "Table" })
@Paths({ "/tables/{tableName}" })
public Representation updateTableConfig(@Parameter(name = "tableName", in = "path", description = "Table name (without type)") String tableName, Representation entity) {
    AbstractTableConfig config = null;
    try {
        config = AbstractTableConfig.init(entity.getText());
    } catch (JSONException e) {
        errorResponseRepresentation(Status.CLIENT_ERROR_BAD_REQUEST, "Invalid json in table configuration");
    } catch (IOException e) {
        LOGGER.error("Failed to read request body while updating configuration for table: {}", tableName, e);
        errorResponseRepresentation(Status.SERVER_ERROR_INTERNAL, "Failed to read request");
    }
    try {
        String tableTypeStr = config.getTableType();
        TableType tableType = TableType.valueOf(tableTypeStr.toUpperCase());
        String configTableName = config.getTableName();
        if (!configTableName.equals(tableName)) {
            errorResponseRepresentation(Status.CLIENT_ERROR_BAD_REQUEST, "Request table name does not match table name in the body");
        }
        String tableNameWithType = null;
        if (config.getTableType().equalsIgnoreCase(TableType.OFFLINE.name())) {
            tableNameWithType = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(tableName);
        } else if (config.getTableType().equalsIgnoreCase(TableType.REALTIME.name())) {
            tableNameWithType = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(tableName);
        }
        _pinotHelixResourceManager.setTableConfig(config, tableNameWithType, tableType);
        return responseRepresentation(Status.SUCCESS_OK, "{\"status\" : \"Success\"}");
    } catch (IOException e) {
        LOGGER.error("Failed to update table configuration for table: {}", tableName, e);
        return errorResponseRepresentation(Status.SERVER_ERROR_INTERNAL, "Internal error while updating table configuration");
    }
}

Example 4 with TableType

use of com.linkedin.pinot.common.utils.CommonConstants.Helix.TableType in project pinot by linkedin.

the class PinotHelixResourceManager method addTable.

/**
   * Table APIs
   */
public void addTable(AbstractTableConfig config) throws JsonGenerationException, JsonMappingException, IOException {
    TenantConfig tenantConfig = null;
    TableType type = TableType.valueOf(config.getTableType().toUpperCase());
    if (isSingleTenantCluster()) {
        tenantConfig = new TenantConfig();
        tenantConfig.setBroker(ControllerTenantNameBuilder.getBrokerTenantNameForTenant(ControllerTenantNameBuilder.DEFAULT_TENANT_NAME));
        switch(type) {
            case OFFLINE:
                tenantConfig.setServer(ControllerTenantNameBuilder.getOfflineTenantNameForTenant(ControllerTenantNameBuilder.DEFAULT_TENANT_NAME));
                break;
            case REALTIME:
                tenantConfig.setServer(ControllerTenantNameBuilder.getRealtimeTenantNameForTenant(ControllerTenantNameBuilder.DEFAULT_TENANT_NAME));
                break;
            default:
                throw new RuntimeException("UnSupported table type");
        }
        config.setTenantConfig(tenantConfig);
    } else {
        tenantConfig = config.getTenantConfig();
        if (tenantConfig.getBroker() == null || tenantConfig.getServer() == null) {
            throw new RuntimeException("missing tenant configs");
        }
    }
    SegmentsValidationAndRetentionConfig segmentsConfig = config.getValidationConfig();
    switch(type) {
        case OFFLINE:
            final String offlineTableName = config.getTableName();
            // now lets build an ideal state
            LOGGER.info("building empty ideal state for table : " + offlineTableName);
            final IdealState offlineIdealState = PinotTableIdealStateBuilder.buildEmptyIdealStateFor(offlineTableName, Integer.parseInt(segmentsConfig.getReplication()));
            LOGGER.info("adding table via the admin");
            _helixAdmin.addResource(_helixClusterName, offlineTableName, offlineIdealState);
            LOGGER.info("successfully added the table : " + offlineTableName + " to the cluster");
            // lets add table configs
            ZKMetadataProvider.setOfflineTableConfig(_propertyStore, offlineTableName, AbstractTableConfig.toZnRecord(config));
            _propertyStore.create(ZKMetadataProvider.constructPropertyStorePathForResource(offlineTableName), new ZNRecord(offlineTableName), AccessOption.PERSISTENT);
            break;
        case REALTIME:
            final String realtimeTableName = config.getTableName();
            // lets add table configs
            ZKMetadataProvider.setRealtimeTableConfig(_propertyStore, realtimeTableName, AbstractTableConfig.toZnRecord(config));
            /*
         * PinotRealtimeSegmentManager sets up watches on table and segment path. When a table gets created,
         * it expects the INSTANCE path in propertystore to be set up so that it can get the kafka group ID and
         * create (high-level consumer) segments for that table.
         * So, we need to set up the instance first, before adding the table resource for HLC new table creation.
         *
         * For low-level consumers, the order is to create the resource first, and set up the propertystore with segments
         * and then tweak the idealstate to add those segments.
         *
         * We also need to support the case when a high-level consumer already exists for a table and we are adding
         * the low-level consumers.
         */
            IndexingConfig indexingConfig = config.getIndexingConfig();
            ensureRealtimeClusterIsSetUp(config, realtimeTableName, indexingConfig);
            LOGGER.info("Successfully added or updated the table {} ", realtimeTableName);
            break;
        default:
            throw new RuntimeException("UnSupported table type");
    }
    handleBrokerResource(config);
}

Example 5 with TableType

use of com.linkedin.pinot.common.utils.CommonConstants.Helix.TableType in project pinot by linkedin.

the class ValidationManager method runValidation.

/**
   * Runs a validation pass over the currently loaded tables.
   */
public void runValidation() {
    if (!_pinotHelixResourceManager.isLeader()) {
        LOGGER.info("Skipping validation, not leader!");
        return;
    }
    LOGGER.info("Starting validation");
    // Fetch the list of tables
    List<String> allTableNames = _pinotHelixResourceManager.getAllPinotTableNames();
    ZkHelixPropertyStore<ZNRecord> propertyStore = _pinotHelixResourceManager.getPropertyStore();
    for (String tableName : allTableNames) {
        List<SegmentMetadata> segmentMetadataList = new ArrayList<SegmentMetadata>();
        TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName);
        AbstractTableConfig tableConfig = null;
        _pinotHelixResourceManager.rebuildBrokerResourceFromHelixTags(tableName);
        // For each table, fetch the metadata for all its segments
        if (tableType.equals(TableType.OFFLINE)) {
            validateOfflineSegmentPush(propertyStore, tableName, segmentMetadataList);
        } else if (tableType.equals(TableType.REALTIME)) {
            LOGGER.info("Starting to validate table {}", tableName);
            List<RealtimeSegmentZKMetadata> realtimeSegmentZKMetadatas = ZKMetadataProvider.getRealtimeSegmentZKMetadataListForTable(propertyStore, tableName);
            // false if this table has ONLY LLC segments (i.e. fully migrated)
            boolean countHLCSegments = true;
            KafkaStreamMetadata streamMetadata = null;
            try {
                tableConfig = _pinotHelixResourceManager.getRealtimeTableConfig(tableName);
                streamMetadata = new KafkaStreamMetadata(tableConfig.getIndexingConfig().getStreamConfigs());
                if (streamMetadata.hasSimpleKafkaConsumerType() && !streamMetadata.hasHighLevelKafkaConsumerType()) {
                    countHLCSegments = false;
                }
                for (RealtimeSegmentZKMetadata realtimeSegmentZKMetadata : realtimeSegmentZKMetadatas) {
                    SegmentMetadata segmentMetadata = new SegmentMetadataImpl(realtimeSegmentZKMetadata);
                    segmentMetadataList.add(segmentMetadata);
                }
                // Update the gauge to contain the total document count in the segments
                _validationMetrics.updateTotalDocumentsGauge(tableName, computeRealtimeTotalDocumentInSegments(segmentMetadataList, countHLCSegments));
                if (streamMetadata.hasSimpleKafkaConsumerType()) {
                    validateLLCSegments(tableName, tableConfig);
                }
            } catch (Exception e) {
                if (tableConfig == null) {
                    LOGGER.warn("Cannot get realtime tableconfig for {}", tableName);
                } else if (streamMetadata == null) {
                    LOGGER.warn("Cannot get streamconfig for {}", tableName);
                } else {
                    LOGGER.error("Exception while validating table {}", tableName, e);
                }
            }
        } else {
            LOGGER.warn("Ignoring table type {} for table {}", tableType, tableName);
        }
    }
    LOGGER.info("Validation completed");
}