Examples with AbstractTableConfig com.linkedin.pinot.common.config.AbstractTableConfig used on opensource projects

Example 6 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class HelixBrokerStarterTest method testResourceAndTagAssignment.

@Test
public void testResourceAndTagAssignment() throws Exception {
    IdealState idealState;
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_BROKER").size(), 6);
    idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
    Assert.assertEquals(idealState.getInstanceSet(DINING_TABLE_NAME).size(), SEGMENT_COUNT);
    ExternalView externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
    Assert.assertEquals(externalView.getStateMap(DINING_TABLE_NAME).size(), SEGMENT_COUNT);
    HelixExternalViewBasedRouting helixExternalViewBasedRouting = _helixBrokerStarter.getHelixExternalViewBasedRouting();
    Field brokerRoutingTableField;
    brokerRoutingTableField = HelixExternalViewBasedRouting.class.getDeclaredField("_brokerRoutingTable");
    brokerRoutingTableField.setAccessible(true);
    final Map<String, List<ServerToSegmentSetMap>> brokerRoutingTable = (Map<String, List<ServerToSegmentSetMap>>) brokerRoutingTableField.get(helixExternalViewBasedRouting);
    // Wait up to 30s for routing table to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return brokerRoutingTable.size() == 1;
        }
    }, 30000L);
    Assert.assertEquals(Arrays.toString(brokerRoutingTable.keySet().toArray()), "[dining_OFFLINE]");
    final String tableName = "coffee";
    JSONObject buildCreateOfflineTableV2JSON = ControllerRequestBuilderUtil.buildCreateOfflineTableJSON(tableName, "testServer", "testBroker", 1);
    AbstractTableConfig config = AbstractTableConfig.init(buildCreateOfflineTableV2JSON.toString());
    _pinotResourceManager.addTable(config);
    Assert.assertEquals(_helixAdmin.getInstancesInClusterWithTag(HELIX_CLUSTER_NAME, "DefaultTenant_BROKER").size(), 6);
    idealState = _helixAdmin.getResourceIdealState(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
    Assert.assertEquals(idealState.getInstanceSet(COFFEE_TABLE_NAME).size(), SEGMENT_COUNT);
    Assert.assertEquals(idealState.getInstanceSet(DINING_TABLE_NAME).size(), SEGMENT_COUNT);
    // Wait up to 30s for broker external view to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE).getStateMap(COFFEE_TABLE_NAME).size() == SEGMENT_COUNT;
        }
    }, 30000L);
    externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, CommonConstants.Helix.BROKER_RESOURCE_INSTANCE);
    Assert.assertEquals(externalView.getStateMap(COFFEE_TABLE_NAME).size(), SEGMENT_COUNT);
    // Wait up to 30s for routing table to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return brokerRoutingTable.size() == 2;
        }
    }, 30000L);
    Object[] tableArray = brokerRoutingTable.keySet().toArray();
    Arrays.sort(tableArray);
    Assert.assertEquals(Arrays.toString(tableArray), "[coffee_OFFLINE, dining_OFFLINE]");
    Set<String> serverSet = brokerRoutingTable.get(DINING_TABLE_NAME).get(0).getServerSet();
    Assert.assertEquals(brokerRoutingTable.get(DINING_TABLE_NAME).get(0).getSegmentSet(serverSet.iterator().next()).size(), 5);
    final String dataResource = DINING_TABLE_NAME;
    addOneSegment(dataResource);
    // Wait up to 30s for external view to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            return _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, DINING_TABLE_NAME).getPartitionSet().size() == SEGMENT_COUNT;
        }
    }, 30000L);
    externalView = _helixAdmin.getResourceExternalView(HELIX_CLUSTER_NAME, DINING_TABLE_NAME);
    Assert.assertEquals(externalView.getPartitionSet().size(), SEGMENT_COUNT);
    tableArray = brokerRoutingTable.keySet().toArray();
    Arrays.sort(tableArray);
    Assert.assertEquals(Arrays.toString(tableArray), "[coffee_OFFLINE, dining_OFFLINE]");
    // Wait up to 30s for routing table to reach the expected size
    waitForPredicate(new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            ServerToSegmentSetMap routingTable = brokerRoutingTable.get(DINING_TABLE_NAME).get(0);
            String firstServer = routingTable.getServerSet().iterator().next();
            return routingTable.getSegmentSet(firstServer).size() == SEGMENT_COUNT;
        }
    }, 30000L);
    serverSet = brokerRoutingTable.get(DINING_TABLE_NAME).get(0).getServerSet();
    Assert.assertEquals(brokerRoutingTable.get(DINING_TABLE_NAME).get(0).getSegmentSet(serverSet.iterator().next()).size(), SEGMENT_COUNT);
}

Example 7 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class PinotHelixResourceManager method rebuildBrokerResourceFromHelixTags.

public PinotResourceManagerResponse rebuildBrokerResourceFromHelixTags(final String tableName) {
    // Get the broker tag for this table
    String brokerTag = null;
    TenantConfig tenantConfig = null;
    try {
        final TableType tableType = TableNameBuilder.getTableTypeFromTableName(tableName);
        AbstractTableConfig tableConfig;
        if (tableType == TableType.OFFLINE) {
            tableConfig = ZKMetadataProvider.getOfflineTableConfig(getPropertyStore(), tableName);
        } else if (tableType == TableType.REALTIME) {
            tableConfig = ZKMetadataProvider.getRealtimeTableConfig(getPropertyStore(), tableName);
        } else {
            return new PinotResourceManagerResponse("Table " + tableName + " does not have a table type", false);
        }
        if (tableConfig == null) {
            return new PinotResourceManagerResponse("Table " + tableName + " does not exist", false);
        }
        tenantConfig = tableConfig.getTenantConfig();
    } catch (Exception e) {
        LOGGER.warn("Caught exception while getting tenant config for table {}", tableName, e);
        return new PinotResourceManagerResponse("Failed to fetch broker tag for table " + tableName + " due to exception: " + e.getMessage(), false);
    }
    brokerTag = tenantConfig.getBroker();
    // Look for all instances tagged with this broker tag
    final Set<String> brokerInstances = getAllInstancesForBrokerTenant(brokerTag);
    // If we add a new broker, we want to rebuild the broker resource.
    HelixAdmin helixAdmin = getHelixAdmin();
    String clusterName = getHelixClusterName();
    IdealState brokerIdealState = HelixHelper.getBrokerIdealStates(helixAdmin, clusterName);
    Set<String> idealStateBrokerInstances = brokerIdealState.getInstanceSet(tableName);
    if (idealStateBrokerInstances.equals(brokerInstances)) {
        return new PinotResourceManagerResponse("Broker resource is not rebuilt because ideal state is the same for table {} " + tableName, false);
    }
    // Reset ideal state with the instance list
    try {
        HelixHelper.updateIdealState(getHelixZkManager(), CommonConstants.Helix.BROKER_RESOURCE_INSTANCE, new Function<IdealState, IdealState>() {

            @Nullable
            @Override
            public IdealState apply(@Nullable IdealState idealState) {
                Map<String, String> instanceStateMap = idealState.getInstanceStateMap(tableName);
                if (instanceStateMap != null) {
                    instanceStateMap.clear();
                }
                for (String brokerInstance : brokerInstances) {
                    idealState.setPartitionState(tableName, brokerInstance, BrokerOnlineOfflineStateModel.ONLINE);
                }
                return idealState;
            }
        }, DEFAULT_RETRY_POLICY);
        LOGGER.info("Successfully rebuilt brokerResource for table {}", tableName);
        return new PinotResourceManagerResponse("Rebuilt brokerResource for table " + tableName, true);
    } catch (Exception e) {
        LOGGER.warn("Caught exception while rebuilding broker resource from Helix tags for table {}", e, tableName);
        return new PinotResourceManagerResponse("Failed to rebuild brokerResource for table " + tableName + " due to exception: " + e.getMessage(), false);
    }
}

Example 8 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class PinotHelixResourceManager method addNewOfflineSegment.

/**
   * Helper method to add the passed in offline segment to the helix cluster.
   * - Gets the segment name and the table name from the passed in segment meta-data.
   * - Identifies the instance set onto which the segment needs to be added, based on
   *   segment assignment strategy and replicas in the table config in the property-store.
   * - Updates ideal state such that the new segment is assigned to required set of instances as per
   *    the segment assignment strategy and replicas.
   *
   * @param segmentMetadata Meta-data for the segment, used to access segmentName and tableName.
   * @throws JsonParseException
   * @throws JsonMappingException
   * @throws JsonProcessingException
   * @throws JSONException
   * @throws IOException
   */
private void addNewOfflineSegment(final SegmentMetadata segmentMetadata) throws JsonParseException, JsonMappingException, JsonProcessingException, JSONException, IOException {
    final AbstractTableConfig offlineTableConfig = ZKMetadataProvider.getOfflineTableConfig(_propertyStore, segmentMetadata.getTableName());
    final String segmentName = segmentMetadata.getName();
    final String offlineTableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(segmentMetadata.getTableName());
    if (!SEGMENT_ASSIGNMENT_STRATEGY_MAP.containsKey(offlineTableName)) {
        SEGMENT_ASSIGNMENT_STRATEGY_MAP.put(offlineTableName, SegmentAssignmentStrategyFactory.getSegmentAssignmentStrategy(offlineTableConfig.getValidationConfig().getSegmentAssignmentStrategy()));
    }
    final SegmentAssignmentStrategy segmentAssignmentStrategy = SEGMENT_ASSIGNMENT_STRATEGY_MAP.get(offlineTableName);
    // Passing a callable to this api to avoid helixHelper having which is in pinot-common having to
    // depend upon pinot-controller.
    Callable<List<String>> getInstancesForSegment = new Callable<List<String>>() {

        @Override
        public List<String> call() throws Exception {
            final IdealState currentIdealState = _helixAdmin.getResourceIdealState(_helixClusterName, offlineTableName);
            final Set<String> currentInstanceSet = currentIdealState.getInstanceSet(segmentName);
            if (currentInstanceSet.isEmpty()) {
                final String serverTenant = ControllerTenantNameBuilder.getOfflineTenantNameForTenant(offlineTableConfig.getTenantConfig().getServer());
                final int replicas = Integer.parseInt(offlineTableConfig.getValidationConfig().getReplication());
                return segmentAssignmentStrategy.getAssignedInstances(_helixAdmin, _helixClusterName, segmentMetadata, replicas, serverTenant);
            } else {
                return new ArrayList<String>(currentIdealState.getInstanceSet(segmentName));
            }
        }
    };
    HelixHelper.addSegmentToIdealState(_helixZkManager, offlineTableName, segmentName, getInstancesForSegment);
}

Example 9 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class PinotSchemaRestletResource method deleteSchema.

@HttpVerb("delete")
@Summary("Deletes a schema")
@Tags({ "schema" })
@Paths({ "/schemas/{schemaName}", "/schemas/{schemaName}/" })
@Responses({ @Response(statusCode = "200", description = "The schema was deleted"), @Response(statusCode = "404", description = "The schema does not exist"), @Response(statusCode = "409", description = "The schema could not be deleted due to being in use"), @Response(statusCode = "500", description = "There was an error while deleting the schema") })
StringRepresentation deleteSchema(@Parameter(name = "schemaName", in = "path", description = "The name of the schema to get", required = true) String schemaName) throws JSONException, IOException {
    Schema schema = _pinotHelixResourceManager.getSchema(schemaName);
    if (schema == null) {
        LOGGER.error("Error: could not find schema {}", schemaName);
        setStatus(Status.CLIENT_ERROR_NOT_FOUND);
        return new StringRepresentation("Error: Could not find schema " + schemaName);
    }
    // If the schema is associated with a table, we should not delete it.
    List<String> tableNames = _pinotHelixResourceManager.getAllRealtimeTables();
    for (String tableName : tableNames) {
        AbstractTableConfig config = _pinotHelixResourceManager.getTableConfig(tableName, CommonConstants.Helix.TableType.REALTIME);
        String tableSchema = config.getValidationConfig().getSchemaName();
        if (schemaName.equals(tableSchema)) {
            LOGGER.error("Cannot delete schema {}, as it is associated with table {}", schemaName, tableName);
            setStatus(Status.CLIENT_ERROR_CONFLICT);
            return new StringRepresentation("Error: Cannot delete schema " + schemaName + " as it is associated with table: " + TableNameBuilder.extractRawTableName(tableName));
        }
    }
    LOGGER.info("Trying to delete schema {}", schemaName);
    if (_pinotHelixResourceManager.deleteSchema(schema)) {
        LOGGER.info("Success: Deleted schema {}", schemaName);
        setStatus(Status.SUCCESS_OK);
        return new StringRepresentation("Success: Deleted schema " + schemaName);
    } else {
        LOGGER.error("Error: could not delete schema {}", schemaName);
        ControllerRestApplication.getControllerMetrics().addMeteredGlobalValue(ControllerMeter.CONTROLLER_SCHEMA_DELETE_ERROR, 1L);
        setStatus(Status.SERVER_ERROR_INTERNAL);
        return new StringRepresentation("Error: Could not delete schema " + schemaName);
    }
}

Example 10 with AbstractTableConfig

use of com.linkedin.pinot.common.config.AbstractTableConfig in project pinot by linkedin.

the class PinotTableMetadataConfigs method updateTableMetadata.

@Deprecated
@HttpVerb("put")
@Summary("DEPRECATED: Updates the metadata configuration for a table")
@Tags({ "table" })
@Paths({ "/tables/{tableName}/metadataConfigs" })
private Representation updateTableMetadata(@Parameter(name = "tableName", in = "path", description = "The name of the table for which to update the metadata configuration", required = true) String tableName, Representation entity) throws Exception {
    AbstractTableConfig config = AbstractTableConfig.init(entity.getText());
    _pinotHelixResourceManager.updateMetadataConfigFor(config.getTableName(), TableType.valueOf(config.getTableType().toUpperCase()), config.getCustomConfigs());
    return new StringRepresentation("done");
}