Examples with ZNRecord org.apache.helix.ZNRecord used on opensource projects

Example 36 with ZNRecord

use of org.apache.helix.ZNRecord in project pinot by linkedin.

the class ValidateConfigCommand method validateTableConfig.

private void validateTableConfig() throws Exception {
    List<String> tableNames = getTableNames();
    LOGGER.info("Validating table config for tables: " + tableNames);
    for (String tableName : tableNames) {
        LOGGER.info("  Validating table config for table: \"{}\"", tableName);
        try {
            ZNRecord record = _helixPropertyStore.get(TABLE_CONFIG_PATH + "/" + tableName, null, 0);
            AbstractTableConfig tableConfig = AbstractTableConfig.fromZnRecord(record);
            if (!TableConfigValidator.validate(tableConfig)) {
                LOGGER.error("    Table config validation failed for table: \"{}\"", tableName);
            }
        } catch (Exception e) {
            LOGGER.error("    Caught exception while validating table config for table: \"{}\"", tableName, e);
        }
    }
}

Example 37 with ZNRecord

use of org.apache.helix.ZNRecord in project pinot by linkedin.

the class ValidateConfigCommand method validateSchema.

private void validateSchema() throws Exception {
    List<String> schemaNames = getSchemaNames();
    LOGGER.info("Validating schemas: " + schemaNames);
    for (String schemaName : schemaNames) {
        LOGGER.info("  Validating schema: \"{}\"", schemaName);
        try {
            ZNRecord record = _helixPropertyStore.get(SCHEMA_PATH + "/" + schemaName, null, 0);
            Schema schema = SchemaUtils.fromZNRecord(record);
            if (!SchemaValidator.validate(schema)) {
                LOGGER.error("    Schema validation failed for schema: \"{}\"", schemaName);
            }
        } catch (Exception e) {
            LOGGER.error("    Caught exception while validating schema: \"{}\"", schemaName, e);
        }
    }
}

Example 38 with ZNRecord

use of org.apache.helix.ZNRecord in project pinot by linkedin.

the class TimeBoundaryServiceTest method beforeTest.

@BeforeTest
public void beforeTest() {
    _zookeeperInstance = ZkStarter.startLocalZkServer();
    _zkClient = new ZkClient(StringUtil.join("/", StringUtils.chomp(ZkStarter.DEFAULT_ZK_STR, "/")), ZkClient.DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer());
    String helixClusterName = "TestTimeBoundaryService";
    _zkClient.deleteRecursive("/" + helixClusterName + "/PROPERTYSTORE");
    _zkClient.createPersistent("/" + helixClusterName + "/PROPERTYSTORE", true);
    _propertyStore = new ZkHelixPropertyStore<ZNRecord>(new ZkBaseDataAccessor<ZNRecord>(_zkClient), "/" + helixClusterName + "/PROPERTYSTORE", null);
}

Example 39 with ZNRecord

use of org.apache.helix.ZNRecord in project pinot by linkedin.

the class DeleteOverlappingSegmentsInPinot method deleteOverlappingSegments.

public static boolean deleteOverlappingSegments(String zkUrl, String zkCluster, String tableName) {
    boolean updateSuccessful = false;
    if (!tableName.endsWith("_OFFLINE")) {
        tableName = tableName + "_OFFLINE";
    }
    ZkClient zkClient = new ZkClient(zkUrl);
    ZNRecordSerializer zkSerializer = new ZNRecordSerializer();
    zkClient.setZkSerializer(zkSerializer);
    BaseDataAccessor<ZNRecord> baseDataAccessor = new ZkBaseDataAccessor<>(zkClient);
    HelixDataAccessor helixDataAccessor = new ZKHelixDataAccessor(zkCluster, baseDataAccessor);
    Builder keyBuilder = helixDataAccessor.keyBuilder();
    PropertyKey idealStateKey = keyBuilder.idealStates(tableName);
    PropertyKey externalViewKey = keyBuilder.externalView(tableName);
    IdealState currentIdealState = helixDataAccessor.getProperty(idealStateKey);
    byte[] serializeIS = zkSerializer.serialize(currentIdealState.getRecord());
    String name = tableName + ".idealstate." + System.currentTimeMillis();
    File outputFile = new File("/tmp", name);
    try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
        IOUtils.write(serializeIS, fileOutputStream);
    } catch (IOException e) {
        LOG.error("Exception in delete overlapping segments", e);
        return updateSuccessful;
    }
    LOG.info("Saved current idealstate to {}", outputFile);
    IdealState newIdealState;
    do {
        newIdealState = computeNewIdealStateAfterDeletingOverlappingSegments(helixDataAccessor, idealStateKey);
        LOG.info("Updating IdealState");
        updateSuccessful = helixDataAccessor.getBaseDataAccessor().set(idealStateKey.getPath(), newIdealState.getRecord(), newIdealState.getRecord().getVersion(), AccessOption.PERSISTENT);
        if (updateSuccessful) {
            int numSegmentsDeleted = currentIdealState.getPartitionSet().size() - newIdealState.getPartitionSet().size();
            LOG.info("Successfully updated IdealState: Removed segments: {}", (numSegmentsDeleted));
        }
    } while (!updateSuccessful);
    try {
        while (true) {
            Thread.sleep(10000);
            ExternalView externalView = helixDataAccessor.getProperty(externalViewKey);
            IdealState idealState = helixDataAccessor.getProperty(idealStateKey);
            Set<String> evPartitionSet = externalView.getPartitionSet();
            Set<String> isPartitionSet = idealState.getPartitionSet();
            if (evPartitionSet.equals(isPartitionSet)) {
                LOG.info("Table {} has reached stable state. i.e segments in external view match idealstates", tableName);
                break;
            }
        }
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    return updateSuccessful;
}

Example 40 with ZNRecord

use of org.apache.helix.ZNRecord in project pinot by linkedin.

the class PinotSegmentRebalancer method rebalanceTable.

/**
   * Rebalances a table
   * @param tableName
   * @throws Exception
   */
public void rebalanceTable(String tableName) throws Exception {
    String tableConfigPath = "/CONFIGS/TABLE/" + tableName;
    Stat stat = new Stat();
    ZNRecord znRecord = propertyStore.get(tableConfigPath, stat, 0);
    AbstractTableConfig tableConfig = AbstractTableConfig.fromZnRecord(znRecord);
    String tenantName = tableConfig.getTenantConfig().getServer().replaceAll(TableType.OFFLINE.toString(), "").replace(TableType.OFFLINE.toString(), "");
    rebalanceTable(tableName, tenantName);
}