Examples with Stat org.apache.flink.shaded.zookeeper3.org.apache.zookeeper.data.Stat used on opensource projects

Example 6 with Stat

use of org.apache.flink.shaded.zookeeper3.org.apache.zookeeper.data.Stat in project pinot by linkedin.

the class PinotRealtimeSegmentManager method refreshWatchers.

/**
   * Helper method to perform idempotent operation to refresh all watches (related to real-time segments):
   * - Data change listener for all existing real-time tables.
   * - Child creation listener for all existing real-time tables.
   * - Data change listener for all existing real-time segments
   *
   * @param path
   */
private void refreshWatchers(String path) {
    LOGGER.info("Received change notification for path: {}", path);
    List<Stat> stats = new ArrayList<>();
    List<ZNRecord> tableConfigs = _pinotHelixResourceManager.getPropertyStore().getChildren(TABLE_CONFIG, stats, 0);
    if (tableConfigs == null) {
        return;
    }
    for (ZNRecord tableConfigZnRecord : tableConfigs) {
        try {
            String znRecordId = tableConfigZnRecord.getId();
            if (TableNameBuilder.getTableTypeFromTableName(znRecordId) == TableType.REALTIME) {
                AbstractTableConfig abstractTableConfig = AbstractTableConfig.fromZnRecord(tableConfigZnRecord);
                KafkaStreamMetadata metadata = new KafkaStreamMetadata(abstractTableConfig.getIndexingConfig().getStreamConfigs());
                if (metadata.hasHighLevelKafkaConsumerType()) {
                    String realtimeTable = abstractTableConfig.getTableName();
                    String realtimeSegmentsPathForTable = _propertyStorePath + SEGMENTS_PATH + "/" + realtimeTable;
                    LOGGER.info("Setting data/child changes watch for real-time table '{}'", realtimeTable);
                    _zkClient.subscribeDataChanges(realtimeSegmentsPathForTable, this);
                    _zkClient.subscribeChildChanges(realtimeSegmentsPathForTable, this);
                    List<String> childNames = _pinotHelixResourceManager.getPropertyStore().getChildNames(SEGMENTS_PATH + "/" + realtimeTable, 0);
                    if (childNames != null && !childNames.isEmpty()) {
                        for (String segmentName : childNames) {
                            if (!SegmentName.isHighLevelConsumerSegmentName(segmentName)) {
                                continue;
                            }
                            String segmentPath = realtimeSegmentsPathForTable + "/" + segmentName;
                            RealtimeSegmentZKMetadata realtimeSegmentZKMetadata = ZKMetadataProvider.getRealtimeSegmentZKMetadata(_pinotHelixResourceManager.getPropertyStore(), abstractTableConfig.getTableName(), segmentName);
                            if (realtimeSegmentZKMetadata == null) {
                                // The segment got deleted by retention manager
                                continue;
                            }
                            if (realtimeSegmentZKMetadata.getStatus() == Status.IN_PROGRESS) {
                                LOGGER.info("Setting data change watch for real-time segment currently being consumed: {}", segmentPath);
                                _zkClient.subscribeDataChanges(segmentPath, this);
                            } else {
                                _zkClient.unsubscribeDataChanges(segmentPath, this);
                            }
                        }
                    }
                }
            }
        } catch (Exception e) {
            // we want to continue setting watches for other tables for any kind of exception here so that
            // errors with one table don't impact others
            LOGGER.error("Caught exception while processing ZNRecord id: {}. Skipping node to continue setting watches", tableConfigZnRecord.getId(), e);
        }
    }
}

Example 7 with Stat

use of org.apache.flink.shaded.zookeeper3.org.apache.zookeeper.data.Stat in project pinot by linkedin.

the class PinotSegmentRebalancer method rebalanceTenantTables.

/**
   * rebalances all tables for the tenant
   * @param tenantName
   */
public void rebalanceTenantTables(String tenantName) throws Exception {
    String tableConfigPath = "/CONFIGS/TABLE";
    List<Stat> stats = new ArrayList<>();
    List<ZNRecord> tableConfigs = propertyStore.getChildren(tableConfigPath, stats, 0);
    String rawTenantName = tenantName.replaceAll("_OFFLINE", "").replace("_REALTIME", "");
    int nRebalances = 0;
    for (ZNRecord znRecord : tableConfigs) {
        AbstractTableConfig tableConfig;
        try {
            tableConfig = AbstractTableConfig.fromZnRecord(znRecord);
        } catch (Exception e) {
            LOGGER.warn("Failed to parse table configuration for ZnRecord id: {}. Skipping", znRecord.getId());
            continue;
        }
        if (tableConfig.getTenantConfig().getServer().equals(rawTenantName)) {
            LOGGER.info(tableConfig.getTableName() + ":" + tableConfig.getTenantConfig().getServer());
            nRebalances++;
            rebalanceTable(tableConfig.getTableName(), tenantName);
        }
    }
    if (nRebalances == 0) {
        LOGGER.info("No tables found for tenant " + tenantName);
    }
}

Example 8 with Stat

use of org.apache.flink.shaded.zookeeper3.org.apache.zookeeper.data.Stat in project pinot by linkedin.

the class HelixExternalViewBasedRouting method processInstanceConfigChange.

public void processInstanceConfigChange() {
    long startTime = System.currentTimeMillis();
    // Get stats for all relevant instance configs
    HelixDataAccessor helixDataAccessor = _helixManager.getHelixDataAccessor();
    PropertyKey.Builder propertyKeyBuilder = helixDataAccessor.keyBuilder();
    List<String> instancesUsed = new ArrayList<>(_tablesForInstance.keySet());
    List<String> instancePaths = new ArrayList<>(instancesUsed.size());
    for (String instanceName : instancesUsed) {
        PropertyKey propertyKey = propertyKeyBuilder.instanceConfig(instanceName);
        instancePaths.add(propertyKey.getPath());
    }
    if (instancePaths.isEmpty()) {
        return;
    }
    long statFetchStart = System.currentTimeMillis();
    Stat[] instanceConfigStats = helixDataAccessor.getBaseDataAccessor().getStats(instancePaths, AccessOption.PERSISTENT);
    long statFetchEnd = System.currentTimeMillis();
    // Make a list of instance configs that changed
    long icConfigCheckStart = System.currentTimeMillis();
    List<String> instancesThatChanged = new ArrayList<>();
    for (int i = 0; i < instanceConfigStats.length; i++) {
        Stat instanceConfigStat = instanceConfigStats[i];
        if (instanceConfigStat != null) {
            String instanceName = instancesUsed.get(i);
            int currentInstanceConfigVersion = instanceConfigStat.getVersion();
            int lastKnownInstanceConfigVersion = _lastKnownInstanceConfigs.get(instanceName).getRecord().getVersion();
            if (currentInstanceConfigVersion != lastKnownInstanceConfigVersion) {
                instancesThatChanged.add(instanceName);
            }
        }
    }
    // Make a list of all tables affected by the instance config changes
    Set<String> affectedTables = new HashSet<>();
    for (String instanceName : instancesThatChanged) {
        affectedTables.addAll(_tablesForInstance.get(instanceName));
    }
    long icConfigCheckEnd = System.currentTimeMillis();
    // Update the routing tables
    long icFetchTime = 0;
    long evFetchTime = 0;
    long rebuildCheckTime = 0;
    long buildTime = 0;
    int routingTablesRebuiltCount = 0;
    if (!affectedTables.isEmpty()) {
        long icFetchStart = System.currentTimeMillis();
        List<InstanceConfig> instanceConfigs = helixDataAccessor.getChildValues(propertyKeyBuilder.instanceConfigs());
        long icFetchEnd = System.currentTimeMillis();
        icFetchTime = icFetchEnd - icFetchStart;
        for (String tableName : affectedTables) {
            long evFetchStart = System.currentTimeMillis();
            ExternalView externalView = helixDataAccessor.getProperty(propertyKeyBuilder.externalView(tableName));
            long evFetchEnd = System.currentTimeMillis();
            evFetchTime += evFetchEnd - evFetchStart;
            long rebuildCheckStart = System.currentTimeMillis();
            final boolean routingTableRebuildRequired = isRoutingTableRebuildRequired(tableName, externalView, instanceConfigs);
            long rebuildCheckEnd = System.currentTimeMillis();
            rebuildCheckTime += rebuildCheckEnd - rebuildCheckStart;
            if (routingTableRebuildRequired) {
                long rebuildStart = System.currentTimeMillis();
                buildRoutingTable(tableName, externalView, instanceConfigs);
                long rebuildEnd = System.currentTimeMillis();
                buildTime += rebuildEnd - rebuildStart;
                routingTablesRebuiltCount++;
            }
        }
    }
    long endTime = System.currentTimeMillis();
    LOGGER.info("Processed instance config change in {} ms (stat {} ms, IC check {} ms, IC fetch {} ms, EV fetch {} ms, rebuild check {} ms, rebuild {} ms), {} / {} routing tables rebuilt", (endTime - startTime), (statFetchEnd - statFetchStart), (icConfigCheckEnd - icConfigCheckStart), icFetchTime, evFetchTime, rebuildCheckTime, buildTime, routingTablesRebuiltCount, _lastKnownExternalViewVersionMap.size());
}

Example 9 with Stat

use of org.apache.flink.shaded.zookeeper3.org.apache.zookeeper.data.Stat in project hbase by apache.

the class ZkSplitLogWorkerCoordination method attemptToOwnTask.

/**
   * Try to own the task by transitioning the zk node data from UNASSIGNED to OWNED.
   * <p>
   * This method is also used to periodically heartbeat the task progress by transitioning the node
   * from OWNED to OWNED.
   * <p>
   * @param isFirstTime shows whther it's the first attempt.
   * @param zkw zk wathcer
   * @param server name
   * @param task to own
   * @param taskZKVersion version of the task in zk
   * @return non-negative integer value when task can be owned by current region server otherwise -1
   */
protected static int attemptToOwnTask(boolean isFirstTime, ZooKeeperWatcher zkw, ServerName server, String task, RecoveryMode mode, int taskZKVersion) {
    int latestZKVersion = FAILED_TO_OWN_TASK;
    try {
        SplitLogTask slt = new SplitLogTask.Owned(server, mode);
        Stat stat = zkw.getRecoverableZooKeeper().setData(task, slt.toByteArray(), taskZKVersion);
        if (stat == null) {
            LOG.warn("zk.setData() returned null for path " + task);
            SplitLogCounters.tot_wkr_task_heartbeat_failed.incrementAndGet();
            return FAILED_TO_OWN_TASK;
        }
        latestZKVersion = stat.getVersion();
        SplitLogCounters.tot_wkr_task_heartbeat.incrementAndGet();
        return latestZKVersion;
    } catch (KeeperException e) {
        if (!isFirstTime) {
            if (e.code().equals(KeeperException.Code.NONODE)) {
                LOG.warn("NONODE failed to assert ownership for " + task, e);
            } else if (e.code().equals(KeeperException.Code.BADVERSION)) {
                LOG.warn("BADVERSION failed to assert ownership for " + task, e);
            } else {
                LOG.warn("failed to assert ownership for " + task, e);
            }
        }
    } catch (InterruptedException e1) {
        LOG.warn("Interrupted while trying to assert ownership of " + task + " " + StringUtils.stringifyException(e1));
        Thread.currentThread().interrupt();
    }
    SplitLogCounters.tot_wkr_task_heartbeat_failed.incrementAndGet();
    return FAILED_TO_OWN_TASK;
}

Example 10 with Stat

use of org.apache.flink.shaded.zookeeper3.org.apache.zookeeper.data.Stat in project hbase by apache.

the class TestZooKeeper method testCreateSilentIsReallySilent.

/**
   * A test for HBASE-3238
   * @throws IOException A connection attempt to zk failed
   * @throws InterruptedException One of the non ZKUtil actions was interrupted
   * @throws KeeperException Any of the zookeeper connections had a
   * KeeperException
   */
@Test
public void testCreateSilentIsReallySilent() throws InterruptedException, KeeperException, IOException {
    Configuration c = TEST_UTIL.getConfiguration();
    String aclZnode = "/aclRoot";
    String quorumServers = ZKConfig.getZKQuorumServersString(c);
    // 5 seconds
    int sessionTimeout = 5 * 1000;
    ZooKeeper zk = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
    zk.addAuthInfo("digest", "hbase:rox".getBytes());
    // Assumes the  root of the ZooKeeper space is writable as it creates a node
    // wherever the cluster home is defined.
    ZooKeeperWatcher zk2 = new ZooKeeperWatcher(TEST_UTIL.getConfiguration(), "testCreateSilentIsReallySilent", null);
    // Save the previous ACL
    Stat s = null;
    List<ACL> oldACL = null;
    while (true) {
        try {
            s = new Stat();
            oldACL = zk.getACL("/", s);
            break;
        } catch (KeeperException e) {
            switch(e.code()) {
                case CONNECTIONLOSS:
                case SESSIONEXPIRED:
                case OPERATIONTIMEOUT:
                    LOG.warn("Possibly transient ZooKeeper exception", e);
                    Threads.sleep(100);
                    break;
                default:
                    throw e;
            }
        }
    }
    // Add retries in case of retryable zk exceptions.
    while (true) {
        try {
            zk.setACL("/", ZooDefs.Ids.CREATOR_ALL_ACL, -1);
            break;
        } catch (KeeperException e) {
            switch(e.code()) {
                case CONNECTIONLOSS:
                case SESSIONEXPIRED:
                case OPERATIONTIMEOUT:
                    LOG.warn("Possibly transient ZooKeeper exception: " + e);
                    Threads.sleep(100);
                    break;
                default:
                    throw e;
            }
        }
    }
    while (true) {
        try {
            zk.create(aclZnode, null, ZooDefs.Ids.CREATOR_ALL_ACL, CreateMode.PERSISTENT);
            break;
        } catch (KeeperException e) {
            switch(e.code()) {
                case CONNECTIONLOSS:
                case SESSIONEXPIRED:
                case OPERATIONTIMEOUT:
                    LOG.warn("Possibly transient ZooKeeper exception: " + e);
                    Threads.sleep(100);
                    break;
                default:
                    throw e;
            }
        }
    }
    zk.close();
    ZKUtil.createAndFailSilent(zk2, aclZnode);
    // Restore the ACL
    ZooKeeper zk3 = new ZooKeeper(quorumServers, sessionTimeout, EmptyWatcher.instance);
    zk3.addAuthInfo("digest", "hbase:rox".getBytes());
    try {
        zk3.setACL("/", oldACL, -1);
    } finally {
        zk3.close();
    }
}