Examples with ZNRecord org.apache.helix.ZNRecord used on opensource projects

Example 6 with ZNRecord

use of org.apache.helix.ZNRecord in project pinot by linkedin.

the class TimeBoundaryServiceTest method beforeTest.

@BeforeTest
public void beforeTest() {
    _zookeeperInstance = ZkStarter.startLocalZkServer();
    _zkClient = new ZkClient(StringUtil.join("/", StringUtils.chomp(ZkStarter.DEFAULT_ZK_STR, "/")), ZkClient.DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer());
    String helixClusterName = "TestTimeBoundaryService";
    _zkClient.deleteRecursive("/" + helixClusterName + "/PROPERTYSTORE");
    _zkClient.createPersistent("/" + helixClusterName + "/PROPERTYSTORE", true);
    _propertyStore = new ZkHelixPropertyStore<ZNRecord>(new ZkBaseDataAccessor<ZNRecord>(_zkClient), "/" + helixClusterName + "/PROPERTYSTORE", null);
}

Example 7 with ZNRecord

use of org.apache.helix.ZNRecord in project pinot by linkedin.

the class DeleteOverlappingSegmentsInPinot method deleteOverlappingSegments.

public static boolean deleteOverlappingSegments(String zkUrl, String zkCluster, String tableName) {
    boolean updateSuccessful = false;
    if (!tableName.endsWith("_OFFLINE")) {
        tableName = tableName + "_OFFLINE";
    }
    ZkClient zkClient = new ZkClient(zkUrl);
    ZNRecordSerializer zkSerializer = new ZNRecordSerializer();
    zkClient.setZkSerializer(zkSerializer);
    BaseDataAccessor<ZNRecord> baseDataAccessor = new ZkBaseDataAccessor<>(zkClient);
    HelixDataAccessor helixDataAccessor = new ZKHelixDataAccessor(zkCluster, baseDataAccessor);
    Builder keyBuilder = helixDataAccessor.keyBuilder();
    PropertyKey idealStateKey = keyBuilder.idealStates(tableName);
    PropertyKey externalViewKey = keyBuilder.externalView(tableName);
    IdealState currentIdealState = helixDataAccessor.getProperty(idealStateKey);
    byte[] serializeIS = zkSerializer.serialize(currentIdealState.getRecord());
    String name = tableName + ".idealstate." + System.currentTimeMillis();
    File outputFile = new File("/tmp", name);
    try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
        IOUtils.write(serializeIS, fileOutputStream);
    } catch (IOException e) {
        LOG.error("Exception in delete overlapping segments", e);
        return updateSuccessful;
    }
    LOG.info("Saved current idealstate to {}", outputFile);
    IdealState newIdealState;
    do {
        newIdealState = computeNewIdealStateAfterDeletingOverlappingSegments(helixDataAccessor, idealStateKey);
        LOG.info("Updating IdealState");
        updateSuccessful = helixDataAccessor.getBaseDataAccessor().set(idealStateKey.getPath(), newIdealState.getRecord(), newIdealState.getRecord().getVersion(), AccessOption.PERSISTENT);
        if (updateSuccessful) {
            int numSegmentsDeleted = currentIdealState.getPartitionSet().size() - newIdealState.getPartitionSet().size();
            LOG.info("Successfully updated IdealState: Removed segments: {}", (numSegmentsDeleted));
        }
    } while (!updateSuccessful);
    try {
        while (true) {
            Thread.sleep(10000);
            ExternalView externalView = helixDataAccessor.getProperty(externalViewKey);
            IdealState idealState = helixDataAccessor.getProperty(idealStateKey);
            Set<String> evPartitionSet = externalView.getPartitionSet();
            Set<String> isPartitionSet = idealState.getPartitionSet();
            if (evPartitionSet.equals(isPartitionSet)) {
                LOG.info("Table {} has reached stable state. i.e segments in external view match idealstates", tableName);
                break;
            }
        }
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    return updateSuccessful;
}

Example 8 with ZNRecord

use of org.apache.helix.ZNRecord in project pinot by linkedin.

the class ZKMetadataProvider method setInstanceZKMetadata.

public static void setInstanceZKMetadata(ZkHelixPropertyStore<ZNRecord> propertyStore, InstanceZKMetadata instanceZKMetadata) {
    ZNRecord znRecord = instanceZKMetadata.toZNRecord();
    propertyStore.set(StringUtil.join("/", PROPERTYSTORE_INSTANCE_CONFIGS_PREFIX, instanceZKMetadata.getId()), znRecord, AccessOption.PERSISTENT);
}

Example 9 with ZNRecord

use of org.apache.helix.ZNRecord in project pinot by linkedin.

the class ZKMetadataProvider method getRealtimeSegmentZKMetadata.

@Nullable
public static RealtimeSegmentZKMetadata getRealtimeSegmentZKMetadata(ZkHelixPropertyStore<ZNRecord> propertyStore, String tableName, String segmentName) {
    String realtimeTableName = TableNameBuilder.REALTIME_TABLE_NAME_BUILDER.forTable(tableName);
    ZNRecord znRecord = propertyStore.get(constructPropertyStorePathForSegment(realtimeTableName, segmentName), null, AccessOption.PERSISTENT);
    // It is possible that the segment metadata has just been deleted due to retention.
    if (znRecord == null) {
        return null;
    }
    if (SegmentName.isHighLevelConsumerSegmentName(segmentName)) {
        return new RealtimeSegmentZKMetadata(znRecord);
    } else {
        return new LLCRealtimeSegmentZKMetadata(znRecord);
    }
}

Example 10 with ZNRecord

use of org.apache.helix.ZNRecord in project pinot by linkedin.

the class ZKMetadataProvider method getOfflineSegmentZKMetadataListForTable.

public static List<OfflineSegmentZKMetadata> getOfflineSegmentZKMetadataListForTable(ZkHelixPropertyStore<ZNRecord> propertyStore, String tableName) {
    List<OfflineSegmentZKMetadata> resultList = new ArrayList<OfflineSegmentZKMetadata>();
    if (propertyStore == null) {
        return resultList;
    }
    String offlineTableName = TableNameBuilder.OFFLINE_TABLE_NAME_BUILDER.forTable(tableName);
    if (propertyStore.exists(constructPropertyStorePathForResource(offlineTableName), AccessOption.PERSISTENT)) {
        List<ZNRecord> znRecordList = propertyStore.getChildren(constructPropertyStorePathForResource(offlineTableName), null, AccessOption.PERSISTENT);
        if (znRecordList != null) {
            for (ZNRecord record : znRecordList) {
                resultList.add(new OfflineSegmentZKMetadata(record));
            }
        }
    }
    return resultList;
}