Examples with ZkClient org.apache.helix.manager.zk.ZkClient used on opensource projects

Example 6 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project pinot by linkedin.

the class DeleteOverlappingSegmentsInPinot method deleteOverlappingSegments.

public static boolean deleteOverlappingSegments(String zkUrl, String zkCluster, String tableName) {
    boolean updateSuccessful = false;
    if (!tableName.endsWith("_OFFLINE")) {
        tableName = tableName + "_OFFLINE";
    }
    ZkClient zkClient = new ZkClient(zkUrl);
    ZNRecordSerializer zkSerializer = new ZNRecordSerializer();
    zkClient.setZkSerializer(zkSerializer);
    BaseDataAccessor<ZNRecord> baseDataAccessor = new ZkBaseDataAccessor<>(zkClient);
    HelixDataAccessor helixDataAccessor = new ZKHelixDataAccessor(zkCluster, baseDataAccessor);
    Builder keyBuilder = helixDataAccessor.keyBuilder();
    PropertyKey idealStateKey = keyBuilder.idealStates(tableName);
    PropertyKey externalViewKey = keyBuilder.externalView(tableName);
    IdealState currentIdealState = helixDataAccessor.getProperty(idealStateKey);
    byte[] serializeIS = zkSerializer.serialize(currentIdealState.getRecord());
    String name = tableName + ".idealstate." + System.currentTimeMillis();
    File outputFile = new File("/tmp", name);
    try (FileOutputStream fileOutputStream = new FileOutputStream(outputFile)) {
        IOUtils.write(serializeIS, fileOutputStream);
    } catch (IOException e) {
        LOG.error("Exception in delete overlapping segments", e);
        return updateSuccessful;
    }
    LOG.info("Saved current idealstate to {}", outputFile);
    IdealState newIdealState;
    do {
        newIdealState = computeNewIdealStateAfterDeletingOverlappingSegments(helixDataAccessor, idealStateKey);
        LOG.info("Updating IdealState");
        updateSuccessful = helixDataAccessor.getBaseDataAccessor().set(idealStateKey.getPath(), newIdealState.getRecord(), newIdealState.getRecord().getVersion(), AccessOption.PERSISTENT);
        if (updateSuccessful) {
            int numSegmentsDeleted = currentIdealState.getPartitionSet().size() - newIdealState.getPartitionSet().size();
            LOG.info("Successfully updated IdealState: Removed segments: {}", (numSegmentsDeleted));
        }
    } while (!updateSuccessful);
    try {
        while (true) {
            Thread.sleep(10000);
            ExternalView externalView = helixDataAccessor.getProperty(externalViewKey);
            IdealState idealState = helixDataAccessor.getProperty(idealStateKey);
            Set<String> evPartitionSet = externalView.getPartitionSet();
            Set<String> isPartitionSet = idealState.getPartitionSet();
            if (evPartitionSet.equals(isPartitionSet)) {
                LOG.info("Table {} has reached stable state. i.e segments in external view match idealstates", tableName);
                break;
            }
        }
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    return updateSuccessful;
}

Example 7 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project pinot by linkedin.

the class PinotThirdEyeClient method fromZookeeper.

/**
   * Creates a new PinotThirdEyeClient using the clusterName and broker tag
   * @param controllerHost
   * @param controllerPort
   * @param zkUrl
   * @param clusterName : required property
   * @return
   */
public static PinotThirdEyeClient fromZookeeper(String controllerHost, int controllerPort, String zkUrl, String clusterName) {
    ZkClient zkClient = new ZkClient(zkUrl);
    zkClient.setZkSerializer(new ZNRecordSerializer());
    zkClient.waitUntilConnected();
    PinotThirdEyeClient pinotThirdEyeClient = new PinotThirdEyeClient(controllerHost, controllerPort);
    LOG.info("Created PinotThirdEyeClient to zookeeper: {} controller: {}:{}", zkUrl, controllerHost, controllerPort);
    return pinotThirdEyeClient;
}

Example 8 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project helix by apache.

the class IdealStateExample method main.

public static void main(String[] args) throws Exception {
    if (args.length < 3) {
        System.err.println("USAGE: IdealStateExample zkAddress clusterName idealStateMode (FULL_AUTO, SEMI_AUTO, or CUSTOMIZED) idealStateJsonFile (required for CUSTOMIZED mode)");
        System.exit(1);
    }
    final String zkAddr = args[0];
    final String clusterName = args[1];
    final String idealStateRebalancerModeStr = args[2].toUpperCase();
    String idealStateJsonFile = null;
    RebalanceMode idealStateRebalancerMode = RebalanceMode.valueOf(idealStateRebalancerModeStr);
    if (idealStateRebalancerMode == RebalanceMode.CUSTOMIZED) {
        if (args.length < 4) {
            System.err.println("Missng idealStateJsonFile for CUSTOMIZED ideal state mode");
            System.exit(1);
        }
        idealStateJsonFile = args[3];
    }
    // add cluster {clusterName}
    ZkClient zkclient = new ZkClient(zkAddr, ZkClient.DEFAULT_SESSION_TIMEOUT, ZkClient.DEFAULT_CONNECTION_TIMEOUT, new ZNRecordSerializer());
    ZKHelixAdmin admin = new ZKHelixAdmin(zkclient);
    admin.addCluster(clusterName, true);
    // add MasterSlave state mode definition
    StateModelConfigGenerator generator = new StateModelConfigGenerator();
    admin.addStateModelDef(clusterName, "MasterSlave", new StateModelDefinition(generator.generateConfigForMasterSlave()));
    // add 3 participants: "localhost:{12918, 12919, 12920}"
    for (int i = 0; i < 3; i++) {
        int port = 12918 + i;
        InstanceConfig config = new InstanceConfig("localhost_" + port);
        config.setHostName("localhost");
        config.setPort(Integer.toString(port));
        config.setInstanceEnabled(true);
        admin.addInstance(clusterName, config);
    }
    // add resource "TestDB" which has 4 partitions and uses MasterSlave state model
    String resourceName = "TestDB";
    if (idealStateRebalancerMode == RebalanceMode.SEMI_AUTO || idealStateRebalancerMode == RebalanceMode.FULL_AUTO) {
        admin.addResource(clusterName, resourceName, 4, "MasterSlave", idealStateRebalancerModeStr);
        // rebalance resource "TestDB" using 3 replicas
        admin.rebalance(clusterName, resourceName, 3);
    } else if (idealStateRebalancerMode == RebalanceMode.CUSTOMIZED) {
        admin.addIdealState(clusterName, resourceName, idealStateJsonFile);
    }
    // start helix controller
    new Thread(new Runnable() {

        @Override
        public void run() {
            try {
                HelixControllerMain.main(new String[] { "--zkSvr", zkAddr, "--cluster", clusterName });
            } catch (Exception e) {
                // TODO Auto-generated catch block
                e.printStackTrace();
            }
        }
    }).start();
    // start 3 dummy participants
    for (int i = 0; i < 3; i++) {
        int port = 12918 + i;
        final String instanceName = "localhost_" + port;
        new Thread(new Runnable() {

            @Override
            public void run() {
                DummyParticipant.main(new String[] { zkAddr, clusterName, instanceName });
            }
        }).start();
    }
}

Example 9 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project helix by apache.

the class ZkPathResource method get.

@Override
public Representation get() {
    StringRepresentation presentation = null;
    String zkPath = getZKPath();
    try {
        ZkClient zkClient = (ZkClient) getContext().getAttributes().get(RestAdminApplication.ZKCLIENT);
        ZNRecord result = readZkDataStatAndChild(zkPath, zkClient);
        presentation = new StringRepresentation(ClusterRepresentationUtil.ZNRecordToJson(result), MediaType.APPLICATION_JSON);
    } catch (Exception e) {
        String error = ClusterRepresentationUtil.getErrorAsJsonStringFromException(e);
        presentation = new StringRepresentation(error, MediaType.APPLICATION_JSON);
        LOG.error("Error in read zkPath: " + zkPath, e);
    }
    return presentation;
}

Example 10 with ZkClient

use of org.apache.helix.manager.zk.ZkClient in project helix by apache.

the class ClusterResource method delete.

/**
 * Remove a cluster
 * <p>
 * Usage: <code> curl -X DELETE http://{host:port}/clusters/{clusterName}
 */
@Override
public Representation delete() {
    try {
        String clusterName = ResourceUtil.getAttributeFromRequest(getRequest(), ResourceUtil.RequestKey.CLUSTER_NAME);
        ZkClient zkClient = ResourceUtil.getAttributeFromCtx(getContext(), ResourceUtil.ContextKey.ZKCLIENT);
        ClusterSetup setupTool = new ClusterSetup(zkClient);
        setupTool.deleteCluster(clusterName);
        getResponse().setStatus(Status.SUCCESS_OK);
    } catch (Exception e) {
        getResponse().setEntity(ClusterRepresentationUtil.getErrorAsJsonStringFromException(e), MediaType.APPLICATION_JSON);
        getResponse().setStatus(Status.SUCCESS_OK);
    }
    return null;
}