use of org.apache.helix.manager.zk.ZNRecordSerializer in project ambry by linkedin.
the class HelixVcrPopulateToolTest method beforeTest.
@Before
public void beforeTest() throws Exception {
try (InputStream input = new ByteArrayInputStream(configData.getBytes())) {
config = new ObjectMapper().readValue(input, HelixVcrUtil.VcrHelixConfig.class);
} catch (IOException ex) {
throw new IllegalStateException("Could not load config from config data: " + configData);
}
srcZkInfo = new com.github.ambry.utils.TestUtils.ZkInfo(TestUtils.getTempDir("helixVcr"), "DC1", (byte) 1, SRC_ZK_SERVER_PORT, true);
SRC_ZK_CONNECT_STRING = SRC_ZK_SERVER_HOSTNAME + ":" + SRC_ZK_SERVER_PORT;
zkClient = SharedZkClientFactory.getInstance().buildZkClient(new HelixZkClient.ZkConnectionConfig(SRC_ZK_CONNECT_STRING));
zkClient.setZkSerializer(new ZNRecordSerializer());
clusterSetup = new ClusterSetup(zkClient);
clusterSetup.addCluster(SRC_CLUSTER_NAME, true);
srcHelixAdmin = new HelixAdminFactory().getHelixAdmin(SRC_ZK_CONNECT_STRING);
String resourceName = "1";
Set<String> partitionSet = new HashSet<>();
for (int i = 0; i < 100; i++) {
partitionSet.add(Integer.toString(i));
}
IdealState idealState = HelixVcrUtil.buildIdealState(resourceName, partitionSet, config.getIdealStateConfigFields());
srcHelixAdmin.addResource(SRC_CLUSTER_NAME, resourceName, idealState);
}
use of org.apache.helix.manager.zk.ZNRecordSerializer in project ambry by linkedin.
the class HelixTaskWorkflowManagerTool method main.
/**
* Runs the cluster wide aggregation tool
* @param args arguments specifying config file. For example: --propsFile /path/AggregationToolConfig
* @throws Exception in case of any error.
*/
public static void main(String[] args) throws Exception {
VerifiableProperties verifiableProperties = ToolUtils.getVerifiableProperties(args);
AggregationToolConfig config = new AggregationToolConfig(verifiableProperties);
Map<String, ClusterMapUtils.DcZkInfo> dataCenterToZKAddress = ClusterMapUtils.parseDcJsonAndPopulateDcInfo(Utils.readStringFromFile(config.zkLayoutFilePath));
String clusterName = config.clusterName;
String workflowName = config.workflowName;
long recurrentIntervalInMinutes = config.recurrentIntervalInMinutes;
boolean isDelete = config.deleteSpecifiedWorkflow;
boolean isRecurrentWorkflow = recurrentIntervalInMinutes != Utils.Infinite_Time;
for (ClusterMapUtils.DcZkInfo zkInfo : dataCenterToZKAddress.values()) {
// If there are multiple ZK endpoints in same dc, we trigger stats aggregation for each of them.
for (String zkAddress : zkInfo.getZkConnectStrs()) {
ZkClient zkClient = new ZkClient(zkAddress, SESSION_TIMEOUT, CONNECTION_TIMEOUT, new ZNRecordSerializer());
TaskDriver taskDriver = new TaskDriver(zkClient, clusterName);
if (isDelete) {
try {
taskDriver.waitToStop(workflowName, TIME_OUT_MILLI_SEC);
taskDriver.delete(workflowName);
System.out.println(String.format("Successfully deleted the workflow: %s in cluster %s at %s", workflowName, clusterName, zkAddress));
} catch (Exception | Error e) {
System.out.println(String.format("Failed to delete %s. Workflow not found in cluster %s at %s", workflowName, clusterName, zkAddress));
}
} else {
Workflow.Builder workflowBuilder = new Workflow.Builder(workflowName);
try {
switch(config.taskType) {
case AGGREGATE_TASK:
buildAggregationTaskWorkflow(workflowBuilder, config, isRecurrentWorkflow);
break;
case DEPRECATED_CONTAINER_CLOUD_SYNC_TASK:
buildDeprecatedContainerCloudSyncTaskWorkflow(workflowBuilder);
break;
default:
throw new IllegalArgumentException("Invalid task type: " + config.taskType);
}
if (isRecurrentWorkflow) {
workflowBuilder.setScheduleConfig(ScheduleConfig.recurringFromNow(TimeUnit.MINUTES, recurrentIntervalInMinutes));
workflowBuilder.setExpiry(TimeUnit.MINUTES.toMillis(recurrentIntervalInMinutes));
}
Workflow workflow = workflowBuilder.build();
taskDriver.start(workflow);
System.out.println(String.format("%s started successfully in cluster %s at %s", workflowName, clusterName, zkAddress));
} catch (Exception | Error e) {
System.out.println(String.format("Failed to start %s in cluster %s at %s", workflowName, clusterName, zkAddress));
}
}
}
}
}
use of org.apache.helix.manager.zk.ZNRecordSerializer in project helix by apache.
the class ClusterSetup method getConstraints.
/**
* get constraints associated with given type
* @param constraintType : constraint-type. e.g. MESSAGE_CONSTRAINT
* @return json-formated constraints
*/
public String getConstraints(String clusterName, String constraintType) {
if (clusterName == null || constraintType == null) {
throw new IllegalArgumentException("fail to get constraint. missing clusterName|constraintType");
}
ConstraintType type = ConstraintType.valueOf(constraintType);
ClusterConstraints constraints = _admin.getConstraints(clusterName, type);
return new String(constraints.serialize(new ZNRecordSerializer()));
}
use of org.apache.helix.manager.zk.ZNRecordSerializer in project helix by apache.
the class ClusterSetup method addIdealState.
// TODO: remove this. has moved to ZkHelixAdmin
public void addIdealState(String clusterName, String resourceName, String idealStateFile) throws IOException {
ZNRecord idealStateRecord = (ZNRecord) (new ZNRecordSerializer().deserialize(readFile(idealStateFile)));
if (idealStateRecord.getId() == null || !idealStateRecord.getId().equals(resourceName)) {
throw new IllegalArgumentException("ideal state must have same id as resource name");
}
_admin.setResourceIdealState(clusterName, resourceName, new IdealState(idealStateRecord));
}
use of org.apache.helix.manager.zk.ZNRecordSerializer in project helix by apache.
the class ClusterSetup method processCommandLineArgs.
public static int processCommandLineArgs(String[] cliArgs) throws Exception {
CommandLineParser cliParser = new GnuParser();
Options cliOptions = constructCommandLineOptions();
CommandLine cmd = null;
try {
cmd = cliParser.parse(cliOptions, cliArgs);
} catch (ParseException pe) {
System.err.println("CommandLineClient: failed to parse command-line options: " + pe.toString());
printUsage(cliOptions);
System.exit(1);
}
ClusterSetup setupTool = new ClusterSetup(cmd.getOptionValue(zkServerAddress));
if (cmd.hasOption(addCluster)) {
String clusterName = cmd.getOptionValue(addCluster);
setupTool.addCluster(clusterName, false);
return 0;
}
if (cmd.hasOption(activateCluster)) {
String clusterName = cmd.getOptionValues(activateCluster)[0];
String grandCluster = cmd.getOptionValues(activateCluster)[1];
boolean enable = Boolean.parseBoolean(cmd.getOptionValues(activateCluster)[2]);
setupTool.activateCluster(clusterName, grandCluster, enable);
return 0;
}
if (cmd.hasOption(dropCluster)) {
String clusterName = cmd.getOptionValue(dropCluster);
setupTool.deleteCluster(clusterName);
return 0;
}
if (cmd.hasOption(addInstance)) {
String clusterName = cmd.getOptionValues(addInstance)[0];
String instanceAddressInfo = cmd.getOptionValues(addInstance)[1];
String[] instanceAddresses = instanceAddressInfo.split(";");
setupTool.addInstancesToCluster(clusterName, instanceAddresses);
return 0;
}
if (cmd.hasOption(addResource)) {
String clusterName = cmd.getOptionValues(addResource)[0];
String resourceName = cmd.getOptionValues(addResource)[1];
int partitions = Integer.parseInt(cmd.getOptionValues(addResource)[2]);
String stateModelRef = cmd.getOptionValues(addResource)[3];
String modeValue = RebalanceMode.SEMI_AUTO.toString();
if (cmd.hasOption(mode)) {
modeValue = cmd.getOptionValues(mode)[0];
}
int bucketSizeVal = 0;
if (cmd.hasOption(bucketSize)) {
bucketSizeVal = Integer.parseInt(cmd.getOptionValues(bucketSize)[0]);
}
int maxPartitionsPerNodeVal = -1;
if (cmd.hasOption(maxPartitionsPerNode)) {
maxPartitionsPerNodeVal = Integer.parseInt(cmd.getOptionValues(maxPartitionsPerNode)[0]);
}
setupTool.addResourceToCluster(clusterName, resourceName, partitions, stateModelRef, modeValue, bucketSizeVal, maxPartitionsPerNodeVal);
return 0;
}
if (cmd.hasOption(rebalance)) {
String clusterName = cmd.getOptionValues(rebalance)[0];
String resourceName = cmd.getOptionValues(rebalance)[1];
int replicas = Integer.parseInt(cmd.getOptionValues(rebalance)[2]);
String keyPrefixVal = "";
String instanceGroupTagVal = "";
if (cmd.hasOption(resourceKeyPrefix)) {
keyPrefixVal = cmd.getOptionValue(resourceKeyPrefix);
}
if (cmd.hasOption(instanceGroupTag)) {
instanceGroupTagVal = cmd.getOptionValue(instanceGroupTag);
}
setupTool.rebalanceCluster(clusterName, resourceName, replicas, keyPrefixVal, instanceGroupTagVal);
return 0;
}
if (cmd.hasOption(expandCluster)) {
String clusterName = cmd.getOptionValues(expandCluster)[0];
setupTool.expandCluster(clusterName);
return 0;
}
if (cmd.hasOption(expandResource)) {
String clusterName = cmd.getOptionValues(expandResource)[0];
String resourceName = cmd.getOptionValues(expandResource)[1];
setupTool.expandResource(clusterName, resourceName);
return 0;
}
if (cmd.hasOption(dropInstance)) {
String clusterName = cmd.getOptionValues(dropInstance)[0];
String instanceAddressInfo = cmd.getOptionValues(dropInstance)[1];
String[] instanceAddresses = instanceAddressInfo.split(";");
setupTool.dropInstancesFromCluster(clusterName, instanceAddresses);
return 0;
}
if (cmd.hasOption(listClusters)) {
List<String> clusters = setupTool.getClusterManagementTool().getClusters();
System.out.println("Existing clusters:");
for (String cluster : clusters) {
System.out.println(cluster);
}
return 0;
}
if (cmd.hasOption(listResources)) {
String clusterName = cmd.getOptionValue(listResources);
List<String> resourceNames = null;
if (cmd.hasOption(tag)) {
String tagValue = cmd.getOptionValues(tag)[0];
resourceNames = setupTool.getClusterManagementTool().getResourcesInClusterWithTag(clusterName, tagValue);
System.out.println("Existing resources in cluster " + clusterName + " with tag " + tagValue + " :");
} else {
resourceNames = setupTool.getClusterManagementTool().getResourcesInCluster(clusterName);
System.out.println("Existing resources in cluster " + clusterName + ":");
}
for (String resourceName : resourceNames) {
System.out.println(resourceName);
}
return 0;
} else if (cmd.hasOption(listClusterInfo)) {
String clusterName = cmd.getOptionValue(listClusterInfo);
List<String> resourceNames = setupTool.getClusterManagementTool().getResourcesInCluster(clusterName);
List<String> instances = setupTool.getClusterManagementTool().getInstancesInCluster(clusterName);
System.out.println("Existing resources in cluster " + clusterName + ":");
for (String resourceName : resourceNames) {
System.out.println(resourceName);
}
System.out.println("Instances in cluster " + clusterName + ":");
for (String InstanceName : instances) {
System.out.println(InstanceName);
}
return 0;
} else if (cmd.hasOption(listInstances)) {
String clusterName = cmd.getOptionValue(listInstances);
List<String> instances;
if (cmd.hasOption(tag)) {
String instanceTag = cmd.getOptionValues(tag)[0];
instances = setupTool.getClusterManagementTool().getInstancesInClusterWithTag(clusterName, instanceTag);
} else {
instances = setupTool.getClusterManagementTool().getInstancesInCluster(clusterName);
}
System.out.println("Instances in cluster " + clusterName + ":");
for (String instanceName : instances) {
System.out.println(instanceName);
}
return 0;
} else if (cmd.hasOption(listInstanceInfo)) {
String clusterName = cmd.getOptionValues(listInstanceInfo)[0];
String instanceName = cmd.getOptionValues(listInstanceInfo)[1];
InstanceConfig config = setupTool.getClusterManagementTool().getInstanceConfig(clusterName, instanceName);
String result = new String(config.serialize(new ZNRecordSerializer()));
System.out.println("InstanceConfig: " + result);
return 0;
} else if (cmd.hasOption(listResourceInfo)) {
// print out partition number, resource name and replication number
// Also the ideal states and current states
String clusterName = cmd.getOptionValues(listResourceInfo)[0];
String resourceName = cmd.getOptionValues(listResourceInfo)[1];
IdealState idealState = setupTool.getClusterManagementTool().getResourceIdealState(clusterName, resourceName);
ExternalView externalView = setupTool.getClusterManagementTool().getResourceExternalView(clusterName, resourceName);
if (idealState != null) {
System.out.println("IdealState for " + resourceName + ":");
System.out.println(new String(idealState.serialize(new ZNRecordSerializer())));
} else {
System.out.println("No idealState for " + resourceName);
}
System.out.println();
if (externalView != null) {
System.out.println("ExternalView for " + resourceName + ":");
System.out.println(new String(externalView.serialize(new ZNRecordSerializer())));
} else {
System.out.println("No externalView for " + resourceName);
}
return 0;
} else if (cmd.hasOption(listPartitionInfo)) {
// print out where the partition master / slaves locates
String clusterName = cmd.getOptionValues(listPartitionInfo)[0];
String resourceName = cmd.getOptionValues(listPartitionInfo)[1];
String partitionName = cmd.getOptionValues(listPartitionInfo)[2];
IdealState idealState = setupTool.getClusterManagementTool().getResourceIdealState(clusterName, resourceName);
ExternalView externalView = setupTool.getClusterManagementTool().getResourceExternalView(clusterName, resourceName);
if (idealState != null) {
ZNRecord partInfo = new ZNRecord(resourceName + "/" + partitionName);
ZNRecord idealStateRec = idealState.getRecord();
partInfo.setSimpleFields(idealStateRec.getSimpleFields());
if (idealStateRec.getMapField(partitionName) != null) {
partInfo.setMapField(partitionName, idealStateRec.getMapField(partitionName));
}
if (idealStateRec.getListField(partitionName) != null) {
partInfo.setListField(partitionName, idealStateRec.getListField(partitionName));
}
System.out.println("IdealState for " + resourceName + "/" + partitionName + ":");
System.out.println(new String(new ZNRecordSerializer().serialize(partInfo)));
} else {
System.out.println("No idealState for " + resourceName + "/" + partitionName);
}
System.out.println();
if (externalView != null) {
ZNRecord partInfo = new ZNRecord(resourceName + "/" + partitionName);
ZNRecord extViewRec = externalView.getRecord();
partInfo.setSimpleFields(extViewRec.getSimpleFields());
if (extViewRec.getMapField(partitionName) != null) {
partInfo.setMapField(partitionName, extViewRec.getMapField(partitionName));
}
if (extViewRec.getListField(partitionName) != null) {
partInfo.setListField(partitionName, extViewRec.getListField(partitionName));
}
System.out.println("ExternalView for " + resourceName + "/" + partitionName + ":");
System.out.println(new String(new ZNRecordSerializer().serialize(partInfo)));
} else {
System.out.println("No externalView for " + resourceName + "/" + partitionName);
}
return 0;
} else if (cmd.hasOption(enableInstance)) {
String clusterName = cmd.getOptionValues(enableInstance)[0];
String instanceName = cmd.getOptionValues(enableInstance)[1];
if (instanceName.contains(":")) {
instanceName = instanceName.replaceAll(":", "_");
}
boolean enabled = Boolean.parseBoolean(cmd.getOptionValues(enableInstance)[2].toLowerCase());
setupTool.getClusterManagementTool().enableInstance(clusterName, instanceName, enabled);
return 0;
} else if (cmd.hasOption(enableResource)) {
String clusterName = cmd.getOptionValues(enableResource)[0];
String resourceName = cmd.getOptionValues(enableResource)[1];
boolean enabled = Boolean.parseBoolean(cmd.getOptionValues(enableResource)[2].toLowerCase());
if (cmd.hasOption(tag)) {
String resourceTag = cmd.getOptionValues(tag)[0];
setupTool.enableResource(clusterName, resourceName, resourceTag, enabled);
} else {
setupTool.getClusterManagementTool().enableResource(clusterName, resourceName, enabled);
}
} else if (cmd.hasOption(enablePartition)) {
String[] args = cmd.getOptionValues(enablePartition);
boolean enabled = Boolean.parseBoolean(args[0].toLowerCase());
String clusterName = args[1];
String instanceName = args[2];
String resourceName = args[3];
List<String> partitionNames = Arrays.asList(Arrays.copyOfRange(args, 4, args.length));
setupTool.getClusterManagementTool().enablePartition(enabled, clusterName, instanceName, resourceName, partitionNames);
return 0;
} else if (cmd.hasOption(resetPartition)) {
String[] args = cmd.getOptionValues(resetPartition);
String clusterName = args[0];
String instanceName = args[1];
String resourceName = args[2];
List<String> partitionNames = Arrays.asList(Arrays.copyOfRange(args, 3, args.length));
setupTool.getClusterManagementTool().resetPartition(clusterName, instanceName, resourceName, partitionNames);
return 0;
} else if (cmd.hasOption(resetInstance)) {
String[] args = cmd.getOptionValues(resetInstance);
String clusterName = args[0];
List<String> instanceNames = Arrays.asList(Arrays.copyOfRange(args, 1, args.length));
setupTool.getClusterManagementTool().resetInstance(clusterName, instanceNames);
return 0;
} else if (cmd.hasOption(resetResource)) {
String[] args = cmd.getOptionValues(resetResource);
String clusterName = args[0];
List<String> resourceNames = Arrays.asList(Arrays.copyOfRange(args, 1, args.length));
setupTool.getClusterManagementTool().resetResource(clusterName, resourceNames);
return 0;
} else if (cmd.hasOption(enableCluster)) {
String[] params = cmd.getOptionValues(enableCluster);
String clusterName = params[0];
boolean enabled = Boolean.parseBoolean(params[1].toLowerCase());
setupTool.getClusterManagementTool().enableCluster(clusterName, enabled);
return 0;
} else if (cmd.hasOption(listStateModels)) {
String clusterName = cmd.getOptionValues(listStateModels)[0];
List<String> stateModels = setupTool.getClusterManagementTool().getStateModelDefs(clusterName);
System.out.println("Existing state models:");
for (String stateModel : stateModels) {
System.out.println(stateModel);
}
return 0;
} else if (cmd.hasOption(listStateModel)) {
String clusterName = cmd.getOptionValues(listStateModel)[0];
String stateModel = cmd.getOptionValues(listStateModel)[1];
StateModelDefinition stateModelDef = setupTool.getClusterManagementTool().getStateModelDef(clusterName, stateModel);
String result = new String(new ZNRecordSerializer().serialize(stateModelDef.getRecord()));
System.out.println("StateModelDefinition: " + result);
return 0;
} else if (cmd.hasOption(addStateModelDef)) {
String clusterName = cmd.getOptionValues(addStateModelDef)[0];
String stateModelFile = cmd.getOptionValues(addStateModelDef)[1];
ZNRecord stateModelRecord = (ZNRecord) (new ZNRecordSerializer().deserialize(readFile(stateModelFile)));
if (stateModelRecord.getId() == null || stateModelRecord.getId().length() == 0) {
throw new IllegalArgumentException("ZNRecord for state model definition must have an id");
}
setupTool.getClusterManagementTool().addStateModelDef(clusterName, stateModelRecord.getId(), new StateModelDefinition(stateModelRecord));
return 0;
} else if (cmd.hasOption(addIdealState)) {
String clusterName = cmd.getOptionValues(addIdealState)[0];
String resourceName = cmd.getOptionValues(addIdealState)[1];
String idealStateFile = cmd.getOptionValues(addIdealState)[2];
setupTool.addIdealState(clusterName, resourceName, idealStateFile);
return 0;
} else if (cmd.hasOption(dropResource)) {
String clusterName = cmd.getOptionValues(dropResource)[0];
String resourceName = cmd.getOptionValues(dropResource)[1];
setupTool.getClusterManagementTool().dropResource(clusterName, resourceName);
} else if (cmd.hasOption(swapInstance)) {
String clusterName = cmd.getOptionValues(swapInstance)[0];
String oldInstanceName = cmd.getOptionValues(swapInstance)[1];
String newInstanceName = cmd.getOptionValues(swapInstance)[2];
setupTool.swapInstance(clusterName, oldInstanceName, newInstanceName);
} else // set/get/remove config options
if (cmd.hasOption(setConfig)) {
String[] values = cmd.getOptionValues(setConfig);
ConfigScopeProperty type = ConfigScopeProperty.valueOf(values[0]);
String scopeArgs = values[1];
String keyValueMap = values[2];
setupTool.setConfig(type, scopeArgs, keyValueMap);
} else if (cmd.hasOption(getConfig)) {
String[] values = cmd.getOptionValues(getConfig);
ConfigScopeProperty type = ConfigScopeProperty.valueOf(values[0]);
String scopeArgs = values[1];
String keys = values[2];
setupTool.getConfig(type, scopeArgs, keys);
} else if (cmd.hasOption(removeConfig)) {
String[] values = cmd.getOptionValues(removeConfig);
ConfigScopeProperty type = ConfigScopeProperty.valueOf(values[0]);
String scoepArgs = values[1];
String keys = values[2];
setupTool.removeConfig(type, scoepArgs, keys);
} else // set/get/remove constraint options
if (cmd.hasOption(setConstraint)) {
String[] values = cmd.getOptionValues(setConstraint);
String clusterName = values[0];
String constraintType = values[1];
String constraintId = values[2];
String constraintAttributesMap = values[3];
setupTool.setConstraint(clusterName, constraintType, constraintId, constraintAttributesMap);
} else if (cmd.hasOption(getConstraints)) {
String[] values = cmd.getOptionValues(getConstraints);
String clusterName = values[0];
String constraintType = values[1];
setupTool.getConstraints(clusterName, constraintType);
} else if (cmd.hasOption(removeConstraint)) {
String[] values = cmd.getOptionValues(removeConstraint);
String clusterName = values[0];
String constraintType = values[1];
String constraintId = values[2];
setupTool.removeConstraint(clusterName, constraintType, constraintId);
} else if (cmd.hasOption(addInstanceTag)) {
String clusterName = cmd.getOptionValues(addInstanceTag)[0];
String instanceName = cmd.getOptionValues(addInstanceTag)[1];
String tag = cmd.getOptionValues(addInstanceTag)[2];
setupTool.getClusterManagementTool().addInstanceTag(clusterName, instanceName, tag);
} else if (cmd.hasOption(removeInstanceTag)) {
String clusterName = cmd.getOptionValues(removeInstanceTag)[0];
String instanceName = cmd.getOptionValues(removeInstanceTag)[1];
String tag = cmd.getOptionValues(removeInstanceTag)[2];
setupTool.getClusterManagementTool().removeInstanceTag(clusterName, instanceName, tag);
} else // help option
if (cmd.hasOption(help)) {
printUsage(cliOptions);
return 0;
} else if (cmd.hasOption(addResourceProperty)) {
String clusterName = cmd.getOptionValues(addResourceProperty)[0];
String resourceName = cmd.getOptionValues(addResourceProperty)[1];
String propertyKey = cmd.getOptionValues(addResourceProperty)[2];
String propertyVal = cmd.getOptionValues(addResourceProperty)[3];
setupTool.addResourceProperty(clusterName, resourceName, propertyKey, propertyVal);
return 0;
} else if (cmd.hasOption(removeResourceProperty)) {
String clusterName = cmd.getOptionValues(removeResourceProperty)[0];
String resourceName = cmd.getOptionValues(removeResourceProperty)[1];
String propertyKey = cmd.getOptionValues(removeResourceProperty)[2];
setupTool.removeResourceProperty(clusterName, resourceName, propertyKey);
return 0;
}
return 0;
}
Aggregations