Examples with Pair com.twitter.heron.common.basics.Pair used on opensource projects

Example 21 with Pair

use of com.twitter.heron.common.basics.Pair in project heron by twitter.

the class UpdateDryRunRenderTest method setUp.

@Before
public void setUp() throws Exception {
    // set up original packing plan
    final String COMPONENT_A = "exclaim1";
    final String COMPONENT_B = "word";
    ContainerPlan containerPlanA = PackingTestUtils.testContainerPlan(1, new Pair<>(COMPONENT_A, 1), new Pair<>(COMPONENT_A, 3), new Pair<>(COMPONENT_B, 5));
    ContainerPlan containerPlanB = PackingTestUtils.testContainerPlan(2, new Pair<>(COMPONENT_A, 2), new Pair<>(COMPONENT_A, 4), new Pair<>(COMPONENT_B, 6));
    Set<ContainerPlan> containerPlans = new HashSet<>();
    containerPlans.add(containerPlanA);
    containerPlans.add(containerPlanB);
    originalPlan = new PackingPlan("ORIG", containerPlans);
    // setup new packing plan A: word:1, exclaim1:9
    Set<ContainerPlan> containerPlansA = new HashSet<>();
    containerPlansA.add(containerPlanA);
    containerPlansA.add(PackingTestUtils.testContainerPlan(2, new Pair<>(COMPONENT_A, 4), new Pair<>(COMPONENT_A, 2), new Pair<>(COMPONENT_A, 6)));
    containerPlansA.add(PackingTestUtils.testContainerPlan(3, new Pair<>(COMPONENT_A, 7), new Pair<>(COMPONENT_A, 8), new Pair<>(COMPONENT_A, 9)));
    containerPlansA.add(PackingTestUtils.testContainerPlan(4, new Pair<>(COMPONENT_A, 10)));
    newPlanA = new PackingPlan("A", containerPlansA);
    // setup new packing plan B: word:1, exclaim1:1
    Set<ContainerPlan> containerPlansB = new HashSet<>();
    containerPlansB.add(PackingTestUtils.testContainerPlan(1, new Pair<>(COMPONENT_A, 3), new Pair<>(COMPONENT_B, 5)));
    newPlanB = new PackingPlan("B", containerPlansB);
}

Example 22 with Pair

use of com.twitter.heron.common.basics.Pair in project incubator-heron by apache.

the class TopologyResource method submit.

@POST
@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@SuppressWarnings({ "IllegalCatch", "JavadocMethod" })
public Response submit(FormDataMultiPart form) throws IOException {
    // verify that all we have all the required params
    final List<String> missingDataKeys = verifyKeys(form.getFields().keySet(), REQUIRED_SUBMIT_TOPOLOGY_PARAMS);
    if (!missingDataKeys.isEmpty()) {
        // return error since we are missing required parameters
        final String message = String.format("Validation failed missing required params: %s", missingDataKeys.toString());
        return Response.status(HTTP_UNPROCESSABLE_ENTITY_CODE).type(MediaType.APPLICATION_JSON).entity(Utils.createValidationError(message, missingDataKeys)).build();
    }
    final String cluster = Forms.getString(form, FORM_KEY_CLUSTER);
    if (!doesClusterMatch(cluster)) {
        return Response.status(HTTP_UNPROCESSABLE_ENTITY_CODE).type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(String.format("Unknown cluster %s expecting '%s'", cluster, getCluster()))).build();
    }
    final String topologyName = Forms.getString(form, FORM_KEY_NAME);
    final String role = Forms.getString(form, FORM_KEY_ROLE);
    final String environment = Forms.getString(form, FORM_KEY_ENVIRONMENT, Constants.DEFAULT_HERON_ENVIRONMENT);
    final String user = Forms.getString(form, FORM_KEY_USER, role);
    // submit overrides are passed key=value
    final Map<String, String> submitOverrides = getSubmitOverrides(form);
    final String topologyDirectory = Files.createTempDirectory(topologyName).toFile().getAbsolutePath();
    try {
        // upload the topology definition file to the topology directory
        final FormDataBodyPart definitionFilePart = form.getField(FORM_KEY_DEFINITION);
        final File topologyDefinitionFile = Forms.uploadFile(definitionFilePart, topologyDirectory);
        // upload the topology binary file to the topology directory
        final FormDataBodyPart topologyFilePart = form.getField(FORM_KEY_TOPOLOGY);
        final File topologyBinaryFile = Forms.uploadFile(topologyFilePart, topologyDirectory);
        final boolean isDryRun = form.getFields().containsKey(PARAM_DRY_RUN);
        // copy configuration files to the sandbox config location
        // topology-dir/<default-heron-sandbox-config>
        FileHelper.copyDirectory(Paths.get(getConfigurationDirectory()), Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG));
        final java.nio.file.Path overridesPath = Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG, Constants.OVERRIDE_FILE);
        // copy override file into topology configuration directory
        FileHelper.copy(Paths.get(getConfigurationOverridePath()), overridesPath);
        // apply submit overrides
        ConfigUtils.applyOverrides(overridesPath, submitOverrides);
        // apply overrides to state manager config
        ConfigUtils.applyOverridesToStateManagerConfig(overridesPath, Paths.get(topologyDirectory, Constants.DEFAULT_HERON_SANDBOX_CONFIG, Constants.STATE_MANAGER_FILE));
        // create tar file from the contents of the topology directory
        final File topologyPackageFile = Paths.get(topologyDirectory, TOPOLOGY_TAR_GZ_FILENAME).toFile();
        FileHelper.createTarGz(topologyPackageFile, FileHelper.getChildren(topologyDirectory));
        // create configs
        Config topologyConfig = ConfigUtils.getTopologyConfig(topologyPackageFile.getAbsolutePath(), topologyBinaryFile.getName(), topologyDefinitionFile.getAbsolutePath());
        List<Pair<String, Object>> val = new LinkedList<>();
        for (Map.Entry<String, Object> entry : topologyConfig.getEntrySet()) {
            val.add(Pair.create(entry.getKey(), entry.getValue()));
        }
        val.addAll(Arrays.asList(Pair.create(Key.CLUSTER.value(), cluster), Pair.create(Key.TOPOLOGY_NAME.value(), topologyName), Pair.create(Key.ROLE.value(), role), Pair.create(Key.ENVIRON.value(), environment), Pair.create(Key.SUBMIT_USER.value(), user), Pair.create(Key.DRY_RUN.value(), isDryRun)));
        final Config config = createConfig(val, submitOverrides);
        // submit the topology
        getActionFactory().createSubmitAction(config, topologyPackageFile.getAbsolutePath(), topologyBinaryFile.getName(), topologyDefinitionFile.getAbsolutePath()).execute();
        return Response.created(URI.create(String.format(TOPOLOGY_PATH_FORMAT, cluster, role, environment, topologyName))).type(MediaType.APPLICATION_JSON).entity(createdResponse(cluster, role, environment, topologyName)).build();
    } catch (SubmitDryRunResponse response) {
        return createDryRunResponse(response, Forms.getString(form, PARAM_DRY_RUN_FORMAT, DEFAULT_DRY_RUN_FORMAT));
    } catch (Exception ex) {
        LOG.error("error submitting topology {}", topologyName, ex);
        return Response.serverError().type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(ex.getMessage())).build();
    } finally {
        FileUtils.deleteDir(topologyDirectory);
    }
}

Example 23 with Pair

use of com.twitter.heron.common.basics.Pair in project incubator-heron by apache.

the class TopologyResource method restart.

@POST
@Path("/{cluster}/{role}/{environment}/{name}/restart")
@Produces(MediaType.APPLICATION_JSON)
@SuppressWarnings("IllegalCatch")
public Response restart(@PathParam("cluster") final String cluster, @PathParam("role") final String role, @PathParam("environment") final String environment, @PathParam("name") final String name, @DefaultValue("-1") @FormParam("container_id") final int containerId) {
    try {
        final List<Pair<String, Object>> keyValues = new ArrayList<>(Arrays.asList(Pair.create(Key.CLUSTER.value(), cluster), Pair.create(Key.ROLE.value(), role), Pair.create(Key.ENVIRON.value(), environment), Pair.create(Key.TOPOLOGY_NAME.value(), name), Pair.create(Key.TOPOLOGY_CONTAINER_ID.value(), containerId)));
        final Config config = createConfig(keyValues);
        getActionFactory().createRuntimeAction(config, ActionType.RESTART).execute();
        return Response.ok().type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(String.format("%s restarted", name))).build();
    } catch (Exception ex) {
        LOG.error("error restarting topology {}", name, ex);
        return Response.serverError().type(MediaType.APPLICATION_JSON).entity(Utils.createMessage(ex.getMessage())).build();
    }
}

Example 24 with Pair

use of com.twitter.heron.common.basics.Pair in project incubator-heron by apache.

the class ZkUtils method setupZkTunnel.

/**
 * Setup the tunnel if needed
 *
 * @param config basing on which we setup the tunnel process
 * @return Pair of (zk_format_connectionString, List of tunneled processes)
 */
public static Pair<String, List<Process>> setupZkTunnel(Config config, NetworkUtils.TunnelConfig tunnelConfig) {
    // Remove all spaces
    String connectionString = Context.stateManagerConnectionString(config).replaceAll("\\s+", "");
    List<Pair<InetSocketAddress, Process>> ret = new ArrayList<>();
    // For zookeeper, connection String can be a list of host:port, separated by comma
    String[] endpoints = connectionString.split(",");
    for (String endpoint : endpoints) {
        InetSocketAddress address = NetworkUtils.getInetSocketAddress(endpoint);
        // Get the tunnel process if needed
        Pair<InetSocketAddress, Process> pair = NetworkUtils.establishSSHTunnelIfNeeded(address, tunnelConfig, NetworkUtils.TunnelType.PORT_FORWARD);
        ret.add(pair);
    }
    // Construct the new ConnectionString and tunnel processes
    StringBuilder connectionStringBuilder = new StringBuilder();
    List<Process> tunnelProcesses = new ArrayList<>();
    String delim = "";
    for (Pair<InetSocketAddress, Process> pair : ret) {
        // Join the list of String with comma as delim
        if (pair.first != null) {
            connectionStringBuilder.append(delim).append(pair.first.getHostName()).append(":").append(pair.first.getPort());
            delim = ",";
            // If tunneled
            if (pair.second != null) {
                tunnelProcesses.add(pair.second);
            }
        }
    }
    String newConnectionString = connectionStringBuilder.toString();
    return new Pair<String, List<Process>>(newConnectionString, tunnelProcesses);
}

Example 25 with Pair

use of com.twitter.heron.common.basics.Pair in project incubator-heron by apache.

the class CommonPackingTests method testScaleDownOneComponentRemoveContainer.

/**
 * Test the scenario where scaling down removes instances from containers that are most imbalanced
 * (i.e., tending towards homogeneity) first. If there is a tie (e.g. AABB, AB), chooses from the
 * container with the fewest instances, to favor ultimately removing  containers. If there is
 * still a tie, favor removing from higher numbered containers
 */
@Test
public void testScaleDownOneComponentRemoveContainer() throws Exception {
    @SuppressWarnings({ "unchecked", "rawtypes" }) Pair<Integer, InstanceId>[] initialComponentInstances = new Pair[] { new Pair<>(1, new InstanceId(A, 1, 0)), new Pair<>(1, new InstanceId(A, 2, 1)), new Pair<>(1, new InstanceId(B, 3, 0)), new Pair<>(3, new InstanceId(B, 4, 1)), new Pair<>(3, new InstanceId(B, 5, 2)), new Pair<>(4, new InstanceId(B, 6, 3)), new Pair<>(4, new InstanceId(B, 7, 4)) };
    Map<String, Integer> componentChanges = new HashMap<>();
    componentChanges.put(B, -2);
    @SuppressWarnings({ "unchecked", "rawtypes" }) Pair<Integer, InstanceId>[] expectedComponentInstances = new Pair[] { new Pair<>(1, new InstanceId(A, 1, 0)), new Pair<>(1, new InstanceId(A, 2, 1)), new Pair<>(1, new InstanceId(B, 3, 0)), new Pair<>(3, new InstanceId(B, 4, 1)), new Pair<>(3, new InstanceId(B, 5, 2)) };
    doScaleDownTest(initialComponentInstances, componentChanges, expectedComponentInstances);
}