Search in sources :

Example 1 with StopWatch

use of org.apache.commons.lang3.time.StopWatch in project azure-sdk-for-java by Azure.

the class ManageSimpleTrafficManager method runSample.

/**
     * Main function which runs the actual sample.
     * @param azure instance of the azure client
     * @return true if sample runs successfully
     */
public static boolean runSample(Azure azure) {
    final String rgName = SdkContext.randomResourceName("rgCOPD", 24);
    final String userName = "tirekicker";
    final String sshKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCfSPC2K7LZcFKEO+/t3dzmQYtrJFZNxOsbVgOVKietqHyvmYGHEC0J2wPdAqQ/63g/hhAEFRoyehM+rbeDri4txB3YFfnOK58jqdkyXzupWqXzOrlKY4Wz9SKjjN765+dqUITjKRIaAip1Ri137szRg71WnrmdP3SphTRlCx1Bk2nXqWPsclbRDCiZeF8QOTi4JqbmJyK5+0UqhqYRduun8ylAwKKQJ1NJt85sYIHn9f1Rfr6Tq2zS0wZ7DHbZL+zB5rSlAr8QyUdg/GQD+cmSs6LvPJKL78d6hMGk84ARtFo4A79ovwX/Fj01znDQkU6nJildfkaolH2rWFG/qttD azjava@javalib.com";
    final int vmCountPerRegion = 2;
    Set<Region> regions = new HashSet<>(Arrays.asList(Region.US_EAST, Region.US_WEST));
    try {
        //=============================================================
        // Create a shared resource group for all the resources so they can all be deleted together
        //
        ResourceGroup resourceGroup = azure.resourceGroups().define(rgName).withRegion(Region.US_EAST).create();
        System.out.println("Created a new resource group - " + resourceGroup.id());
        // Prepare a batch of creatable VM definitions to put behind the traffic manager
        //
        List<Creatable<VirtualMachine>> creatableVirtualMachines = new ArrayList<>();
        for (Region region : regions) {
            String linuxVMNamePrefix = SdkContext.randomResourceName("vm", 15);
            for (int i = 0; i < vmCountPerRegion; i++) {
                //=============================================================
                // Create a virtual machine in its own virtual network
                String vmName = String.format("%s-%d", linuxVMNamePrefix, i);
                Creatable<VirtualMachine> vmDefinition = azure.virtualMachines().define(vmName).withRegion(region).withExistingResourceGroup(resourceGroup).withNewPrimaryNetwork("10.0.0.0/29").withPrimaryPrivateIPAddressDynamic().withNewPrimaryPublicIPAddress(vmName).withPopularLinuxImage(KnownLinuxVirtualMachineImage.UBUNTU_SERVER_16_04_LTS).withRootUsername(userName).withSsh(sshKey).withSize(VirtualMachineSizeTypes.STANDARD_A1);
                creatableVirtualMachines.add(vmDefinition);
            }
        }
        //=============================================================
        // Create the VMs !!
        StopWatch stopwatch = new StopWatch();
        System.out.println("Creating the virtual machines...");
        stopwatch.start();
        Collection<VirtualMachine> virtualMachines = azure.virtualMachines().create(creatableVirtualMachines).values();
        stopwatch.stop();
        System.out.println(String.format("Created virtual machines in %d seconds.", stopwatch.getTime() / 1000));
        //=============================================================
        // Create 1 traffic manager profile
        //
        String trafficManagerName = SdkContext.randomResourceName("tra", 15);
        TrafficManagerProfile.DefinitionStages.WithEndpoint profileWithEndpoint = azure.trafficManagerProfiles().define(trafficManagerName).withExistingResourceGroup(resourceGroup).withLeafDomainLabel(trafficManagerName).withPerformanceBasedRouting();
        TrafficManagerProfile.DefinitionStages.WithCreate profileWithCreate = null;
        int routingPriority = 1;
        for (VirtualMachine vm : virtualMachines) {
            String endpointName = SdkContext.randomResourceName("ep", 15);
            profileWithCreate = profileWithEndpoint.defineAzureTargetEndpoint(endpointName).toResourceId(vm.getPrimaryPublicIPAddressId()).withRoutingPriority(routingPriority++).attach();
        }
        stopwatch.reset();
        stopwatch.start();
        TrafficManagerProfile trafficManagerProfile = profileWithCreate.create();
        stopwatch.stop();
        System.out.println(String.format("Created a traffic manager profile %s\n in %d seconds.", trafficManagerProfile.id(), stopwatch.getTime() / 1000));
        //=============================================================
        // Modify the traffic manager to use priority based routing
        //
        trafficManagerProfile.update().withPriorityBasedRouting().apply();
        System.out.println("Modified the traffic manager to use priority-based routing.");
        return true;
    } catch (Exception f) {
        System.out.println(f.getMessage());
        f.printStackTrace();
    } finally {
        try {
            System.out.println("Deleting Resource Group: " + rgName);
            azure.resourceGroups().deleteByName(rgName);
            System.out.println("Deleted Resource Group: " + rgName);
        } catch (NullPointerException npe) {
            System.out.println("Did not create any resources in Azure. No clean up is necessary");
        } catch (Exception g) {
            g.printStackTrace();
        }
    }
    return false;
}
Also used : TrafficManagerProfile(com.microsoft.azure.management.trafficmanager.TrafficManagerProfile) ArrayList(java.util.ArrayList) StopWatch(org.apache.commons.lang3.time.StopWatch) Region(com.microsoft.azure.management.resources.fluentcore.arm.Region) Creatable(com.microsoft.azure.management.resources.fluentcore.model.Creatable) ResourceGroup(com.microsoft.azure.management.resources.ResourceGroup) HashSet(java.util.HashSet) VirtualMachine(com.microsoft.azure.management.compute.VirtualMachine)

Example 2 with StopWatch

use of org.apache.commons.lang3.time.StopWatch in project azure-sdk-for-java by Azure.

the class ManageInternalLoadBalancer method runSample.

/**
     * Main function which runs the actual sample.
     * @param azure instance of the azure client
     * @return true if sample runs successfully
     */
public static boolean runSample(Azure azure) {
    final String rgName = SdkContext.randomResourceName("rgNEML", 15);
    final String vnetName = SdkContext.randomResourceName("vnet", 24);
    final String loadBalancerName3 = SdkContext.randomResourceName("intlb3" + "-", 18);
    final String loadBalancerName4 = SdkContext.randomResourceName("intlb4" + "-", 18);
    final String privateFrontEndName = loadBalancerName3 + "-BE";
    final String backendPoolName3 = loadBalancerName3 + "-BAP3";
    final int orcaleSQLNodePort = 1521;
    final String httpProbe = "httpProbe";
    final String tcpLoadBalancingRule = "tcpRule";
    final String natRule6000to22forVM3 = "nat6000to22forVM3";
    final String natRule6001to23forVM3 = "nat6001to23forVM3";
    final String natRule6002to22forVM4 = "nat6002to22forVM4";
    final String natRule6003to23forVM4 = "nat6003to23forVM4";
    final String networkInterfaceName3 = SdkContext.randomResourceName("nic3", 24);
    final String networkInterfaceName4 = SdkContext.randomResourceName("nic4", 24);
    final String availSetName = SdkContext.randomResourceName("av2", 24);
    final String vmName3 = SdkContext.randomResourceName("lVM3", 24);
    final String vmName4 = SdkContext.randomResourceName("lVM4", 24);
    final String userName = "tirekicker";
    final String sshKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCfSPC2K7LZcFKEO+/t3dzmQYtrJFZNxOsbVgOVKietqHyvmYGHEC0J2wPdAqQ/63g/hhAEFRoyehM+rbeDri4txB3YFfnOK58jqdkyXzupWqXzOrlKY4Wz9SKjjN765+dqUITjKRIaAip1Ri137szRg71WnrmdP3SphTRlCx1Bk2nXqWPsclbRDCiZeF8QOTi4JqbmJyK5+0UqhqYRduun8ylAwKKQJ1NJt85sYIHn9f1Rfr6Tq2zS0wZ7DHbZL+zB5rSlAr8QyUdg/GQD+cmSs6LvPJKL78d6hMGk84ARtFo4A79ovwX/Fj01znDQkU6nJildfkaolH2rWFG/qttD azjava@javalib.com";
    try {
        //=============================================================
        // Create a virtual network with a frontend and a backend subnets
        System.out.println("Creating virtual network with a frontend and a backend subnets...");
        Network network = azure.networks().define(vnetName).withRegion(Region.US_EAST).withNewResourceGroup(rgName).withAddressSpace("172.16.0.0/16").defineSubnet("Front-end").withAddressPrefix("172.16.1.0/24").attach().defineSubnet("Back-end").withAddressPrefix("172.16.3.0/24").attach().create();
        System.out.println("Created a virtual network");
        // Print the virtual network details
        Utils.print(network);
        //=============================================================
        // Create an internal load balancer
        // Create a frontend IP address
        // Two backend address pools which contain network interfaces for the virtual
        //  machines to receive HTTP and HTTPS network traffic from the load balancer
        // Two load balancing rules for HTTP and HTTPS to map public ports on the load
        //  balancer to ports in the backend address pool
        // Two probes which contain HTTP and HTTPS health probes used to check availability
        //  of virtual machines in the backend address pool
        // Two inbound NAT rules which contain rules that map a public port on the load
        //  balancer to a port for a specific virtual machine in the backend address pool
        //  - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23
        System.out.println("Creating an internal facing load balancer with ...");
        System.out.println("- A private IP address");
        System.out.println("- One backend address pool which contain network interfaces for the virtual\n" + "  machines to receive 1521 network traffic from the load balancer");
        System.out.println("- One load balancing rules for 1521 to map public ports on the load\n" + "  balancer to ports in the backend address pool");
        System.out.println("- One probe which contains HTTP health probe used to check availability\n" + "  of virtual machines in the backend address pool");
        System.out.println("- Two inbound NAT rules which contain rules that map a port on the load\n" + "  balancer to a port for a specific virtual machine in the backend address pool\n" + "  - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23");
        LoadBalancer loadBalancer3 = azure.loadBalancers().define(loadBalancerName3).withRegion(Region.US_EAST).withExistingResourceGroup(rgName).definePrivateFrontend(privateFrontEndName).withExistingSubnet(network, "Back-end").withPrivateIPAddressStatic("172.16.3.5").attach().defineBackend(backendPoolName3).attach().defineHttpProbe("httpProbe").withRequestPath("/").attach().defineLoadBalancingRule(tcpLoadBalancingRule).withProtocol(TransportProtocol.TCP).withFrontend(privateFrontEndName).withFrontendPort(orcaleSQLNodePort).withProbe(httpProbe).withBackend(backendPoolName3).attach().defineInboundNatRule(natRule6000to22forVM3).withProtocol(TransportProtocol.TCP).withFrontend(privateFrontEndName).withFrontendPort(6000).withBackendPort(22).attach().defineInboundNatRule(natRule6001to23forVM3).withProtocol(TransportProtocol.TCP).withFrontend(privateFrontEndName).withFrontendPort(6001).withBackendPort(23).attach().defineInboundNatRule(natRule6002to22forVM4).withProtocol(TransportProtocol.TCP).withFrontend(privateFrontEndName).withFrontendPort(6002).withBackendPort(22).attach().defineInboundNatRule(natRule6003to23forVM4).withProtocol(TransportProtocol.TCP).withFrontend(privateFrontEndName).withFrontendPort(6003).withBackendPort(23).attach().create();
        // Print load balancer details
        System.out.println("Created an internal load balancer");
        Utils.print(loadBalancer3);
        //=============================================================
        // Define two network interfaces in the backend subnet
        // associate network interfaces to NAT rules, backend pools
        Creatable<NetworkInterface> networkInterface3Creatable = azure.networkInterfaces().define(networkInterfaceName3).withRegion(Region.US_EAST).withNewResourceGroup(rgName).withExistingPrimaryNetwork(network).withSubnet("Back-end").withPrimaryPrivateIPAddressDynamic().withExistingLoadBalancerBackend(loadBalancer3, backendPoolName3).withExistingLoadBalancerInboundNatRule(loadBalancer3, natRule6000to22forVM3).withExistingLoadBalancerInboundNatRule(loadBalancer3, natRule6001to23forVM3);
        Creatable<NetworkInterface> networkInterface4Creatable = azure.networkInterfaces().define(networkInterfaceName4).withRegion(Region.US_EAST).withNewResourceGroup(rgName).withExistingPrimaryNetwork(network).withSubnet("Back-end").withPrimaryPrivateIPAddressDynamic().withExistingLoadBalancerBackend(loadBalancer3, backendPoolName3).withExistingLoadBalancerInboundNatRule(loadBalancer3, natRule6002to22forVM4).withExistingLoadBalancerInboundNatRule(loadBalancer3, natRule6003to23forVM4);
        //=============================================================
        // Define an availability set
        Creatable<AvailabilitySet> availSet2Definition = azure.availabilitySets().define(availSetName).withRegion(Region.US_EAST).withNewResourceGroup(rgName).withFaultDomainCount(2).withUpdateDomainCount(4).withSku(AvailabilitySetSkuTypes.MANAGED);
        //=============================================================
        // Create two virtual machines and assign network interfaces
        System.out.println("Creating two virtual machines in the frontend subnet ...");
        System.out.println("- And assigning network interfaces");
        List<Creatable<VirtualMachine>> virtualMachineCreateables2 = new ArrayList<Creatable<VirtualMachine>>();
        Creatable<VirtualMachine> virtualMachine3Creatable = azure.virtualMachines().define(vmName3).withRegion(Region.US_EAST).withExistingResourceGroup(rgName).withNewPrimaryNetworkInterface(networkInterface3Creatable).withPopularLinuxImage(KnownLinuxVirtualMachineImage.UBUNTU_SERVER_16_04_LTS).withRootUsername(userName).withSsh(sshKey).withSize(VirtualMachineSizeTypes.STANDARD_D3_V2).withNewAvailabilitySet(availSet2Definition);
        virtualMachineCreateables2.add(virtualMachine3Creatable);
        Creatable<VirtualMachine> virtualMachine4Creatable = azure.virtualMachines().define(vmName4).withRegion(Region.US_EAST).withExistingResourceGroup(rgName).withNewPrimaryNetworkInterface(networkInterface4Creatable).withPopularLinuxImage(KnownLinuxVirtualMachineImage.UBUNTU_SERVER_16_04_LTS).withRootUsername(userName).withSsh(sshKey).withSize(VirtualMachineSizeTypes.STANDARD_D3_V2).withNewAvailabilitySet(availSet2Definition);
        virtualMachineCreateables2.add(virtualMachine4Creatable);
        StopWatch stopwatch = new StopWatch();
        stopwatch.start();
        Collection<VirtualMachine> virtualMachines = azure.virtualMachines().create(virtualMachineCreateables2).values();
        stopwatch.stop();
        System.out.println("Created 2 Linux VMs: (took " + (stopwatch.getTime() / 1000) + " seconds) ");
        System.out.println();
        // Print virtual machine details
        for (VirtualMachine vm : virtualMachines) {
            Utils.print(vm);
            System.out.println();
        }
        //=============================================================
        // Update a load balancer
        //  configure TCP idle timeout to 15 minutes
        System.out.println("Updating the load balancer ...");
        loadBalancer3.update().updateLoadBalancingRule(tcpLoadBalancingRule).withIdleTimeoutInMinutes(15).parent().apply();
        System.out.println("Update the load balancer with a TCP idle timeout to 15 minutes");
        //=============================================================
        // Create another internal load balancer
        // Create a frontend IP address
        // Two backend address pools which contain network interfaces for the virtual
        //  machines to receive HTTP and HTTPS network traffic from the load balancer
        // Two load balancing rules for HTTP and HTTPS to map public ports on the load
        //  balancer to ports in the backend address pool
        // Two probes which contain HTTP and HTTPS health probes used to check availability
        //  of virtual machines in the backend address pool
        // Two inbound NAT rules which contain rules that map a public port on the load
        //  balancer to a port for a specific virtual machine in the backend address pool
        //  - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23
        System.out.println("Creating another internal facing load balancer with ...");
        System.out.println("- A private IP address");
        System.out.println("- One backend address pool which contain network interfaces for the virtual\n" + "  machines to receive 1521 network traffic from the load balancer");
        System.out.println("- One load balancing rules for 1521 to map public ports on the load\n" + "  balancer to ports in the backend address pool");
        System.out.println("- One probe which contains HTTP health probe used to check availability\n" + "  of virtual machines in the backend address pool");
        System.out.println("- Two inbound NAT rules which contain rules that map a port on the load\n" + "  balancer to a port for a specific virtual machine in the backend address pool\n" + "  - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23");
        LoadBalancer loadBalancer4 = azure.loadBalancers().define(loadBalancerName4).withRegion(Region.US_EAST).withExistingResourceGroup(rgName).definePrivateFrontend(privateFrontEndName).withExistingSubnet(network, "Back-end").withPrivateIPAddressStatic("172.16.3.15").attach().defineBackend(backendPoolName3).attach().defineHttpProbe("httpProbe").withRequestPath("/").attach().defineLoadBalancingRule(tcpLoadBalancingRule).withProtocol(TransportProtocol.TCP).withFrontend(privateFrontEndName).withFrontendPort(orcaleSQLNodePort).withProbe(httpProbe).withBackend(backendPoolName3).attach().defineInboundNatRule(natRule6000to22forVM3).withProtocol(TransportProtocol.TCP).withFrontend(privateFrontEndName).withFrontendPort(6000).withBackendPort(22).attach().defineInboundNatRule(natRule6001to23forVM3).withProtocol(TransportProtocol.TCP).withFrontend(privateFrontEndName).withFrontendPort(6001).withBackendPort(23).attach().defineInboundNatRule(natRule6002to22forVM4).withProtocol(TransportProtocol.TCP).withFrontend(privateFrontEndName).withFrontendPort(6002).withBackendPort(22).attach().defineInboundNatRule(natRule6003to23forVM4).withProtocol(TransportProtocol.TCP).withFrontend(privateFrontEndName).withFrontendPort(6003).withBackendPort(23).attach().create();
        // Print load balancer details
        System.out.println("Created an internal load balancer");
        Utils.print(loadBalancer4);
        //=============================================================
        // List load balancers
        List<LoadBalancer> loadBalancers = azure.loadBalancers().list();
        System.out.println("Walking through the list of load balancers");
        for (LoadBalancer loadBalancer : loadBalancers) {
            Utils.print(loadBalancer);
            System.out.println();
        }
        //=============================================================
        // Remove a load balancer
        System.out.println("Deleting load balancer " + loadBalancerName4 + "(" + loadBalancer4.id() + ")");
        azure.loadBalancers().deleteById(loadBalancer4.id());
        System.out.println("Deleted load balancer" + loadBalancerName4);
        return true;
    } catch (Exception f) {
        System.out.println(f.getMessage());
        f.printStackTrace();
    } finally {
        try {
            System.out.println("Deleting Resource Group: " + rgName);
            azure.resourceGroups().deleteByName(rgName);
            System.out.println("Deleted Resource Group: " + rgName);
        } catch (NullPointerException npe) {
            System.out.println("Did not create any resources in Azure. No clean up is necessary");
        } catch (Exception g) {
            g.printStackTrace();
        }
    }
    return false;
}
Also used : ArrayList(java.util.ArrayList) LoadBalancer(com.microsoft.azure.management.network.LoadBalancer) NetworkInterface(com.microsoft.azure.management.network.NetworkInterface) StopWatch(org.apache.commons.lang3.time.StopWatch) Network(com.microsoft.azure.management.network.Network) Creatable(com.microsoft.azure.management.resources.fluentcore.model.Creatable) AvailabilitySet(com.microsoft.azure.management.compute.AvailabilitySet) VirtualMachine(com.microsoft.azure.management.compute.VirtualMachine)

Example 3 with StopWatch

use of org.apache.commons.lang3.time.StopWatch in project azure-sdk-for-java by Azure.

the class ManageInternetFacingLoadBalancer method runSample.

/**
     * Main function which runs the actual sample.
     * @param azure instance of the azure client
     * @return true if sample runs successfully
     */
public static boolean runSample(Azure azure) {
    final String rgName = SdkContext.randomResourceName("rgNEML", 15);
    final String vnetName = SdkContext.randomResourceName("vnet", 24);
    final String loadBalancerName1 = SdkContext.randomResourceName("intlb1" + "-", 18);
    final String loadBalancerName2 = SdkContext.randomResourceName("intlb2" + "-", 18);
    final String publicIpName1 = "pip1-" + loadBalancerName1;
    final String publicIpName2 = "pip2-" + loadBalancerName1;
    final String frontendName = loadBalancerName1 + "-FE1";
    final String backendPoolName1 = loadBalancerName1 + "-BAP1";
    final String backendPoolName2 = loadBalancerName1 + "-BAP2";
    final String httpProbe = "httpProbe";
    final String httpsProbe = "httpsProbe";
    final String httpLoadBalancingRule = "httpRule";
    final String httpsLoadBalancingRule = "httpsRule";
    final String natRule5000to22forVM1 = "nat5000to22forVM1";
    final String natRule5001to23forVM1 = "nat5001to23forVM1";
    final String natRule5002to22forVM2 = "nat5002to22forVM2";
    final String natRule5003to23forVM2 = "nat5003to23forVM2";
    final String networkInterfaceName1 = SdkContext.randomResourceName("nic1", 24);
    final String networkInterfaceName2 = SdkContext.randomResourceName("nic2", 24);
    final String availSetName = SdkContext.randomResourceName("av", 24);
    final String userName = "tirekicker";
    final String sshKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCfSPC2K7LZcFKEO+/t3dzmQYtrJFZNxOsbVgOVKietqHyvmYGHEC0J2wPdAqQ/63g/hhAEFRoyehM+rbeDri4txB3YFfnOK58jqdkyXzupWqXzOrlKY4Wz9SKjjN765+dqUITjKRIaAip1Ri137szRg71WnrmdP3SphTRlCx1Bk2nXqWPsclbRDCiZeF8QOTi4JqbmJyK5+0UqhqYRduun8ylAwKKQJ1NJt85sYIHn9f1Rfr6Tq2zS0wZ7DHbZL+zB5rSlAr8QyUdg/GQD+cmSs6LvPJKL78d6hMGk84ARtFo4A79ovwX/Fj01znDQkU6nJildfkaolH2rWFG/qttD azjava@javalib.com";
    try {
        //=============================================================
        // Create a virtual network with a frontend and a backend subnets
        System.out.println("Creating virtual network with a frontend and a backend subnets...");
        Network network = azure.networks().define(vnetName).withRegion(Region.US_EAST).withNewResourceGroup(rgName).withAddressSpace("172.16.0.0/16").defineSubnet("Front-end").withAddressPrefix("172.16.1.0/24").attach().defineSubnet("Back-end").withAddressPrefix("172.16.3.0/24").attach().create();
        System.out.println("Created a virtual network");
        // Print the virtual network details
        Utils.print(network);
        //=============================================================
        // Create a public IP address
        System.out.println("Creating a public IP address...");
        PublicIPAddress publicIPAddress = azure.publicIPAddresses().define(publicIpName1).withRegion(Region.US_EAST).withExistingResourceGroup(rgName).withLeafDomainLabel(publicIpName1).create();
        System.out.println("Created a public IP address");
        // Print the virtual network details
        Utils.print(publicIPAddress);
        //=============================================================
        // Create an Internet facing load balancer
        // Create a frontend IP address
        // Two backend address pools which contain network interfaces for the virtual
        //  machines to receive HTTP and HTTPS network traffic from the load balancer
        // Two load balancing rules for HTTP and HTTPS to map public ports on the load
        //  balancer to ports in the backend address pool
        // Two probes which contain HTTP and HTTPS health probes used to check availability
        //  of virtual machines in the backend address pool
        // Two inbound NAT rules which contain rules that map a public port on the load
        //  balancer to a port for a specific virtual machine in the backend address pool
        //  - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23
        System.out.println("Creating a Internet facing load balancer with ...");
        System.out.println("- A frontend IP address");
        System.out.println("- Two backend address pools which contain network interfaces for the virtual\n" + "  machines to receive HTTP and HTTPS network traffic from the load balancer");
        System.out.println("- Two load balancing rules for HTTP and HTTPS to map public ports on the load\n" + "  balancer to ports in the backend address pool");
        System.out.println("- Two probes which contain HTTP and HTTPS health probes used to check availability\n" + "  of virtual machines in the backend address pool");
        System.out.println("- Two inbound NAT rules which contain rules that map a public port on the load\n" + "  balancer to a port for a specific virtual machine in the backend address pool\n" + "  - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23");
        LoadBalancer loadBalancer1 = azure.loadBalancers().define(loadBalancerName1).withRegion(Region.US_EAST).withExistingResourceGroup(rgName).definePublicFrontend(frontendName).withExistingPublicIPAddress(publicIPAddress).attach().defineBackend(backendPoolName1).attach().defineBackend(backendPoolName2).attach().defineHttpProbe(httpProbe).withRequestPath("/").withPort(80).attach().defineHttpProbe(httpsProbe).withRequestPath("/").withPort(443).attach().defineLoadBalancingRule(httpLoadBalancingRule).withProtocol(TransportProtocol.TCP).withFrontend(frontendName).withFrontendPort(80).withProbe(httpProbe).withBackend(backendPoolName1).attach().defineLoadBalancingRule(httpsLoadBalancingRule).withProtocol(TransportProtocol.TCP).withFrontend(frontendName).withFrontendPort(443).withProbe(httpsProbe).withBackend(backendPoolName2).attach().defineInboundNatRule(natRule5000to22forVM1).withProtocol(TransportProtocol.TCP).withFrontend(frontendName).withFrontendPort(5000).withBackendPort(22).attach().defineInboundNatRule(natRule5001to23forVM1).withProtocol(TransportProtocol.TCP).withFrontend(frontendName).withFrontendPort(5001).withBackendPort(23).attach().defineInboundNatRule(natRule5002to22forVM2).withProtocol(TransportProtocol.TCP).withFrontend(frontendName).withFrontendPort(5002).withBackendPort(22).attach().defineInboundNatRule(natRule5003to23forVM2).withProtocol(TransportProtocol.TCP).withFrontend(frontendName).withFrontendPort(5003).withBackendPort(23).attach().create();
        // Print load balancer details
        System.out.println("Created a load balancer");
        Utils.print(loadBalancer1);
        //=============================================================
        // Define two network interfaces in the frontend subnet
        // associate network interfaces to NAT rules, backend pools
        System.out.println("Creating two network interfaces in the frontend subnet ...");
        System.out.println("- And associating network interfaces to backend pools and NAT rules");
        List<Creatable<NetworkInterface>> networkInterfaceCreatables = new ArrayList<Creatable<NetworkInterface>>();
        Creatable<NetworkInterface> networkInterface1Creatable = azure.networkInterfaces().define(networkInterfaceName1).withRegion(Region.US_EAST).withNewResourceGroup(rgName).withExistingPrimaryNetwork(network).withSubnet("Front-end").withPrimaryPrivateIPAddressDynamic().withExistingLoadBalancerBackend(loadBalancer1, backendPoolName1).withExistingLoadBalancerBackend(loadBalancer1, backendPoolName2).withExistingLoadBalancerInboundNatRule(loadBalancer1, natRule5000to22forVM1).withExistingLoadBalancerInboundNatRule(loadBalancer1, natRule5001to23forVM1);
        networkInterfaceCreatables.add(networkInterface1Creatable);
        Creatable<NetworkInterface> networkInterface2Creatable = azure.networkInterfaces().define(networkInterfaceName2).withRegion(Region.US_EAST).withNewResourceGroup(rgName).withExistingPrimaryNetwork(network).withSubnet("Front-end").withPrimaryPrivateIPAddressDynamic().withExistingLoadBalancerBackend(loadBalancer1, backendPoolName1).withExistingLoadBalancerBackend(loadBalancer1, backendPoolName2).withExistingLoadBalancerInboundNatRule(loadBalancer1, natRule5002to22forVM2).withExistingLoadBalancerInboundNatRule(loadBalancer1, natRule5003to23forVM2);
        networkInterfaceCreatables.add(networkInterface2Creatable);
        //=============================================================
        // Define an availability set
        Creatable<AvailabilitySet> availSet1Definition = azure.availabilitySets().define(availSetName).withRegion(Region.US_EAST).withNewResourceGroup(rgName).withFaultDomainCount(2).withUpdateDomainCount(4).withSku(AvailabilitySetSkuTypes.MANAGED);
        //=============================================================
        // Create two virtual machines and assign network interfaces
        System.out.println("Creating two virtual machines in the frontend subnet ...");
        System.out.println("- And assigning network interfaces");
        List<Creatable<VirtualMachine>> virtualMachineCreateables1 = new ArrayList<Creatable<VirtualMachine>>();
        for (Creatable<NetworkInterface> nicDefinition : networkInterfaceCreatables) {
            virtualMachineCreateables1.add(azure.virtualMachines().define(SdkContext.randomResourceName("lVM1", 24)).withRegion(Region.US_EAST).withExistingResourceGroup(rgName).withNewPrimaryNetworkInterface(nicDefinition).withPopularLinuxImage(KnownLinuxVirtualMachineImage.UBUNTU_SERVER_16_04_LTS).withRootUsername(userName).withSsh(sshKey).withSize(VirtualMachineSizeTypes.STANDARD_D3_V2).withNewAvailabilitySet(availSet1Definition));
        }
        StopWatch stopwatch = new StopWatch();
        stopwatch.start();
        Collection<VirtualMachine> virtualMachines = azure.virtualMachines().create(virtualMachineCreateables1).values();
        stopwatch.stop();
        System.out.println("Created 2 Linux VMs: (took " + (stopwatch.getTime() / 1000) + " seconds) ");
        System.out.println();
        // Print virtual machine details
        for (VirtualMachine vm : virtualMachines) {
            Utils.print(vm);
            System.out.println();
        }
        //=============================================================
        // Update a load balancer
        //  configure TCP idle timeout to 15 minutes
        System.out.println("Updating the load balancer ...");
        loadBalancer1.update().updateLoadBalancingRule(httpLoadBalancingRule).withIdleTimeoutInMinutes(15).parent().updateLoadBalancingRule(httpsLoadBalancingRule).withIdleTimeoutInMinutes(15).parent().apply();
        System.out.println("Update the load balancer with a TCP idle timeout to 15 minutes");
        //=============================================================
        // Create another public IP address
        System.out.println("Creating another public IP address...");
        PublicIPAddress publicIPAddress2 = azure.publicIPAddresses().define(publicIpName2).withRegion(Region.US_EAST).withExistingResourceGroup(rgName).withLeafDomainLabel(publicIpName2).create();
        System.out.println("Created another public IP address");
        // Print the virtual network details
        Utils.print(publicIPAddress2);
        //=============================================================
        // Create another Internet facing load balancer
        // Create a frontend IP address
        // Two backend address pools which contain network interfaces for the virtual
        //  machines to receive HTTP and HTTPS network traffic from the load balancer
        // Two load balancing rules for HTTP and HTTPS to map public ports on the load
        //  balancer to ports in the backend address pool
        // Two probes which contain HTTP and HTTPS health probes used to check availability
        //  of virtual machines in the backend address pool
        // Two inbound NAT rules which contain rules that map a public port on the load
        //  balancer to a port for a specific virtual machine in the backend address pool
        //  - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23
        System.out.println("Creating another Internet facing load balancer with ...");
        System.out.println("- A frontend IP address");
        System.out.println("- Two backend address pools which contain network interfaces for the virtual\n" + "  machines to receive HTTP and HTTPS network traffic from the load balancer");
        System.out.println("- Two load balancing rules for HTTP and HTTPS to map public ports on the load\n" + "  balancer to ports in the backend address pool");
        System.out.println("- Two probes which contain HTTP and HTTPS health probes used to check availability\n" + "  of virtual machines in the backend address pool");
        System.out.println("- Two inbound NAT rules which contain rules that map a public port on the load\n" + "  balancer to a port for a specific virtual machine in the backend address pool\n" + "  - this provides direct VM connectivity for SSH to port 22 and TELNET to port 23");
        LoadBalancer loadBalancer2 = azure.loadBalancers().define(loadBalancerName2).withRegion(Region.US_EAST).withExistingResourceGroup(rgName).definePublicFrontend(frontendName).withExistingPublicIPAddress(publicIPAddress2).attach().defineBackend(backendPoolName1).attach().defineBackend(backendPoolName2).attach().defineHttpProbe(httpProbe).withRequestPath("/").withPort(80).attach().defineHttpProbe(httpsProbe).withRequestPath("/").withPort(443).attach().defineLoadBalancingRule(httpLoadBalancingRule).withProtocol(TransportProtocol.TCP).withFrontend(frontendName).withFrontendPort(80).withProbe(httpProbe).withBackend(backendPoolName1).attach().defineLoadBalancingRule(httpsLoadBalancingRule).withProtocol(TransportProtocol.TCP).withFrontend(frontendName).withFrontendPort(443).withProbe(httpsProbe).withBackend(backendPoolName2).attach().defineInboundNatRule(natRule5000to22forVM1).withProtocol(TransportProtocol.TCP).withFrontend(frontendName).withFrontendPort(5000).withBackendPort(22).attach().defineInboundNatRule(natRule5001to23forVM1).withProtocol(TransportProtocol.TCP).withFrontend(frontendName).withFrontendPort(5001).withBackendPort(23).attach().defineInboundNatRule(natRule5002to22forVM2).withProtocol(TransportProtocol.TCP).withFrontend(frontendName).withFrontendPort(5002).withBackendPort(22).attach().defineInboundNatRule(natRule5003to23forVM2).withProtocol(TransportProtocol.TCP).withFrontend(frontendName).withFrontendPort(5003).withBackendPort(23).attach().create();
        // Print load balancer details
        System.out.println("Created another load balancer");
        Utils.print(loadBalancer2);
        //=============================================================
        // List load balancers
        List<LoadBalancer> loadBalancers = azure.loadBalancers().list();
        System.out.println("Walking through the list of load balancers");
        for (LoadBalancer loadBalancer : loadBalancers) {
            Utils.print(loadBalancer);
        }
        //=============================================================
        // Remove a load balancer
        System.out.println("Deleting load balancer " + loadBalancerName2 + "(" + loadBalancer2.id() + ")");
        azure.loadBalancers().deleteById(loadBalancer2.id());
        System.out.println("Deleted load balancer" + loadBalancerName2);
        return true;
    } catch (Exception f) {
        System.out.println(f.getMessage());
        f.printStackTrace();
    } finally {
        try {
            System.out.println("Deleting Resource Group: " + rgName);
            azure.resourceGroups().deleteByName(rgName);
            System.out.println("Deleted Resource Group: " + rgName);
        } catch (NullPointerException npe) {
            System.out.println("Did not create any resources in Azure. No clean up is necessary");
        } catch (Exception g) {
            g.printStackTrace();
        }
    }
    return false;
}
Also used : ArrayList(java.util.ArrayList) LoadBalancer(com.microsoft.azure.management.network.LoadBalancer) NetworkInterface(com.microsoft.azure.management.network.NetworkInterface) PublicIPAddress(com.microsoft.azure.management.network.PublicIPAddress) StopWatch(org.apache.commons.lang3.time.StopWatch) Network(com.microsoft.azure.management.network.Network) Creatable(com.microsoft.azure.management.resources.fluentcore.model.Creatable) AvailabilitySet(com.microsoft.azure.management.compute.AvailabilitySet) VirtualMachine(com.microsoft.azure.management.compute.VirtualMachine)

Example 4 with StopWatch

use of org.apache.commons.lang3.time.StopWatch in project azure-sdk-for-java by Azure.

the class ManageApplicationGateway method runSample.

/**
     * Main function which runs the actual sample.
     * @param azure instance of the azure client
     * @return true if sample runs successfully
     */
public static boolean runSample(Azure azure) {
    final String rgName = SdkContext.randomResourceName("rgNEAG", 15);
    final String pipName = SdkContext.randomResourceName("pip" + "-", 18);
    final String userName = "tirekicker";
    final String sshKey = "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCfSPC2K7LZcFKEO+/t3dzmQYtrJFZNxOsbVgOVKietqHyvmYGHEC0J2wPdAqQ/63g/hhAEFRoyehM+rbeDri4txB3YFfnOK58jqdkyXzupWqXzOrlKY4Wz9SKjjN765+dqUITjKRIaAip1Ri137szRg71WnrmdP3SphTRlCx1Bk2nXqWPsclbRDCiZeF8QOTi4JqbmJyK5+0UqhqYRduun8ylAwKKQJ1NJt85sYIHn9f1Rfr6Tq2zS0wZ7DHbZL+zB5rSlAr8QyUdg/GQD+cmSs6LvPJKL78d6hMGk84ARtFo4A79ovwX/Fj01znDQkU6nJildfkaolH2rWFG/qttD azjava@javalib.com";
    int backendPools = 2;
    int vmCountInAPool = 4;
    Region[] regions = { Region.US_EAST, Region.UK_WEST };
    String[] addressSpaces = { "172.16.0.0/16", "172.17.0.0/16" };
    String[][] publicIpCreatableKeys = new String[backendPools][vmCountInAPool];
    String[][] ipAddresses = new String[backendPools][vmCountInAPool];
    try {
        //=============================================================
        // Create a resource group (Where all resources gets created)
        //
        ResourceGroup resourceGroup = azure.resourceGroups().define(rgName).withRegion(Region.US_EAST).create();
        System.out.println("Created a new resource group - " + resourceGroup.id());
        //=============================================================
        // Create a public IP address for the Application Gateway
        System.out.println("Creating a public IP address for the application gateway ...");
        PublicIPAddress publicIPAddress = azure.publicIPAddresses().define(pipName).withRegion(Region.US_EAST).withExistingResourceGroup(rgName).create();
        System.out.println("Created a public IP address");
        // Print the virtual network details
        Utils.print(publicIPAddress);
        //=============================================================
        // Create backend pools
        // Prepare a batch of Creatable definitions
        List<Creatable<VirtualMachine>> creatableVirtualMachines = new ArrayList<>();
        for (int i = 0; i < backendPools; i++) {
            //=============================================================
            // Create 1 network creatable per region
            // Prepare Creatable Network definition (Where all the virtual machines get added to)
            String networkName = SdkContext.randomResourceName("vnetNEAG-", 20);
            Creatable<Network> networkCreatable = azure.networks().define(networkName).withRegion(regions[i]).withExistingResourceGroup(resourceGroup).withAddressSpace(addressSpaces[i]);
            //=============================================================
            // Create 1 storage creatable per region (For storing VMs disk)
            String storageAccountName = SdkContext.randomResourceName("stgneag", 20);
            Creatable<StorageAccount> storageAccountCreatable = azure.storageAccounts().define(storageAccountName).withRegion(regions[i]).withExistingResourceGroup(resourceGroup);
            String linuxVMNamePrefix = SdkContext.randomResourceName("vm-", 15);
            for (int j = 0; j < vmCountInAPool; j++) {
                //=============================================================
                // Create 1 public IP address creatable
                Creatable<PublicIPAddress> publicIPAddressCreatable = azure.publicIPAddresses().define(String.format("%s-%d", linuxVMNamePrefix, j)).withRegion(regions[i]).withExistingResourceGroup(resourceGroup).withLeafDomainLabel(String.format("%s-%d", linuxVMNamePrefix, j));
                publicIpCreatableKeys[i][j] = publicIPAddressCreatable.key();
                //=============================================================
                // Create 1 virtual machine creatable
                Creatable<VirtualMachine> virtualMachineCreatable = azure.virtualMachines().define(String.format("%s-%d", linuxVMNamePrefix, j)).withRegion(regions[i]).withExistingResourceGroup(resourceGroup).withNewPrimaryNetwork(networkCreatable).withPrimaryPrivateIPAddressDynamic().withNewPrimaryPublicIPAddress(publicIPAddressCreatable).withPopularLinuxImage(KnownLinuxVirtualMachineImage.UBUNTU_SERVER_16_04_LTS).withRootUsername(userName).withSsh(sshKey).withSize(VirtualMachineSizeTypes.STANDARD_DS3_V2).withNewStorageAccount(storageAccountCreatable);
                creatableVirtualMachines.add(virtualMachineCreatable);
            }
        }
        //=============================================================
        // Create two backend pools of virtual machines
        StopWatch stopwatch = new StopWatch();
        System.out.println("Creating virtual machines (two backend pools)");
        stopwatch.start();
        CreatedResources<VirtualMachine> virtualMachines = azure.virtualMachines().create(creatableVirtualMachines);
        stopwatch.stop();
        System.out.println("Created virtual machines (two backend pools)");
        for (VirtualMachine virtualMachine : virtualMachines.values()) {
            System.out.println(virtualMachine.id());
        }
        System.out.println("Virtual machines created: (took " + (stopwatch.getTime() / 1000) + " seconds) to create == " + virtualMachines.size() + " == virtual machines (4 virtual machines per backend pool)");
        //=======================================================================
        // Get IP addresses from created resources
        System.out.println("IP Addresses in the backend pools are - ");
        for (int i = 0; i < backendPools; i++) {
            for (int j = 0; j < vmCountInAPool; j++) {
                PublicIPAddress pip = (PublicIPAddress) virtualMachines.createdRelatedResource(publicIpCreatableKeys[i][j]);
                pip.refresh();
                ipAddresses[i][j] = pip.ipAddress();
                System.out.println(String.format("[backend pool = %d][vm = %d] = %s", i, j, ipAddresses[i][j]));
            }
            System.out.println("======");
        }
        //=======================================================================
        // Create an application gateway
        System.out.println("================= CREATE ======================");
        System.out.println("Creating an application gateway");
        stopwatch.reset();
        stopwatch.start();
        final String sslCertificatePfxPath = ManageApplicationGateway.class.getClassLoader().getResource("myTest._pfx").getPath();
        final String sslCertificatePfxPath2 = ManageApplicationGateway.class.getClassLoader().getResource("myTest2._pfx").getPath();
        ApplicationGateway applicationGateway = azure.applicationGateways().define("myFirstAppGateway").withRegion(Region.US_EAST).withExistingResourceGroup(resourceGroup).defineRequestRoutingRule("HTTP-80-to-8080").fromPublicFrontend().fromFrontendHttpPort(80).toBackendHttpPort(8080).toBackendIPAddress(ipAddresses[0][0]).toBackendIPAddress(ipAddresses[0][1]).toBackendIPAddress(ipAddresses[0][2]).toBackendIPAddress(ipAddresses[0][3]).attach().defineRequestRoutingRule("HTTPs-443-to-8080").fromPublicFrontend().fromFrontendHttpsPort(443).withSslCertificateFromPfxFile(new File(sslCertificatePfxPath)).withSslCertificatePassword("Abc123").toBackendHttpPort(8080).toBackendIPAddress(ipAddresses[1][0]).toBackendIPAddress(ipAddresses[1][1]).toBackendIPAddress(ipAddresses[1][2]).toBackendIPAddress(ipAddresses[1][3]).attach().withExistingPublicIPAddress(publicIPAddress).create();
        stopwatch.stop();
        System.out.println("Application gateway created: (took " + (stopwatch.getTime() / 1000) + " seconds)");
        Utils.print(applicationGateway);
        //=======================================================================
        // Update an application gateway
        // configure the first routing rule for SSL offload
        System.out.println("================= UPDATE ======================");
        System.out.println("Updating the application gateway");
        stopwatch.reset();
        stopwatch.start();
        applicationGateway.update().withoutRequestRoutingRule("HTTP-80-to-8080").defineRequestRoutingRule("HTTPs-1443-to-8080").fromPublicFrontend().fromFrontendHttpsPort(1443).withSslCertificateFromPfxFile(new File(sslCertificatePfxPath2)).withSslCertificatePassword("Abc123").toBackendHttpPort(8080).toBackendIPAddress(ipAddresses[0][0]).toBackendIPAddress(ipAddresses[0][1]).toBackendIPAddress(ipAddresses[0][2]).toBackendIPAddress(ipAddresses[0][3]).withHostName("www.contoso.com").withCookieBasedAffinity().attach().apply();
        stopwatch.stop();
        System.out.println("Application gateway updated: (took " + (stopwatch.getTime() / 1000) + " seconds)");
        Utils.print(applicationGateway);
        return true;
    } catch (Exception f) {
        System.out.println(f.getMessage());
        f.printStackTrace();
    } finally {
        try {
            System.out.println("Deleting Resource Group: " + rgName);
            azure.resourceGroups().deleteByName(rgName);
            System.out.println("Deleted Resource Group: " + rgName);
        } catch (NullPointerException npe) {
            System.out.println("Did not create any resources in Azure. No clean up is necessary");
        } catch (Exception g) {
            g.printStackTrace();
        }
    }
    return false;
}
Also used : ApplicationGateway(com.microsoft.azure.management.network.ApplicationGateway) ArrayList(java.util.ArrayList) PublicIPAddress(com.microsoft.azure.management.network.PublicIPAddress) StopWatch(org.apache.commons.lang3.time.StopWatch) StorageAccount(com.microsoft.azure.management.storage.StorageAccount) Network(com.microsoft.azure.management.network.Network) Region(com.microsoft.azure.management.resources.fluentcore.arm.Region) Creatable(com.microsoft.azure.management.resources.fluentcore.model.Creatable) File(java.io.File) ResourceGroup(com.microsoft.azure.management.resources.ResourceGroup) VirtualMachine(com.microsoft.azure.management.compute.VirtualMachine)

Example 5 with StopWatch

use of org.apache.commons.lang3.time.StopWatch in project flink by apache.

the class RollingSink method handleRestoredBucketState.

private void handleRestoredBucketState(BucketState bucketState) {
    // we can clean all the pending files since they were renamed to
    // final files after this checkpoint was successful
    // (we re-start from the last **successful** checkpoint)
    bucketState.pendingFiles.clear();
    if (bucketState.currentFile != null) {
        // We were writing to a file when the last checkpoint occurred. This file can either
        // be still in-progress or became a pending file at some point after the checkpoint.
        // Either way, we have to truncate it back to a valid state (or write a .valid-length
        // file that specifies up to which length it is valid) and rename it to the final name
        // before starting a new bucket file.
        Path partPath = new Path(bucketState.currentFile);
        try {
            Path partPendingPath = getPendingPathFor(partPath);
            Path partInProgressPath = getInProgressPathFor(partPath);
            if (fs.exists(partPendingPath)) {
                LOG.debug("In-progress file {} has been moved to pending after checkpoint, moving to final location.", partPath);
                // has been moved to pending in the mean time, rename to final location
                fs.rename(partPendingPath, partPath);
            } else if (fs.exists(partInProgressPath)) {
                LOG.debug("In-progress file {} is still in-progress, moving to final location.", partPath);
                // it was still in progress, rename to final path
                fs.rename(partInProgressPath, partPath);
            } else if (fs.exists(partPath)) {
                LOG.debug("In-Progress file {} was already moved to final location {}.", bucketState.currentFile, partPath);
            } else {
                LOG.debug("In-Progress file {} was neither moved to pending nor is still in progress. Possibly, " + "it was moved to final location by a previous snapshot restore", bucketState.currentFile);
            }
            if (this.refTruncate == null) {
                this.refTruncate = reflectTruncate(fs);
            }
            // truncate it or write a ".valid-length" file to specify up to which point it is valid
            if (refTruncate != null) {
                LOG.debug("Truncating {} to valid length {}", partPath, bucketState.currentFileValidLength);
                // recovering, after all ...
                if (fs instanceof DistributedFileSystem) {
                    DistributedFileSystem dfs = (DistributedFileSystem) fs;
                    LOG.debug("Trying to recover file lease {}", partPath);
                    dfs.recoverLease(partPath);
                    boolean isclosed = dfs.isFileClosed(partPath);
                    StopWatch sw = new StopWatch();
                    sw.start();
                    while (!isclosed) {
                        if (sw.getTime() > asyncTimeout) {
                            break;
                        }
                        try {
                            Thread.sleep(500);
                        } catch (InterruptedException e1) {
                        // ignore it
                        }
                        isclosed = dfs.isFileClosed(partPath);
                    }
                }
                Boolean truncated = (Boolean) refTruncate.invoke(fs, partPath, bucketState.currentFileValidLength);
                if (!truncated) {
                    LOG.debug("Truncate did not immediately complete for {}, waiting...", partPath);
                    // we must wait for the asynchronous truncate operation to complete
                    StopWatch sw = new StopWatch();
                    sw.start();
                    long newLen = fs.getFileStatus(partPath).getLen();
                    while (newLen != bucketState.currentFileValidLength) {
                        if (sw.getTime() > asyncTimeout) {
                            break;
                        }
                        try {
                            Thread.sleep(500);
                        } catch (InterruptedException e1) {
                        // ignore it
                        }
                        newLen = fs.getFileStatus(partPath).getLen();
                    }
                    if (newLen != bucketState.currentFileValidLength) {
                        throw new RuntimeException("Truncate did not truncate to right length. Should be " + bucketState.currentFileValidLength + " is " + newLen + ".");
                    }
                }
            } else {
                LOG.debug("Writing valid-length file for {} to specify valid length {}", partPath, bucketState.currentFileValidLength);
                Path validLengthFilePath = getValidLengthPathFor(partPath);
                if (!fs.exists(validLengthFilePath) && fs.exists(partPath)) {
                    FSDataOutputStream lengthFileOut = fs.create(validLengthFilePath);
                    lengthFileOut.writeUTF(Long.toString(bucketState.currentFileValidLength));
                    lengthFileOut.close();
                }
            }
            // invalidate in the state object
            bucketState.currentFile = null;
            bucketState.currentFileValidLength = -1;
            isWriterOpen = false;
        } catch (IOException e) {
            LOG.error("Error while restoring RollingSink state.", e);
            throw new RuntimeException("Error while restoring RollingSink state.", e);
        } catch (InvocationTargetException | IllegalAccessException e) {
            LOG.error("Could not invoke truncate.", e);
            throw new RuntimeException("Could not invoke truncate.", e);
        }
    }
    // Move files that are confirmed by a checkpoint but did not get moved to final location
    // because the checkpoint notification did not happen before a failure
    Set<Long> pastCheckpointIds = bucketState.pendingFilesPerCheckpoint.keySet();
    LOG.debug("Moving pending files to final location on restore.");
    for (Long pastCheckpointId : pastCheckpointIds) {
        // to their final name
        for (String filename : bucketState.pendingFilesPerCheckpoint.get(pastCheckpointId)) {
            Path finalPath = new Path(filename);
            Path pendingPath = getPendingPathFor(finalPath);
            try {
                if (fs.exists(pendingPath)) {
                    LOG.debug("(RESTORE) Moving pending file {} to final location after complete checkpoint {}.", pendingPath, pastCheckpointId);
                    fs.rename(pendingPath, finalPath);
                }
            } catch (IOException e) {
                LOG.error("(RESTORE) Error while renaming pending file {} to final path {}: {}", pendingPath, finalPath, e);
                throw new RuntimeException("Error while renaming pending file " + pendingPath + " to final path " + finalPath, e);
            }
        }
    }
    synchronized (bucketState.pendingFilesPerCheckpoint) {
        bucketState.pendingFilesPerCheckpoint.clear();
    }
}
Also used : Path(org.apache.hadoop.fs.Path) IOException(java.io.IOException) DistributedFileSystem(org.apache.hadoop.hdfs.DistributedFileSystem) InvocationTargetException(java.lang.reflect.InvocationTargetException) StopWatch(org.apache.commons.lang3.time.StopWatch) FSDataOutputStream(org.apache.hadoop.fs.FSDataOutputStream)

Aggregations

StopWatch (org.apache.commons.lang3.time.StopWatch)500 Test (org.junit.Test)149 EventResult (org.alfresco.bm.event.EventResult)97 DBObject (com.mongodb.DBObject)90 Event (org.alfresco.bm.event.Event)87 FolderData (org.alfresco.bm.cm.FolderData)75 File (java.io.File)72 ArrayList (java.util.ArrayList)48 HashSet (java.util.HashSet)31 Gene (ubic.gemma.model.genome.Gene)31 Vertex (org.apache.tinkerpop.gremlin.structure.Vertex)26 BaseTest (org.umlg.sqlg.test.BaseTest)26 Element (org.w3c.dom.Element)25 LoadSingleComponentUnitTest (org.alfresco.bm.dataload.LoadSingleComponentUnitTest)23 UserModel (org.alfresco.utility.model.UserModel)23 IOException (java.io.IOException)20 Collectors (java.util.stream.Collectors)19 HashMap (java.util.HashMap)18 List (java.util.List)18 ExpressionExperiment (ubic.gemma.model.expression.experiment.ExpressionExperiment)18