Examples with Queue java.util.Queue used on opensource projects

Example 1 with Queue

use of java.util.Queue in project che by eclipse.

the class CheEnvironmentEngine method startEnvironmentQueue.

/**
     * Starts all machine from machine queue of environment.
     */
private void startEnvironmentQueue(String namespace, String workspaceId, String devMachineName, String networkId, boolean recover, MachineStartedHandler startedHandler) throws ServerException, EnvironmentException {
    // Starting all machines in environment one by one by getting configs
    // from the corresponding starting queue.
    // Config will be null only if there are no machines left in the queue
    String envName;
    MessageConsumer<MachineLogMessage> envLogger;
    String creator = EnvironmentContext.getCurrent().getSubject().getUserId();
    try (@SuppressWarnings("unused") Unlocker u = stripedLocks.readLock(workspaceId)) {
        EnvironmentHolder environmentHolder = environments.get(workspaceId);
        if (environmentHolder == null) {
            throw new ServerException("Environment start is interrupted.");
        }
        envName = environmentHolder.name;
        envLogger = environmentHolder.logger;
    }
    try {
        machineProvider.createNetwork(networkId);
        String machineName = queuePeekOrFail(workspaceId);
        while (machineName != null) {
            boolean isDev = devMachineName.equals(machineName);
            // Environment start is failed when any machine start is failed, so if any error
            // occurs during machine creation then environment start fail is reported and
            // start resources such as queue and descriptor must be cleaned up
            CheServiceImpl service;
            @Nullable ExtendedMachine extendedMachine;
            try (@SuppressWarnings("unused") Unlocker u = stripedLocks.readLock(workspaceId)) {
                EnvironmentHolder environmentHolder = environments.get(workspaceId);
                if (environmentHolder == null) {
                    throw new ServerException("Environment start is interrupted.");
                }
                service = environmentHolder.environment.getServices().get(machineName);
                extendedMachine = environmentHolder.environmentConfig.getMachines().get(machineName);
            }
            // should not happen
            if (service == null) {
                LOG.error("Start of machine with name {} in workspace {} failed. Machine not found in start queue", machineName, workspaceId);
                throw new ServerException(format("Environment of workspace with ID '%s' failed due to internal error", workspaceId));
            }
            final String finalMachineName = machineName;
            // needed to reuse startInstance method and
            // create machine instances by different implementation-specific providers
            MachineStarter machineStarter = (machineLogger, machineSource) -> {
                CheServiceImpl serviceWithNormalizedSource = normalizeServiceSource(service, machineSource);
                return machineProvider.startService(namespace, workspaceId, envName, finalMachineName, isDev, networkId, serviceWithNormalizedSource, machineLogger);
            };
            MachineImpl machine = MachineImpl.builder().setConfig(MachineConfigImpl.builder().setDev(isDev).setLimits(new MachineLimitsImpl(bytesToMB(service.getMemLimit()))).setType("docker").setName(machineName).setEnvVariables(service.getEnvironment()).build()).setId(service.getId()).setWorkspaceId(workspaceId).setStatus(MachineStatus.CREATING).setEnvName(envName).setOwner(creator).build();
            checkInterruption(workspaceId, envName);
            Instance instance = startInstance(recover, envLogger, machine, machineStarter);
            checkInterruption(workspaceId, envName);
            startedHandler.started(instance, extendedMachine);
            checkInterruption(workspaceId, envName);
            // Machine destroying is an expensive operation which must be
            // performed outside of the lock, this section checks if
            // the environment wasn't stopped while it is starting and sets
            // polled flag to true if the environment wasn't stopped.
            // Also polls the proceeded machine configuration from the queue
            boolean queuePolled = false;
            try (@SuppressWarnings("unused") Unlocker u = stripedLocks.writeLock(workspaceId)) {
                ensurePreDestroyIsNotExecuted();
                EnvironmentHolder environmentHolder = environments.get(workspaceId);
                if (environmentHolder != null) {
                    final Queue<String> queue = environmentHolder.startQueue;
                    if (queue != null) {
                        queue.poll();
                        queuePolled = true;
                    }
                }
            }
            // must be destroyed
            if (!queuePolled) {
                try {
                    eventService.publish(newDto(MachineStatusEvent.class).withEventType(MachineStatusEvent.EventType.DESTROYING).withDev(isDev).withMachineName(machineName).withMachineId(instance.getId()).withWorkspaceId(workspaceId));
                    instance.destroy();
                    removeMachine(workspaceId, instance.getId());
                    eventService.publish(newDto(MachineStatusEvent.class).withEventType(MachineStatusEvent.EventType.DESTROYED).withDev(isDev).withMachineName(machineName).withMachineId(instance.getId()).withWorkspaceId(workspaceId));
                } catch (MachineException e) {
                    LOG.error(e.getLocalizedMessage(), e);
                }
                throw new ServerException("Workspace '" + workspaceId + "' start interrupted. Workspace stopped before all its machines started");
            }
            machineName = queuePeekOrFail(workspaceId);
        }
    } catch (RuntimeException | ServerException | EnvironmentStartInterruptedException e) {
        boolean interrupted = Thread.interrupted();
        EnvironmentHolder env;
        try (@SuppressWarnings("unused") Unlocker u = stripedLocks.writeLock(workspaceId)) {
            env = environments.remove(workspaceId);
        }
        try {
            destroyEnvironment(env.networkId, env.machines);
        } catch (Exception remEx) {
            LOG.error(remEx.getLocalizedMessage(), remEx);
        }
        if (interrupted) {
            throw new EnvironmentStartInterruptedException(workspaceId, envName);
        }
        try {
            throw e;
        } catch (ServerException | EnvironmentStartInterruptedException rethrow) {
            throw rethrow;
        } catch (Exception wrap) {
            throw new ServerException(wrap.getMessage(), wrap);
        }
    }
}

Example 2 with Queue

use of java.util.Queue in project jetty.project by eclipse.

the class AsyncRestServlet method doGet.

@Override
protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException {
    Long start = System.nanoTime();
    // Do we have results yet?
    Queue<Map<String, String>> results = (Queue<Map<String, String>>) request.getAttribute(RESULTS_ATTR);
    // If no results, this must be the first dispatch, so send the REST request(s)
    if (results == null) {
        // define results data structures
        final Queue<Map<String, String>> resultsQueue = new ConcurrentLinkedQueue<>();
        request.setAttribute(RESULTS_ATTR, results = resultsQueue);
        // suspend the request
        // This is done before scheduling async handling to avoid race of
        // dispatch before startAsync!
        final AsyncContext async = request.startAsync();
        async.setTimeout(30000);
        // extract keywords to search for
        String[] keywords = sanitize(request.getParameter(ITEMS_PARAM)).split(",");
        final AtomicInteger outstanding = new AtomicInteger(keywords.length);
        // Send request each keyword
        for (final String item : keywords) {
            _client.newRequest(restURL(item)).method(HttpMethod.GET).send(new AsyncRestRequest() {

                @Override
                void onAuctionFound(Map<String, String> auction) {
                    resultsQueue.add(auction);
                }

                @Override
                void onComplete() {
                    if (outstanding.decrementAndGet() <= 0)
                        async.dispatch();
                }
            });
        }
        // save timing info and return
        request.setAttribute(START_ATTR, start);
        request.setAttribute(DURATION_ATTR, System.nanoTime() - start);
        return;
    }
    // We have results!
    // Generate the response
    String thumbs = generateThumbs(results);
    response.setContentType("text/html");
    PrintWriter out = response.getWriter();
    out.println("<html><head>");
    out.println(STYLE);
    out.println("</head><body><small>");
    long initial = (Long) request.getAttribute(DURATION_ATTR);
    long start0 = (Long) request.getAttribute(START_ATTR);
    long now = System.nanoTime();
    long total = now - start0;
    long generate = now - start;
    long thread = initial + generate;
    out.print("<b>Asynchronous: " + sanitize(request.getParameter(ITEMS_PARAM)) + "</b><br/>");
    out.print("Total Time: " + ms(total) + "ms<br/>");
    out.print("Thread held (<span class='red'>red</span>): " + ms(thread) + "ms (" + ms(initial) + " initial + " + ms(generate) + " generate )<br/>");
    out.print("Async wait (<span class='green'>green</span>): " + ms(total - thread) + "ms<br/>");
    out.println("<img border='0px' src='asyncrest/red.png'   height='20px' width='" + width(initial) + "px'>" + "<img border='0px' src='asyncrest/green.png' height='20px' width='" + width(total - thread) + "px'>" + "<img border='0px' src='asyncrest/red.png'   height='20px' width='" + width(generate) + "px'>");
    out.println("<hr />");
    out.println(thumbs);
    out.println("</small>");
    out.println("</body></html>");
    out.close();
}

Example 3 with Queue

use of java.util.Queue in project storm by apache.

the class DefaultResourceAwareStrategy method orderExecutors.

/**
     * Order executors based on how many in and out connections it will potentially need to make.
     * First order components by the number of in and out connections it will have.  Then iterate through the sorted list of components.
     * For each component sort the neighbors of that component by how many connections it will have to make with that component.
     * Add an executor from this component and then from each neighboring component in sorted order.  Do this until there is nothing left to schedule
     *
     * @param td                  The topology the executors belong to
     * @param unassignedExecutors a collection of unassigned executors that need to be unassigned. Should only try to assign executors from this list
     * @return a list of executors in sorted order
     */
private List<ExecutorDetails> orderExecutors(TopologyDetails td, Collection<ExecutorDetails> unassignedExecutors) {
    Map<String, Component> componentMap = td.getComponents();
    List<ExecutorDetails> execsScheduled = new LinkedList<>();
    Map<String, Queue<ExecutorDetails>> compToExecsToSchedule = new HashMap<>();
    for (Component component : componentMap.values()) {
        compToExecsToSchedule.put(component.id, new LinkedList<ExecutorDetails>());
        for (ExecutorDetails exec : component.execs) {
            if (unassignedExecutors.contains(exec)) {
                compToExecsToSchedule.get(component.id).add(exec);
            }
        }
    }
    Set<Component> sortedComponents = sortComponents(componentMap);
    sortedComponents.addAll(componentMap.values());
    for (Component currComp : sortedComponents) {
        Map<String, Component> neighbors = new HashMap<String, Component>();
        for (String compId : (List<String>) ListUtils.union(currComp.children, currComp.parents)) {
            neighbors.put(compId, componentMap.get(compId));
        }
        Set<Component> sortedNeighbors = sortNeighbors(currComp, neighbors);
        Queue<ExecutorDetails> currCompExesToSched = compToExecsToSchedule.get(currComp.id);
        boolean flag = false;
        do {
            flag = false;
            if (!currCompExesToSched.isEmpty()) {
                execsScheduled.add(currCompExesToSched.poll());
                flag = true;
            }
            for (Component neighborComp : sortedNeighbors) {
                Queue<ExecutorDetails> neighborCompExesToSched = compToExecsToSchedule.get(neighborComp.id);
                if (!neighborCompExesToSched.isEmpty()) {
                    execsScheduled.add(neighborCompExesToSched.poll());
                    flag = true;
                }
            }
        } while (flag);
    }
    return execsScheduled;
}

Example 4 with Queue

use of java.util.Queue in project pinot by linkedin.

the class InnerSegmentSelectionMultiValueQueriesTest method testSelectionOrderBy.

@Test
public void testSelectionOrderBy() {
    String query = "SELECT" + SELECTION + " FROM testTable" + ORDER_BY;
    // Test query without filter.
    MSelectionOrderByOperator selectionOrderByOperator = getOperatorForQuery(query);
    IntermediateResultsBlock resultsBlock = (IntermediateResultsBlock) selectionOrderByOperator.nextBlock();
    ExecutionStatistics executionStatistics = selectionOrderByOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 100000L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 0L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 400000L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L);
    DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 4);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column5");
    Assert.assertEquals(selectionDataSchema.getColumnName(3), "column6");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.STRING);
    Assert.assertEquals(selectionDataSchema.getColumnType(3), FieldSpec.DataType.INT_ARRAY);
    Queue<Serializable[]> selectionResult = (Queue<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    Serializable[] lastRow = selectionResult.peek();
    Assert.assertEquals(lastRow.length, 4);
    Assert.assertEquals((String) lastRow[0], "AKXcXcIqsqOJFsdwxZ");
    Assert.assertEquals(lastRow[3], new int[] { 1252 });
    // Test query with filter.
    selectionOrderByOperator = getOperatorForQueryWithFilter(query);
    resultsBlock = (IntermediateResultsBlock) selectionOrderByOperator.nextBlock();
    executionStatistics = selectionOrderByOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 15620L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 282430L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 62480L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 100000L);
    selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 4);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column5");
    Assert.assertEquals(selectionDataSchema.getColumnName(3), "column6");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.STRING);
    Assert.assertEquals(selectionDataSchema.getColumnType(3), FieldSpec.DataType.INT_ARRAY);
    selectionResult = (Queue<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    lastRow = selectionResult.peek();
    Assert.assertEquals(lastRow.length, 4);
    Assert.assertEquals((String) lastRow[0], "AKXcXcIqsqOJFsdwxZ");
    Assert.assertEquals(lastRow[3], new int[] { 2147483647 });
}

Example 5 with Queue

use of java.util.Queue in project pinot by linkedin.

the class InnerSegmentSelectionSingleValueQueriesTest method testSelectionOrderBy.

@Test
public void testSelectionOrderBy() {
    String query = "SELECT" + SELECTION + " FROM testTable" + ORDER_BY;
    // Test query without filter.
    MSelectionOrderByOperator selectionOrderByOperator = getOperatorForQuery(query);
    IntermediateResultsBlock resultsBlock = (IntermediateResultsBlock) selectionOrderByOperator.nextBlock();
    ExecutionStatistics executionStatistics = selectionOrderByOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 30000L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 0L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 120000L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L);
    DataSchema selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 4);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column6");
    Assert.assertEquals(selectionDataSchema.getColumnName(1), "column1");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.INT);
    Assert.assertEquals(selectionDataSchema.getColumnType(1), FieldSpec.DataType.INT);
    Queue<Serializable[]> selectionResult = (Queue<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    Serializable[] lastRow = selectionResult.peek();
    Assert.assertEquals(lastRow.length, 4);
    Assert.assertEquals(((Integer) lastRow[0]).intValue(), 6043515);
    Assert.assertEquals(((Integer) lastRow[1]).intValue(), 10542595);
    // Test query with filter.
    selectionOrderByOperator = getOperatorForQueryWithFilter(query);
    resultsBlock = (IntermediateResultsBlock) selectionOrderByOperator.nextBlock();
    executionStatistics = selectionOrderByOperator.getExecutionStatistics();
    Assert.assertEquals(executionStatistics.getNumDocsScanned(), 6129L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedInFilter(), 84134L);
    Assert.assertEquals(executionStatistics.getNumEntriesScannedPostFilter(), 24516L);
    Assert.assertEquals(executionStatistics.getNumTotalRawDocs(), 30000L);
    selectionDataSchema = resultsBlock.getSelectionDataSchema();
    Assert.assertEquals(selectionDataSchema.size(), 4);
    Assert.assertEquals(selectionDataSchema.getColumnName(0), "column6");
    Assert.assertEquals(selectionDataSchema.getColumnName(1), "column1");
    Assert.assertEquals(selectionDataSchema.getColumnType(0), FieldSpec.DataType.INT);
    Assert.assertEquals(selectionDataSchema.getColumnType(1), FieldSpec.DataType.INT);
    selectionResult = (Queue<Serializable[]>) resultsBlock.getSelectionResult();
    Assert.assertEquals(selectionResult.size(), 10);
    lastRow = selectionResult.peek();
    Assert.assertEquals(lastRow.length, 4);
    Assert.assertEquals(((Integer) lastRow[0]).intValue(), 6043515);
    Assert.assertEquals(((Integer) lastRow[1]).intValue(), 462769197);
}