Examples with LinkedList java.util.LinkedList used on opensource projects

Example 41 with LinkedList

use of java.util.LinkedList in project hbase by apache.

the class TestFromClientSide method doAppend.

private List<Result> doAppend(final boolean walUsed) throws IOException {
    LOG.info("Starting testAppend, walUsed is " + walUsed);
    final TableName TABLENAME = TableName.valueOf(walUsed ? "testAppendWithWAL" : "testAppendWithoutWAL");
    Table t = TEST_UTIL.createTable(TABLENAME, FAMILY);
    final byte[] row1 = Bytes.toBytes("c");
    final byte[] row2 = Bytes.toBytes("b");
    final byte[] row3 = Bytes.toBytes("a");
    final byte[] qual = Bytes.toBytes("qual");
    Put put_0 = new Put(row2);
    put_0.addColumn(FAMILY, qual, Bytes.toBytes("put"));
    Put put_1 = new Put(row3);
    put_1.addColumn(FAMILY, qual, Bytes.toBytes("put"));
    Append append_0 = new Append(row1);
    append_0.add(FAMILY, qual, Bytes.toBytes("i"));
    Append append_1 = new Append(row1);
    append_1.add(FAMILY, qual, Bytes.toBytes("k"));
    Append append_2 = new Append(row1);
    append_2.add(FAMILY, qual, Bytes.toBytes("e"));
    if (!walUsed) {
        append_2.setDurability(Durability.SKIP_WAL);
    }
    Append append_3 = new Append(row1);
    append_3.add(FAMILY, qual, Bytes.toBytes("a"));
    Scan s = new Scan();
    s.setCaching(1);
    t.append(append_0);
    t.put(put_0);
    t.put(put_1);
    List<Result> results = new LinkedList<>();
    try (ResultScanner scanner = t.getScanner(s)) {
        t.append(append_1);
        t.append(append_2);
        t.append(append_3);
        for (Result r : scanner) {
            results.add(r);
        }
    }
    TEST_UTIL.deleteTable(TABLENAME);
    return results;
}

Example 42 with LinkedList

use of java.util.LinkedList in project buck by facebook.

the class FileClassPathRunner method getClasspathFiles.

// @VisibileForTesting
@SuppressWarnings("PMD.EmptyCatchBlock")
static List<Path> getClasspathFiles(URL[] urls) {
    List<Path> paths = new LinkedList<>();
    for (URL url : urls) {
        if (!"file".equals(url.getProtocol())) {
            continue;
        }
        // Segment the path, looking for a section that starts with "@". If one is found, reconstruct
        // the rest of the path.
        // WARNING: While the classfile's path can contain directories that include "@", the path
        // cannot contain a directory that starts with "@".
        String path = url.getPath();
        int found, splitIndex = -1;
        if (path == null || path.length() == 0) {
            continue;
        } else if (path.charAt(0) == '@') {
            // path starts with '@'
            splitIndex = 1;
        } else if ((found = path.indexOf("/@")) >= 0) {
            // found first section that begins with '@'
            splitIndex = found + 2;
        }
        if (splitIndex > 0 && splitIndex < path.length()) {
            try {
                paths.add(Paths.get(path.substring(splitIndex)));
            } catch (InvalidPathException e) {
            // Carry on regardless
            }
        }
    }
    return paths;
}

Example 43 with LinkedList

use of java.util.LinkedList in project buck by facebook.

the class DistBuildClientExecutor method executeAndPrintFailuresToEventBus.

public int executeAndPrintFailuresToEventBus(final WeightedListeningExecutorService executorService, ProjectFilesystem projectFilesystem, FileHashCache fileHashCache, BuckEventBus eventBus) throws IOException, InterruptedException {
    BuildJob job = distBuildService.createBuild();
    final StampedeId id = job.getStampedeId();
    LOG.info("Created job. Build id = " + id.getId());
    logDebugInfo(job);
    List<ListenableFuture<Void>> asyncJobs = new LinkedList<>();
    LOG.info("Uploading local changes.");
    asyncJobs.add(distBuildService.uploadMissingFiles(buildJobState.fileHashes, executorService));
    LOG.info("Uploading target graph.");
    asyncJobs.add(distBuildService.uploadTargetGraph(buildJobState, id, executorService));
    LOG.info("Uploading buck dot-files.");
    asyncJobs.add(distBuildService.uploadBuckDotFiles(id, projectFilesystem, fileHashCache, executorService));
    try {
        Futures.allAsList(asyncJobs).get();
    } catch (ExecutionException e) {
        LOG.error("Upload failed.");
        throw new RuntimeException(e);
    }
    distBuildService.setBuckVersion(id, buckVersion);
    LOG.info("Set Buck Version. Build status: " + job.getStatus().toString());
    job = distBuildService.startBuild(id);
    LOG.info("Started job. Build status: " + job.getStatus().toString());
    logDebugInfo(job);
    Stopwatch stopwatch = Stopwatch.createStarted();
    // Keep polling until the build is complete or failed.
    do {
        job = distBuildService.getCurrentBuildJobState(id);
        LOG.info("Got build status: " + job.getStatus().toString());
        DistBuildStatus distBuildStatus = prepareStatusFromJob(job).setETAMillis(MAX_BUILD_DURATION_MILLIS - stopwatch.elapsed(TimeUnit.MILLISECONDS)).build();
        eventBus.post(new DistBuildStatusEvent(distBuildStatus));
        List<LogLineBatchRequest> newLogLineRequests = distBuildLogStateTracker.createRealtimeLogRequests(job.getSlaveInfoByRunId().values());
        MultiGetBuildSlaveRealTimeLogsResponse slaveLogsResponse = distBuildService.fetchSlaveLogLines(job.stampedeId, newLogLineRequests);
        distBuildLogStateTracker.processStreamLogs(slaveLogsResponse.getMultiStreamLogs());
        try {
            // TODO(shivanker): Get rid of sleeps in methods which we want to unit test
            Thread.sleep(millisBetweenStatusPoll);
        } catch (InterruptedException e) {
            LOG.error(e, "BuildStatus polling sleep call has been interrupted unexpectedly.");
        }
    } while (!(job.getStatus().equals(BuildStatus.FINISHED_SUCCESSFULLY) || job.getStatus().equals(BuildStatus.FAILED)));
    try {
        MultiGetBuildSlaveLogDirResponse logDirsResponse = distBuildService.fetchBuildSlaveLogDir(job.stampedeId, distBuildLogStateTracker.runIdsToMaterializeLogDirsFor(job.slaveInfoByRunId.values()));
        distBuildLogStateTracker.materializeLogDirs(logDirsResponse.getLogDirs());
    } catch (IOException ex) {
        LOG.error(ex);
    }
    LOG.info("Build was " + (job.getStatus().equals(BuildStatus.FINISHED_SUCCESSFULLY) ? "" : "not ") + "successful!");
    logDebugInfo(job);
    DistBuildStatus distBuildStatus = prepareStatusFromJob(job).setETAMillis(0).build();
    eventBus.post(new DistBuildStatusEvent(distBuildStatus));
    return job.getStatus().equals(BuildStatus.FINISHED_SUCCESSFULLY) ? 0 : 1;
}

Example 44 with LinkedList

use of java.util.LinkedList in project buck by facebook.

the class DistBuildService method uploadBuckDotFiles.

public ListenableFuture<Void> uploadBuckDotFiles(final StampedeId id, final ProjectFilesystem filesystem, FileHashCache fileHashCache, ListeningExecutorService executorService) throws IOException {
    ListenableFuture<Pair<List<FileInfo>, List<PathInfo>>> filesFuture = executorService.submit(() -> {
        List<Path> buckDotFilesExceptConfig = Lists.newArrayList();
        for (Path path : filesystem.getDirectoryContents(filesystem.getRootPath())) {
            String fileName = path.getFileName().toString();
            if (!filesystem.isDirectory(path) && !filesystem.isSymLink(path) && fileName.startsWith(".") && fileName.contains("buck") && !fileName.startsWith(".buckconfig")) {
                buckDotFilesExceptConfig.add(path);
            }
        }
        List<FileInfo> fileEntriesToUpload = new LinkedList<>();
        List<PathInfo> pathEntriesToUpload = new LinkedList<>();
        for (Path path : buckDotFilesExceptConfig) {
            FileInfo fileInfoObject = new FileInfo();
            fileInfoObject.setContent(filesystem.readFileIfItExists(path).get().getBytes());
            fileInfoObject.setContentHash(fileHashCache.get(path.toAbsolutePath()).toString());
            fileEntriesToUpload.add(fileInfoObject);
            PathInfo pathInfoObject = new PathInfo();
            pathInfoObject.setPath(path.toString());
            pathInfoObject.setContentHash(fileHashCache.get(path.toAbsolutePath()).toString());
            pathEntriesToUpload.add(pathInfoObject);
        }
        return new Pair<>(fileEntriesToUpload, pathEntriesToUpload);
    });
    ListenableFuture<Void> setFilesFuture = Futures.transformAsync(filesFuture, filesAndPaths -> {
        setBuckDotFiles(id, filesAndPaths.getSecond());
        return Futures.immediateFuture(null);
    }, executorService);
    ListenableFuture<Void> uploadFilesFuture = Futures.transformAsync(filesFuture, filesAndPaths -> {
        uploadMissingFilesFromList(filesAndPaths.getFirst(), executorService);
        return Futures.immediateFuture(null);
    }, executorService);
    return Futures.transform(Futures.allAsList(ImmutableList.of(setFilesFuture, uploadFilesFuture)), new Function<List<Void>, Void>() {

        @Nullable
        @Override
        public Void apply(@Nullable List<Void> input) {
            return null;
        }
    });
}

Example 45 with LinkedList

use of java.util.LinkedList in project buck by facebook.

the class OfflineScribeLogger method sendStoredLogs.

private synchronized void sendStoredLogs() {
    ImmutableSortedSet<Path> logsPaths;
    try {
        if (!filesystem.isDirectory(logDir)) {
            // No logs to submit to Scribe.
            return;
        }
        logsPaths = filesystem.getMtimeSortedMatchingDirectoryContents(logDir, LOGFILE_PATTERN);
    } catch (Exception e) {
        LOG.error(e, "Fetching stored logs list failed.");
        return;
    }
    long totalBytesToSend = 0;
    for (Path logPath : logsPaths) {
        // Sending should be ceased if storing has been initiated or closing was started.
        if (startedStoring || startedClosing) {
            break;
        }
        // Get iterator.
        Iterator<ScribeData> it;
        File logFile;
        try {
            logFile = logPath.toFile();
            totalBytesToSend += logFile.length();
            if (totalBytesToSend > maxScribeOfflineLogsBytes) {
                LOG.warn("Total size of offline logs exceeds the limit. Ceasing to send them to Scribe.");
                return;
            }
            InputStream logFileStream;
            try {
                logFileStream = new BufferedInputStream(new FileInputStream(logFile), BUFFER_SIZE);
            } catch (FileNotFoundException e) {
                LOG.info(e, "There was a problem getting stream for logfile: %s. Likely logfile was resent and" + "deleted by a concurrent Buck command.", logPath);
                continue;
            }
            it = new ObjectMapper().readValues(new JsonFactory().createParser(logFileStream), ScribeData.class);
        } catch (Exception e) {
            LOG.error(e, "Failed to initiate reading from: %s. File may be corrupted.", logPath);
            continue;
        }
        // Read and submit.
        int scribeLinesInFile = 0;
        List<ListenableFuture<Void>> logFutures = new LinkedList<>();
        Map<String, CategoryData> logReadData = new HashMap<>();
        try {
            boolean interrupted = false;
            // Read data and build per category clusters - dispatch if needed.
            while (it.hasNext()) {
                if (startedStoring || startedClosing) {
                    interrupted = true;
                    break;
                }
                ScribeData newData = it.next();
                // Prepare map entry for new data (dispatch old data if needed).
                if (!logReadData.containsKey(newData.getCategory())) {
                    logReadData.put(newData.getCategory(), new CategoryData());
                }
                CategoryData categoryData = logReadData.get(newData.getCategory());
                if (categoryData.getLinesBytes() > CLUSTER_DISPATCH_SIZE) {
                    logFutures.add(scribeLogger.log(newData.getCategory(), categoryData.getLines()));
                    categoryData.clearData();
                }
                // Add new data to the cluster for the category.
                for (String line : newData.getLines()) {
                    categoryData.addLine(line);
                    scribeLinesInFile++;
                }
            }
            // Send remaining data from per category clusters.
            if (!interrupted) {
                for (Map.Entry<String, CategoryData> logReadDataEntry : logReadData.entrySet()) {
                    if (startedStoring || startedClosing) {
                        interrupted = true;
                        break;
                    }
                    List<String> categoryLines = logReadDataEntry.getValue().getLines();
                    if (categoryLines.size() > 0) {
                        logFutures.add(scribeLogger.log(logReadDataEntry.getKey(), categoryLines));
                    }
                }
            }
            if (interrupted) {
                LOG.info("Stopped while sending from offline log (it will not be removed): %s.", logPath);
                logFutures.clear();
                break;
            }
        } catch (Exception e) {
            LOG.error(e, "Error while reading offline log from: %s. This log will not be removed now. If this " + "error reappears in further runs, the file may be corrupted and should be deleted. ", logPath);
            logFutures.clear();
            continue;
        } finally {
            logReadData.clear();
        }
        // Confirm data was successfully sent and remove logfile.
        try {
            Futures.allAsList(logFutures).get(LOG_TIMEOUT, LOG_TIMEOUT_UNIT);
            totalBytesResent.inc(logFile.length());
            totalLinesResent.inc(scribeLinesInFile);
            logfilesResent.inc();
            try {
                filesystem.deleteFileAtPathIfExists(logPath);
            } catch (Exception e) {
                LOG.error(e, "Failed to remove successfully resent offline log. Stopping sending.");
                break;
            }
        } catch (Exception e) {
            LOG.info("Failed to send all data from offline log: %s. Log will not be removed.", logPath);
            // Do not attempt to send data from further logfiles - likely there are network issues.
            break;
        } finally {
            logFutures.clear();
        }
    }
}