Examples with FileReference com.yahoo.config.FileReference used on opensource projects

Example 31 with FileReference

use of com.yahoo.config.FileReference in project vespa by vespa-engine.

the class FileDirectoryTest method requireThatFileReferenceWithSubDirectoriesWorks.

@Test
public void requireThatFileReferenceWithSubDirectoriesWorks() throws IOException {
    FileDirectory fileDirectory = new FileDirectory(temporaryFolder.getRoot());
    String subdirName = "subdir";
    File subDirectory = new File(temporaryFolder.getRoot(), subdirName);
    createFileInSubDir(subDirectory, "foo");
    FileReference fileReference = fileDirectory.addFile(subDirectory);
    File dir = fileDirectory.getFile(fileReference);
    assertTrue(dir.exists());
    assertTrue(new File(dir, "foo").exists());
    assertFalse(new File(dir, "doesnotexist").exists());
    assertEquals("1315a322fc323608", fileReference.value());
    // Add a file, should be available and file reference should have another value
    createFileInSubDir(subDirectory, "bar");
    fileReference = fileDirectory.addFile(subDirectory);
    dir = fileDirectory.getFile(fileReference);
    assertTrue(new File(dir, "foo").exists());
    assertTrue(new File(dir, "bar").exists());
    assertEquals("9ca074b47a4b510c", fileReference.value());
}

Example 32 with FileReference

use of com.yahoo.config.FileReference in project vespa by vespa-engine.

the class FileDownloader method getFutureFile.

private Future<Optional<File>> getFutureFile(FileReferenceDownload fileReferenceDownload) {
    FileReference fileReference = fileReferenceDownload.fileReference();
    Objects.requireNonNull(fileReference, "file reference cannot be null");
    File directory = new File(downloadDirectory, fileReference.value());
    log.log(LogLevel.DEBUG, () -> "Checking if there is a file in '" + directory.getAbsolutePath() + "' ");
    Optional<File> file = getFileFromFileSystem(fileReference, directory);
    if (file.isPresent()) {
        SettableFuture<Optional<File>> future = SettableFuture.create();
        future.set(file);
        return future;
    } else {
        log.log(LogLevel.DEBUG, () -> "File reference '" + fileReference.value() + "' not found in " + directory.getAbsolutePath() + ", starting download");
        return queueForAsyncDownload(fileReferenceDownload, timeout);
    }
}

Example 33 with FileReference

use of com.yahoo.config.FileReference in project vespa by vespa-engine.

the class FileReceiver method receiveFilePart.

@SuppressWarnings({ "UnusedDeclaration" })
public final void receiveFilePart(Request req) {
    log.log(LogLevel.DEBUG, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters());
    FileReference reference = new FileReference(req.parameters().get(0).asString());
    int sessionId = req.parameters().get(1).asInt32();
    int partId = req.parameters().get(2).asInt32();
    byte[] part = req.parameters().get(3).asData();
    Session session = getSession(sessionId);
    int retval = verifySession(session, sessionId, reference);
    try {
        session.addPart(partId, part);
    } catch (Exception e) {
        log.severe("Got exception " + e);
        retval = 1;
    }
    double completeness = (double) session.currentFileSize / (double) session.fileSize;
    log.log(LogLevel.DEBUG, () -> String.format("%.1f percent of '%s' downloaded", completeness * 100, reference.value()));
    downloader.setDownloadStatus(reference, completeness);
    req.returnValues().add(new Int32Value(retval));
}

Example 34 with FileReference

use of com.yahoo.config.FileReference in project vespa by vespa-engine.

the class FileReceiver method receiveFileMeta.

@SuppressWarnings({ "UnusedDeclaration" })
public final void receiveFileMeta(Request req) {
    log.log(LogLevel.DEBUG, () -> "Received method call '" + req.methodName() + "' with parameters : " + req.parameters());
    FileReference reference = new FileReference(req.parameters().get(0).asString());
    String fileName = req.parameters().get(1).asString();
    String type = req.parameters().get(2).asString();
    long fileSize = req.parameters().get(3).asInt64();
    int sessionId = nextSessionId.getAndIncrement();
    int retval = 0;
    synchronized (sessions) {
        if (sessions.containsKey(sessionId)) {
            retval = 1;
            log.severe("Session id " + sessionId + " already exist, impossible. Request from(" + req.target() + ")");
        } else {
            try {
                sessions.put(sessionId, new Session(downloadDirectory, tmpDirectory, sessionId, reference, FileReferenceData.Type.valueOf(type), fileName, fileSize));
            } catch (Exception e) {
                retval = 1;
            }
        }
    }
    req.returnValues().add(new Int32Value(retval));
    req.returnValues().add(new Int32Value(sessionId));
}

Example 35 with FileReference

use of com.yahoo.config.FileReference in project vespa by vespa-engine.

the class FileReferenceDownloader method startDownload.

private void startDownload(Duration timeout, FileReferenceDownload fileReferenceDownload) {
    FileReference fileReference = fileReferenceDownload.fileReference();
    long end = System.currentTimeMillis() + timeout.toMillis();
    boolean downloadStarted = false;
    while ((System.currentTimeMillis() < end) && !downloadStarted) {
        try {
            if (startDownloadRpc(fileReferenceDownload)) {
                downloadStarted = true;
            } else {
                Thread.sleep(sleepBetweenRetries.toMillis());
            }
        } catch (InterruptedException e) {
        /* ignored */
        }
    }
    if (!downloadStarted) {
        fileReferenceDownload.future().setException(new RuntimeException("Failed getting file reference '" + fileReference.value() + "'"));
        synchronized (downloads) {
            downloads.remove(fileReference);
        }
    }
}