Examples with DigestInputStream java.security.DigestInputStream used on opensource projects

Example 46 with DigestInputStream

use of java.security.DigestInputStream in project hadoop by apache.

the class Util method receiveFile.

/**
   * Receives file at the url location from the input stream and puts them in
   * the specified destination storage location.
   */
public static MD5Hash receiveFile(String url, List<File> localPaths, Storage dstStorage, boolean getChecksum, long advertisedSize, MD5Hash advertisedDigest, String fsImageName, InputStream stream, DataTransferThrottler throttler) throws IOException {
    long startTime = Time.monotonicNow();
    Map<FileOutputStream, File> streamPathMap = new HashMap<>();
    StringBuilder xferStats = new StringBuilder();
    double xferCombined = 0;
    if (localPaths != null) {
        // If the local paths refer to directories, use the server-provided header
        // as the filename within that directory
        List<File> newLocalPaths = new ArrayList<>();
        for (File localPath : localPaths) {
            if (localPath.isDirectory()) {
                if (fsImageName == null) {
                    throw new IOException("No filename header provided by server");
                }
                newLocalPaths.add(new File(localPath, fsImageName));
            } else {
                newLocalPaths.add(localPath);
            }
        }
        localPaths = newLocalPaths;
    }
    long received = 0;
    MessageDigest digester = null;
    if (getChecksum) {
        digester = MD5Hash.getDigester();
        stream = new DigestInputStream(stream, digester);
    }
    boolean finishedReceiving = false;
    List<FileOutputStream> outputStreams = Lists.newArrayList();
    try {
        if (localPaths != null) {
            for (File f : localPaths) {
                try {
                    if (f.exists()) {
                        LOG.warn("Overwriting existing file " + f + " with file downloaded from " + url);
                    }
                    FileOutputStream fos = new FileOutputStream(f);
                    outputStreams.add(fos);
                    streamPathMap.put(fos, f);
                } catch (IOException ioe) {
                    LOG.warn("Unable to download file " + f, ioe);
                    // outside of an NNStorage directory.
                    if (dstStorage != null && (dstStorage instanceof StorageErrorReporter)) {
                        ((StorageErrorReporter) dstStorage).reportErrorOnFile(f);
                    }
                }
            }
            if (outputStreams.isEmpty()) {
                throw new IOException("Unable to download to any storage directory");
            }
        }
        int num = 1;
        byte[] buf = new byte[IO_FILE_BUFFER_SIZE];
        while (num > 0) {
            num = stream.read(buf);
            if (num > 0) {
                received += num;
                for (FileOutputStream fos : outputStreams) {
                    fos.write(buf, 0, num);
                }
                if (throttler != null) {
                    throttler.throttle(num);
                }
            }
        }
        finishedReceiving = true;
        double xferSec = Math.max(((float) (Time.monotonicNow() - startTime)) / 1000.0, 0.001);
        long xferKb = received / 1024;
        xferCombined += xferSec;
        xferStats.append(String.format(" The file download took %.2fs at %.2f KB/s.", xferSec, xferKb / xferSec));
    } finally {
        stream.close();
        for (FileOutputStream fos : outputStreams) {
            long flushStartTime = Time.monotonicNow();
            fos.getChannel().force(true);
            fos.close();
            double writeSec = Math.max(((float) (flushStartTime - Time.monotonicNow())) / 1000.0, 0.001);
            xferCombined += writeSec;
            xferStats.append(String.format(" Synchronous (fsync) write to disk of " + streamPathMap.get(fos).getAbsolutePath() + " took %.2fs.", writeSec));
        }
        // Remove the temporary files.
        if (!finishedReceiving) {
            deleteTmpFiles(localPaths);
        }
        if (finishedReceiving && received != advertisedSize) {
            // only throw this exception if we think we read all of it on our end
            // -- otherwise a client-side IOException would be masked by this
            // exception that makes it look like a server-side problem!
            deleteTmpFiles(localPaths);
            throw new IOException("File " + url + " received length " + received + " is not of the advertised size " + advertisedSize);
        }
    }
    xferStats.insert(0, String.format("Combined time for file download and" + " fsync to all disks took %.2fs.", xferCombined));
    LOG.info(xferStats.toString());
    if (digester != null) {
        MD5Hash computedDigest = new MD5Hash(digester.digest());
        if (advertisedDigest != null && !computedDigest.equals(advertisedDigest)) {
            deleteTmpFiles(localPaths);
            throw new IOException("File " + url + " computed digest " + computedDigest + " does not match advertised digest " + advertisedDigest);
        }
        return computedDigest;
    } else {
        return null;
    }
}

Example 47 with DigestInputStream

use of java.security.DigestInputStream in project android_frameworks_base by ParanoidAndroid.

the class ManifestDigest method fromInputStream.

static ManifestDigest fromInputStream(InputStream fileIs) {
    if (fileIs == null) {
        return null;
    }
    final MessageDigest md;
    try {
        md = MessageDigest.getInstance(DIGEST_ALGORITHM);
    } catch (NoSuchAlgorithmException e) {
        throw new RuntimeException(DIGEST_ALGORITHM + " must be available", e);
    }
    final DigestInputStream dis = new DigestInputStream(new BufferedInputStream(fileIs), md);
    try {
        byte[] readBuffer = new byte[8192];
        while (dis.read(readBuffer, 0, readBuffer.length) != -1) {
        // not using
        }
    } catch (IOException e) {
        Slog.w(TAG, "Could not read manifest");
        return null;
    } finally {
        IoUtils.closeQuietly(dis);
    }
    final byte[] digest = md.digest();
    return new ManifestDigest(digest);
}

Example 48 with DigestInputStream

use of java.security.DigestInputStream in project mobile-android by photo.

the class SHA1Utils method computeSha1ForFile.

public static String computeSha1ForFile(String filePath) throws IOException, NoSuchAlgorithmException {
    MessageDigest md = MessageDigest.getInstance("SHA1");
    // reads the file path & file name As a argument
    FileInputStream fis = new FileInputStream(filePath);
    DigestInputStream dis = new DigestInputStream(fis, md);
    BufferedInputStream bis = new BufferedInputStream(dis);
    try {
        // Read the bis so SHA1 is auto calculated at dis
        while (true) {
            int b = bis.read();
            if (b == -1)
                break;
        }
    } finally {
        bis.close();
    }
    return byteArray2Hex(md.digest());
}

Example 49 with DigestInputStream

use of java.security.DigestInputStream in project robovm by robovm.

the class DigestInputStreamTest method testReadbyteArrayintint01.

/**
     * Test #1 for <code>read(byte[],int,int)</code> method<br>
     *
     * Assertion: returns the number of bytes read<br>
     *
     * Assertion: put bytes read into specified array at specified offset<br>
     *
     * Assertion: updates associated digest<br>
     */
public final void testReadbyteArrayintint01() throws IOException {
    for (int ii = 0; ii < algorithmName.length; ii++) {
        try {
            MessageDigest md = MessageDigest.getInstance(algorithmName[ii]);
            InputStream is = new ByteArrayInputStream(myMessage);
            DigestInputStream dis = new DigestInputStream(is, md);
            byte[] bArray = new byte[MY_MESSAGE_LEN];
            // check that read(byte[],int,int) returns valid value
            assertTrue("retval", dis.read(bArray, 0, bArray.length) == MY_MESSAGE_LEN);
            // check that bArray has been filled properly
            assertTrue("bArray", Arrays.equals(myMessage, bArray));
            // check that associated digest has been updated properly
            assertTrue("update", Arrays.equals(dis.getMessageDigest().digest(), MDGoldenData.getDigest(algorithmName[ii])));
            return;
        } catch (NoSuchAlgorithmException e) {
        // allowed failure
        }
    }
    fail(getName() + ": no MessageDigest algorithms available - test not performed");
}

Example 50 with DigestInputStream

use of java.security.DigestInputStream in project robovm by robovm.

the class DigestInputStreamTest method testRead05.

/**
     * Test #5 for <code>read()</code> method<br>
     *
     * Assertion: broken <code>DigestInputStream</code>instance:
     * associated <code>MessageDigest</code> not set.
     * <code>read()</code> must not work when digest
     * functionality is on
     */
public final void testRead05() {
    InputStream is = new ByteArrayInputStream(myMessage);
    DigestInputStream dis = new DigestInputStream(is, null);
    // must result in an exception
    try {
        for (int i = 0; i < MY_MESSAGE_LEN; i++) {
            dis.read();
        }
        fail("read() must not work when digest functionality is on");
    } catch (Exception e) {
    // Expected.
    }
}