Examples with PDRI nl.uva.cs.lobcder.resources.PDRI used on opensource projects

Example 6 with PDRI

use of nl.uva.cs.lobcder.resources.PDRI in project lobcder by skoulouzis.

the class DeleteSweep method run.

@Override
public void run() {
    PDRIFactory factory;
    try (Connection connection = datasource.getConnection()) {
        // This query interferes with the assimilator
        try {
            connection.setAutoCommit(false);
            try (Statement s1 = connection.createStatement(java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_UPDATABLE)) {
                ResultSet rs1 = s1.executeQuery("SELECT pdriGroupId FROM pdrigroup_table WHERE refCount = 0 ");
                while (rs1.next()) {
                    Long groupId = rs1.getLong(1);
                    try (PreparedStatement ps2 = connection.prepareStatement("SELECT pdriId, fileName, storageSiteRef FROM pdri_table WHERE pdriGroupRef = ?", java.sql.ResultSet.TYPE_FORWARD_ONLY, java.sql.ResultSet.CONCUR_UPDATABLE)) {
                        ps2.setLong(1, groupId);
                        ResultSet rs2 = ps2.executeQuery();
                        while (rs2.next()) {
                            // here better to use a kind a local cache, i.e. hashmap
                            try (PreparedStatement ps3 = connection.prepareStatement("SELECT resourceUri, username, password FROM storage_site_table " + "JOIN credential_table ON credentialRef = credintialId " + "WHERE storageSiteId = ?", java.sql.ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY)) {
                                String fileName = rs2.getString(2);
                                Long storageSiteRef = rs2.getLong(3);
                                ps3.setLong(1, storageSiteRef);
                                ResultSet rs3 = ps3.executeQuery();
                                if (rs3.next()) {
                                    String resourceUri = rs3.getString(1);
                                    String username = rs3.getString(2);
                                    String password = rs3.getString(3);
                                    // For deleteing we don't care for encryption
                                    PDRIDescr pdriDescr = new PDRIDescr(fileName, storageSiteRef, resourceUri, username, password, false, null, null, null);
                                    DeleteSweep.log.log(Level.FINE, "PDRI Description: {0}, {1}, {2}, {3}, {4}", new Object[] { fileName, storageSiteRef, resourceUri, username, password });
                                    DeleteSweep.log.log(Level.FINE, "PDRI pdriDescr: {0}", new Object[] { pdriDescr });
                                    factory = PDRIFactory.getFactory();
                                    DeleteSweep.log.log(Level.FINE, "PDRIFactory: {0}", factory);
                                    PDRI pdri = factory.createInstance(pdriDescr, false);
                                    DeleteSweep.log.log(Level.FINE, "pdri: {0}", pdri);
                                    DeleteSweep.log.log(Level.FINE, "PDRI Instance file name: {0}", new Object[] { pdri.getFileName() });
                                    pdri.delete();
                                    DeleteSweep.log.log(Level.FINE, "DELETE:", pdri.getURI());
                                    rs2.deleteRow();
                                    connection.commit();
                                }
                            }
                        }
                        rs1.deleteRow();
                        connection.commit();
                    }
                }
            }
        } catch (SQLException | IOException e) {
            DeleteSweep.log.log(Level.SEVERE, null, e);
            connection.rollback();
        }
    } catch (SQLException e) {
        DeleteSweep.log.log(Level.SEVERE, null, e);
    }
}

Example 7 with PDRI

use of nl.uva.cs.lobcder.resources.PDRI in project lobcder by skoulouzis.

the class WebDataFileResource method transfer.

private PDRI transfer(List<PDRIDescr> pdris, OutputStream out, int tryCount, boolean doCircularStreamBufferTransferer) throws IOException, NotFoundException {
    InputStream in = null;
    PDRI pdri = null;
    double start = 0;
    Logger.getLogger(WebDataFileResource.class.getName()).log(Level.FINEST, "Start for {0}", getLogicalData().getName());
    try {
        PDRIDescr pdriDescr = selectBestPDRI(pdris);
        pdri = PDRIFactory.getFactory().createInstance(pdriDescr, false);
        Logger.getLogger(WebDataFileResource.class.getName()).log(Level.FINEST, "pdri: {0}", pdri.getURI());
        if (pdri != null) {
            start = System.currentTimeMillis();
            in = pdri.getData();
            if (!pdri.getEncrypted()) {
                if (doCircularStreamBufferTransferer) {
                    CircularStreamBufferTransferer cBuff = new CircularStreamBufferTransferer((Constants.BUF_SIZE), in, out);
                    cBuff.startTransfer((long) -1);
                } else {
                    int read;
                    byte[] copyBuffer = new byte[Constants.BUF_SIZE];
                    while ((read = in.read(copyBuffer, 0, copyBuffer.length)) != -1) {
                        out.write(copyBuffer, 0, read);
                    // tos.write(copyBuffer, 0, read);
                    }
                }
            } else {
                DesEncrypter encrypter = new DesEncrypter(pdri.getKeyInt());
                encrypter.decrypt(in, out);
            }
        } else {
            sleepTime = 5;
            throw new NotFoundException("Physical resource not found");
        }
    } catch (Exception ex) {
        if (ex instanceof NotFoundException) {
            throw (NotFoundException) ex;
        }
        if (ex.getMessage().contains("Resource not found")) {
            throw new NotFoundException(ex.getMessage());
        }
        if (pdri != null) {
            Double speed = weightPDRIMap.get(pdri.getHost());
            if (speed != null) {
                speed = speed - 10;
            } else {
                speed = Double.valueOf(-10);
            }
            weightPDRIMap.put(pdri.getHost(), speed);
        }
        try {
            sleepTime = sleepTime + 5;
            Thread.sleep(sleepTime);
            if (ex instanceof nl.uva.vlet.exception.VlInterruptedException && ++tryCount < Constants.RECONNECT_NTRY) {
                transfer(pdris, out, tryCount, false);
            } else if (++tryCount < Constants.RECONNECT_NTRY) {
                transfer(pdris, out, tryCount, false);
            } else {
                transfer(pdris, out, 0, false);
            }
        } catch (InterruptedException ex1) {
            sleepTime = 5;
            throw new IOException(ex1);
        }
    } finally {
        if (in != null) {
            in.close();
        }
    }
    sleepTime = 5;
    double elapsed = System.currentTimeMillis() - start;
    double speed = ((pdri.getLength() * 8.0) * 1000.0) / (elapsed * 1000.0);
    try {
        String msg = "File: " + pdri.getFileName() + " Destination: " + new URI(pdri.getURI()).getScheme() + "://" + pdri.getHost() + " Rx_Speed: " + speed + " Kbites/sec Rx_Size: " + (pdri.getLength()) + " bytes Elapsed_Time: " + elapsed + " ms";
        Logger.getLogger(WebDataFileResource.class.getName()).log(Level.INFO, msg);
    } catch (URISyntaxException ex) {
    }
    return pdri;
}

Example 8 with PDRI

use of nl.uva.cs.lobcder.resources.PDRI in project lobcder by skoulouzis.

the class WebDataFileResource method sendContent.

@Override
public void sendContent(OutputStream out, Range range, Map<String, String> params, String contentType) throws IOException, NotAuthorizedException, BadRequestException, NotFoundException {
    double start = System.currentTimeMillis();
    PDRI pdri;
    Iterator<PDRIDescr> it;
    try {
        List<PDRIDescr> pdris = getCatalogue().getPdriDescrByGroupId(getLogicalData().getPdriGroupId());
        if (pdris.size() <= 0) {
            throw new NotFoundException("File inconsistency! Could not find physical file!");
        }
        // it = getCatalogue().getPdriDescrByGroupId(getLogicalData().getPdriGroupId()).iterator();
        if (range != null) {
            if (range.getFinish() == null) {
                range = new Range(range.getStart(), (getLogicalData().getLength() - 1));
            }
            it = pdris.iterator();
            Logger.getLogger(WebDataFileResource.class.getName()).log(Level.FINEST, "Start: {0} end: {1} range: {2}", new Object[] { range.getStart(), range.getFinish(), range.getRange() });
            pdri = transfererRange(it, out, 0, null, range);
        } else {
            // pdri = transfer(it, out, 0, null, false);
            pdri = transfer(pdris, out, 0, false);
        }
    } catch (SQLException ex) {
        throw new BadRequestException(this, ex.getMessage());
    } catch (IOException ex) {
        if (ex.getMessage().contains("Resource not found") || ex.getMessage().contains("Couldn't locate path")) {
            throw new NotFoundException(ex.getMessage());
        } else {
            throw new BadRequestException(this, ex.getMessage());
        }
    } finally {
    // Don't close the output, we need it to send back the response
    // if (out != null) {
    // out.flush();
    // out.close();
    // }
    }
    double elapsed = System.currentTimeMillis() - start;
    long len;
    if (range != null) {
        len = range.getFinish() - range.getStart() + 1;
    } else {
        len = this.getLogicalData().getLength();
    }
    double speed = ((len * 8.0) * 1000.0) / (elapsed * 1000.0);
    Double oldSpeed = weightPDRIMap.get(pdri.getHost());
    if (oldSpeed == null) {
        oldSpeed = speed;
    }
    Integer numOfGets = numOfGetsMap.get(pdri.getHost());
    if (numOfGets == null) {
        numOfGets = 1;
    }
    double averagre = (speed + oldSpeed) / (double) numOfGets;
    numOfGetsMap.put(pdri.getHost(), numOfGets++);
    weightPDRIMap.put(pdri.getHost(), averagre);
    getCatalogue().addViewForRes(getLogicalData().getUid());
    Stats stats = new Stats();
    stats.setSource(pdri.getHost());
    stats.setDestination(fromAddress);
    stats.setSpeed(speed);
    stats.setSize(getLogicalData().getLength());
    String msg = "Source: " + stats.getSource() + " Destination: " + stats.getDestination() + " Tx_Speed: " + speed + " Kbites/sec Tx_Size: " + getLogicalData().getLength() + " bytes Elapsed_Time: " + elapsed + " ms";
    try {
        if (!pdri.isCahce()) {
            getCatalogue().setSpeed(stats);
        }
    } catch (SQLException ex) {
        Logger.getLogger(WebDataFileResource.class.getName()).log(Level.SEVERE, null, ex);
    }
    Logger.getLogger(WebDataFileResource.class.getName()).log(Level.INFO, msg);
}

Example 9 with PDRI

use of nl.uva.cs.lobcder.resources.PDRI in project lobcder by skoulouzis.

the class StorageSitesService method updateLogicalDataAndPdri.

@PUT
@Path("update_pdri/{uid}")
@Consumes({ MediaType.APPLICATION_XML, MediaType.APPLICATION_JSON })
public void updateLogicalDataAndPdri(PDRIDescrWrapperList pdris, @PathParam("uid") Long uid) throws SQLException, IOException {
    MyPrincipal mp = (MyPrincipal) request.getAttribute("myprincipal");
    if (mp.isAdmin()) {
        try (Connection cn = getCatalogue().getConnection()) {
            for (PDRIDescr pdriDescr : pdris.getPdris()) {
                LogicalData logicalData = getCatalogue().getLogicalDataByUid(uid, cn);
                PDRI pdri = PDRIFactory.getFactory().createInstance(pdriDescr, false);
                logicalData.setLength(pdri.getLength());
                getCatalogue().updatePdri(logicalData, pdri, cn);
            }
            cn.commit();
            cn.close();
        }
    }
}

Example 10 with PDRI

use of nl.uva.cs.lobcder.resources.PDRI in project lobcder by skoulouzis.

the class Catalogue method replicate.

boolean replicate(Pair<File, String> p) throws IOException {
    boolean result = true;
    PDRIDescr sourceDescr = createPdriDescr(p.getLeft());
    PDRI sourcePdri = PDRIFactory.getFactory().createInstance(sourceDescr);
    String[] sites = p.getRight().split("-");
    Long fileUID = Long.valueOf(sites[0]);
    Long pdriGroupId = Long.valueOf(sites[1]);
    List<PDRIDescr> destinationDescrList = new ArrayList<>();
    for (int i = 2; i < sites.length; i++) {
        try {
            StorageSiteWrapper ss = storageSiteCache.get(Long.valueOf(sites[i]));
            if (ss == null) {
                getStorageSites(String.valueOf(Long.valueOf(sites[i])));
                ss = storageSiteCache.get(Long.valueOf(sites[i]));
            }
            BigInteger pdriKey = nl.uva.cs.lobcder.util.DesEncrypter.generateKey();
            PDRIDescr destinationDescr = new PDRIDescr(sourceDescr.getName(), ss.getStorageSiteId(), ss.getResourceURI(), ss.getCredential().getStorageSiteUsername(), ss.getCredential().getStorageSitePassword(), ss.isEncrypt(), pdriKey, pdriGroupId, null, ss.isCache());
            PDRI destinationPdri = PDRIFactory.getFactory().createInstance(destinationDescr);
            destinationPdri.replicate(sourcePdri);
            result = destinationPdri.exists(destinationPdri.getFileName());
            long srcLen = sourcePdri.getLength();
            if (result == false || destinationPdri.getLength() != srcLen) {
                result = false;
            }
            destinationDescrList.add(destinationDescr);
        } catch (Exception e) {
            result = false;
        } catch (Throwable e) {
            result = false;
        }
    }
    result = updateMaster(destinationDescrList, fileUID);
    return result;
}