Examples with DeletedObservation ca.nrc.cadc.caom2.DeletedObservation used on opensource projects

Example 6 with DeletedObservation

use of ca.nrc.cadc.caom2.DeletedObservation in project caom2db by opencadc.

the class AbstractDeletedEntityDAOTest method testPutGetDelete.

@Test
public void testPutGetDelete() {
    try {
        UUID id1 = new UUID(0L, 100L);
        Date d1 = new Date();
        ObservationURI u1 = new ObservationURI("FOO", "bar1");
        DeletedObservation o1 = new DeletedObservation(id1, u1);
        log.info("put: " + o1);
        dao.put(o1);
        Assert.assertNotNull("side effect: DeletedObservation.lastModified", o1.lastModified);
        DeletedEntity per = dao.get(DeletedObservation.class, id1);
        Assert.assertNotNull("found DeletedObservation " + id1, per);
        Assert.assertNotNull("DeletedObservation.lastModified", per.lastModified);
        dao.delete(per);
        DeletedEntity gone = dao.get(DeletedObservation.class, id1);
        Assert.assertNull(gone);
    } catch (Exception unexpected) {
        log.error("unexpected exception", unexpected);
        Assert.fail("unexpected exception: " + unexpected);
    }
}

Example 7 with DeletedObservation

use of ca.nrc.cadc.caom2.DeletedObservation in project caom2db by opencadc.

the class AbstractDeletedEntityDAOTest method testGetListDeletedObservation.

@Test
public void testGetListDeletedObservation() {
    try {
        UUID id1 = new UUID(0L, 100L);
        UUID id2 = new UUID(0L, 200L);
        UUID id3 = new UUID(0L, 300L);
        UUID id4 = new UUID(0L, 400L);
        UUID id5 = new UUID(0L, 500L);
        Date d1 = new Date();
        Date d2 = new Date(d1.getTime() + 10L);
        Date d3 = new Date(d2.getTime() + 10L);
        Date d4 = new Date(d3.getTime() + 10L);
        Date d5 = new Date(d4.getTime() + 10L);
        ObservationURI u1 = new ObservationURI("FOO", "bar1");
        ObservationURI u2 = new ObservationURI("FOO", "bar2");
        ObservationURI u3 = new ObservationURI("FOO", "bar3");
        ObservationURI u4 = new ObservationURI("FOO", "bar4");
        ObservationURI u5 = new ObservationURI("FOO", "bar5");
        DeletedObservation o1 = new DeletedObservation(id1, u1);
        DeletedObservation o2 = new DeletedObservation(id2, u2);
        DeletedObservation o3 = new DeletedObservation(id3, u3);
        DeletedObservation o4 = new DeletedObservation(id4, u4);
        DeletedObservation o5 = new DeletedObservation(id5, u5);
        log.info("put: \n" + o1 + "\n" + o2 + "\n" + o3 + "\n" + o4 + "\n" + o5 + "\n");
        dao.put(o1);
        Thread.sleep(10L);
        dao.put(o2);
        Thread.sleep(10L);
        dao.put(o3);
        Thread.sleep(10L);
        dao.put(o4);
        Thread.sleep(10L);
        dao.put(o5);
        Assert.assertNotNull("DeletedObservation.lastModified", o1.lastModified);
        // past
        Date start = new Date(o1.lastModified.getTime() - 100L);
        Date end = null;
        Integer batchSize = new Integer(3);
        List<DeletedEntity> dels;
        // get first batch
        dels = dao.getList(DeletedObservation.class, start, end, batchSize);
        Assert.assertNotNull("deleted list", dels);
        Assert.assertEquals(3, dels.size());
        Assert.assertEquals(o1.getID(), dels.get(0).getID());
        Assert.assertEquals(o2.getID(), dels.get(1).getID());
        Assert.assertEquals(o3.getID(), dels.get(2).getID());
        // get next batch
        dels = dao.getList(DeletedObservation.class, o3.lastModified, end, batchSize);
        Assert.assertNotNull("deleted list/next batch", dels);
        // o3 gets picked up by the >=
        Assert.assertEquals(3, dels.size());
        Assert.assertEquals(o3.getID(), dels.get(0).getID());
        Assert.assertEquals(o4.getID(), dels.get(1).getID());
        Assert.assertEquals(o5.getID(), dels.get(2).getID());
    } catch (Exception unexpected) {
        log.error("unexpected exception", unexpected);
        Assert.fail("unexpected exception: " + unexpected);
    }
}

Example 8 with DeletedObservation

use of ca.nrc.cadc.caom2.DeletedObservation in project caom2db by opencadc.

the class Main method main.

public static void main(String[] args) {
    try {
        ArgumentMap am = new ArgumentMap(args);
        if (am.isSet("d") || am.isSet("debug")) {
            Log4jInit.setLevel("ca.nrc.cadc.caom2.repo.client", Level.DEBUG);
            Log4jInit.setLevel("ca.nrc.cadc.reg.client", Level.DEBUG);
        } else if (am.isSet("v") || am.isSet("verbose")) {
            Log4jInit.setLevel("ca.nrc.cadc.caom2.repo.client", Level.INFO);
        } else {
            Log4jInit.setLevel("ca.nrc.cadc", Level.WARN);
            Log4jInit.setLevel("ca.nrc.cadc.caom2.repo.client", Level.WARN);
        }
        if (am.isSet("h") || am.isSet("help")) {
            usage();
            System.exit(0);
        }
        // Subject subject = AuthenticationUtil.getAnonSubject();
        // if (am.isSet("netrc")) {
        // subject = AuthenticationUtil.getSubject(new NetrcAuthenticator(true));
        // } else if (am.isSet("cert")) {
        // subject = CertCmdArgUtil.initSubject(am);
        // }
        // AuthMethod meth = AuthenticationUtil.getAuthMethodFromCredentials(subject);
        // log.info("authentication using: " + meth);
        // 
        // log.info("AuthenticationUtil.getCurrentSubject(): " + AuthenticationUtil.getCurrentSubject());
        RepoClient rc = new RepoClient(new URI("ivo://cadc.nrc.ca/caom2repo"), 1);
        List<DeletedObservation> list = rc.getDeleted("HSTHLA", null, null, 100);
    // for (DeletedObservation deleted : list) {
    // log.info(deleted.getURI().getObservationID());
    // }
    } catch (Throwable uncaught) {
        log.error("uncaught exception", uncaught);
        System.exit(-1);
    }
}

Example 9 with DeletedObservation

use of ca.nrc.cadc.caom2.DeletedObservation in project caom2db by opencadc.

the class RepoClient method readDeletedEntityList.

private List<DeletedObservation> readDeletedEntityList(DeletionListReader transformer, String collection, Date start, Date end, Integer maxrec) {
    List<DeletedObservation> accList = new ArrayList<>();
    List<DeletedObservation> partialList = null;
    boolean tooBigRequest = maxrec == null || maxrec > DEFAULT_BATCH_SIZE;
    Integer rec = maxrec;
    Integer recCounter;
    if (tooBigRequest) {
        rec = DEFAULT_BATCH_SIZE;
    }
    // Use HttpDownload to make the http GET calls (because it handles a lot
    // of the
    // authentication stuff)
    boolean go = true;
    String surlCommon = baseDeletionURL.toExternalForm() + File.separator + collection;
    while (go) {
        ByteArrayOutputStream bos = new ByteArrayOutputStream();
        if (!tooBigRequest) {
            // only one go
            go = false;
        }
        String surl = surlCommon;
        surl = surl + "?maxRec=" + (rec + 1);
        if (start != null) {
            surl = surl + "&start=" + df.format(start);
        }
        if (end != null) {
            surl = surl + "&end=" + df.format(end);
        }
        URL url;
        log.debug("URL: " + surl);
        try {
            url = new URL(surl);
            HttpDownload get = new HttpDownload(url, bos);
            get.setFollowRedirects(true);
            get.run();
            int responseCode = get.getResponseCode();
            log.debug("RESPONSE CODE: '" + responseCode + "'");
            if (responseCode == 302) {
                // redirected url
                url = get.getRedirectURL();
                log.debug("REDIRECTED URL: " + url);
                bos = new ByteArrayOutputStream();
                get = new HttpDownload(url, bos);
                responseCode = get.getResponseCode();
                log.debug("RESPONSE CODE (REDIRECTED URL): '" + responseCode + "'");
            }
            if (get.getThrowable() != null) {
                if (get.getThrowable() instanceof AccessControlException) {
                    throw (AccessControlException) get.getThrowable();
                }
                throw new RuntimeException("failed to get observation list", get.getThrowable());
            }
        } catch (MalformedURLException e) {
            throw new RuntimeException("BUG: failed to generate observation list url", e);
        }
        try {
            // log.debug("RESPONSE = '" + bos.toString() + "'");
            partialList = transformer.read(new ByteArrayInputStream(bos.toByteArray()));
            // df, '\t', '\n');
            if (partialList != null && !partialList.isEmpty() && !accList.isEmpty() && accList.get(accList.size() - 1).equals(partialList.get(0))) {
                partialList.remove(0);
            }
            if (partialList != null) {
                accList.addAll(partialList);
                log.debug("adding " + partialList.size() + " elements to accList. Now there are " + accList.size());
            }
            bos.close();
        } catch (ParseException | URISyntaxException | IOException e) {
            throw new RuntimeException("Unable to list of ObservationState from " + bos.toString(), e);
        }
        if (accList.size() > 0) {
            start = accList.get(accList.size() - 1).getLastModified();
        }
        recCounter = accList.size();
        if (maxrec != null && maxrec - recCounter > 0 && maxrec - recCounter < rec) {
            rec = maxrec - recCounter;
        }
        int i = 0;
        for (DeletedObservation de : accList) {
            log.debug("accList.get( " + i++ + ") = " + de.getLastModified());
        }
        log.debug("accList.size() = " + accList.size());
        log.debug("dynamic batch (rec): " + rec);
        log.debug("maxrec: " + maxrec);
        log.debug("start: " + start);
        log.debug("end: " + end);
        if (partialList != null) {
            log.debug("partialList.size(): " + partialList.size());
            if (partialList.size() < rec || (end != null && start != null && start.equals(end))) {
                log.debug("************** go false");
                go = false;
            }
        }
    }
    return partialList;
}

Example 10 with DeletedObservation

use of ca.nrc.cadc.caom2.DeletedObservation in project caom2db by opencadc.

the class DeletionHarvester method doit.

/**
 * Does the work
 *
 * @return progress status
 */
private Progress doit() {
    log.info("batch: " + entityClass.getSimpleName());
    Progress ret = new Progress();
    int expectedNum = Integer.MAX_VALUE;
    if (batchSize != null) {
        expectedNum = batchSize.intValue();
    }
    boolean correct = true;
    try {
        HarvestState state = harvestStateDAO.get(source, cname);
        log.info("last harvest: " + format(state.curLastModified));
        if (initHarvestState && state.curLastModified == null) {
            state.curLastModified = initDate;
            harvestStateDAO.put(state);
            state = harvestStateDAO.get(source, cname);
            log.info("harvest state initialised to: " + df.format(state.curLastModified));
        }
        startDate = state.curLastModified;
        if (firstIteration) {
            if (super.minDate != null) {
                // override state
                startDate = super.minDate;
            }
            endDate = super.maxDate;
            // harvest up to a little in the past because the head of the
            // sequence may be volatile
            long fiveMinAgo = System.currentTimeMillis() - 5 * 60000L;
            if (endDate == null) {
                endDate = new Date(fiveMinAgo);
            } else {
                endDate = new Date(Math.min(fiveMinAgo, endDate.getTime()));
            }
        }
        firstIteration = false;
        List<DeletedObservation> entityList = null;
        String source = null;
        if (deletedDAO != null) {
            source = "deletedDAO";
            entityList = deletedDAO.getList(src.getCollection(), startDate, endDate, batchSize);
        } else {
            source = "repoClient";
            entityList = repoClient.getDeleted(src.getCollection(), startDate, endDate, batchSize);
        }
        if (entityList == null) {
            throw new RuntimeException("Error gathering deleted observations from " + source);
        }
        if (entityList.size() == expectedNum) {
            detectLoop(entityList);
        }
        ret.found = entityList.size();
        log.info("found: " + entityList.size());
        ListIterator<DeletedObservation> iter = entityList.listIterator();
        while (iter.hasNext()) {
            DeletedObservation de = iter.next();
            // allow garbage collection asap
            iter.remove();
            log.debug("Observation read from deletion end-point: " + de.getID() + " date = " + de.getLastModified());
            if (!dryrun) {
                txnManager.startTransaction();
            }
            boolean ok = false;
            try {
                if (!dryrun) {
                    state.curLastModified = de.getLastModified();
                    state.curID = de.getID();
                    ObservationState cur = obsDAO.getState(de.getID());
                    if (cur != null) {
                        log.debug("Observation: " + de.getID() + " found in DB");
                        Date lastUpdate = cur.getMaxLastModified();
                        Date deleted = de.getLastModified();
                        log.debug("to be deleted: " + de.getClass().getSimpleName() + " " + de.getURI() + " " + de.getID() + "deleted date " + format(de.getLastModified()) + " modified date " + format(cur.getMaxLastModified()));
                        if (deleted.after(lastUpdate)) {
                            log.info("delete: " + de.getClass().getSimpleName() + " " + de.getURI() + " " + de.getID());
                            obsDAO.delete(de.getID());
                            ret.deleted++;
                        } else {
                            log.info("skip out-of-date delete: " + de.getClass().getSimpleName() + " " + de.getURI() + " " + de.getID() + " " + format(de.getLastModified()));
                            ret.skipped++;
                        }
                    } else {
                        log.debug("Observation: " + de.getID() + " not found in DB");
                    }
                    // track progress
                    harvestStateDAO.put(state);
                    log.debug("committing transaction");
                    txnManager.commitTransaction();
                    log.debug("commit: OK");
                }
                ok = true;
            } catch (Throwable t) {
                log.error("unexpected exception", t);
            } finally {
                if (!ok && !dryrun) {
                    log.warn("failed to process " + de + ": trying to rollback the transaction");
                    txnManager.rollbackTransaction();
                    log.warn("rollback: OK");
                    ret.abort = true;
                }
            }
        }
        if (ret.found < expectedNum) {
            ret.done = true;
            if (state != null && state.curLastModified != null && ret.found > 0) {
                // tweak HarvestState so we don't keep picking up the same
                // one
                // 1
                Date n = new Date(state.curLastModified.getTime() + 1L);
                // ms
                // ahead
                Date now = new Date();
                if (now.getTime() - n.getTime() > 600 * 1000L) {
                    n = new Date(state.curLastModified.getTime() + 100L);
                }
                // ahead
                state.curLastModified = n;
                log.info("reached last " + entityClass.getSimpleName() + ": setting curLastModified to " + format(state.curLastModified));
                harvestStateDAO.put(state);
            }
        }
    } catch (Throwable t) {
        log.error("unexpected exception", t);
        ret.abort = true;
        correct = false;
    } finally {
        if (correct) {
            log.debug("DONE");
        }
    }
    return ret;
}