Examples with DeletedObservation ca.nrc.cadc.caom2.DeletedObservation used on opensource projects

Example 1 with DeletedObservation

use of ca.nrc.cadc.caom2.DeletedObservation in project caom2db by opencadc.

the class AbstractObservationDAOTest method testPutGetDelete.

@Test
public void testPutGetDelete() {
    try {
        Observation orig = getTestObservation(false, 5, false, true);
        // !EXISTS
        Assert.assertNull(dao.getState(orig.getURI()));
        log.info("not-exists");
        ObservationResponse notFound = dao.getObservationResponse(orig.getURI());
        Assert.assertNotNull("wrapper", notFound);
        Assert.assertNotNull("wrapper", notFound.observationState);
        Assert.assertNull("wrapped", notFound.observation);
        // PUT
        dao.put(orig);
        // this is so we can detect incorrect timestamp round trips
        // caused by assigning something other than what was stored
        Thread.sleep(2 * TIME_TOLERANCE);
        log.info("put + sleep");
        // EXISTS
        Assert.assertNotNull("exists-by-uri", dao.getState(orig.getURI()));
        Assert.assertNotNull("exists-by-uuid", dao.getState(orig.getID()));
        log.info("exists");
        // GET by URI
        ObservationState st = dao.getState(orig.getURI());
        log.info("found: " + st);
        Assert.assertNotNull("found ObservationState by URI", st);
        ObservationResponse rs = dao.getObservationResponse(st);
        log.info("found: " + rs);
        Assert.assertNotNull("found ObservationResponse by URI", rs);
        Observation retrieved = rs.observation;
        log.info("found: " + retrieved);
        Assert.assertNotNull("found Observation by URI", retrieved);
        log.info("retrieved by URI");
        testEqual(orig, retrieved);
        log.info("equal");
        // GET by ID
        retrieved = dao.get(orig.getID());
        Assert.assertNotNull("found by ID", retrieved);
        log.info("retrieved by UUID");
        testEqual(orig, retrieved);
        log.info("equal");
        // DELETE by ID
        dao.delete(orig.getID());
        // EXISTS
        Assert.assertNull(dao.getState(orig.getURI()));
        log.info("delete & not-exists");
        log.info("check deletion track: " + orig.getID());
        DeletedEntity de = ded.get(DeletedObservation.class, orig.getID());
        Assert.assertNotNull("deletion tracker", de);
        Assert.assertEquals("deleted.id", orig.getID(), de.getID());
        Assert.assertNotNull("deleted.lastModified", de.lastModified);
        DeletedObservation doe = (DeletedObservation) de;
        Assert.assertEquals("deleted.uri", orig.getURI(), doe.getURI());
        Assert.assertFalse("open transaction", txnManager.isOpen());
    } catch (Exception unexpected) {
        log.error("unexpected exception", unexpected);
        Assert.fail("unexpected exception: " + unexpected);
    }
}

Example 2 with DeletedObservation

use of ca.nrc.cadc.caom2.DeletedObservation in project caom2db by opencadc.

the class DeletedEntityDAO method getList.

/**
 * Get list of deleted observations.
 *
 * @param collection
 * @param minLastModified
 * @param maxLastModified
 * @param batchSize
 * @return
 */
public List<DeletedObservation> getList(String collection, Date minLastModified, Date maxLastModified, Integer batchSize) {
    checkInit();
    log.debug("GET: " + batchSize);
    long t = System.currentTimeMillis();
    try {
        JdbcTemplate jdbc = new JdbcTemplate(dataSource);
        String sql = gen.getSelectSQL(DeletedObservation.class, minLastModified, maxLastModified, batchSize, true, collection);
        if (log.isDebugEnabled()) {
            log.debug("GET SQL: " + Util.formatSQL(sql));
        }
        Object result = jdbc.query(sql, gen.getDeletedEntityMapper(DeletedObservation.class));
        if (result == null) {
            return new ArrayList<DeletedObservation>(0);
        }
        if (result instanceof List) {
            List obs = (List) result;
            List<DeletedObservation> ret = new ArrayList<DeletedObservation>(obs.size());
            ret.addAll(obs);
            return ret;
        }
        throw new RuntimeException("BUG: query returned an unexpected type " + result.getClass().getName());
    } finally {
        long dt = System.currentTimeMillis() - t;
        log.debug("GET: " + batchSize + " " + dt + "ms");
    }
}

Example 3 with DeletedObservation

use of ca.nrc.cadc.caom2.DeletedObservation in project caom2db by opencadc.

the class GetDeletedAction method doList.

protected void doList(int maxRec, Date start, Date end, boolean isAscending) throws Exception {
    log.debug("START: " + getCollection());
    checkReadPermission();
    DeletedEntityDAO dao = getDeletedDAO();
    List<DeletedObservation> dels = dao.getList(getCollection(), start, end, maxRec);
    if (dels == null) {
        throw new ResourceNotFoundException("Collection not found: " + getCollection());
    }
    long byteCount = writeDeleted(dels);
    logInfo.setBytes(byteCount);
    log.debug("DONE: " + getCollection());
}

Example 4 with DeletedObservation

use of ca.nrc.cadc.caom2.DeletedObservation in project caom2db by opencadc.

the class ObservationRemover method run.

@Override
public void run() {
    // Get HarvestState record to see if this collection has actually been harvested before
    HarvestStateDAO harvestStateDAO = new PostgresqlHarvestStateDAO(obsDAO.getDataSource(), target.getDatabase(), target.getSchema());
    HarvestState harvestStateRec = null;
    if (src != null) {
        harvestStateRec = harvestStateDAO.get(src.getIdentifier(), Observation.class.getSimpleName());
        // in this case, getLastModified is null
        if (harvestStateRec.getLastModified() == null) {
            log.error("no found: HarvestState record for " + src.getIdentifier());
            return;
        }
    }
    log.info("Using batchSize: " + batchSize);
    log.info("Removing Observation(s) ...");
    Progress prog = null;
    int total = 0;
    boolean go = true;
    while (go) {
        prog = deleteObservations();
        if (prog.found > 0) {
            log.info("finished batch: " + prog.toString());
        }
        if (prog.abort) {
            log.error("batch aborted");
        }
        total += prog.found;
        go = (prog.found > 0 && !prog.abort);
    }
    log.info("Removed " + total + " observations");
    if (prog.abort) {
        log.warn("Problem removing observations. Quitting...");
        return;
    }
    log.info("Removing DeletedObservation(s)...");
    total = 0;
    go = true;
    while (go) {
        prog = deleteDeletedObservations();
        if (prog.found > 0) {
            log.info("finished batch: " + prog.toString());
        }
        if (prog.abort) {
            log.error("batch aborted");
        }
        total += prog.found;
        go = (prog.found > 0 && !prog.abort);
    }
    log.info("Removed " + total + " DeletedObservation(s)");
    if (prog.abort) {
        log.warn("Problem removing DeletedObservation(s). Quitting...");
        return;
    }
    if (src != null) {
        log.info("Removing HarvesetSkipURI(s)...");
        total = 0;
        go = true;
        while (go) {
            prog = deleteHarvestSkipURI();
            if (prog.found > 0) {
                log.info("finished batch: " + prog.toString());
            }
            if (prog.abort) {
                log.error("batch aborted");
            }
            total += prog.found;
            go = (prog.found > 0 && !prog.abort);
        }
        log.info("Removed " + total + " HarvesetSkipURI(s)");
        if (prog.abort) {
            log.warn("Problem removing HarvesetSkipURI(s). Quitting...");
            return;
        }
    }
    if (src != null) {
        log.info("Deleting harvest state records...");
        harvestStateDAO.delete(harvestStateRec);
        harvestStateRec = harvestStateDAO.get(src.getIdentifier(), DeletedObservation.class.getSimpleName());
        if (harvestStateRec.getLastModified() != null) {
            harvestStateDAO.delete(harvestStateRec);
        }
    }
}

Example 5 with DeletedObservation

use of ca.nrc.cadc.caom2.DeletedObservation in project caom2db by opencadc.

the class DeletionListReader method read.

@Override
public List<DeletedObservation> read(Reader in) throws ParseException, IOException, URISyntaxException {
    final List<DeletedObservation> ret = new ArrayList<>(1000);
    LineNumberReader reader = new LineNumberReader(in, 16384);
    String line = reader.readLine();
    while (line != null) {
        try {
            line = line.trim();
            if (!line.isEmpty()) {
                // <Observation.id> <Observation.collection> <Observation.observationID> <deletion timestamp>
                // ICD says tabs but be generous and split of any whitespace
                String[] tokens = line.split("\\s+");
                String sid = tokens[0];
                UUID uuid = UUID.fromString(sid);
                String collection = tokens[1];
                String observationID = tokens[2];
                ObservationURI uri = new ObservationURI(collection, observationID);
                DeletedObservation dd = new DeletedObservation(uuid, uri);
                // 
                if (tokens.length > 3 && tokens[3].length() > 0) {
                    dd.lastModified = DateUtil.flexToDate(tokens[3], dateFormat);
                }
                ret.add(dd);
            }
            line = reader.readLine();
        } finally {
            if (reader.getLineNumber() % 100 == 0) {
                log.debug("read: line " + reader.getLineNumber());
            }
        }
    }
    return ret;
}