Examples with ObservationState ca.nrc.cadc.caom2.ObservationState used on opensource projects

Example 26 with ObservationState

use of ca.nrc.cadc.caom2.ObservationState in project caom2db by opencadc.

the class ObservationRemover method deleteObservations.

private Progress deleteObservations() {
    Progress ret = new Progress();
    List<ObservationState> obsList;
    if (obsDAO != null) {
        try {
            obsList = obsDAO.getObservationList(target.getCollection(), null, null, batchSize);
            ret.found = obsList.size();
            for (ObservationState obsState : obsList) {
                ObservationURI obsURI = obsState.getURI();
                obsDAO.delete(obsURI);
                log.info("removed: observation: " + obsURI.toString());
                ret.removed++;
            }
        } catch (Exception e) {
            log.error("failed to list && delete observations", e);
            ret.abort = true;
        }
    } else {
        log.error("destination DAO is null: Quitting....");
        ret.abort = true;
    }
    return ret;
}

Example 27 with ObservationState

use of ca.nrc.cadc.caom2.ObservationState in project caom2db by opencadc.

the class AbstractObservationDAOTest method testNonOriginPut.

@Test
public void testNonOriginPut() {
    try {
        dao.setOrigin(false);
        Observation orig = getTestObservation(false, 5, false, true);
        UUID externalID = orig.getID();
        // !EXISTS
        // txnManager.startTransaction();
        Assert.assertNull(dao.getState(orig.getURI()));
        // txnManager.commitTransaction();
        // PUT
        // txnManager.startTransaction();
        dao.put(orig);
        // txnManager.commitTransaction();
        // this is so we can detect incorrect timestamp round trips
        // caused by assigning something other than what was stored
        Thread.sleep(2 * TIME_TOLERANCE);
        // EXISTS
        // txnManager.startTransaction();
        ObservationState os1 = dao.getState(orig.getURI());
        Assert.assertNotNull("exists", os1);
        // txnManager.commitTransaction();
        // GET by URI
        ObservationState st = dao.getState(orig.getURI());
        Assert.assertNotNull("found by URI", st);
        ObservationResponse rs = dao.getObservationResponse(st);
        Assert.assertNotNull("found by URI", rs);
        Observation retrieved = rs.observation;
        Assert.assertNotNull("found by URI", retrieved);
        testEqual(orig, retrieved);
        // GET by ID
        retrieved = dao.get(orig.getID());
        Assert.assertNotNull("found by ID", retrieved);
        testEqual(orig, retrieved);
        // non-origin: make sure UUID did not change
        Assert.assertEquals("non-origin UUID", externalID, retrieved.getID());
        // DELETE by ID
        // txnManager.startTransaction();
        dao.delete(orig.getID());
        // txnManager.commitTransaction();
        // EXISTS
        // txnManager.startTransaction();
        ObservationState os2 = dao.getState(orig.getURI());
        Assert.assertNull("!exists", os2);
        // txnManager.commitTransaction();
        log.info("check deletion track: " + orig.getID());
        DeletedEntity de = ded.get(DeletedObservation.class, orig.getID());
        Assert.assertNotNull("deletion tracker", de);
        Assert.assertEquals("deleted.id", orig.getID(), de.getID());
        Assert.assertNotNull("deleted.lastModified", de.lastModified);
        DeletedObservation doe = (DeletedObservation) de;
        Assert.assertEquals("deleted.uri", orig.getURI(), doe.getURI());
        Assert.assertFalse("open transaction", txnManager.isOpen());
    } catch (Exception unexpected) {
        log.error("unexpected exception", unexpected);
        Assert.fail("unexpected exception: " + unexpected);
    } finally {
        dao.setOrigin(true);
    }
}

Example 28 with ObservationState

use of ca.nrc.cadc.caom2.ObservationState in project caom2db by opencadc.

the class AbstractObservationDAOTest method testGetState.

@Test
public void testGetState() {
    try {
        String collection = "FOO";
        Observation obs = new SimpleObservation(collection, "bar1");
        dao.put(obs);
        ObservationState o = dao.getState(obs.getID());
        Assert.assertNotNull(o);
        log.info("state-by-id: " + o);
        ObservationState o2 = dao.getState(obs.getURI());
        Assert.assertNotNull(o);
        log.info("state-by-uri: " + o);
    } catch (Exception unexpected) {
        log.error("unexpected exception", unexpected);
        Assert.fail("unexpected exception: " + unexpected);
    }
}

Example 29 with ObservationState

use of ca.nrc.cadc.caom2.ObservationState in project caom2db by opencadc.

the class ObservationHarvester method doit.

private Progress doit() {
    Progress ret = new Progress();
    if (!ready) {
        log.error("Observation Harvester not ready");
        ret.abort = true;
        return ret;
    }
    long t = System.currentTimeMillis();
    long timeState = -1;
    long timeQuery = -1;
    long timeTransaction = -1;
    int expectedNum = Integer.MAX_VALUE;
    if (batchSize != null) {
        expectedNum = batchSize.intValue();
    }
    try {
        // hint
        System.gc();
        t = System.currentTimeMillis();
        HarvestState state = null;
        if (!skipped) {
            state = harvestStateDAO.get(source, Observation.class.getSimpleName());
            startDate = state.curLastModified;
            log.debug("state " + state);
        }
        timeState = System.currentTimeMillis() - t;
        t = System.currentTimeMillis();
        if (firstIteration) {
            if (full) {
                startDate = null;
            } else if (super.minDate != null) {
                startDate = super.minDate;
            }
            endDate = super.maxDate;
            if (!skipped) {
                // harvest up to a little in the past because the head of
                // the sequence may be volatile
                long fiveMinAgo = System.currentTimeMillis() - 5 * 60000L;
                if (endDate == null) {
                    endDate = new Date(fiveMinAgo);
                } else {
                    endDate = new Date(Math.min(fiveMinAgo, endDate.getTime()));
                }
            }
        }
        firstIteration = false;
        List<SkippedWrapperURI<ObservationResponse>> entityList;
        if (skipped) {
            entityList = getSkipped(startDate);
        } else {
            log.info("harvest window: " + format(startDate) + " :: " + format(endDate) + " [" + batchSize + "]");
            List<ObservationResponse> obsList;
            if (srcObservationDAO != null) {
                obsList = srcObservationDAO.getList(src.getCollection(), startDate, endDate, batchSize + 1);
            } else {
                obsList = srcObservationService.getList(src.getCollection(), startDate, endDate, batchSize + 1);
            }
            entityList = wrap(obsList);
        }
        // HarvestState (normal case because query: >= startDate)
        if (!entityList.isEmpty() && !skipped) {
            ListIterator<SkippedWrapperURI<ObservationResponse>> iter = entityList.listIterator();
            Observation curBatchLeader = iter.next().entity.observation;
            if (curBatchLeader != null) {
                log.debug("currentBatch: " + curBatchLeader.getURI() + " " + format(curBatchLeader.getMaxLastModified()));
                log.debug("harvestState: " + format(state.curID) + " " + format(state.curLastModified));
                if (curBatchLeader.getID().equals(state.curID) && curBatchLeader.getMaxLastModified().equals(state.curLastModified)) {
                    iter.remove();
                    expectedNum--;
                }
            }
        }
        ret.found = entityList.size();
        log.debug("found: " + entityList.size());
        timeQuery = System.currentTimeMillis() - t;
        t = System.currentTimeMillis();
        ListIterator<SkippedWrapperURI<ObservationResponse>> iter1 = entityList.listIterator();
        // int i = 0;
        while (iter1.hasNext()) {
            SkippedWrapperURI<ObservationResponse> ow = iter1.next();
            Observation o = null;
            if (ow.entity != null) {
                o = ow.entity.observation;
            }
            HarvestSkipURI hs = ow.skip;
            // allow garbage collection during loop
            iter1.remove();
            String skipMsg = null;
            if (!dryrun) {
                if (destObservationDAO.getTransactionManager().isOpen()) {
                    throw new RuntimeException("BUG: found open transaction at start of next observation");
                }
                log.debug("starting transaction");
                destObservationDAO.getTransactionManager().startTransaction();
            }
            boolean ok = false;
            try {
                // o could be null in skip mode cleanup
                if (o != null) {
                    String treeSize = computeTreeSize(o);
                    log.info("put: " + o.getClass().getSimpleName() + " " + o.getURI() + " " + format(o.getMaxLastModified()) + " " + treeSize);
                } else if (hs != null) {
                    log.info("put (retry error): " + hs.getName() + " " + hs.getSkipID() + " " + format(hs.getLastModified()));
                } else {
                    log.info("put (error): Observation " + ow.entity.observationState.getURI() + " " + format(ow.entity.observationState.maxLastModified));
                }
                if (!dryrun) {
                    if (skipped) {
                        startDate = hs.getTryAfter();
                    }
                    if (o != null) {
                        if (state != null) {
                            state.curLastModified = o.getMaxLastModified();
                            state.curID = o.getID();
                        }
                        // try to avoid DataIntegrityViolationException due
                        // to missed deletion followed by insert with new
                        // UUID
                        ObservationState cur = destObservationDAO.getState(o.getURI());
                        if (cur != null && !cur.getID().equals(o.getID())) {
                            // missed harvesting a deletion: trust source
                            log.info("delete: " + o.getClass().getSimpleName() + " " + cur.getURI() + " " + cur.getID() + " (ObservationURI conflict avoided)");
                            destObservationDAO.delete(cur.getID());
                        }
                        // verify we retrieved the observation intact
                        if (!nochecksum) {
                            validateChecksum(o);
                        }
                        // extended content verification
                        CaomValidator.validate(o);
                        for (Plane p : o.getPlanes()) {
                            for (Artifact a : p.getArtifacts()) {
                                CaomWCSValidator.validate(a);
                            }
                        }
                        // optionally augment the observation
                        if (computePlaneMetadata) {
                            log.debug("computePlaneMetadata: " + o.getURI());
                            for (Plane p : o.getPlanes()) {
                                ComputeUtil.computeTransientState(o, p);
                            }
                        }
                        if (acGenerator != null) {
                            log.debug("generateReadAccessTuples: " + o.getURI());
                            acGenerator.generateTuples(o);
                        }
                        // everything is OK
                        destObservationDAO.put(o);
                        if (!skipped) {
                            harvestStateDAO.put(state);
                        }
                        if (hs == null) {
                            // normal harvest mode: try to cleanup skip
                            // records immediately
                            hs = harvestSkipDAO.get(source, cname, o.getURI().getURI());
                        }
                        if (hs != null) {
                            log.info("delete: " + hs + " " + format(hs.getLastModified()));
                            harvestSkipDAO.delete(hs);
                        }
                    } else if (skipped && ow.entity == null) {
                        // observation was simply missing from source == missed deletion
                        ObservationURI uri = new ObservationURI(hs.getSkipID());
                        log.info("delete: " + uri);
                        destObservationDAO.delete(uri);
                        log.info("delete: " + hs + " " + format(hs.getLastModified()));
                        harvestSkipDAO.delete(hs);
                    } else if (ow.entity.error != null) {
                        // try to make progress on failures
                        if (state != null && ow.entity.observationState.maxLastModified != null) {
                            state.curLastModified = ow.entity.observationState.maxLastModified;
                            // unknown
                            state.curID = null;
                        }
                        throw new HarvestReadException(ow.entity.error);
                    }
                    log.debug("committing transaction");
                    destObservationDAO.getTransactionManager().commitTransaction();
                    log.debug("commit: OK");
                }
                ok = true;
                ret.ingested++;
            } catch (Throwable oops) {
                log.debug("exception during harvest", oops);
                skipMsg = null;
                String str = oops.toString();
                if (oops instanceof HarvestReadException) {
                    // unwrap HarvestReadException from above
                    oops = oops.getCause();
                    // unwrap intervening RuntimeException(s)
                    while (oops.getCause() != null && oops instanceof RuntimeException) {
                        oops = oops.getCause();
                    }
                    log.error("HARVEST PROBLEM - failed to read observation: " + ow.entity.observationState.getURI() + " - " + oops.getMessage());
                    ret.handled++;
                } else if (oops instanceof IllegalStateException) {
                    if (oops.getMessage().contains("XML failed schema validation")) {
                        log.error("CONTENT PROBLEM - XML failed schema validation: " + oops.getMessage());
                        ret.handled++;
                    } else if (oops.getMessage().contains("failed to read")) {
                        log.error("CONTENT PROBLEM - " + oops.getMessage(), oops.getCause());
                        ret.handled++;
                    }
                } else if (oops instanceof IllegalArgumentException) {
                    log.error("CONTENT PROBLEM - invalid observation: " + ow.entity.observationState.getURI() + " - " + oops.getMessage());
                    if (oops.getCause() != null) {
                        log.error("cause: " + oops.getCause());
                    }
                    ret.handled++;
                } else if (oops instanceof MismatchedChecksumException) {
                    log.error("CONTENT PROBLEM - mismatching checksums: " + ow.entity.observationState.getURI());
                    ret.handled++;
                } else if (str.contains("duplicate key value violates unique constraint \"i_observationuri\"")) {
                    log.error("CONTENT PROBLEM - duplicate observation: " + ow.entity.observationState.getURI());
                    ret.handled++;
                } else if (oops instanceof TransientException) {
                    log.error("CONTENT PROBLEM - " + oops.getMessage());
                    ret.handled++;
                } else if (oops instanceof Error) {
                    log.error("FATAL - probably installation or environment", oops);
                    ret.abort = true;
                } else if (oops instanceof NullPointerException) {
                    log.error("BUG", oops);
                    ret.abort = true;
                } else if (oops instanceof BadSqlGrammarException) {
                    log.error("BUG", oops);
                    BadSqlGrammarException bad = (BadSqlGrammarException) oops;
                    SQLException sex1 = bad.getSQLException();
                    if (sex1 != null) {
                        log.error("CAUSE", sex1);
                        SQLException sex2 = sex1.getNextException();
                        log.error("NEXT CAUSE", sex2);
                    }
                    ret.abort = true;
                } else if (oops instanceof DataAccessResourceFailureException) {
                    log.error("SEVERE PROBLEM - probably out of space in database", oops);
                    ret.abort = true;
                } else if (str.contains("spherepoly_from_array")) {
                    log.error("CONTENT PROBLEM - failed to persist: " + ow.entity.observationState.getURI() + " - " + oops.getMessage());
                    oops = new IllegalArgumentException("invalid polygon (spoly): " + oops.getMessage(), oops);
                    ret.handled++;
                } else {
                    log.error("unexpected exception", oops);
                }
                // message for HarvestSkipURI record
                skipMsg = oops.getMessage();
            } finally {
                if (!ok && !dryrun) {
                    destObservationDAO.getTransactionManager().rollbackTransaction();
                    log.debug("rollback: OK");
                    timeTransaction += System.currentTimeMillis() - t;
                    try {
                        log.debug("starting HarvestSkipURI transaction");
                        HarvestSkipURI skip = null;
                        if (o != null) {
                            skip = harvestSkipDAO.get(source, cname, o.getURI().getURI());
                        } else {
                            skip = harvestSkipDAO.get(source, cname, ow.entity.observationState.getURI().getURI());
                        }
                        Date tryAfter = ow.entity.observationState.maxLastModified;
                        if (o != null) {
                            tryAfter = o.getMaxLastModified();
                        }
                        if (skip == null) {
                            if (o != null) {
                                skip = new HarvestSkipURI(source, cname, o.getURI().getURI(), tryAfter, skipMsg);
                            } else {
                                skip = new HarvestSkipURI(source, cname, ow.entity.observationState.getURI().getURI(), tryAfter, skipMsg);
                            }
                        } else {
                            skip.errorMessage = skipMsg;
                            skip.setTryAfter(tryAfter);
                        }
                        log.debug("starting HarvestSkipURI transaction");
                        destObservationDAO.getTransactionManager().startTransaction();
                        if (!skipped) {
                            // track the harvest state progress
                            harvestStateDAO.put(state);
                        }
                        // track the fail
                        log.info("put: " + skip);
                        harvestSkipDAO.put(skip);
                        if (!src.getIdentifier().equals(dest.getIdentifier())) {
                            // delete previous version of observation (if any)
                            log.info("delete: " + ow.entity.observationState.getURI());
                            destObservationDAO.delete(ow.entity.observationState.getURI());
                        }
                        log.debug("committing HarvestSkipURI transaction");
                        destObservationDAO.getTransactionManager().commitTransaction();
                        log.debug("commit HarvestSkipURI: OK");
                    } catch (Throwable oops) {
                        log.warn("failed to insert HarvestSkipURI", oops);
                        destObservationDAO.getTransactionManager().rollbackTransaction();
                        log.debug("rollback HarvestSkipURI: OK");
                        ret.abort = true;
                    }
                    ret.failed++;
                }
            }
            if (ret.abort) {
                return ret;
            }
        }
        if (ret.found < expectedNum) {
            ret.done = true;
        }
    } catch (InterruptedException | ExecutionException e) {
        log.error("SEVERE PROBLEM - ThreadPool harvesting Observations failed: " + e.getMessage());
        ret.abort = true;
    } finally {
        timeTransaction = System.currentTimeMillis() - t;
        log.debug("time to get HarvestState: " + timeState + "ms");
        log.debug("time to run ObservationListQuery: " + timeQuery + "ms");
        log.debug("time to run transactions: " + timeTransaction + "ms");
    }
    return ret;
}

Example 30 with ObservationState

use of ca.nrc.cadc.caom2.ObservationState in project caom2db by opencadc.

the class AbstractObservationDAOTest method testGetObservationStateList.

@Test
public void testGetObservationStateList() {
    try {
        String collection = "FOO";
        Thread.sleep(10);
        Observation obs = new SimpleObservation(collection, "bar1");
        dao.put(obs);
        ObservationState o = dao.getState(obs.getID());
        Assert.assertNotNull(o);
        log.info("created: " + o);
        // before 1
        Date start = new Date(o.getMaxLastModified().getTime() - 2 * TIME_TOLERANCE);
        Thread.sleep(10);
        obs = new SimpleObservation(collection, "bar2");
        dao.put(obs);
        o = dao.getState(obs.getID());
        Assert.assertNotNull(o);
        log.info("created: " + o);
        // after 2
        Date mid = new Date(o.getMaxLastModified().getTime() + 2 * TIME_TOLERANCE);
        Thread.sleep(10);
        obs = new SimpleObservation(collection, "bar3");
        dao.put(obs);
        Assert.assertNotNull(dao.getState(obs.getURI()));
        log.info("created: " + obs);
        Thread.sleep(10);
        obs = new SimpleObservation(collection, "bar4");
        dao.put(obs);
        o = dao.getState(obs.getID());
        Assert.assertNotNull(o);
        log.info("created: " + o);
        // after 4
        Date end = new Date(o.getMaxLastModified().getTime() + 2 * TIME_TOLERANCE);
        Thread.sleep(10);
        Integer batchSize = 100;
        DateFormat df = DateUtil.getDateFormat(DateUtil.IVOA_DATE_FORMAT, DateUtil.UTC);
        List<ObservationState> result = dao.getObservationList(collection, start, end, batchSize);
        for (int i = 0; i < result.size(); i++) {
            ObservationState os = result.get(i);
            log.info("found: " + df.format(os.maxLastModified) + " " + os);
            // 1 2 3 4
            ObservationURI exp = new ObservationURI(collection, "bar" + (i + 1));
            Assert.assertEquals(exp, os.getURI());
        }
        Assert.assertEquals("start-end", 4, result.size());
        // descending order
        result = dao.getObservationList(collection, start, end, batchSize, false);
        for (int i = 0; i < result.size(); i++) {
            ObservationState os = result.get(i);
            log.info("start-end found: " + df.format(os.maxLastModified) + " " + os);
            // 4 3 2 1
            ObservationURI exp = new ObservationURI(collection, "bar" + (4 - i));
            Assert.assertEquals(exp, os.getURI());
        }
        Assert.assertEquals("start-end", 4, result.size());
        result = dao.getObservationList(collection, start, mid, batchSize);
        for (ObservationState os : result) {
            log.info("start-mid found: " + df.format(os.maxLastModified) + " " + os);
        }
        Assert.assertEquals("start-mid", 2, result.size());
        result = dao.getObservationList(collection, mid, end, batchSize);
        for (ObservationState os : result) {
            log.info("mid-end found: " + df.format(os.maxLastModified) + " " + os);
        }
        Assert.assertEquals(2, result.size());
        try {
            result = dao.getObservationList(null, start, end, batchSize);
            Assert.fail("expected IllegalArgumentException for null collection, got results");
        } catch (IllegalArgumentException ex) {
            log.info("caught expected exception: " + ex);
        }
        result = dao.getObservationList(collection, null, end, batchSize);
        Assert.assertEquals("-end", 4, result.size());
        result = dao.getObservationList(collection, start, null, batchSize);
        Assert.assertEquals("start-", 4, result.size());
        result = dao.getObservationList(collection, null, null, batchSize);
        result = dao.getObservationList(collection, start, null, null);
    } catch (Exception unexpected) {
        log.error("unexpected exception", unexpected);
        Assert.fail("unexpected exception: " + unexpected);
    }
}