use of net.geoprism.registry.io.IgnoreRowException in project geoprism-registry by terraframe.
the class BusinessObjectImporter method getGeoObject.
/**
* Returns the entity as defined by the 'parent' and 'parentType' attributes
* of the given feature. If an entity is not found then Earth is returned by
* default. The 'parent' value of the feature must define an entity name or a
* geo oid. The 'parentType' value of the feature must define the localized
* display label of the universal.
*
* This algorithm resolves parent contexts starting at the top of the
* hierarchy and traversing downward, resolving hierarchy inheritance as
* needed. If at any point a location cannot be found, a SmartException will
* be thrown which varies depending on the ParentMatchStrategy.
*
* @param feature
* Shapefile feature used to determine the parent
* @return Parent entity
*/
private ServerGeoObjectIF getGeoObject(FeatureRow feature) {
List<Location> locations = this.configuration.getLocations();
ServerGeoObjectIF parent = null;
JSONArray context = new JSONArray();
ArrayList<String> parentKeyBuilder = new ArrayList<String>();
for (Location location : locations) {
Object label = getParentCode(feature, location);
if (label != null) {
String key = parent != null ? parent.getCode() + "-" + label : label.toString();
parentKeyBuilder.add(label.toString());
if (this.configuration.isExclusion(GeoObjectImportConfiguration.PARENT_EXCLUSION, key)) {
throw new IgnoreRowException();
}
// Check the parent cache
String parentChainKey = StringUtils.join(parentKeyBuilder, parentConcatToken);
if (this.cache.containsKey(parentChainKey)) {
parent = this.cache.get(parentChainKey);
JSONObject element = new JSONObject();
element.put("label", label.toString());
element.put("type", location.getType().getLabel().getValue());
context.put(element);
continue;
}
final ParentMatchStrategy ms = location.getMatchStrategy();
// Search
ServerGeoObjectQuery query = new ServerGeoObjectService().createQuery(location.getType(), this.configuration.getDate());
query.setLimit(20);
if (ms.equals(ParentMatchStrategy.CODE)) {
query.setRestriction(new ServerCodeRestriction(label.toString()));
} else if (ms.equals(ParentMatchStrategy.EXTERNAL)) {
query.setRestriction(new ServerExternalIdRestriction(this.getConfiguration().getExternalSystem(), label.toString()));
} else if (ms.equals(ParentMatchStrategy.DHIS2_PATH)) {
String path = label.toString();
String dhis2Parent;
try {
if (path.startsWith("/")) {
path = path.substring(1);
}
String[] pathArr = path.split("/");
dhis2Parent = pathArr[pathArr.length - 2];
} catch (Throwable t) {
InvalidDhis2PathException ex = new InvalidDhis2PathException(t);
ex.setDhis2Path(path);
throw ex;
}
query.setRestriction(new ServerExternalIdRestriction(this.getConfiguration().getExternalSystem(), dhis2Parent));
} else {
query.setRestriction(new ServerSynonymRestriction(label.toString(), this.configuration.getDate(), parent, location.getHierarchy()));
}
List<ServerGeoObjectIF> results = query.getResults();
if (results != null && results.size() > 0) {
ServerGeoObjectIF result = null;
// into the query SQL.
for (ServerGeoObjectIF loop : results) {
if (result != null && !result.getCode().equals(loop.getCode())) {
AmbiguousParentException ex = new AmbiguousParentException();
ex.setParentLabel(label.toString());
ex.setContext(context.toString());
throw ex;
}
result = loop;
}
parent = result;
JSONObject element = new JSONObject();
element.put("label", label.toString());
element.put("type", location.getType().getLabel().getValue());
context.put(element);
this.cache.put(parentChainKey, parent);
} else {
if (ms.equals(ParentMatchStrategy.CODE)) {
final ParentCodeException ex = new ParentCodeException();
ex.setParentCode(label.toString());
ex.setParentType(location.getType().getLabel().getValue());
ex.setContext(context.toString());
throw ex;
} else if (ms.equals(ParentMatchStrategy.EXTERNAL)) {
final ExternalParentReferenceException ex = new ExternalParentReferenceException();
ex.setExternalId(label.toString());
ex.setParentType(location.getType().getLabel().getValue());
ex.setContext(context.toString());
throw ex;
} else {
String parentCode = (parent == null) ? null : parent.getCode();
ParentReferenceProblem prp = new ParentReferenceProblem(location.getType().getCode(), label.toString(), parentCode, context.toString());
prp.addAffectedRowNumber(this.progressListener.getWorkProgress() + 1);
prp.setHistoryId(this.configuration.historyId);
this.progressListener.addReferenceProblem(prp);
}
return null;
}
}
}
return parent;
}
use of net.geoprism.registry.io.IgnoreRowException in project geoprism-registry by terraframe.
the class BusinessObjectImporter method validateRow.
@Transaction
public void validateRow(FeatureRow row) throws InterruptedException {
try {
// Refresh the session because it might expire on long imports
final long curWorkProgress = this.progressListener.getWorkProgress();
if ((this.lastValidateSessionRefresh + BusinessObjectImporter.refreshSessionRecordCount) < curWorkProgress) {
SessionFacade.renewSession(Session.getCurrentSession().getOid());
this.lastValidateSessionRefresh = curWorkProgress;
}
try {
if (this.configuration.getHierarchy() != null && this.configuration.getLocations().size() > 0) {
this.getGeoObject(row);
}
/*
* 2. Check for serialization and term problems
*/
String code = this.getCode(row);
if (code == null || code.length() <= 0) {
RequiredMappingException ex = new RequiredMappingException();
ex.setAttributeLabel(GeoObjectTypeMetadata.getAttributeDisplayLabel(DefaultAttribute.CODE.getName()));
throw ex;
}
BusinessObject entity;
entity = BusinessObject.newInstance(this.configuration.getType());
Map<String, AttributeType> attributes = this.getConfiguration().getType().getAttributeMap();
Set<Entry<String, AttributeType>> entries = attributes.entrySet();
for (Entry<String, AttributeType> entry : entries) {
String attributeName = entry.getKey();
if (!attributeName.equals(GeoObject.CODE)) {
ShapefileFunction function = this.configuration.getFunction(attributeName);
if (function != null) {
Object value = function.getValue(row);
if (value != null && !this.isEmptyString(value)) {
AttributeType attributeType = entry.getValue();
this.setValue(entity, attributeType, attributeName, value);
}
}
}
}
} catch (IgnoreRowException e) {
// Do nothing
} catch (Throwable t) {
RowValidationProblem problem = new RowValidationProblem(t);
problem.addAffectedRowNumber(curWorkProgress + 1);
problem.setHistoryId(this.configuration.historyId);
this.progressListener.addRowValidationProblem(problem);
}
this.progressListener.setWorkProgress(curWorkProgress + 1);
if (Thread.interrupted()) {
throw new InterruptedException();
}
Thread.yield();
} catch (Throwable e) {
e.printStackTrace();
}
}
use of net.geoprism.registry.io.IgnoreRowException in project geoprism-registry by terraframe.
the class GeoObjectImporter method getParent.
/**
* Returns the entity as defined by the 'parent' and 'parentType' attributes
* of the given feature. If an entity is not found then Earth is returned by
* default. The 'parent' value of the feature must define an entity name or a
* geo oid. The 'parentType' value of the feature must define the localized
* display label of the universal.
*
* This algorithm resolves parent contexts starting at the top of the
* hierarchy and traversing downward, resolving hierarchy inheritance as
* needed. If at any point a location cannot be found, a SmartException will
* be thrown which varies depending on the ParentMatchStrategy.
*
* @param feature
* Shapefile feature used to determine the parent
* @return Parent entity
*/
private ServerGeoObjectIF getParent(FeatureRow feature) {
List<Location> locations = this.configuration.getLocations();
ServerGeoObjectIF parent = null;
JSONArray context = new JSONArray();
ArrayList<String> parentKeyBuilder = new ArrayList<String>();
for (Location location : locations) {
Object label = getParentCode(feature, location);
if (label != null) {
String key = parent != null ? parent.getCode() + "-" + label : label.toString();
parentKeyBuilder.add(label.toString());
if (this.configuration.isExclusion(GeoObjectImportConfiguration.PARENT_EXCLUSION, key)) {
throw new IgnoreRowException();
}
// Check the parent cache
String parentChainKey = StringUtils.join(parentKeyBuilder, parentConcatToken);
if (this.parentCache.containsKey(parentChainKey)) {
parent = this.parentCache.get(parentChainKey);
JSONObject element = new JSONObject();
element.put("label", label.toString());
element.put("type", location.getType().getLabel().getValue());
context.put(element);
continue;
}
final ParentMatchStrategy ms = location.getMatchStrategy();
// Search
ServerGeoObjectQuery query = this.service.createQuery(location.getType(), this.configuration.getStartDate());
query.setLimit(20);
if (ms.equals(ParentMatchStrategy.CODE)) {
query.setRestriction(new ServerCodeRestriction(label.toString()));
} else if (ms.equals(ParentMatchStrategy.EXTERNAL)) {
query.setRestriction(new ServerExternalIdRestriction(this.getConfiguration().getExternalSystem(), label.toString()));
} else if (ms.equals(ParentMatchStrategy.DHIS2_PATH)) {
String path = label.toString();
String dhis2Parent;
try {
if (path.startsWith("/")) {
path = path.substring(1);
}
String[] pathArr = path.split("/");
dhis2Parent = pathArr[pathArr.length - 2];
} catch (Throwable t) {
InvalidDhis2PathException ex = new InvalidDhis2PathException(t);
ex.setDhis2Path(path);
throw ex;
}
query.setRestriction(new ServerExternalIdRestriction(this.getConfiguration().getExternalSystem(), dhis2Parent));
} else {
query.setRestriction(new ServerSynonymRestriction(label.toString(), this.configuration.getStartDate(), parent, location.getHierarchy()));
}
List<ServerGeoObjectIF> results = query.getResults();
if (results != null && results.size() > 0) {
ServerGeoObjectIF result = null;
// into the query SQL.
for (ServerGeoObjectIF loop : results) {
if (result != null && !result.getCode().equals(loop.getCode())) {
AmbiguousParentException ex = new AmbiguousParentException();
ex.setParentLabel(label.toString());
ex.setContext(context.toString());
throw ex;
}
result = loop;
}
parent = result;
JSONObject element = new JSONObject();
element.put("label", label.toString());
element.put("type", location.getType().getLabel().getValue());
context.put(element);
this.parentCache.put(parentChainKey, parent);
} else {
if (context.length() == 0) {
GeoObject root = this.configuration.getRoot();
if (root != null) {
JSONObject element = new JSONObject();
element.put("label", root.getLocalizedDisplayLabel());
element.put("type", root.getType().getLabel().getValue());
context.put(element);
}
}
if (ms.equals(ParentMatchStrategy.CODE)) {
final ParentCodeException ex = new ParentCodeException();
ex.setParentCode(label.toString());
ex.setParentType(location.getType().getLabel().getValue());
ex.setContext(context.toString());
throw ex;
} else if (ms.equals(ParentMatchStrategy.EXTERNAL)) {
final ExternalParentReferenceException ex = new ExternalParentReferenceException();
ex.setExternalId(label.toString());
ex.setParentType(location.getType().getLabel().getValue());
ex.setContext(context.toString());
throw ex;
} else {
String parentCode = (parent == null) ? null : parent.getCode();
ParentReferenceProblem prp = new ParentReferenceProblem(location.getType().getCode(), label.toString(), parentCode, context.toString());
prp.addAffectedRowNumber(this.progressListener.getWorkProgress() + 1);
prp.setHistoryId(this.configuration.historyId);
this.progressListener.addReferenceProblem(prp);
}
return null;
}
}
}
return parent;
}
use of net.geoprism.registry.io.IgnoreRowException in project geoprism-registry by terraframe.
the class GeoObjectImporter method validateRow.
@Transaction
public void validateRow(FeatureRow row) throws InterruptedException {
try {
// Refresh the session because it might expire on long imports
final long curWorkProgress = this.progressListener.getWorkProgress();
if ((this.lastValidateSessionRefresh + GeoObjectImporter.refreshSessionRecordCount) < curWorkProgress) {
SessionFacade.renewSession(Session.getCurrentSession().getOid());
this.lastValidateSessionRefresh = curWorkProgress;
}
try {
/*
* 1. Check for location problems
*/
if (this.configuration.isPostalCode() && PostalCodeFactory.isAvailable(this.configuration.getType())) {
// Skip location synonym check
} else if (this.configuration.getHierarchy() != null && this.configuration.getLocations().size() > 0) {
this.getParent(row);
}
/*
* 2. Check for serialization and term problems
*/
String code = this.getCode(row);
ServerGeoObjectIF entity;
if (code == null || code.length() <= 0) {
RequiredMappingException ex = new RequiredMappingException();
ex.setAttributeLabel(GeoObjectTypeMetadata.getAttributeDisplayLabel(DefaultAttribute.CODE.getName()));
throw ex;
}
entity = service.newInstance(this.configuration.getType());
entity.setCode(code);
entity.setInvalid(false);
try {
LocalizedValue entityName = this.getName(row);
if (entityName != null && this.hasValue(entityName)) {
entity.setDisplayLabel(entityName, this.configuration.getStartDate(), this.configuration.getEndDate());
}
Geometry geometry = (Geometry) this.getFormatSpecificImporter().getGeometry(row);
if (geometry != null) {
// geometry.getSRID().
if (geometry.isValid()) {
entity.setGeometry(geometry, this.configuration.getStartDate(), this.configuration.getEndDate());
} else {
InvalidGeometryException geomEx = new InvalidGeometryException();
throw geomEx;
}
}
Map<String, AttributeType> attributes = this.configuration.getType().getAttributeMap();
Set<Entry<String, AttributeType>> entries = attributes.entrySet();
for (Entry<String, AttributeType> entry : entries) {
String attributeName = entry.getKey();
if (!attributeName.equals(GeoObject.CODE)) {
ShapefileFunction function = this.configuration.getFunction(attributeName);
if (function != null) {
Object value = function.getValue(row);
if (value != null && !this.isEmptyString(value)) {
AttributeType attributeType = entry.getValue();
this.setValue(entity, attributeType, attributeName, value);
}
}
}
}
GeoObjectOverTime go = entity.toGeoObjectOverTime(false);
go.toJSON().toString();
if (this.configuration.isExternalImport()) {
ShapefileFunction function = this.configuration.getExternalIdFunction();
Object value = function.getValue(row);
if (value == null || !(value instanceof String || value instanceof Integer || value instanceof Long) || (value instanceof String && ((String) value).length() == 0)) {
throw new InvalidExternalIdException();
}
}
} finally {
entity.unlock();
}
} catch (IgnoreRowException e) {
// Do nothing
} catch (Throwable t) {
RowValidationProblem problem = new RowValidationProblem(t);
problem.addAffectedRowNumber(curWorkProgress + 1);
problem.setHistoryId(this.configuration.historyId);
this.progressListener.addRowValidationProblem(problem);
}
this.progressListener.setWorkProgress(curWorkProgress + 1);
if (Thread.interrupted()) {
throw new InterruptedException();
}
Thread.yield();
} catch (Throwable e) {
e.printStackTrace();
}
}
use of net.geoprism.registry.io.IgnoreRowException in project geoprism-registry by terraframe.
the class BusinessObjectImporter method importRowInTrans.
@Transaction
public void importRowInTrans(FeatureRow row, RowData data) {
// Refresh the session because it might expire on long imports
final long curWorkProgress = this.progressListener.getWorkProgress();
if ((this.lastImportSessionRefresh + BusinessObjectImporter.refreshSessionRecordCount) < curWorkProgress) {
SessionFacade.renewSession(Session.getCurrentSession().getOid());
this.lastImportSessionRefresh = curWorkProgress;
}
BusinessObject businessObject = null;
ServerGeoObjectIF geoObject = null;
boolean isNew = false;
GeoObjectErrorBuilder builder = new GeoObjectErrorBuilder();
try {
String code = this.getCode(row);
if (code == null || code.length() <= 0) {
RequiredMappingException ex = new RequiredMappingException();
ex.setAttributeLabel(GeoObjectTypeMetadata.getAttributeDisplayLabel(DefaultAttribute.CODE.getName()));
throw ex;
}
if (this.configuration.getImportStrategy().equals(ImportStrategy.UPDATE_ONLY) || this.configuration.getImportStrategy().equals(ImportStrategy.NEW_AND_UPDATE)) {
businessObject = BusinessObject.getByCode(this.configuration.getType(), code);
}
if (businessObject == null) {
if (this.configuration.getImportStrategy().equals(ImportStrategy.UPDATE_ONLY)) {
net.geoprism.registry.DataNotFoundException ex = new net.geoprism.registry.DataNotFoundException();
ex.setTypeLabel(GeoObjectMetadata.get().getClassDisplayLabel());
ex.setDataIdentifier(code);
ex.setAttributeLabel(GeoObjectMetadata.get().getAttributeDisplayLabel(DefaultAttribute.CODE.getName()));
throw ex;
}
isNew = true;
businessObject = BusinessObject.newInstance(this.configuration.getType());
businessObject.setCode(code);
}
builder.setObject(businessObject);
Map<String, AttributeType> attributes = this.configuration.getType().getAttributeMap();
Set<Entry<String, AttributeType>> entries = attributes.entrySet();
for (Entry<String, AttributeType> entry : entries) {
String attributeName = entry.getKey();
ShapefileFunction function = this.configuration.getFunction(attributeName);
if (function != null) {
Object value = function.getValue(row);
AttributeType attributeType = entry.getValue();
if (value != null && !this.isEmptyString(value)) {
this.setValue(businessObject, attributeType, attributeName, value);
} else if (this.configuration.getCopyBlank()) {
this.setValue(businessObject, attributeType, attributeName, null);
}
}
/*
* Try to get the parent and ensure that this row is not ignored. The
* getParent method will throw a IgnoreRowException if the parent is
* configured to be ignored.
*/
if (this.configuration.getHierarchy() != null && this.configuration.getLocations().size() > 0) {
geoObject = this.getGeoObject(row);
}
builder.setGeoObject(geoObject);
if (this.progressListener.hasValidationProblems()) {
throw new RuntimeException("Did not expect to encounter validation problems during import.");
}
data.setGoJson(businessObject.toJSON().toString());
data.setNew(isNew);
data.setParentBuilder(builder);
businessObject.setGeoObject(geoObject);
businessObject.apply();
}
// We must ensure that any problems created during the transaction are
// logged now instead of when the request returns. As such, if any
// problems exist immediately throw a ProblemException so that normal
// exception handling can occur.
List<ProblemIF> existingProblems = RequestState.getProblemsInCurrentRequest();
if (existingProblems.size() != 0) {
throw new ProblemException(null, new LinkedList<ProblemIF>(existingProblems));
}
this.progressListener.setImportedRecords(this.progressListener.getImportedRecords() + 1);
} catch (IgnoreRowException e) {
// Do nothing
} catch (Throwable t) {
buildRecordException(businessObject.toJSON().toString(), isNew, builder, t);
}
this.progressListener.setWorkProgress(curWorkProgress + 1);
}
Aggregations