use of net.geoprism.data.importer.BasicColumnFunction in project geoprism-registry by terraframe.
the class BusinessObjectImportConfiguration method fromJSON.
@Request
public BusinessObjectImportConfiguration fromJSON(String json, boolean includeCoordinates) {
super.fromJSON(json);
SimpleDateFormat format = new SimpleDateFormat(BusinessObjectImportConfiguration.DATE_FORMAT);
format.setTimeZone(GeoRegistryUtil.SYSTEM_TIMEZONE);
JSONObject config = new JSONObject(json);
JSONObject type = config.getJSONObject(TYPE);
JSONArray locations = config.has(LOCATIONS) ? config.getJSONArray(LOCATIONS) : new JSONArray();
JSONArray attributes = type.getJSONArray(GeoObjectType.JSON_ATTRIBUTES);
String code = type.getString(GeoObjectType.JSON_CODE);
BusinessType businessType = BusinessType.getByCode(code);
this.setType(businessType);
try {
if (config.has(BusinessObjectImportConfiguration.DATE)) {
this.setDate(format.parse(config.getString(BusinessObjectImportConfiguration.DATE)));
}
} catch (ParseException e) {
throw new ProgrammingErrorException(e);
}
if (config.has(HIERARCHY)) {
String hCode = config.getString(HIERARCHY);
if (hCode.length() > 0) {
ServerHierarchyType hierarchyType = ServerHierarchyType.get(hCode);
this.setHierarchy(hierarchyType);
}
}
if (config.has(EXCLUSIONS)) {
JSONArray exclusions = config.getJSONArray(EXCLUSIONS);
for (int i = 0; i < exclusions.length(); i++) {
JSONObject exclusion = exclusions.getJSONObject(i);
String attributeName = exclusion.getString(AttributeType.JSON_CODE);
String value = exclusion.getString(VALUE);
this.addExclusion(attributeName, value);
}
}
for (int i = 0; i < attributes.length(); i++) {
JSONObject attribute = attributes.getJSONObject(i);
if (attribute.has(TARGET)) {
String attributeName = attribute.getString(AttributeType.JSON_CODE);
// In the case of a spreadsheet, this ends up being the column header
String target = attribute.getString(TARGET);
if (attribute.has("locale")) {
String locale = attribute.getString("locale");
if (this.getFunction(attributeName) == null) {
this.setFunction(attributeName, new LocalizedValueFunction());
}
LocalizedValueFunction function = (LocalizedValueFunction) this.getFunction(attributeName);
function.add(locale, new BasicColumnFunction(target));
} else {
this.setFunction(attributeName, new BasicColumnFunction(target));
}
}
}
for (int i = 0; i < locations.length(); i++) {
JSONObject location = locations.getJSONObject(i);
if (location.has(TARGET) && location.getString(TARGET).length() > 0 && location.has(MATCH_STRATEGY) && location.getString(MATCH_STRATEGY).length() > 0) {
String pCode = location.getString(AttributeType.JSON_CODE);
ServerGeoObjectType pType = ServerGeoObjectType.get(pCode);
String target = location.getString(TARGET);
ParentMatchStrategy matchStrategy = ParentMatchStrategy.valueOf(location.getString(MATCH_STRATEGY));
// coming in with use BasicColumnFunctions
if (location.has("type") && location.getString("type").equals(ConstantShapefileFunction.class.getName())) {
this.addLocation(new Location(pType, this.hierarchy, new ConstantShapefileFunction(target), matchStrategy));
} else {
this.addLocation(new Location(pType, this.hierarchy, new BasicColumnFunction(target), matchStrategy));
}
}
}
return this;
}
use of net.geoprism.data.importer.BasicColumnFunction in project geoprism-registry by terraframe.
the class ShapefileImporter method process.
/**
* Imports the entities from the shapefile
*
* @param writer
* Log file writer
* @throws InvocationTargetException
* @throws IOException
* @throws InterruptedException
*/
@Request
private void process(ImportStage stage, File shp) throws InvocationTargetException, IOException, InterruptedException {
/*
* Check permissions
*/
ImportConfiguration config = this.getObjectImporter().getConfiguration();
config.enforceCreatePermissions();
FileDataStore myData = FileDataStoreFinder.getDataStore(shp);
SimpleFeatureSource source = myData.getFeatureSource();
SimpleFeatureCollection featCol = source.getFeatures();
this.progressListener.setWorkTotal((long) featCol.size());
if (this.getStartIndex() > 0) {
Query query = new Query();
query.setStartIndex(Math.toIntExact(this.getStartIndex()));
featCol = source.getFeatures(query);
}
SimpleFeatureIterator featIt = featCol.features();
SimpleFeatureReader fr = new DelegateSimpleFeatureReader(source.getSchema(), featIt);
FilterFactory ff = CommonFactoryFinder.getFilterFactory(null);
// We want to sort the features by the parentId column for better lookup
// performance (more cache hits) and
// also so that we have predictable ordering if we want to resume the import
// later.
List<SortBy> sortBy = new ArrayList<SortBy>();
// We also sort by featureId because it's
sortBy.add(SortBy.NATURAL_ORDER);
// guaranteed to be unique.
if (this.config.getLocations().size() > 0) {
ShapefileFunction loc = this.config.getLocations().get(0).getFunction();
if (loc instanceof BasicColumnFunction) {
// TODO
sortBy.add(ff.sort(loc.toJson().toString(), SortOrder.ASCENDING));
// :
// This
// assumes
// loc.tojson()
// returns
// only
// the
// attribute
// name.
}
}
try (SimpleFeatureReader sr = new SortedFeatureReader(fr, sortBy.toArray(new SortBy[sortBy.size()]), 5000)) {
while (sr.hasNext()) {
SimpleFeature feature = sr.next();
if (stage.equals(ImportStage.VALIDATE)) {
this.objectImporter.validateRow(new SimpleFeatureRow(feature));
} else {
this.objectImporter.importRow(new SimpleFeatureRow(feature));
}
}
} catch (Throwable t) {
t.printStackTrace();
} finally {
myData.dispose();
}
}
use of net.geoprism.data.importer.BasicColumnFunction in project geoprism-registry by terraframe.
the class BusinessObjectImporterTest method testImportValueUpdateAndNew.
@Test
@Request
public void testImportValueUpdateAndNew() throws InterruptedException {
String value = "Test Text";
String rowAttribute = "Bad";
HashMap<String, Object> row = new HashMap<String, Object>();
row.put(rowAttribute, value);
row.put(BusinessObject.CODE, TEST_CODE);
BusinessObjectImportConfiguration configuration = new BusinessObjectImportConfiguration();
configuration.setImportStrategy(ImportStrategy.NEW_AND_UPDATE);
configuration.setType(type);
configuration.setDate(FastTestDataset.DEFAULT_END_TIME_DATE);
configuration.setCopyBlank(false);
configuration.setFunction(attributeType.getName(), new BasicColumnFunction(rowAttribute));
configuration.setFunction(BusinessObject.CODE, new BasicColumnFunction(BusinessObject.CODE));
BusinessObjectImporter importer = new BusinessObjectImporter(configuration, new NullImportProgressListener());
importer.importRow(new MapFeatureRow(row));
BusinessObject result = BusinessObject.get(type, attributeType.getName(), value);
try {
Assert.assertNotNull(result);
} finally {
if (result != null) {
result.delete();
}
}
}
use of net.geoprism.data.importer.BasicColumnFunction in project geoprism-registry by terraframe.
the class ShapefileServiceTest method testImportShapefileInteger.
@Test
@Request
public void testImportShapefileInteger() throws Throwable {
InputStream istream = this.getClass().getResourceAsStream("/cb_2017_us_state_500k.zip.test");
Assert.assertNotNull(istream);
ShapefileService service = new ShapefileService();
GeoObjectImportConfiguration config = this.getTestConfiguration(istream, service, null, ImportStrategy.NEW_AND_UPDATE);
ServerHierarchyType hierarchyType = ServerHierarchyType.get(USATestData.HIER_ADMIN.getCode());
config.setFunction(testInteger.getName(), new BasicColumnFunction("ALAND"));
config.setHierarchy(hierarchyType);
ImportHistory hist = mockImport(config);
Assert.assertTrue(hist.getStatus().get(0).equals(AllJobStatus.SUCCESS));
hist = ImportHistory.get(hist.getOid());
Assert.assertEquals(new Long(56), hist.getWorkTotal());
Assert.assertEquals(new Long(56), hist.getWorkProgress());
Assert.assertEquals(new Long(56), hist.getImportedRecords());
Assert.assertEquals(ImportStage.COMPLETE, hist.getStage().get(0));
GeoObject object = ServiceFactory.getRegistryService().getGeoObjectByCode(testData.clientRequest.getSessionId(), "01", USATestData.STATE.getCode(), TestDataSet.DEFAULT_OVER_TIME_DATE);
Assert.assertNotNull(object);
Assert.assertNotNull(object.getGeometry());
Assert.assertEquals("Alabama", object.getLocalizedDisplayLabel());
Assert.assertEquals(131174431216L, object.getValue(testInteger.getName()));
}
use of net.geoprism.data.importer.BasicColumnFunction in project geoprism-registry by terraframe.
the class ShapefileServiceTest method testBadParentSynonymAndResume.
@Test
@Request
public void testBadParentSynonymAndResume() throws Throwable {
InputStream istream = this.getClass().getResourceAsStream("/cb_2017_us_state_500k.zip.test");
Assert.assertNotNull(istream);
ShapefileService service = new ShapefileService();
GeoObjectImportConfiguration config = this.getTestConfiguration(istream, service, null, ImportStrategy.NEW_AND_UPDATE);
ServerHierarchyType hierarchyType = ServerHierarchyType.get(USATestData.HIER_ADMIN.getCode());
config.setHierarchy(hierarchyType);
config.addParent(new Location(USATestData.COUNTRY.getServerObject(), hierarchyType, new BasicColumnFunction("LSAD"), ParentMatchStrategy.ALL));
// ImportHistory hist = mockImport(config);
// Assert.assertTrue(hist.getStatus().get(0).equals(AllJobStatus.FEEDBACK));
ImportHistory hist = importShapefile(testData.clientRequest.getSessionId(), config.toJSON().toString());
SchedulerTestUtils.waitUntilStatus(hist.getOid(), AllJobStatus.FEEDBACK);
hist = ImportHistory.get(hist.getOid());
Assert.assertEquals(new Long(56), hist.getWorkTotal());
Assert.assertEquals(new Long(56), hist.getWorkProgress());
Assert.assertEquals(new Long(0), hist.getImportedRecords());
Assert.assertEquals(ImportStage.VALIDATION_RESOLVE, hist.getStage().get(0));
JSONObject page = new JSONObject(new ETLService().getValidationProblems(testData.clientRequest.getSessionId(), hist.getOid(), false, 100, 1).toString());
JSONArray results = page.getJSONArray("resultSet");
Assert.assertEquals(1, results.length());
// Ensure the geo objects were not created
ServerGeoObjectQuery query = new ServerGeoObjectService().createQuery(USATestData.STATE.getServerObject(), config.getStartDate());
query.setRestriction(new ServerCodeRestriction("01"));
Assert.assertNull(query.getSingleResult());
// Resolve the import problem with a synonym
GeoObject geoObj = ServiceFactory.getRegistryService().newGeoObjectInstance(testData.clientRequest.getSessionId(), USATestData.COUNTRY.getCode());
geoObj.setCode("99");
geoObj.setDisplayLabel(LocalizedValue.DEFAULT_LOCALE, "Test Label99");
geoObj.setUid(ServiceFactory.getIdService().getUids(1)[0]);
ServerGeoObjectIF serverGo = new ServerGeoObjectService(new AllowAllGeoObjectPermissionService()).apply(geoObj, TestDataSet.DEFAULT_OVER_TIME_DATE, TestDataSet.DEFAULT_END_TIME_DATE, true, false);
JSONObject valRes = new JSONObject();
valRes.put("validationProblemId", results.getJSONObject(0).getString("id"));
valRes.put("resolution", ValidationResolution.SYNONYM);
valRes.put("code", serverGo.getCode());
valRes.put("typeCode", serverGo.getType().getCode());
valRes.put("label", "00");
new ETLService().submitValidationProblemResolution(testData.clientRequest.getSessionId(), valRes.toString());
ValidationProblem vp = ValidationProblem.get(results.getJSONObject(0).getString("id"));
Assert.assertEquals(ValidationResolution.SYNONYM.name(), vp.getResolution());
Assert.assertEquals(ParentReferenceProblem.DEFAULT_SEVERITY, vp.getSeverity());
ImportHistory hist2 = importShapefile(testData.clientRequest.getSessionId(), hist.getConfigJson());
Assert.assertEquals(hist.getOid(), hist2.getOid());
SchedulerTestUtils.waitUntilStatus(hist.getOid(), AllJobStatus.SUCCESS);
hist = ImportHistory.get(hist.getOid());
Assert.assertEquals(ImportStage.COMPLETE, hist.getStage().get(0));
Assert.assertEquals(new Long(56), hist.getWorkTotal());
Assert.assertEquals(new Long(56), hist.getWorkProgress());
Assert.assertEquals(new Long(56), hist.getImportedRecords());
String sessionId = testData.clientRequest.getSessionId();
GeoObject go = ServiceFactory.getRegistryService().getGeoObjectByCode(sessionId, "01", USATestData.STATE.getCode(), TestDataSet.DEFAULT_OVER_TIME_DATE);
Assert.assertEquals("01", go.getCode());
ParentTreeNode nodes = ServiceFactory.getRegistryService().getParentGeoObjects(sessionId, go.getCode(), config.getType().getCode(), new String[] { USATestData.COUNTRY.getCode() }, false, TestDataSet.DEFAULT_OVER_TIME_DATE);
List<ParentTreeNode> parents = nodes.getParents();
Assert.assertEquals(1, parents.size());
JSONObject page2 = new JSONObject(new ETLService().getValidationProblems(testData.clientRequest.getSessionId(), hist.getOid(), false, 100, 1).toString());
JSONArray results2 = page2.getJSONArray("resultSet");
Assert.assertEquals(0, results2.length());
Assert.assertEquals(0, page2.getInt("count"));
}
Aggregations