Examples with ImportService com.baremaps.osm.repository.ImportService used on opensource projects

Example 1 with ImportService

use of com.baremaps.osm.repository.ImportService in project baremaps by baremaps.

the class ImportUpdateMonacoTest method monaco.

@Test
@Tag("integration")
void monaco() throws Exception {
    LongDataMap<Coordinate> coordinates = new LongDataOpenHashMap<>(new DataStore<>(new CoordinateDataType(), new OnHeapMemory()));
    LongDataMap<List<Long>> references = new LongDataOpenHashMap<>(new DataStore<>(new LongListDataType(), new OnHeapMemory()));
    // Import data
    new ImportService(new URI("res://monaco/monaco-210801.osm.pbf"), blobStore, coordinates, references, headerRepository, nodeRepository, wayRepository, relationRepository, 3857).call();
    assertEquals(3047l, headerRepository.selectLatest().getReplicationSequenceNumber());
    // Fix the replicationUrl so that we can update the database with local files
    headerRepository.delete(3047l);
    headerRepository.put(new Header(3047l, LocalDateTime.of(2021, 8, 01, 20, 21, 41, 0), "res://monaco/monaco-updates", "", ""));
    coordinates = new PostgresCoordinateMap(dataSource);
    references = new PostgresReferenceMap(dataSource);
    // Generate the diff and update the database
    long replicationSequenceNumber = headerRepository.selectLatest().getReplicationSequenceNumber();
    while (replicationSequenceNumber < 3075) {
        new DiffService(blobStore, coordinates, references, headerRepository, nodeRepository, wayRepository, relationRepository, 3857, 14).call();
        new UpdateService(blobStore, coordinates, references, headerRepository, nodeRepository, wayRepository, relationRepository, 3857).call();
        long nextReplicationSequenceNumber = headerRepository.selectLatest().getReplicationSequenceNumber();
        assertEquals(replicationSequenceNumber + 1, nextReplicationSequenceNumber);
        replicationSequenceNumber = nextReplicationSequenceNumber;
    }
}

Example 2 with ImportService

use of com.baremaps.osm.repository.ImportService in project baremaps by baremaps.

the class Import method call.

@Override
public Integer call() throws Exception {
    BlobStore blobStore = options.blobStore();
    DataSource datasource = PostgresUtils.datasource(database);
    HeaderRepository headerRepository = new PostgresHeaderRepository(datasource);
    Repository<Long, Node> nodeRepository = new PostgresNodeRepository(datasource);
    Repository<Long, Way> wayRepository = new PostgresWayRepository(datasource);
    Repository<Long, Relation> relationRepository = new PostgresRelationRepository(datasource);
    Path directory = Files.createTempDirectory(Paths.get("."), "baremaps_");
    Path nodes = Files.createDirectories(directory.resolve("nodes"));
    Path referencesKeys = Files.createDirectories(directory.resolve("references_keys"));
    Path referencesValues = Files.createDirectories(directory.resolve("references_values"));
    LongDataMap<Coordinate> coordinates = new LongAlignedDataDenseMap<>(new LonLatDataType(), new OnDiskMemory(nodes));
    LongDataMap<List<Long>> references = new LongDataSortedMap<>(new AlignedDataList<>(new PairDataType<>(new LongDataType(), new LongDataType()), new OnDiskMemory(referencesKeys)), new DataStore<>(new LongListDataType(), new OnDiskMemory(referencesValues)));
    logger.info("Importing data");
    new ImportService(file, blobStore, coordinates, references, headerRepository, nodeRepository, wayRepository, relationRepository, srid).call();
    logger.info("Done");
    return 0;
}

Example 3 with ImportService

use of com.baremaps.osm.repository.ImportService in project baremaps by baremaps.

the class ImportUpdateDataTest method data.

@Test
@Tag("integration")
void data() throws Exception {
    LongDataMap<Coordinate> coordinates = new LongDataOpenHashMap<>(new DataStore<>(new CoordinateDataType(), new OnHeapMemory()));
    LongDataMap<List<Long>> references = new LongDataOpenHashMap<>(new DataStore<>(new LongListDataType(), new OnHeapMemory()));
    // Import data
    new ImportService(new URI("res://simple/data.osm.pbf"), blobStore, coordinates, references, headerRepository, nodeRepository, wayRepository, relationRepository, 3857).call();
    headerRepository.put(new Header(0l, LocalDateTime.of(2020, 1, 1, 0, 0, 0, 0), "res://simple", "", ""));
    // Check node importation
    assertNull(nodeRepository.get(0l));
    assertNotNull(nodeRepository.get(1l));
    assertNotNull(nodeRepository.get(2l));
    assertNotNull(nodeRepository.get(3l));
    assertNull(nodeRepository.get(4l));
    // Check way importation
    assertNull(wayRepository.get(0l));
    assertNotNull(wayRepository.get(1l));
    assertNull(wayRepository.get(2l));
    // Check relation importation
    assertNull(relationRepository.get(0l));
    assertNotNull(relationRepository.get(1l));
    assertNull(relationRepository.get(2l));
    // Check node properties
    Node node = nodeRepository.get(1l);
    Assertions.assertEquals(1, node.getLon());
    Assertions.assertEquals(1, node.getLat());
    // Check way properties
    Way way = wayRepository.get(1l);
    assertNotNull(way);
    // Update the database
    new UpdateService(blobStore, new PostgresCoordinateMap(dataSource), new PostgresReferenceMap(dataSource), headerRepository, nodeRepository, wayRepository, relationRepository, 3857).call();
    // Check deletions
    assertNull(nodeRepository.get(0l));
    assertNull(nodeRepository.get(1l));
    // Check insertions
    assertNotNull(nodeRepository.get(2l));
    assertNotNull(nodeRepository.get(3l));
    assertNotNull(nodeRepository.get(4l));
}

Example 4 with ImportService

use of com.baremaps.osm.repository.ImportService in project baremaps by baremaps.

the class ImportUpdateLiechtensteinTest method liechtenstein.

@Test
@Tag("integration")
void liechtenstein() throws Exception {
    LongDataMap<Coordinate> coordinates = new LongDataOpenHashMap<>(new DataStore<>(new CoordinateDataType(), new OnHeapMemory()));
    LongDataMap<List<Long>> references = new LongDataOpenHashMap<>(new DataStore<>(new LongListDataType(), new OnHeapMemory()));
    // Import data
    new ImportService(new URI("res://liechtenstein/liechtenstein.osm.pbf"), blobStore, coordinates, references, headerRepository, nodeRepository, wayRepository, relationRepository, 3857).call();
    assertEquals(2434l, headerRepository.selectLatest().getReplicationSequenceNumber());
    // Fix the replicationUrl so that we can update the database with local files
    headerRepository.put(new Header(2434l, LocalDateTime.of(2019, 11, 18, 21, 19, 5, 0), "res://liechtenstein", "", ""));
    coordinates = new PostgresCoordinateMap(dataSource);
    references = new PostgresReferenceMap(dataSource);
    assertEquals(0, new DiffService(blobStore, coordinates, references, headerRepository, nodeRepository, wayRepository, relationRepository, 3857, 14).call().size());
    // Update the database
    new UpdateService(blobStore, coordinates, references, headerRepository, nodeRepository, wayRepository, relationRepository, 3857).call();
    assertEquals(2435l, headerRepository.selectLatest().getReplicationSequenceNumber());
    assertEquals(2, new DiffService(blobStore, coordinates, references, headerRepository, nodeRepository, wayRepository, relationRepository, 3857, 14).call().size());
    new UpdateService(blobStore, coordinates, references, headerRepository, nodeRepository, wayRepository, relationRepository, 3857).call();
    assertEquals(2436l, headerRepository.selectLatest().getReplicationSequenceNumber());
    assertEquals(0, new DiffService(blobStore, coordinates, references, headerRepository, nodeRepository, wayRepository, relationRepository, 3857, 14).call().size());
    new UpdateService(blobStore, coordinates, references, headerRepository, nodeRepository, wayRepository, relationRepository, 3857).call();
    assertEquals(2437l, headerRepository.selectLatest().getReplicationSequenceNumber());
}