From b351edc4964676ebaf99bda6b747559912f6fb96 Mon Sep 17 00:00:00 2001 From: leolo Date: Fri, 23 Sep 2022 07:00:36 +0000 Subject: [PATCH] Switching library Co-authored-by: LO Kam Tao Leo Reviewed-on: https://git.leolo.org/leolo/osm-extract/pulls/4 --- pom.xml | 7 +- .../org/leolo/map/osm/extract/ExtractElement.java | 155 +++++++++++++++++++-- 2 files changed, 144 insertions(+), 18 deletions(-) diff --git a/pom.xml b/pom.xml index a1d9d3b..7547ffd 100644 --- a/pom.xml +++ b/pom.xml @@ -51,11 +51,6 @@ 3.0.5 - org.openstreetmap.pbf - osmpbf - 1.5.0 - - junit junit 4.13.2 @@ -68,7 +63,7 @@ de.topobyte osm4j-pbf - 0.1.0 + 1.2.0 diff --git a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java index 4ad5749..358e846 100644 --- a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java +++ b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java @@ -1,8 +1,11 @@ package org.leolo.map.osm.extract; -import crosby.binary.BinaryParser; -import crosby.binary.Osmformat; -import crosby.binary.file.BlockInputStream; +import de.topobyte.osm4j.core.access.DefaultOsmHandler; +import de.topobyte.osm4j.core.access.OsmInputException; +import de.topobyte.osm4j.core.access.OsmIterator; +import de.topobyte.osm4j.core.access.OsmReader; +import de.topobyte.osm4j.core.model.iface.*; +import de.topobyte.osm4j.pbf.seq.PbfReader; import org.apache.commons.cli.*; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -137,8 +140,11 @@ public class ExtractElement { actionFile = InputUtil.getInputFile(actionFilePath); log.atInfo().log("Database file size {}MB", dbFile.length()/1_000_000); log.atInfo().log("Action file size {}kB", actionFile.length()/1_000); - doExtract(dbFile, actionFile, outputStream); - + try { + doExtract(dbFile, actionFile, outputStream); + }catch(IOException e){ + log.atError().withThrowable(e).log("Exception when extracting the data!"); + } } catch(MissingOptionException|MissingArgumentException moe) { System.out.println("FATAL ERROR : "+moe.getMessage()); printHelp(1); @@ -152,7 +158,7 @@ public class ExtractElement { } } - private void doExtract(InputFile dbFile, InputFile actionFile, PrintStream outputStream) { + private void doExtract(InputFile dbFile, InputFile actionFile, PrintStream outputStream) throws IOException{ //Step 1: parse the actionFile ActionFile af; try { @@ -162,10 +168,20 @@ public class ExtractElement { System.exit(3); return; } + for(ExtractItem ei:af.getRelation()){ + pendingRelations.add(ei.getId()); + } + for(ExtractItem ei:af.getWay()){ + pendingWays.add(ei.getId()); + } + for(ExtractItem ei:af.getNode()){ + pendingNodes.add(ei.getId()); + } doSearch(dbFile, af); - expandRelations(dbFile); + expandRelations(dbFile, 1); processWay(dbFile); processNode(dbFile); + log.atInfo().log("Done!"); } /** @@ -173,23 +189,138 @@ public class ExtractElement { * @param dbFile The database file going to be searched * @param af The file which defines the action to be performed */ - private void doSearch(InputFile dbFile, ActionFile af) { + private void doSearch(InputFile dbFile, ActionFile af) throws IOException{ //Placeholder } - private void processNode(InputFile dbFile) { + private void processNode(InputFile dbFile) throws IOException{ if(pendingNodes.isEmpty()) return; + log.atInfo().log("Processing {} node(s)", pendingNodes.size()); + OsmReader reader = new PbfReader(dbFile.getInputStream(), true); + reader.setHandler(new DefaultOsmHandler() { + @Override + public void handle(OsmNode node) throws IOException { + super.handle(node); + if(pendingNodes.contains(node.getId())){ + log.atDebug().log("Found node #{}", node.getId()); + Node n = new Node(node.getLatitude(), node.getLongitude()); + for(int i=0;i{}", t.getKey(), t.getValue()); + n.putTag(t.getKey(), t.getValue()); + } + allEntry.put(node.getId(), n); + nodes.put(node.getId(), n); + } + } + + @Override + public void complete() throws IOException { + super.complete(); + log.atInfo().log("Finish reading the DB file"); + } + }); + try { + reader.read(); + } catch (OsmInputException e) { + log.atError().withThrowable(e).log("Error when reading the input!"); + } } - private void processWay(InputFile dbFile) { + private void processWay(InputFile dbFile) throws IOException{ if(pendingWays.isEmpty()) return; + log.atInfo().log("Processing {} way(s)", pendingWays.size()); + OsmReader reader = new PbfReader(dbFile.getInputStream(), true); + reader.setHandler(new DefaultOsmHandler() { + @Override + public void handle(OsmWay w) throws IOException { + super.handle(w); + if(pendingWays.contains(w.getId())){ + Way way = new Way(); + log.atDebug().log("Found way #{}", w.getId()); + for(int i=0;i{}", tag.getKey(), tag.getValue()); + way.putTag(tag.getKey(), tag.getValue()); + } + allEntry.put(w.getId(), way); + ways.put(w.getId(), way); + } + } + + @Override + public void complete() throws IOException { + super.complete(); + log.atInfo().log("Finish reading the DB file"); + } + }); + try { + reader.read(); + } catch (OsmInputException e) { + log.atError().withThrowable(e).log("Error when reading the input!"); + } } - private void expandRelations(InputFile dbFile) { - if(pendingRelations.isEmpty()) + private void expandRelations(InputFile dbFile,final int round) throws IOException{ + if(pendingRelations.isEmpty()) { + log.atInfo().log("No relation to be expanded"); return; + } + HashSet nextRound = new HashSet<>(); + log.atInfo().log("Expanding relations round {} with {} relation(s)", round, pendingRelations.size()); + OsmReader reader = new PbfReader(dbFile.getInputStream(), true); + reader.setHandler(new DefaultOsmHandler() { + @Override + public void handle(OsmRelation relation) throws IOException { + super.handle(relation); + if(pendingRelations.contains(relation.getId())){ + log.atDebug().log("Relation {} found with {} member(s) and {} tag(s).", + relation.getId(), + relation.getNumberOfMembers(), + relation.getNumberOfTags() + ); + Relation rel = new Relation(); + for(int i=0;i nextRound.add(orm.getId()); + case Way -> pendingWays.add(orm.getId()); + case Node -> pendingNodes.add(orm.getId()); + } + rel.addMember(orm.getId()); + } + for(int i=0;i{}", tag.getKey(), tag.getValue()); + rel.putTag(tag.getKey(), tag.getValue()); + } + allEntry.put(relation.getId(), rel); + relations.put(relation.getId(), rel); + } + } + + @Override + public void complete() throws IOException { + super.complete(); + log.atInfo().log("Finish reading round {} when expanding relation", round); + } + }); + try { + reader.read(); + } catch (OsmInputException e) { + log.atError().withThrowable(e).log("Error when reading the input!"); + } + log.atInfo().log("There are {} relations to be expanded in next round", nextRound.size()); + pendingRelations = nextRound; + if(pendingRelations.size()>0){ + expandRelations(dbFile, round+1); + } } private void printHelp(int returnValue){