commit cab3646fcd6ad7950d2b56fde181055bc326008a Author: LO Kam Tao Leo Date: Tue Sep 20 06:42:18 2022 +0100 Skeleton diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4e338ee --- /dev/null +++ b/.gitignore @@ -0,0 +1,40 @@ +target/ +!.mvn/wrapper/maven-wrapper.jar +!**/src/main/**/target/ +!**/src/test/**/target/ + +### IntelliJ IDEA ### +.idea/modules.xml +.idea/jarRepositories.xml +.idea/compiler.xml +.idea/libraries/ +*.iws +*.iml +*.ipr + +### Eclipse ### +.apt_generated +.classpath +.factorypath +.project +.settings +.springBeans +.sts4-cache + +### NetBeans ### +/nbproject/private/ +/nbbuild/ +/dist/ +/nbdist/ +/.nb-gradle/ +build/ +!**/src/main/**/build/ +!**/src/test/**/build/ + +### VS Code ### +.vscode/ + +### Mac OS ### +.DS_Store +ri_log +action.xml diff --git a/.idea/aws.xml b/.idea/aws.xml new file mode 100644 index 0000000..b63b642 --- /dev/null +++ b/.idea/aws.xml @@ -0,0 +1,11 @@ + + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..6372894 --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,14 @@ + + + + + + + + + + \ No newline at end of file diff --git a/.idea/uiDesigner.xml b/.idea/uiDesigner.xml new file mode 100644 index 0000000..2b63946 --- /dev/null +++ b/.idea/uiDesigner.xml @@ -0,0 +1,124 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..3f817f5 --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,193 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1663575347472 + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml new file mode 100644 index 0000000..6ce400a --- /dev/null +++ b/pom.xml @@ -0,0 +1,39 @@ + + + 4.0.0 + + org.leolo + OSM-extract + 1.0-SNAPSHOT + + + 18 + 18 + 3.21.6 + + + + org.apache.logging.log4j + log4j-core + 2.18.0 + + + commons-cli + commons-cli + 1.5.0 + + + eu.infomas + annotation-detector + 3.0.5 + + + org.openstreetmap.pbf + osmpbf + 1.5.0 + + + + \ No newline at end of file diff --git a/src/main/java/org/leolo/map/osm/extract/Constants.java b/src/main/java/org/leolo/map/osm/extract/Constants.java new file mode 100644 index 0000000..589b1c4 --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/Constants.java @@ -0,0 +1,7 @@ +package org.leolo.map.osm.extract; + +public class Constants { + public class CLI{ +// public static final String HELP = "help"; + } +} diff --git a/src/main/java/org/leolo/map/osm/extract/ExtractElement.java b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java new file mode 100644 index 0000000..c69edd1 --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/ExtractElement.java @@ -0,0 +1,185 @@ +package org.leolo.map.osm.extract; + +import crosby.binary.BinaryParser; +import crosby.binary.Osmformat; +import crosby.binary.file.BlockInputStream; +import org.apache.commons.cli.*; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.leolo.map.osm.extract.model.*; +import org.xml.sax.SAXException; + +import javax.xml.parsers.ParserConfigurationException; +import java.io.*; +import java.util.*; +import java.util.concurrent.*; + +public class ExtractElement { + + private Logger log = LogManager.getLogger(); + private Options options; + + private class CliOpt{ + Option help = Option.builder() + .longOpt("help") + .hasArg(false) + .desc("Print this help message") + .build(); + Option dbFile = Option.builder() + .longOpt("dbFile") + .option("d") + .required(true) + .hasArg(true) + .argName("pbf File") + .desc("The PBF file from OSM") + .build(); + Option outputFile = Option.builder() + .option("o") + .longOpt("output") + .argName("output file") + .hasArg(true) + .required(false) + .desc("Output file") + .build(); + + Option actionFile = Option.builder() + .option("a") + .longOpt("actionFile") + .argName("action file") + .hasArg(true) + .required(true) + .desc("Action files") + .build(); + Option outputFormat = Option.builder() + .option("f") + .longOpt("format") + .argName("format") + .hasArg(true) + .required(true) + .desc("output format") + .build(); + } + + private Map allEntry = new Hashtable<>(); + private Map relations = new Hashtable<>(); + private Map ways = new Hashtable<>(); + private Map nodes = new Hashtable<>(); + private Set strings = new HashSet<>(); + + + private CliOpt cliOpt = new CliOpt(); + + public static void main(String [] args){ + new ExtractElement().run(args); + } + + private void run(String[] args) { + CommandLineParser parser = new DefaultParser(); + String dbFilePath; + String outputFilePath = null; + String actionFilePath; + String format; + try { + CommandLine cmd = parser.parse(getOptions(), args); + if(args.length == 0 || cmd.hasOption(cliOpt.help)){ + printHelp(0); + } + dbFilePath = cmd.getOptionValue(cliOpt.dbFile); + actionFilePath = cmd.getOptionValue(cliOpt.actionFile); + format = cmd.getOptionValue(cliOpt.outputFormat); + if(cmd.hasOption(cliOpt.outputFile)){ + outputFilePath = cmd.getOptionValue(cliOpt.outputFile); + } + log.always().log("Database file : {}", dbFilePath); + log.always().log("Action file : {}", actionFilePath); + log.always().log("Output : {}", outputFilePath==null?"STDOUT":outputFilePath); + log.always().log("Format : {}", format); + log.atInfo().log("{} formatter found, with {} format names", FormatterManager.getInstance().getOutputFormatterCount(), FormatterManager.getInstance().size()); + if(!FormatterManager.getInstance().containsKey(format)){ + log.atError().log("Unknown output format {}", format); + StringBuilder sb = new StringBuilder(); + Iterator iFormat = FormatterManager.getInstance().keySet().iterator(); + while(iFormat.hasNext()){ + sb.append(iFormat.next()); + if(iFormat.hasNext()){ + sb.append(" ,"); + } + } + log.atError().log("Valid format names: {}", sb); + System.exit(2); + } + PrintStream outputStream = outputFilePath==null?System.out:new PrintStream(new File(outputFilePath)); + File dbFile = new File(dbFilePath); + if(!dbFile.exists() || !dbFile.canRead()){ + log.atError().log("Unable to read db file {}", dbFilePath); + System.exit(2); + } + File actionFile = new File(actionFilePath); + if(!actionFile.exists() || !actionFile.canRead()){ + log.atError().log("Unable to read action file {}", actionFilePath); + System.exit(2); + } + log.atInfo().log("Database file size {}MB", dbFile.length()/1_000_000); + log.atInfo().log("Action file size {}kB", actionFile.length()/1_000); + doExtract(dbFile, actionFile, outputStream); + + } catch(MissingOptionException|MissingArgumentException moe) { + System.out.println("FATAL ERROR : "+moe.getMessage()); + printHelp(1); + }catch (ParseException e) { + System.err.println("Fatal error - unable to parse command line input!"); + e.printStackTrace(); + System.exit(1); + } catch (FileNotFoundException e) { + log.atError().withThrowable(e).log("Unable to locate the specified file"); + System.exit(1); + } + } + + private void doExtract(File dbFile, File actionFile, PrintStream outputStream) { + //Step 1: parse the actionFile + ActionFile af; + try { + af = new ActionFile(actionFile); + } catch (IOException|SAXException|ParserConfigurationException e) { + log.atError().withThrowable(e).log("Unable to parse the action file"); + System.exit(3); + return; + } + expandRelations(dbFile, af, af.getRelation()); + processWay(dbFile, af); + processNode(dbFile, af); + } + + private void processNode(File dbFile, ActionFile af) { + + } + + private void processWay(File dbFile, ActionFile af) { + + } + + private void expandRelations(File dbFile, ActionFile af, Collection relations) { + + } + + private void printHelp(int returnValue){ + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("java "+ExtractElement.class.getName(), "", options, "", true); + if(returnValue!=Integer.MIN_VALUE) { + System.exit(returnValue); + } + } + + private synchronized Options getOptions(){ + if(options==null) { + options = new Options(); + options.addOption(cliOpt.dbFile); + options.addOption(cliOpt.outputFile); + options.addOption(cliOpt.actionFile); + options.addOption(cliOpt.help); + options.addOption(cliOpt.outputFormat); + } + return options; + } +} diff --git a/src/main/java/org/leolo/map/osm/extract/FormatterManager.java b/src/main/java/org/leolo/map/osm/extract/FormatterManager.java new file mode 100644 index 0000000..72ee228 --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/FormatterManager.java @@ -0,0 +1,106 @@ +package org.leolo.map.osm.extract; + +import eu.infomas.annotation.AnnotationDetector; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.lang.annotation.Annotation; +import java.util.*; +import java.util.function.BiConsumer; + +public class FormatterManager { + + private static FormatterManager instance; + private Logger log = LogManager.getLogger(); + private Hashtable outformats; + + public static synchronized FormatterManager getInstance(){ + if(instance==null){ + instance = new FormatterManager(); + } + return instance; + } + + private FormatterManager(){ + log.atDebug().log("Looking for output formatter"); + final AnnotationDetector.TypeReporter reporter = new AnnotationDetector.TypeReporter() { + @Override + public void reportTypeAnnotation(Class annoClass, String clazz) { + log.atDebug().log("Class {} has annotation {}", clazz, annoClass.getName()); + try { + Class typeClass = Class.forName(clazz); + if(OutputFormat.class.isAssignableFrom(typeClass)){ + log.atDebug().log("Class {} is an output formatter", clazz); + OutputFormatter of = (OutputFormatter) typeClass.getAnnotation(OutputFormatter.class); + for(String format: of.formatName()){ + if(outformats.containsKey(format)){ + log.atWarn().log("Duplicated format {}. Shared by (at least) {} and {}", + format, + typeClass.getName(), + outformats.get(format).getName() + ); + }else{ + outformats.put(format, typeClass); + log.atInfo().log("Registering class {} for output format {}", typeClass, format); + } + } + }else{ + log.atError().log("Class {} does not implements OutputFormat but have annotation @OutputFormatter", clazz); + } + } catch (ClassNotFoundException e) { + log.atError().withThrowable(e).log(e.getMessage()); + } + + } + + @Override + public Class[] annotations() { + return new Class[]{OutputFormatter.class}; + } + }; + final AnnotationDetector cf = new AnnotationDetector(reporter); + outformats = new Hashtable<>(); + try { + cf.detect(); + } catch (IOException e) { + log.atError().withThrowable(e).log(e.getMessage()); + } + } + + public int size() { + return outformats.size(); + } + + public boolean isEmpty() { + return outformats.isEmpty(); + } + + public Enumeration keys() { + return outformats.keys(); + } + + public boolean containsKey(Object key) { + return outformats.containsKey(key); + } + + public Class get(Object key) { + return outformats.get(key); + } + + public Set keySet() { + return outformats.keySet(); + } + + public void forEach(BiConsumer action) { + outformats.forEach(action); + } + + public int getOutputFormatterCount(){ + HashSet set = new HashSet<>(); + for(Class clazz:outformats.values()){ + set.add(clazz); + } + return set.size(); + } +} diff --git a/src/main/java/org/leolo/map/osm/extract/OutputFormat.java b/src/main/java/org/leolo/map/osm/extract/OutputFormat.java new file mode 100644 index 0000000..ef844bc --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/OutputFormat.java @@ -0,0 +1,4 @@ +package org.leolo.map.osm.extract; + +public interface OutputFormat { +} diff --git a/src/main/java/org/leolo/map/osm/extract/OutputFormatter.java b/src/main/java/org/leolo/map/osm/extract/OutputFormatter.java new file mode 100644 index 0000000..7c7ef46 --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/OutputFormatter.java @@ -0,0 +1,12 @@ +package org.leolo.map.osm.extract; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface OutputFormatter { + String[] formatName(); +} diff --git a/src/main/java/org/leolo/map/osm/extract/format/KMLFormatter.java b/src/main/java/org/leolo/map/osm/extract/format/KMLFormatter.java new file mode 100644 index 0000000..8129604 --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/format/KMLFormatter.java @@ -0,0 +1,8 @@ +package org.leolo.map.osm.extract.format; + +import org.leolo.map.osm.extract.OutputFormat; +import org.leolo.map.osm.extract.OutputFormatter; + +@OutputFormatter(formatName = {"kml","application/vnd.google-earth.kml+xml"}) +public class KMLFormatter implements OutputFormat { +} diff --git a/src/main/java/org/leolo/map/osm/extract/format/PlainFormatter.java b/src/main/java/org/leolo/map/osm/extract/format/PlainFormatter.java new file mode 100644 index 0000000..2c3266d --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/format/PlainFormatter.java @@ -0,0 +1,8 @@ +package org.leolo.map.osm.extract.format; + +import org.leolo.map.osm.extract.OutputFormat; +import org.leolo.map.osm.extract.OutputFormatter; + +@OutputFormatter(formatName = "plain") +public class PlainFormatter implements OutputFormat { +} diff --git a/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java b/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java new file mode 100644 index 0000000..6179d27 --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/model/ActionFile.java @@ -0,0 +1,121 @@ +package org.leolo.map.osm.extract.model; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.SAXException; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import java.io.File; +import java.io.IOException; +import java.util.*; + +public class ActionFile { + + Logger log = LogManager.getLogger(); + + private Map relation = new Hashtable<>(); + private Map way = new Hashtable<>(); + private Map node = new Hashtable<>(); + + public ActionFile(File path) throws IOException, SAXException, ParserConfigurationException { + log.atInfo().log("Parsing action file {}", path.getAbsolutePath()); + DocumentBuilder builder = DocumentBuilderFactory.newInstance().newDocumentBuilder(); + Document doc = builder.parse(path); + doc.getDocumentElement().normalize(); + Element rootElement = doc.getDocumentElement(); + log.atDebug().log("Root element : {}", rootElement.getTagName()); + Node extract = rootElement.getElementsByTagName("extracts").item(0); + NodeList extractItems = extract.getChildNodes(); + for(int i=0;i v = new Vector<>(); + v.add(ei.getParameter().get(param.getNodeName())); + v.add(param.getTextContent()); + ei.getParameter().put(param.getNodeName(), v); + } + }else{ + ei.getParameter().put(param.getNodeName(), param.getTextContent()); + } + } + } + ei.setFromFile(true); + if("relation".equals(child.getNodeName())){ + addRelation(ei); + }else if("way".equals(child.getNodeName())){ + addWay(ei); + }else if("node".equals(child.getNodeName())){ + addNode(ei); + }else{ + log.atWarn().log("Unknown node name {}", child.getNodeName()); + } + } + Node properties = rootElement.getElementsByTagName("properties").item(0); + NodeList props = properties.getChildNodes(); + for(int i=0;i getRelation() { + return relation.values(); + } + + public Collection getWay() { + return way.values(); + } + + public Collection getNode() { + return node.values(); + } + + @Deprecated + public void sortList(){ + + } + public void addRelation(ExtractItem ei) { + if(!relation.containsKey(ei.getId())) + relation.put(ei.getId(), ei); + } + public void addNode(ExtractItem ei) { + if(!node.containsKey(ei.getId())) + node.put(ei.getId(), ei); + } + + public void addWay(ExtractItem ei) { + if(!way.containsKey(ei.getId())) + way.put(ei.getId(), ei); + } +} diff --git a/src/main/java/org/leolo/map/osm/extract/model/ExtractItem.java b/src/main/java/org/leolo/map/osm/extract/model/ExtractItem.java new file mode 100644 index 0000000..b2c2ef2 --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/model/ExtractItem.java @@ -0,0 +1,44 @@ +package org.leolo.map.osm.extract.model; + +import java.util.Hashtable; +import java.util.Map; + +public class ExtractItem implements Comparable{ + private long id; + private String label; + private Map parameter = new Hashtable<>(); + private boolean fromFile = false; + + public boolean isFromFile() { + return fromFile; + } + + public void setFromFile(boolean fromFile) { + this.fromFile = fromFile; + } + + public long getId() { + return id; + } + + public void setId(long id) { + this.id = id; + } + + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + public Map getParameter() { + return parameter; + } + + @Override + public int compareTo(ExtractItem o) { + return Long.compare(id, o.id); + } +} diff --git a/src/main/java/org/leolo/map/osm/extract/model/MapEntry.java b/src/main/java/org/leolo/map/osm/extract/model/MapEntry.java new file mode 100644 index 0000000..8134cb7 --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/model/MapEntry.java @@ -0,0 +1,33 @@ +package org.leolo.map.osm.extract.model; + +import java.util.Hashtable; +import java.util.Map; +import java.util.Set; + +public class MapEntry { + private Map tags = new Hashtable<>(); + + public int tagCount() { + return tags.size(); + } + + public boolean hasNoTags() { + return tags.isEmpty(); + } + + public boolean containsTag(Object key) { + return tags.containsKey(key); + } + + public String getTag(Object key) { + return tags.get(key); + } + + public void putTag(String key, String value) { + tags.put(key, value); + } + + public Set tagSet() { + return tags.keySet(); + } +} diff --git a/src/main/java/org/leolo/map/osm/extract/model/Node.java b/src/main/java/org/leolo/map/osm/extract/model/Node.java new file mode 100644 index 0000000..453d75d --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/model/Node.java @@ -0,0 +1,12 @@ +package org.leolo.map.osm.extract.model; + + +public class Node extends MapEntry{ + private double lat; + private double lon; + + public Node(double lat, double lon) { + this.lat = lat; + this.lon = lon; + } +} diff --git a/src/main/java/org/leolo/map/osm/extract/model/Relation.java b/src/main/java/org/leolo/map/osm/extract/model/Relation.java new file mode 100644 index 0000000..1d0d93f --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/model/Relation.java @@ -0,0 +1,28 @@ +package org.leolo.map.osm.extract.model; + +import java.util.*; + +public class Relation extends MapEntry{ + private List members = new Vector<>(); + + + public int memberCount() { + return members.size(); + } + + public boolean addMember(Long aLong) { + return members.add(aLong); + } + + public boolean removeMember(Object o) { + return members.remove(o); + } + + public boolean containsAllMembers(Collection c) { + return members.containsAll(c); + } + + public Long getMember(int index) { + return members.get(index); + } +} diff --git a/src/main/java/org/leolo/map/osm/extract/model/Way.java b/src/main/java/org/leolo/map/osm/extract/model/Way.java new file mode 100644 index 0000000..4f9c48a --- /dev/null +++ b/src/main/java/org/leolo/map/osm/extract/model/Way.java @@ -0,0 +1,30 @@ +package org.leolo.map.osm.extract.model; + +import java.util.Collection; +import java.util.List; +import java.util.Vector; + +public class Way extends MapEntry{ + + private List members = new Vector<>(); + public int memberCount() { + return members.size(); + } + + public boolean addMember(Long aLong) { + return members.add(aLong); + } + + public boolean removeMember(Object o) { + return members.remove(o); + } + + public boolean containsAllMembers(Collection c) { + return members.containsAll(c); + } + + public Long getMember(int index) { + return members.get(index); + } + +} diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml new file mode 100644 index 0000000..36b41b5 --- /dev/null +++ b/src/main/resources/log4j2.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file