jomu/shared
1
0
Fork 0
Code Issues Pull Requests 1 Packages Projects Releases 7 Activity

improved Whois parsing

Browse Source
This commit is contained in:
jomu
2013-04-14 15:27:02 +00:00
parent 923dba1c1b
commit 9785326a1d
15 changed files with 764 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
network/src/main/java/de/muehlencord/shared/network/AbstractWhoisParser.java Normal file
View File

@ -0,0 +1,45 @@
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package de.muehlencord.shared.network;
import java.util.HashMap;
import java.util.Map;
/**
*
* @author jomu
*/
public abstract class AbstractWhoisParser implements WhoisParser {
protected Map<String, String> getValueMap(String blockString) {
Map<String, String> returnMap = new HashMap<>();
String[] lines = blockString.split("\n");
for (String line : lines) {
String key;
String value;
if (line.indexOf(" ") == -1) {
key = line;
value = "";
} else {
key = line.substring(0, line.indexOf(" "));
value = line.substring(line.indexOf(" "));
value = value.trim();
}
returnMap.put(key, value);
}
return returnMap;
}
protected String getValue(String key, Map<String, String> valueMap) {
if (valueMap.containsKey(key)) {
return valueMap.get(key);
} else {
return null;
}
}
}

130
network/src/main/java/de/muehlencord/shared/network/ArinWhoisParser.java Normal file
View File

@ -0,0 +1,130 @@
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package de.muehlencord.shared.network;
import de.muehlencord.shared.util.StringUtil;
import java.text.ParseException;
import java.util.Map;
import org.apache.log4j.Logger;
/**
*
* @author joern@muehlencord.de
*/
public class ArinWhoisParser extends AbstractWhoisParser implements WhoisParser {
/** logger object */
private final static Logger logger = Logger.getLogger(DefaultWhoisParser.class);
/** information to be returned */
private WhoisInformation whoisInformation;
@Override
public WhoisInformation parseWhoIsString(String whoisString) throws WhoisException {
whoisInformation = new WhoisInformation();
if (whoisString.contains("# start")) { // multiple values
String multipleWhoisString = whoisString;
while (multipleWhoisString.contains("# start")) {
String whoisPartString;
try {
whoisPartString = StringUtil.getValueBetweenKeywords(multipleWhoisString, "# start", "# end");
} catch (ParseException ex) {
throw new WhoisException("Error while reading whois part elements. Reason: " + ex.getMessage(), ex);
}
analyse(whoisPartString);
multipleWhoisString = multipleWhoisString.substring(multipleWhoisString.indexOf("# end")+4);
}
} else {
analyse(whoisString);
}
whoisInformation.validate();
return whoisInformation;
}
private void analyse(String whoisString) throws WhoisException {
String[] blocks;
if (whoisString.contains("\r\n")) {
blocks = whoisString.split("\r\n\r\n");
} else {
blocks = whoisString.split("\n\n");
}
for (String block : blocks) {
analyseBlock(block);
}
}
private void analyseBlock(String block) throws WhoisException {
logger.debug("Start analysing block");
logger.debug("\n---\n" + block + "\n---\n");
if ((block == null) || (block.equals("")) || (!block.contains(" "))) {
logger.debug("Skippig empty block");
return;
}
String startString = block.substring(0, block.indexOf(" "));
if (startString.equals("%")) {
startString = block.substring(0, block.indexOf(" ", 2));
}
Map<String, String> valueMap = getValueMap(block);
switch (startString) {
case "NetRange:":
if (valueMap.containsKey("NetRange:")) {
String netRange = valueMap.get("NetRange:");
// convert inetnum to CDIR
String[] ipAddresses = netRange.split(" "); // TODO - what happens if not 3 parts are found
// FIXME add CDIR notation support
String startIpAddress = ipAddresses[0];
String endIPAddress = ipAddresses[2];
whoisInformation.getNetwork().addAll(CidrTool.rangeToCidrList(startIpAddress, endIPAddress));
logger.info("Network:" + whoisInformation.getNetwork().toString());
} else {
throw new WhoisException("Cannot identify netrange value");
}
break;
case "OrgName:":
String orgName = getValue("OrgName:", valueMap);
String orgId = getValue("OrgId:", valueMap);
String country = getValue("Country:", valueMap);
if ((orgName == null) || (country == null)) {
throw new WhoisException("Cannot identify OrgName and/or country value");
}
if (orgId == null) {
orgId = orgName;
}
whoisInformation.setNetworkInformation(new NetworkInformation(orgName, orgId, country));
logger.info("Networkinformation:" + whoisInformation.getNetworkInformation().toString());
break;
case "OrgAbuseHandle:":
// TODO add abuse handler
logger.info("Skipping OrgAbuseHandle block");
break;
case "OrgTechHandle:":
// admin person of network server belongs to
logger.info("Skipping OrgTechHandle block");
break;
case "#":
logger.info("Skipping comment block");
break;
default:
logger.info("Unknown block found");
}
}
}

119
network/src/main/java/de/muehlencord/shared/network/DefaultWhoisParser.java Normal file
View File

@ -0,0 +1,119 @@
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package de.muehlencord.shared.network;
import de.muehlencord.shared.util.StringUtil;
import java.text.ParseException;
import java.util.Map;
import org.apache.log4j.Logger;
/**
*
* @author jomu
*/
public class DefaultWhoisParser extends AbstractWhoisParser implements WhoisParser {
/** logger object */
private final static Logger logger = Logger.getLogger(DefaultWhoisParser.class);
/** information to be returned */
private WhoisInformation whoisInformation;
@Override
public WhoisInformation parseWhoIsString(String whoisString) throws WhoisException {
whoisInformation = new WhoisInformation();
// split by blank lines to identify blocks
String[] blocks;
if (whoisString.contains("\r\n")) {
blocks = whoisString.split("\r\n\r\n");
} else {
blocks = whoisString.split("\n\n");
}
for (String block : blocks) {
analyseBlock(block);
}
whoisInformation.validate();
return whoisInformation;
}
private void analyseBlock(String block) throws WhoisException {
logger.debug("Start analysing block");
logger.debug("\n---\n" + block + "\n---\n");
if ((block == null) || (block.equals("")) || (!block.contains(" "))) {
logger.debug("Skippig empty block");
return;
}
String startString = block.substring(0, block.indexOf(" "));
if (startString.equals("%")) {
startString = block.substring(0, block.indexOf(" ", 2));
}
Map<String, String> valueMap = getValueMap(block);
switch (startString) {
case "inetnum:":
String inetnum = getValue("inetnum:", valueMap);
String country = getValue("country:", valueMap);
String netname = getValue("descr:", valueMap);
String descr = getValue("netname:", valueMap);
if ((inetnum == null) || (country == null)) {
throw new WhoisException("Cannot identify inetnum and/or country value");
}
if (inetnum.contains(" ")) {
// convert inetnum to CDIR
String[] ipAddresses = inetnum.split(" "); // TODO - what happens if not 3 parts are found
// FIXME add CDIR notation support
String startIpAddress = ipAddresses[0];
String endIPAddress = ipAddresses[2];
whoisInformation.getNetwork().addAll(CidrTool.rangeToCidrList(startIpAddress, endIPAddress));
} else {
whoisInformation.getNetwork().add(inetnum);
}
logger.info("Network:" + whoisInformation.getNetwork().toString());
whoisInformation.setNetworkInformation(new NetworkInformation(netname, descr, country));
break;
case "route:":
// get information on level-2 network
String route = getValue("route:", valueMap);
/*
try {
route = StringUtil.getValueBetweenKeywords(block, "route:", "descr:");
} catch (ParseException ex) {
throw new WhoisException ("Error while reading route information: "+ex.getMessage(), ex);
}
*/
whoisInformation.getRootNetwork().add(route);
logger.info("Root network " + whoisInformation.getRootNetwork().toString());
break;
case "role:":
// admin company of network server belongs to
logger.info("Skipping role block");
break;
case "person:":
// admin person of network server belongs to
logger.info("Skipping person block");
break;
case "% Information":
case "% Note:":
case "% This":
case "%":
logger.info("Skipping comment block");
break;
default:
logger.info("Unknown block found");
}
}
}

5
network/src/main/java/de/muehlencord/shared/network/NetworkInformation.java
View File

@ -30,6 +30,11 @@ public class NetworkInformation implements Serializable {
this.status = null;
this.maintainedBy = null;
}
@Override
public String toString() {
return this.netname+", "+this.country;
}
/* *** getter / setter *** */
/**

163
network/src/main/java/de/muehlencord/shared/network/Whois.java
View File

@ -1,18 +1,11 @@
package de.muehlencord.shared.network;
import de.muehlencord.shared.util.StringUtil;
import java.io.IOException;
import java.net.SocketException;
import java.net.UnknownHostException;
import java.text.ParseException;
import java.util.LinkedList;
import java.util.List;
import org.apache.commons.net.whois.WhoisClient;
import org.apache.log4j.Logger;
/**
*
* @author jomu
* @author joern@muehlencord.de
*/
public class Whois {
@ -21,138 +14,48 @@ public class Whois {
/** whois client from commons-net package to execute commands with */
WhoisClient whois;
/** the complete output */
String whoIsString = null;
/** network list in CIDR form the host belongs to */
private List<String> network;
/** network list in CIDDR form network belongs to */
private List<String> rootNetwork;
/** network information of host */
private NetworkInformation networkInformation;
String whoisString = null;
/**
* creates a new instance of the whois client
*/
public Whois() {
whois = new WhoisClient();
whois.setDefaultTimeout(60000);
network = null;
rootNetwork = new LinkedList<>();
networkInformation = null;
}
}
public void execute(String whoIsServer, String ip) throws UnknownHostException, SocketException, IOException, ParseException {
whois.connect(whoIsServer);
whoIsString = whois.query(ip);
// split by blank lines to identify blocks
String[] blocks = whoIsString.split("\n\n");
for (String block : blocks) {
analyseBlock(block);
}
whois.disconnect();
}
// TODO introduce parer per RIR
private void analyseBlock(String block) throws ParseException {
logger.debug("Start analysing block");
logger.debug("\n---\n" + block + "\n---\n");
if ((block == null) || (block.equals("")) || (!block.contains(" "))) {
logger.debug("Skippig empty block");
return;
/**
* reads and parses the inforomation for the given ip address from the given whoisserver
* @param whoIsServer the whois server to use
* @param ip the ip to query whois data for
* @return whoisInformation parsed from whois string
* @throws WhoisException if the parsing fails
*/
public WhoisInformation execute(String whoIsServer, String ip) throws WhoisException {
String parameter = "";
if (whoIsServer.equals("whois.arin.net")) {
parameter = "n + ";
}
String startString = block.substring(0, block.indexOf(" "));
if (startString.equals("%")) {
startString = block.substring(0, block.indexOf(" ", 2));
// get whois information
try {
whois.connect(whoIsServer);
whoisString = whois.query(parameter + ip);
whois.disconnect();
} catch (Exception ex) {
throw new WhoisException("Error while reading whois data from " + whoIsServer + ". Reason: " + ex.getMessage(), ex);
}
switch (startString) {
case "inetnum:":
String inetnum = null;
String country = null;
String netname = null;
String descr = null;
String[] lines = block.split ("\n");
for (String line : lines) {
String lineStartString;
String value;
if (line.indexOf (" ") == -1) {
lineStartString = line;
value = "";
} else {
lineStartString = line.substring(0, line.indexOf(" "));
value = line.substring(line.indexOf(" "));
value = value.trim();
}
switch (lineStartString) {
case "inetnum:":
inetnum = value;
break;
case "country:":
country = value;
break;
case "descr":
descr = value;
break;
case "netname":
netname = value;
break;
}
}
if (inetnum == null) {
inetnum = ""; // FIXME throw exception
}
// convert inetnum to CDIR
String[] ipAddresses = inetnum.split(" "); // TODO - what happens if not 3 parts are found
// FIXME add CDIR notation support
String startIpAddress = ipAddresses[0];
String endIPAddress = ipAddresses[2];
network = CidrTool.rangeToCidrList(startIpAddress, endIPAddress);
logger.info("Network:" + network.toString());
this.networkInformation = new NetworkInformation(netname, descr, country);
break;
case "route:":
// get information on level-2 network
String route = StringUtil.getValueBetweenKeywords(block, "route:", "descr:");
this.rootNetwork.add (route);
logger.info ("Root network "+this.rootNetwork.toString());
break;
case "role:":
// admin company of network server belongs to
logger.info("Skipping role block");
break;
case "person:":
// admin person of network server belongs to
logger.info("Skipping person block");
break;
case "% Information":
case "% Note:":
case "% This":
case "%":
logger.info("Skipping comment block");
break;
default:
logger.info("Unknown block found");
// identify RIR and select correct parser
WhoisParser parser;
if (whoisString.toLowerCase().contains("arin.net")) {
parser = new ArinWhoisParser();
} else {
parser = new DefaultWhoisParser();
}
}
public List<String> getNetwork() {
return this.network;
}
public List<String> getRootNetwork() {
return this.rootNetwork;
}
public NetworkInformation getNetworkInformation() {
return this.networkInformation;
return parser.parseWhoIsString(whoisString);
}
}

40
network/src/main/java/de/muehlencord/shared/network/WhoisException.java Normal file
View File

@ -0,0 +1,40 @@
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package de.muehlencord.shared.network;
/**
*
* @author jomu
*/
public class WhoisException extends Exception {
/**
* Creates a new instance of
* <code>WhoisException</code> without detail message.
*/
public WhoisException() {
}
/**
* Constructs an instance of
* <code>WhoisException</code> with the specified detail message.
*
* @param msg the detail message.
*/
public WhoisException(String msg) {
super(msg);
}
/**
* Constructs an instance of
* <code>WhoisException</code> with the specified detail message.
*
* @param msg the detail message.
* @param th the root cause of this exception
*/
public WhoisException(String msg, Throwable th) {
super(msg, th);
}
}

76
network/src/main/java/de/muehlencord/shared/network/WhoisInformation.java Normal file
View File

@ -0,0 +1,76 @@
package de.muehlencord.shared.network;
import java.util.LinkedList;
import java.util.List;
/**
*
* @author joern@muehlencord.de
*/
public class WhoisInformation {
/** network list in CIDR form the host belongs to */
private List<String> network;
/** network list in CIDDR form network belongs to */
private List<String> rootNetwork;
/** network information of host */
private NetworkInformation networkInformation;
public WhoisInformation() {
network = new LinkedList<>();
rootNetwork = new LinkedList<>();
networkInformation = null;
}
/**
* @return the network
*/
public List<String> getNetwork() {
return network;
}
/**
* @param network the network to set
*/
public void setNetwork(List<String> network) {
this.network = network;
}
/**
* @return the rootNetwork
*/
public List<String> getRootNetwork() {
return rootNetwork;
}
/**
* @param rootNetwork the rootNetwork to set
*/
public void setRootNetwork(List<String> rootNetwork) {
this.rootNetwork = rootNetwork;
}
/**
* @return the networkInformation
*/
public NetworkInformation getNetworkInformation() {
return networkInformation;
}
/**
* @param networkInformation the networkInformation to set
*/
public void setNetworkInformation(NetworkInformation networkInformation) {
this.networkInformation = networkInformation;
}
void validate() throws WhoisException {
if (network.isEmpty()) {
throw new WhoisException("no network information found");
}
if (networkInformation == null) {
throw new WhoisException("network information not set");
}
}
}

15
network/src/main/java/de/muehlencord/shared/network/WhoisParser.java Normal file
View File

@ -0,0 +1,15 @@
/*
* To change this template, choose Tools | Templates
* and open the template in the editor.
*/
package de.muehlencord.shared.network;
/**
*
* @author jomu
*/
public interface WhoisParser {
WhoisInformation parseWhoIsString(String whoIsString) throws WhoisException;
}