|
|
|
|
@ -26,6 +26,7 @@ import java.util.List;
|
|
|
|
|
import java.util.Map;
|
|
|
|
|
import java.util.regex.Matcher;
|
|
|
|
|
import java.util.regex.Pattern;
|
|
|
|
|
import org.apache.commons.codec.binary.Base64;
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Web browser with simple cookies support
|
|
|
|
|
@ -33,7 +34,7 @@ import java.util.regex.Pattern;
|
|
|
|
|
public final class WebBrowser {
|
|
|
|
|
|
|
|
|
|
private static Map<String, String> browserProperties = new HashMap<String, String>();
|
|
|
|
|
private static Map<String, Map<String, String>> cookies;
|
|
|
|
|
private static Map<String, Map<String, String>> cookies = new HashMap<String, Map<String, String>>();
|
|
|
|
|
private static String proxyHost = null;
|
|
|
|
|
private static String proxyPort = null;
|
|
|
|
|
private static String proxyUsername = null;
|
|
|
|
|
@ -42,15 +43,32 @@ public final class WebBrowser {
|
|
|
|
|
private static int webTimeoutConnect = 25000; // 25 second timeout
|
|
|
|
|
private static int webTimeoutRead = 90000; // 90 second timeout
|
|
|
|
|
|
|
|
|
|
static {
|
|
|
|
|
/**
|
|
|
|
|
* Constructor for WebBrowser.
|
|
|
|
|
* Does instantiates the browser properties.
|
|
|
|
|
*/
|
|
|
|
|
public WebBrowser() {
|
|
|
|
|
if (browserProperties.isEmpty()) {
|
|
|
|
|
browserProperties.put("User-Agent", "Mozilla/5.25 Netscape/5.0 (Windows; I; Win95)");
|
|
|
|
|
cookies = new HashMap<String, Map<String, String>>();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Request the web page at the specified URL
|
|
|
|
|
* @param url
|
|
|
|
|
* @return
|
|
|
|
|
* @throws IOException
|
|
|
|
|
*/
|
|
|
|
|
public static String request(String url) throws IOException {
|
|
|
|
|
return request(new URL(url));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Open a connection using proxy parameters if they exist.
|
|
|
|
|
* @param url
|
|
|
|
|
* @return
|
|
|
|
|
* @throws IOException
|
|
|
|
|
*/
|
|
|
|
|
public static URLConnection openProxiedConnection(URL url) throws IOException {
|
|
|
|
|
if (proxyHost != null) {
|
|
|
|
|
System.getProperties().put("proxySet", "true");
|
|
|
|
|
@ -67,11 +85,14 @@ public final class WebBrowser {
|
|
|
|
|
return cnx;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Request the web page at the specified URL
|
|
|
|
|
* @param url
|
|
|
|
|
* @return
|
|
|
|
|
* @throws IOException
|
|
|
|
|
*/
|
|
|
|
|
public static String request(URL url) throws IOException {
|
|
|
|
|
StringWriter content = null;
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
content = new StringWriter();
|
|
|
|
|
StringBuilder content = new StringBuilder();
|
|
|
|
|
|
|
|
|
|
BufferedReader in = null;
|
|
|
|
|
URLConnection cnx = null;
|
|
|
|
|
@ -84,7 +105,7 @@ public final class WebBrowser {
|
|
|
|
|
in = new BufferedReader(new InputStreamReader(cnx.getInputStream(), getCharset(cnx)));
|
|
|
|
|
String line;
|
|
|
|
|
while ((line = in.readLine()) != null) {
|
|
|
|
|
content.write(line);
|
|
|
|
|
content.append(line);
|
|
|
|
|
}
|
|
|
|
|
} finally {
|
|
|
|
|
if (in != null) {
|
|
|
|
|
@ -97,13 +118,12 @@ public final class WebBrowser {
|
|
|
|
|
|
|
|
|
|
}
|
|
|
|
|
return content.toString();
|
|
|
|
|
} finally {
|
|
|
|
|
if (content != null) {
|
|
|
|
|
content.close();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Set the header information for the connection
|
|
|
|
|
* @param cnx
|
|
|
|
|
*/
|
|
|
|
|
private static void sendHeader(URLConnection cnx) {
|
|
|
|
|
// send browser properties
|
|
|
|
|
for (Map.Entry<String, String> browserProperty : browserProperties.entrySet()) {
|
|
|
|
|
@ -116,6 +136,11 @@ public final class WebBrowser {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Create the cookies for the header
|
|
|
|
|
* @param cnx
|
|
|
|
|
* @return
|
|
|
|
|
*/
|
|
|
|
|
private static String createCookieHeader(URLConnection cnx) {
|
|
|
|
|
String host = cnx.getURL().getHost();
|
|
|
|
|
StringBuilder cookiesHeader = new StringBuilder();
|
|
|
|
|
@ -136,6 +161,10 @@ public final class WebBrowser {
|
|
|
|
|
return cookiesHeader.toString();
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Read the header information into the cookies
|
|
|
|
|
* @param cnx
|
|
|
|
|
*/
|
|
|
|
|
private static void readHeader(URLConnection cnx) {
|
|
|
|
|
// read new cookies and update our cookies
|
|
|
|
|
for (Map.Entry<String, List<String>> header : cnx.getHeaderFields().entrySet()) {
|
|
|
|
|
@ -172,6 +201,11 @@ public final class WebBrowser {
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Determine the charset for the connection
|
|
|
|
|
* @param cnx
|
|
|
|
|
* @return
|
|
|
|
|
*/
|
|
|
|
|
private static Charset getCharset(URLConnection cnx) {
|
|
|
|
|
Charset charset = null;
|
|
|
|
|
// content type will be string like "text/html; charset=UTF-8" or "text/html"
|
|
|
|
|
@ -195,55 +229,104 @@ public final class WebBrowser {
|
|
|
|
|
return charset;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Return the proxy host name
|
|
|
|
|
* @return
|
|
|
|
|
*/
|
|
|
|
|
public static String getProxyHost() {
|
|
|
|
|
return proxyHost;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Set the proxy host name
|
|
|
|
|
* @param tvdbProxyHost
|
|
|
|
|
*/
|
|
|
|
|
public static void setProxyHost(String tvdbProxyHost) {
|
|
|
|
|
WebBrowser.proxyHost = tvdbProxyHost;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Get the proxy port
|
|
|
|
|
* @return
|
|
|
|
|
*/
|
|
|
|
|
public static String getProxyPort() {
|
|
|
|
|
return proxyPort;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static void setProxyPort(String tvdbProxyPort) {
|
|
|
|
|
WebBrowser.proxyPort = tvdbProxyPort;
|
|
|
|
|
/**
|
|
|
|
|
* Set the proxy port
|
|
|
|
|
* @param proxyPort
|
|
|
|
|
*/
|
|
|
|
|
public static void setProxyPort(String proxyPort) {
|
|
|
|
|
WebBrowser.proxyPort = proxyPort;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static String getTvdbProxyUsername() {
|
|
|
|
|
/**
|
|
|
|
|
* Get the proxy username
|
|
|
|
|
* @return
|
|
|
|
|
*/
|
|
|
|
|
public static String getProxyUsername() {
|
|
|
|
|
return proxyUsername;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static void setProxyUsername(String tvdbProxyUsername) {
|
|
|
|
|
WebBrowser.proxyUsername = tvdbProxyUsername;
|
|
|
|
|
/**
|
|
|
|
|
* Set the proxy username
|
|
|
|
|
* @param proxyUsername
|
|
|
|
|
*/
|
|
|
|
|
public static void setProxyUsername(String proxyUsername) {
|
|
|
|
|
WebBrowser.proxyUsername = proxyUsername;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Get the proxy password
|
|
|
|
|
* @return
|
|
|
|
|
*/
|
|
|
|
|
public static String getProxyPassword() {
|
|
|
|
|
return proxyPassword;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
public static void setProxyPassword(String tvdbProxyPassword) {
|
|
|
|
|
WebBrowser.proxyPassword = tvdbProxyPassword;
|
|
|
|
|
/**
|
|
|
|
|
* Set the proxy password.
|
|
|
|
|
* Note this will automatically encode the password
|
|
|
|
|
* @param proxyPassword
|
|
|
|
|
*/
|
|
|
|
|
public static void setProxyPassword(String proxyPassword) {
|
|
|
|
|
WebBrowser.proxyPassword = proxyPassword;
|
|
|
|
|
|
|
|
|
|
if (proxyUsername != null) {
|
|
|
|
|
proxyEncodedPassword = proxyUsername + ":" + tvdbProxyPassword;
|
|
|
|
|
proxyEncodedPassword = Base64.base64Encode(proxyEncodedPassword);
|
|
|
|
|
if (proxyUsername != null && !proxyPassword.isEmpty()) {
|
|
|
|
|
proxyEncodedPassword = proxyUsername + ":" + proxyPassword;
|
|
|
|
|
proxyEncodedPassword = "Basic " + new String(Base64.encodeBase64((proxyUsername + ":" + proxyPassword).getBytes()));
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Get the current web connect timeout value
|
|
|
|
|
* @return
|
|
|
|
|
*/
|
|
|
|
|
public static int getWebTimeoutConnect() {
|
|
|
|
|
return webTimeoutConnect;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Get the current web read timeout value
|
|
|
|
|
* @return
|
|
|
|
|
*/
|
|
|
|
|
public static int getWebTimeoutRead() {
|
|
|
|
|
return webTimeoutRead;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Set the web connect timeout value
|
|
|
|
|
* @param webTimeoutConnect
|
|
|
|
|
*/
|
|
|
|
|
public static void setWebTimeoutConnect(int webTimeoutConnect) {
|
|
|
|
|
WebBrowser.webTimeoutConnect = webTimeoutConnect;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Set the web read timeout value
|
|
|
|
|
* @param webTimeoutRead
|
|
|
|
|
*/
|
|
|
|
|
public static void setWebTimeoutRead(int webTimeoutRead) {
|
|
|
|
|
WebBrowser.webTimeoutRead = webTimeoutRead;
|
|
|
|
|
}
|
|
|
|
|
|