diff --git a/protocols/xml/src/main/java/org/opennms/protocols/http/HttpUrlConnection.java b/protocols/xml/src/main/java/org/opennms/protocols/http/HttpUrlConnection.java index 4d036ec..dc7a9ed 100644 --- a/protocols/xml/src/main/java/org/opennms/protocols/http/HttpUrlConnection.java +++ b/protocols/xml/src/main/java/org/opennms/protocols/http/HttpUrlConnection.java @@ -28,20 +28,26 @@ package org.opennms.protocols.http; -import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; import java.net.URL; import java.net.URLConnection; +import org.apache.http.Header; +import org.apache.http.HeaderElement; +import org.apache.http.HttpEntity; +import org.apache.http.HttpException; +import org.apache.http.HttpRequest; +import org.apache.http.HttpRequestInterceptor; import org.apache.http.HttpResponse; +import org.apache.http.HttpResponseInterceptor; import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.HttpClient; +import org.apache.http.client.entity.GzipDecompressingEntity; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.utils.URIUtils; import org.apache.http.impl.auth.BasicScheme; import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.util.EntityUtils; +import org.apache.http.protocol.HttpContext; import org.opennms.core.utils.ThreadCategory; /** @@ -54,6 +60,9 @@ public class HttpUrlConnection extends URLConnection { /** The URL. */ private URL m_url; + /** The HTTP Client. */ + private DefaultHttpClient m_client; + /** * Instantiates a new SFTP URL connection. * @@ -68,7 +77,39 @@ public class HttpUrlConnection extends URLConnection { * @see java.net.URLConnection#connect() */ @Override - public void connect() throws IOException {} + public void connect() throws IOException { + if (m_client != null) { + return; + } + m_client = new DefaultHttpClient(); + m_client.addRequestInterceptor(new HttpRequestInterceptor() { + @Override + public void process(HttpRequest request, HttpContext context) + throws HttpException, IOException { + if (!request.containsHeader("Accept-Encoding")) { + request.addHeader("Accept-Encoding", "gzip"); + } + } + }); + m_client.addResponseInterceptor(new HttpResponseInterceptor() { + public void process(final HttpResponse response, final HttpContext context) + throws HttpException, IOException { + HttpEntity entity = response.getEntity(); + if (entity != null) { + Header ceheader = entity.getContentEncoding(); + if (ceheader != null) { + HeaderElement[] codecs = ceheader.getElements(); + for (int i = 0; i < codecs.length; i++) { + if (codecs[i].getName().equalsIgnoreCase("gzip")) { + response.setEntity(new GzipDecompressingEntity(response.getEntity())); + return; + } + } + } + } + } + }); + } /* (non-Javadoc) * @see java.net.URLConnection#getInputStream() @@ -76,18 +117,30 @@ public class HttpUrlConnection extends URLConnection { @Override public InputStream getInputStream() throws IOException { try { + if (m_client == null) { + connect(); + } int port = m_url.getPort() > 0 ? m_url.getPort() : m_url.getDefaultPort(); String[] userInfo = m_url.getUserInfo() == null ? null : m_url.getUserInfo().split(":"); - HttpClient client = new DefaultHttpClient(); + HttpGet request = new HttpGet(URIUtils.createURI(m_url.getProtocol(), m_url.getHost(), port, m_url.getPath(), m_url.getQuery(), null)); if (userInfo != null) { UsernamePasswordCredentials credentials = new UsernamePasswordCredentials(userInfo[0], userInfo[1]); request.addHeader(BasicScheme.authenticate(credentials, "UTF-8", false)); } - HttpResponse response = client.execute(request); - return new ByteArrayInputStream(EntityUtils.toByteArray(response.getEntity())); + HttpResponse response = m_client.execute(request); + return response.getEntity().getContent(); } catch (Exception e) { - throw new IOException("Can't retrieve " + m_url.getPath() + " from " + m_url.getHost() + " because " + e.getMessage()); + throw new IOException("Can't retrieve " + m_url.getPath() + " from " + m_url.getHost() + " because " + e.getMessage(), e); + } + } + + /** + * Disconnect + */ + public void disconnect() { + if (m_client != null) { + m_client.getConnectionManager().shutdown(); } } diff --git a/protocols/xml/src/main/java/org/opennms/protocols/xml/collector/AbstractXmlCollectionHandler.java b/protocols/xml/src/main/java/org/opennms/protocols/xml/collector/AbstractXmlCollectionHandler.java index 37d8599..36cb0c8 100644 --- a/protocols/xml/src/main/java/org/opennms/protocols/xml/collector/AbstractXmlCollectionHandler.java +++ b/protocols/xml/src/main/java/org/opennms/protocols/xml/collector/AbstractXmlCollectionHandler.java @@ -45,6 +45,7 @@ import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathExpressionException; import javax.xml.xpath.XPathFactory; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; import org.joda.time.DateTime; import org.joda.time.format.DateTimeFormat; @@ -301,10 +302,11 @@ public abstract class AbstractXmlCollectionHandler implements XmlCollectionHandl * @return the XML document */ protected Document getXmlDocument(String urlString) { + InputStream is = null; try { URL url = UrlFactory.getUrl(urlString); URLConnection c = url.openConnection(); - InputStream is = c.getInputStream(); + is = c.getInputStream(); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); factory.setIgnoringComments(true); DocumentBuilder builder = factory.newDocumentBuilder(); @@ -313,6 +315,8 @@ public abstract class AbstractXmlCollectionHandler implements XmlCollectionHandl return doc; } catch (Exception e) { throw new XmlCollectorException(e.getMessage(), e); + } finally { + IOUtils.closeQuietly(is); } } diff --git a/protocols/xml/src/main/java/org/opennms/protocols/xml/collector/Sftp3gppXmlCollectionHandler.java b/protocols/xml/src/main/java/org/opennms/protocols/xml/collector/Sftp3gppXmlCollectionHandler.java index 141afaa..1f66ced 100644 --- a/protocols/xml/src/main/java/org/opennms/protocols/xml/collector/Sftp3gppXmlCollectionHandler.java +++ b/protocols/xml/src/main/java/org/opennms/protocols/xml/collector/Sftp3gppXmlCollectionHandler.java @@ -44,6 +44,7 @@ import java.util.regex.Pattern; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; +import org.apache.commons.io.IOUtils; import org.opennms.netmgt.collectd.CollectionAgent; import org.opennms.netmgt.collectd.CollectionException; import org.opennms.netmgt.collectd.ServiceCollector; @@ -114,6 +115,7 @@ public class Sftp3gppXmlCollectionHandler extends AbstractXmlCollectionHandler { log().debug("collect(multiple): retrieving file " + fileName + " from " + agent.getHostAddress()); InputStream is = connection.getFile(fileName); Document doc = builder.parse(is); + IOUtils.closeQuietly(is); fillCollectionSet(agent, collectionSet, source, doc); setLastFilename(resourceDir, url.getPath(), fileName); deleteFile(connection, fileName); diff --git a/protocols/xml/src/main/java/org/opennms/protocols/xml/collector/UrlFactory.java b/protocols/xml/src/main/java/org/opennms/protocols/xml/collector/UrlFactory.java index a9fc27e..196698b 100644 --- a/protocols/xml/src/main/java/org/opennms/protocols/xml/collector/UrlFactory.java +++ b/protocols/xml/src/main/java/org/opennms/protocols/xml/collector/UrlFactory.java @@ -33,6 +33,7 @@ import java.net.MalformedURLException; import java.net.URL; import java.net.URLConnection; +import org.opennms.protocols.http.HttpUrlConnection; import org.opennms.protocols.http.HttpUrlHandler; import org.opennms.protocols.sftp.Sftp3gppUrlHandler; import org.opennms.protocols.sftp.SftpUrlConnection; @@ -81,5 +82,7 @@ public class UrlFactory { public static void disconnect(URLConnection connection) throws IOException { if (connection != null && connection instanceof SftpUrlConnection) // We need to be sure to close the connections for SFTP ((SftpUrlConnection)connection).disconnect(); + if (connection != null && connection instanceof HttpUrlConnection) + ((HttpUrlConnection)connection).disconnect(); } }