Initial revision
mercury
commit
80c48012cd
|
|
@ -0,0 +1,26 @@
|
|||
WebLech license information.
|
||||
============================
|
||||
|
||||
This is the MIT license, see also http://www.opensource.org/licenses/mit-license.html
|
||||
|
||||
Copyright (c) 2001 Brian Pitcher
|
||||
Copyright (c) 2004 Andrew Coleman
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a
|
||||
copy of this software and associated documentation files (the "Software"),
|
||||
to deal in the Software without restriction, including without limitation
|
||||
the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
and/or sell copies of the Software, and to permit persons to whom the
|
||||
Software is furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in
|
||||
all copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
|
||||
|
|
@ -0,0 +1,31 @@
|
|||
<project name="WebLech" default="dist" basedir=".">
|
||||
<description>
|
||||
WebLech - a tool for downloading the web
|
||||
</description>
|
||||
|
||||
<property name="src" location="weblech" />
|
||||
<property name="build" location="build" />
|
||||
<property name="dist" location="jars" />
|
||||
|
||||
<target name="init">
|
||||
<tstamp />
|
||||
<mkdir dir="${build}" />
|
||||
</target>
|
||||
|
||||
<target name="compile" depends="init">
|
||||
<javac srcdir="${src}" destdir="${build}" />
|
||||
</target>
|
||||
|
||||
<target name="dist" depends="compile">
|
||||
<mkdir dir="${dist}" />
|
||||
<manifest file="${dist}/MANIFEST.MF">
|
||||
<attribute name="Main-Class" value="weblech.ui.Troll" />
|
||||
</manifest>
|
||||
<jar manifest="${dist}/MANIFEST.MF" jarfile="${dist}/WebLech-${DSTAMP}.jar" basedir="${build}" />
|
||||
</target>
|
||||
|
||||
<target name="clean">
|
||||
<delete dir="${build}" />
|
||||
<delete dir="${dist}" />
|
||||
</target>
|
||||
</project>
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
/*
|
||||
* Created by IntelliJ IDEA.
|
||||
* User: Michael Mason
|
||||
* Date: Jun 5, 2002
|
||||
* Time: 6:43:04 PM
|
||||
* To change template for new interface use
|
||||
* Code Style | Class Templates options (Tools | IDE Options).
|
||||
*/
|
||||
package weblech.spider;
|
||||
|
||||
public interface Constants
|
||||
{
|
||||
|
||||
/** How often to check the queue status */
|
||||
int QUEUE_CHECK_INTERVAL = 500;
|
||||
/** How long to pause for threads to finish before exitting */
|
||||
int SPIDER_STOP_PAUSE = 500;
|
||||
}
|
||||
|
|
@ -0,0 +1,143 @@
|
|||
/*
|
||||
* This is the MIT license, see also http://www.opensource.org/licenses/mit-license.html
|
||||
*
|
||||
* Copyright (c) 2001 Brian Pitcher
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
// $Header: /home/andrew/Projects/penguincoder/cvs/WebLech/weblech/spider/DownloadQueue.java,v 1.1 2004/03/07 20:51:05 mercury Exp $
|
||||
|
||||
package weblech.spider;
|
||||
|
||||
import java.util.*;
|
||||
import java.net.URL;
|
||||
import java.io.Serializable;
|
||||
|
||||
public class DownloadQueue implements Serializable
|
||||
{
|
||||
private SpiderConfig config;
|
||||
|
||||
private List interestingURLsToDownload;
|
||||
private List averageURLsToDownload;
|
||||
private List boringURLsToDownload;
|
||||
private Set urlsInQueue;
|
||||
|
||||
public DownloadQueue(SpiderConfig config)
|
||||
{
|
||||
this.config = config;
|
||||
interestingURLsToDownload = new ArrayList();
|
||||
averageURLsToDownload = new ArrayList();
|
||||
boringURLsToDownload = new ArrayList();
|
||||
urlsInQueue = new HashSet();
|
||||
}
|
||||
|
||||
public void queueURL(URLToDownload url)
|
||||
{
|
||||
URL u = url.getURL();
|
||||
if(urlsInQueue.contains(u))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if(config.isInteresting(u))
|
||||
{
|
||||
if(config.isDepthFirstSearch())
|
||||
{
|
||||
interestingURLsToDownload.add(0, url);
|
||||
}
|
||||
else
|
||||
{
|
||||
interestingURLsToDownload.add(url);
|
||||
}
|
||||
}
|
||||
else if(config.isBoring(u))
|
||||
{
|
||||
if(config.isDepthFirstSearch())
|
||||
{
|
||||
boringURLsToDownload.add(0, url);
|
||||
}
|
||||
else
|
||||
{
|
||||
boringURLsToDownload.add(url);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
if(config.isDepthFirstSearch())
|
||||
{
|
||||
averageURLsToDownload.add(0, url);
|
||||
}
|
||||
else
|
||||
{
|
||||
averageURLsToDownload.add(url);
|
||||
}
|
||||
}
|
||||
|
||||
urlsInQueue.add(u);
|
||||
}
|
||||
|
||||
public void queueURLs(Collection urls)
|
||||
{
|
||||
for(Iterator i = urls.iterator(); i.hasNext(); )
|
||||
{
|
||||
URLToDownload u2d = (URLToDownload) i.next();
|
||||
queueURL(u2d);
|
||||
}
|
||||
}
|
||||
|
||||
public URLToDownload getNextInQueue()
|
||||
{
|
||||
if(interestingURLsToDownload.size() > 0)
|
||||
{
|
||||
return returnURLFrom(interestingURLsToDownload);
|
||||
}
|
||||
else if(averageURLsToDownload.size() > 0)
|
||||
{
|
||||
return returnURLFrom(averageURLsToDownload);
|
||||
}
|
||||
else if(boringURLsToDownload.size() > 0)
|
||||
{
|
||||
return returnURLFrom(boringURLsToDownload);
|
||||
}
|
||||
else
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private URLToDownload returnURLFrom(List urlList)
|
||||
{
|
||||
URLToDownload u2d = (URLToDownload) urlList.get(0);
|
||||
urlList.remove(0);
|
||||
urlsInQueue.remove(u2d.getURL());
|
||||
return u2d;
|
||||
}
|
||||
|
||||
public int size()
|
||||
{
|
||||
return interestingURLsToDownload.size() + averageURLsToDownload.size() + boringURLsToDownload.size();
|
||||
}
|
||||
|
||||
public String toString()
|
||||
{
|
||||
return size() + " URLs";
|
||||
}
|
||||
|
||||
} // End class DownloadQueue
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
/*
|
||||
* This is the MIT license, see also http://www.opensource.org/licenses/mit-license.html
|
||||
*
|
||||
* Copyright (c) 2001 Brian Pitcher
|
||||
* Copyright (c) 2004 Andrew Coleman
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
package weblech.spider;
|
||||
|
||||
import java.net.Authenticator;
|
||||
import java.net.PasswordAuthentication;
|
||||
|
||||
import weblech.ui.LechLogger;
|
||||
|
||||
public class DumbAuthenticator extends Authenticator
|
||||
{
|
||||
private final String user;
|
||||
private final String password;
|
||||
|
||||
public DumbAuthenticator(String user, String password)
|
||||
{
|
||||
LechLogger.debug("DumbAuthenticator(" + user + ", ***)");
|
||||
this.user = user;
|
||||
this.password = password;
|
||||
}
|
||||
|
||||
public PasswordAuthentication getPasswordAuthentication()
|
||||
{
|
||||
LechLogger.debug("getPasswordAuthentication()");
|
||||
return new PasswordAuthentication(user, password.toCharArray());
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,188 @@
|
|||
/*
|
||||
* This is the MIT license, see also http://www.opensource.org/licenses/mit-license.html
|
||||
*
|
||||
* Copyright (c) 2001 Brian Pitcher
|
||||
* Copyright (c) 2004 Andrew Coleman
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
package weblech.spider;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashSet;
|
||||
import java.util.Set;
|
||||
import java.net.URL;
|
||||
import java.net.MalformedURLException;
|
||||
import java.io.ByteArrayInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.FileWriter;
|
||||
import java.io.PrintWriter;
|
||||
|
||||
import weblech.ui.LechLogger;
|
||||
|
||||
public class HTMLParser
|
||||
{
|
||||
private SpiderConfig config;
|
||||
|
||||
public HTMLParser(SpiderConfig config)
|
||||
{
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public List parseLinksInDocument(URL sourceURL, String textContent)
|
||||
{
|
||||
return parseAsHTML(sourceURL, textContent);
|
||||
}
|
||||
|
||||
private List parseAsHTML(URL sourceURL, String textContent)
|
||||
{
|
||||
LechLogger.debug("parseAsHTML()");
|
||||
ArrayList newURLs = new ArrayList();
|
||||
HashSet newURLSet = new HashSet();
|
||||
|
||||
/* note from coleman:
|
||||
* I had to add a few tags into this, namely the link and embeds. weblech should download flash
|
||||
* movies, mpegs, avis, and anything else that it finds on the page. even stylesheets :)
|
||||
*/
|
||||
extractAttributesFromTags("img", "src", sourceURL, newURLs, newURLSet, textContent);
|
||||
extractAttributesFromTags("a", "href", sourceURL, newURLs, newURLSet, textContent);
|
||||
extractAttributesFromTags("body", "background", sourceURL, newURLs, newURLSet, textContent);
|
||||
extractAttributesFromTags("frame", "src", sourceURL, newURLs, newURLSet, textContent);
|
||||
extractAttributesFromTags("link", "href", sourceURL, newURLs, newURLSet, textContent);
|
||||
extractAttributesFromTags("embed", "src", sourceURL, newURLs, newURLSet, textContent);
|
||||
extractAttributesFromTags("IMG", "SRC", sourceURL, newURLs, newURLSet, textContent);
|
||||
extractAttributesFromTags("A", "HREF", sourceURL, newURLs, newURLSet, textContent);
|
||||
extractAttributesFromTags("BODY", "BACKGROUND", sourceURL, newURLs, newURLSet, textContent);
|
||||
extractAttributesFromTags("FRAME", "SRC", sourceURL, newURLs, newURLSet, textContent);
|
||||
extractAttributesFromTags("LINK", "HREF", sourceURL, newURLs, newURLSet, textContent);
|
||||
extractAttributesFromTags("EMBED", "SRC", sourceURL, newURLs, newURLSet, textContent);
|
||||
|
||||
if(newURLs.size() == 0)
|
||||
{
|
||||
LechLogger.debug("Got 0 new URLs from HTML parse, check HTML\n" + textContent);
|
||||
}
|
||||
LechLogger.debug("Returning " + newURLs.size() + " urls extracted from page");
|
||||
return newURLs;
|
||||
}
|
||||
|
||||
private void extractAttributesFromTags(String tag, String attr, URL sourceURL, List newURLs, Set newURLSet, String input)
|
||||
{
|
||||
LechLogger.debug("extractAttributesFromTags(" + tag + ", " + attr + ", ...)");
|
||||
|
||||
int startPos = 0;
|
||||
String startTag = "<" + tag + " ";
|
||||
String attrStr = attr + "=\"";
|
||||
while(true)
|
||||
{
|
||||
int tagPos = input.indexOf(startTag, startPos);
|
||||
if(tagPos < 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
int attrPos = input.indexOf(attrStr, tagPos + 1);
|
||||
if(attrPos < 0)
|
||||
{
|
||||
startPos = tagPos + 1;
|
||||
continue;
|
||||
}
|
||||
int nextClosePos = input.indexOf(">", tagPos + 1);
|
||||
if(attrPos < nextClosePos)
|
||||
{
|
||||
// Ooh, found one
|
||||
int closeQuotePos = input.indexOf("\"", attrPos + attrStr.length() + 1);
|
||||
if(closeQuotePos > 0)
|
||||
{
|
||||
String urlStr = input.substring(attrPos + attrStr.length(), closeQuotePos);
|
||||
if(urlStr.indexOf('#') != -1)
|
||||
{
|
||||
urlStr = urlStr.substring(0, urlStr.indexOf('#'));
|
||||
}
|
||||
//LechLogger.debug("Found possible URL string: " + URL);
|
||||
|
||||
if(isMailTo(urlStr))
|
||||
{
|
||||
logMailURL(urlStr);
|
||||
}
|
||||
else
|
||||
{
|
||||
try
|
||||
{
|
||||
|
||||
URL u = new URL(sourceURL, urlStr);
|
||||
if(newURLSet.contains(u))
|
||||
{
|
||||
//LechLogger.debug("Already found URL on page: " + u);
|
||||
}
|
||||
else
|
||||
{
|
||||
newURLs.add(u);
|
||||
newURLSet.add(u);
|
||||
//LechLogger.debug("Found new URL on page: " + u);
|
||||
}
|
||||
}
|
||||
catch(MalformedURLException murle)
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
startPos = tagPos + 1;
|
||||
continue;
|
||||
}
|
||||
else
|
||||
{
|
||||
startPos = tagPos + 1;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void logMailURL(String url)
|
||||
{
|
||||
LechLogger.debug("logMailURL()");
|
||||
|
||||
try
|
||||
{
|
||||
FileWriter appendedFile = new FileWriter(config.getMailtoLogFile().toString(), true);
|
||||
PrintWriter pW = new PrintWriter(appendedFile);
|
||||
pW.println(url);
|
||||
pW.flush();
|
||||
pW.close();
|
||||
}
|
||||
catch(IOException ioe)
|
||||
{
|
||||
LechLogger.warn("Caught IO exception writing mailto URL:" + ioe.getMessage(), ioe);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a particular URL looks like it's a mailto: style link.
|
||||
*/
|
||||
private boolean isMailTo(String url)
|
||||
{
|
||||
if(url == null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
url = url.toUpperCase();
|
||||
return (url.indexOf("MAILTO:") != -1);
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,333 @@
|
|||
/*
|
||||
* This is the MIT license, see also http://www.opensource.org/licenses/mit-license.html
|
||||
*
|
||||
* Copyright (c) 2001 Brian Pitcher
|
||||
* Copyright (c) 2004 Andrew Coleman
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
package weblech.spider;
|
||||
|
||||
import weblech.ui.LechLogger;
|
||||
|
||||
import java.util.*;
|
||||
import java.io.*;
|
||||
import java.net.URL;
|
||||
|
||||
public class Spider implements Runnable, Constants
|
||||
{
|
||||
/** Config for the spider */
|
||||
private SpiderConfig config;
|
||||
/**
|
||||
* Download queue.
|
||||
* Thread safety: To access the queue, first synchronize on it.
|
||||
*/
|
||||
private DownloadQueue queue;
|
||||
/**
|
||||
* Set of URLs downloaded or scheduled, so we don't download a
|
||||
* URL more than once.
|
||||
* Thread safety: To access the set, first synchronize on it.
|
||||
*/
|
||||
private Set urlsDownloadedOrScheduled;
|
||||
/**
|
||||
* Set of URLs currently being downloaded by Spider threads.
|
||||
* Thread safety: To access the set, first synchronize on it.
|
||||
*/
|
||||
private Set urlsDownloading;
|
||||
/**
|
||||
* Number of downloads currently taking place.
|
||||
* Thread safety: To modify this value, first synchronize on
|
||||
* the download queue.
|
||||
*/
|
||||
private int downloadsInProgress;
|
||||
/** Whether the spider should quit */
|
||||
private boolean quit;
|
||||
/** Count of running Spider threads. */
|
||||
private int running;
|
||||
/** Time we last checkpointed. */
|
||||
private long lastCheckpoint;
|
||||
|
||||
public Spider(SpiderConfig config)
|
||||
{
|
||||
this.config = config;
|
||||
queue = new DownloadQueue(config);
|
||||
queue.queueURL(new URLToDownload(config.getStartLocation(), 0));
|
||||
urlsDownloadedOrScheduled = new HashSet();
|
||||
urlsDownloading = new HashSet();
|
||||
downloadsInProgress = 0;
|
||||
lastCheckpoint = 0;
|
||||
}
|
||||
|
||||
public void start()
|
||||
{
|
||||
quit = false;
|
||||
running = 0;
|
||||
|
||||
for(int i = 0; i < config.getSpiderThreads(); i++)
|
||||
{
|
||||
LechLogger.info("Starting Spider thread");
|
||||
Thread t = new Thread(this, "Spider-Thread-" + (i + 1));
|
||||
t.start();
|
||||
running++;
|
||||
}
|
||||
}
|
||||
|
||||
public void stop()
|
||||
{
|
||||
quit = true;
|
||||
}
|
||||
|
||||
public boolean isRunning()
|
||||
{
|
||||
return running == 0;
|
||||
}
|
||||
|
||||
private void checkpointIfNeeded()
|
||||
{
|
||||
if(config.getCheckpointInterval() == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if(System.currentTimeMillis() - lastCheckpoint > config.getCheckpointInterval())
|
||||
{
|
||||
synchronized(queue)
|
||||
{
|
||||
if(System.currentTimeMillis() - lastCheckpoint > config.getCheckpointInterval())
|
||||
{
|
||||
writeCheckpoint();
|
||||
lastCheckpoint = System.currentTimeMillis();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void writeCheckpoint()
|
||||
{
|
||||
LechLogger.debug("writeCheckpoint()");
|
||||
try
|
||||
{
|
||||
FileOutputStream fos = new FileOutputStream("spider.checkpoint", false);
|
||||
ObjectOutputStream oos = new ObjectOutputStream(fos);
|
||||
oos.writeObject(queue);
|
||||
oos.writeObject(urlsDownloading);
|
||||
oos.close();
|
||||
}
|
||||
catch(IOException ioe)
|
||||
{
|
||||
LechLogger.warn("IO Exception attempting checkpoint: " + ioe.getMessage(), ioe);
|
||||
}
|
||||
}
|
||||
|
||||
public void readCheckpoint()
|
||||
{
|
||||
try
|
||||
{
|
||||
FileInputStream fis = new FileInputStream("spider.checkpoint");
|
||||
ObjectInputStream ois = new ObjectInputStream(fis);
|
||||
queue = (DownloadQueue) ois.readObject();
|
||||
urlsDownloading = (Set) ois.readObject();
|
||||
queue.queueURLs(urlsDownloading);
|
||||
urlsDownloading.clear();
|
||||
}
|
||||
catch(Exception e)
|
||||
{
|
||||
LechLogger.error("Caught exception reading checkpoint: " + e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
public void run()
|
||||
{
|
||||
HTMLParser htmlParser = new HTMLParser(config);
|
||||
URLGetter urlGetter = new URLGetter(config);
|
||||
|
||||
while((queueSize() > 0 || downloadsInProgress > 0) && quit == false)
|
||||
{
|
||||
checkpointIfNeeded();
|
||||
if(queueSize() == 0 && downloadsInProgress > 0)
|
||||
{
|
||||
// Wait for a download to finish before seeing if this thread should stop
|
||||
try
|
||||
{
|
||||
Thread.sleep(QUEUE_CHECK_INTERVAL);
|
||||
}
|
||||
catch(InterruptedException ignored)
|
||||
{
|
||||
}
|
||||
// Have another go at the loop
|
||||
continue;
|
||||
}
|
||||
else if(queueSize() == 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
URLToDownload nextURL;
|
||||
synchronized(queue)
|
||||
{
|
||||
nextURL = queue.getNextInQueue();
|
||||
downloadsInProgress++;
|
||||
}
|
||||
synchronized(urlsDownloading)
|
||||
{
|
||||
urlsDownloading.add(nextURL);
|
||||
}
|
||||
int newDepth = nextURL.getDepth() + 1;
|
||||
int maxDepth = config.getMaxDepth();
|
||||
synchronized(urlsDownloading)
|
||||
{
|
||||
urlsDownloading.remove(nextURL);
|
||||
}
|
||||
List newURLs = downloadURL(nextURL, urlGetter, htmlParser);
|
||||
|
||||
newURLs = filterURLs(newURLs);
|
||||
|
||||
ArrayList u2dsToQueue = new ArrayList();
|
||||
for(Iterator i = newURLs.iterator(); i.hasNext(); )
|
||||
{
|
||||
URL u = (URL) i.next();
|
||||
// Download if not yet downloaded, and the new depth is less than the maximum
|
||||
synchronized(urlsDownloadedOrScheduled)
|
||||
{
|
||||
if(!urlsDownloadedOrScheduled.contains(u)
|
||||
&& (maxDepth == 0 || newDepth <= maxDepth))
|
||||
{
|
||||
u2dsToQueue.add(new URLToDownload(u, nextURL.getURL(), newDepth));
|
||||
urlsDownloadedOrScheduled.add(u);
|
||||
}
|
||||
}
|
||||
}
|
||||
synchronized(queue)
|
||||
{
|
||||
queue.queueURLs(u2dsToQueue);
|
||||
downloadsInProgress--;
|
||||
}
|
||||
}
|
||||
LechLogger.info("Spider thread stopping [" + config.getStartLocation() + "]" );
|
||||
running--;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the size of the download queue in a thread-safe manner.
|
||||
*/
|
||||
private int queueSize()
|
||||
{
|
||||
synchronized(queue)
|
||||
{
|
||||
return queue.size();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get a URL, and return new URLs that are referenced from it.
|
||||
*
|
||||
* @return A List of URL objects.
|
||||
*/
|
||||
private List downloadURL(URLToDownload url, URLGetter urlGetter, HTMLParser htmlParser)
|
||||
{
|
||||
LechLogger.debug("downloadURL(" + url + ")");
|
||||
|
||||
// Bail out early if image and already on disk
|
||||
URLObject obj = new URLObject(url.getURL(), config);
|
||||
if(obj.existsOnDisk())
|
||||
{
|
||||
if(config.refreshHTMLs() && (obj.isHTML() || obj.isXML()))
|
||||
{
|
||||
LechLogger.info("Q: [" + queue + "] " + url);
|
||||
obj = urlGetter.getURL(url);
|
||||
}
|
||||
else if(config.refreshImages() && obj.isImage())
|
||||
{
|
||||
LechLogger.info("Q: [" + queue + "] " + url);
|
||||
obj = urlGetter.getURL(url);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
LechLogger.info("Q: [" + queue + "] " + url);
|
||||
obj = urlGetter.getURL(url);
|
||||
}
|
||||
|
||||
if(obj == null)
|
||||
{
|
||||
return new ArrayList();
|
||||
}
|
||||
|
||||
if(!obj.existsOnDisk())
|
||||
{
|
||||
obj.writeToFile();
|
||||
}
|
||||
|
||||
if(obj.isHTML() || obj.isXML())
|
||||
{
|
||||
return htmlParser.parseLinksInDocument(url.getURL(), obj.getStringContent());
|
||||
}
|
||||
else if(obj.isImage())
|
||||
{
|
||||
return new ArrayList();
|
||||
}
|
||||
else
|
||||
{
|
||||
LechLogger.warn("Unknown content type received: " + obj.getContentType());
|
||||
LechLogger.info("URL was " + url);
|
||||
return new ArrayList();
|
||||
}
|
||||
}
|
||||
|
||||
private List filterURLs(List URLs)
|
||||
{
|
||||
String match = config.getURLMatch();
|
||||
ArrayList retVal = new ArrayList();
|
||||
|
||||
synchronized(urlsDownloadedOrScheduled)
|
||||
{
|
||||
for(Iterator i = URLs.iterator(); i.hasNext(); )
|
||||
{
|
||||
URL u = (URL) i.next();
|
||||
if(urlsDownloadedOrScheduled.contains(u))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
String s = u.toString();
|
||||
if(s.indexOf(match) != -1)
|
||||
{
|
||||
retVal.add(u);
|
||||
}
|
||||
}
|
||||
}
|
||||
return retVal;
|
||||
}
|
||||
|
||||
/* Method By Coleman
|
||||
* A basic check to see if there is another spider downloading the same thing
|
||||
*/
|
||||
protected boolean compareSpiderConfig ( SpiderConfig sc ) {
|
||||
return config.getStartLocation().equals ( sc.getStartLocation() );
|
||||
}
|
||||
|
||||
/* Method By Coleman
|
||||
* A method to determine if one spider is downloading the same file as another spider
|
||||
*/
|
||||
public boolean equals ( Object o ) {
|
||||
if ( !o.getClass().isInstance ( this ) ) return false;
|
||||
return ((Spider) o).compareSpiderConfig ( config );
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -0,0 +1,464 @@
|
|||
/*
|
||||
* This is the MIT license, see also http://www.opensource.org/licenses/mit-license.html
|
||||
*
|
||||
* Copyright (c) 2001 Brian Pitcher
|
||||
* Copyright (c) 2004 Andrew Coleman
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
package weblech.spider;
|
||||
|
||||
import weblech.ui.LechLogger;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.Serializable;
|
||||
import java.util.*;
|
||||
import java.net.URL;
|
||||
import java.net.MalformedURLException;
|
||||
|
||||
public class SpiderConfig implements Serializable
|
||||
{
|
||||
private File saveRootDirectory;
|
||||
private File mailtoLogFile;
|
||||
|
||||
private boolean refreshHTMLs;
|
||||
private boolean refreshImages;
|
||||
private boolean refreshOthers;
|
||||
|
||||
private Set htmlExtensions;
|
||||
private Set imageExtensions;
|
||||
|
||||
private URL startLocation;
|
||||
private String urlMatch;
|
||||
|
||||
private List interestingURLSubstrings;
|
||||
private List boringURLSubstrings;
|
||||
|
||||
private boolean depthFirst;
|
||||
private int maxDepth;
|
||||
|
||||
private String userAgent;
|
||||
|
||||
private String basicAuthUser;
|
||||
private String basicAuthPassword;
|
||||
|
||||
private int spiderThreads;
|
||||
|
||||
private long checkpointInterval;
|
||||
|
||||
/**
|
||||
* Create a default config.
|
||||
*/
|
||||
public SpiderConfig()
|
||||
{
|
||||
LechLogger.debug("SpiderConfig()");
|
||||
|
||||
saveRootDirectory = new File(".");
|
||||
mailtoLogFile = new File("mailto.txt");
|
||||
|
||||
refreshHTMLs = true;
|
||||
refreshImages = false;
|
||||
refreshOthers = false;
|
||||
|
||||
htmlExtensions = new HashSet();
|
||||
htmlExtensions.add("htm");
|
||||
htmlExtensions.add("html");
|
||||
htmlExtensions.add("shtml");
|
||||
|
||||
imageExtensions = new HashSet();
|
||||
imageExtensions.add("jpg");
|
||||
imageExtensions.add("gif");
|
||||
imageExtensions.add("png");
|
||||
/* Added a few image extensions -- Coleman */
|
||||
imageExtensions.add("tiff");
|
||||
imageExtensions.add("bmp");
|
||||
|
||||
urlMatch = null;
|
||||
interestingURLSubstrings = new ArrayList();
|
||||
boringURLSubstrings = new ArrayList();
|
||||
depthFirst = false;
|
||||
maxDepth = 0;
|
||||
|
||||
userAgent = "WebLech Spider [Release C]";
|
||||
basicAuthUser = "";
|
||||
basicAuthPassword = "";
|
||||
|
||||
spiderThreads = 1;
|
||||
|
||||
checkpointInterval = 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a config from a java.util.Properties object.
|
||||
*/
|
||||
public SpiderConfig(Properties props)
|
||||
{
|
||||
LechLogger.debug("SpiderConfig(props)");
|
||||
|
||||
saveRootDirectory = new File(props.getProperty("saveRootDirectory", "."));
|
||||
if(!saveRootDirectory.exists())
|
||||
{
|
||||
if(!saveRootDirectory.mkdirs())
|
||||
{
|
||||
LechLogger.error("Couldn't create root directory: " + saveRootDirectory);
|
||||
LechLogger.info("Defaulting to . instead");
|
||||
saveRootDirectory = new File(".");
|
||||
}
|
||||
}
|
||||
else if(!saveRootDirectory.isDirectory())
|
||||
{
|
||||
LechLogger.error("Save root is not a directory: " + saveRootDirectory);
|
||||
LechLogger.info("Defaulting to . instead");
|
||||
saveRootDirectory = new File(".");
|
||||
}
|
||||
|
||||
String mailtoFileStr = props.getProperty("mailtoLogFile", "mailto.txt");
|
||||
// Check if absolute or relative name given
|
||||
if(mailtoFileStr.indexOf(":") != -1 || mailtoFileStr.startsWith("/") || mailtoFileStr.startsWith("\\"))
|
||||
{
|
||||
LechLogger.debug("Using absolute file name " + mailtoFileStr);
|
||||
mailtoLogFile = new File(mailtoFileStr);
|
||||
}
|
||||
else
|
||||
{
|
||||
LechLogger.debug("Constructing relative file name " + saveRootDirectory.getPath() + "/" + mailtoFileStr);
|
||||
mailtoLogFile = new File(saveRootDirectory.getPath() + "/" + mailtoFileStr);
|
||||
}
|
||||
|
||||
refreshHTMLs = Boolean.valueOf(props.getProperty("refreshHTMLs", "true")).booleanValue();
|
||||
refreshImages = Boolean.valueOf(props.getProperty("refreshImages", "false")).booleanValue();
|
||||
refreshOthers = Boolean.valueOf(props.getProperty("refreshOthers", "false")).booleanValue();
|
||||
|
||||
htmlExtensions = parseSet(props.getProperty("htmlExtensions", "htm,html,shtml"));
|
||||
imageExtensions = parseSet(props.getProperty("imageExtensions", "jpg,gif,png"));
|
||||
|
||||
String startLocStr = props.getProperty("startLocation");
|
||||
if(startLocStr != null)
|
||||
{
|
||||
try
|
||||
{
|
||||
startLocation = new URL(startLocStr);
|
||||
}
|
||||
catch(MalformedURLException murle)
|
||||
{
|
||||
LechLogger.error("Caught MalformedURLException parsing start URL '" + startLocStr + "' : " + murle.getMessage(), murle);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
LechLogger.warn("startLocation not found in properties");
|
||||
}
|
||||
|
||||
urlMatch = props.getProperty("urlMatch");
|
||||
|
||||
interestingURLSubstrings = parsePropCommaSeparated(props.getProperty("interestingURLs"));
|
||||
boringURLSubstrings = parsePropCommaSeparated(props.getProperty("boringURLs"));
|
||||
|
||||
depthFirst = Boolean.valueOf(props.getProperty("depthFirst", "false")).booleanValue();
|
||||
try
|
||||
{
|
||||
String maxDepthStr = props.getProperty("maxDepth", "0");
|
||||
maxDepth = Integer.parseInt(maxDepthStr);
|
||||
}
|
||||
catch(NumberFormatException nfe)
|
||||
{
|
||||
LechLogger.error("Caught number format exception parsing max depth, defaulting to 1", nfe);
|
||||
maxDepth = 1;
|
||||
}
|
||||
|
||||
userAgent = props.getProperty("userAgent", "WebLech Spider [Version C]");
|
||||
basicAuthUser = props.getProperty("basicAuthUser", "");
|
||||
basicAuthPassword = props.getProperty("basicAuthPassword", "");
|
||||
|
||||
try
|
||||
{
|
||||
String threadsStr = props.getProperty("spiderThreads", "1");
|
||||
spiderThreads = Integer.parseInt(threadsStr);
|
||||
}
|
||||
catch(NumberFormatException nfe)
|
||||
{
|
||||
LechLogger.error("Caught number format exception parsing number of threads, defaulting to 1", nfe);
|
||||
spiderThreads = 1;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
String intervalStr = props.getProperty("checkpointInterval", "0");
|
||||
checkpointInterval = Long.parseLong(intervalStr);
|
||||
}
|
||||
catch(NumberFormatException nfe)
|
||||
{
|
||||
LechLogger.error("Caught number format exception parsing checkpoint interval, defaulting to 0", nfe);
|
||||
spiderThreads = 1;
|
||||
}
|
||||
}
|
||||
|
||||
private List parsePropCommaSeparated(String str)
|
||||
{
|
||||
ArrayList result = new ArrayList();
|
||||
if(str != null && str.length() > 0)
|
||||
{
|
||||
StringTokenizer tok = new StringTokenizer(str, ",");
|
||||
while(tok.hasMoreTokens())
|
||||
{
|
||||
result.add(tok.nextToken());
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
|
||||
public void setRefreshHTMLs(boolean refreshHTMLs)
|
||||
{
|
||||
this.refreshHTMLs = refreshHTMLs;
|
||||
}
|
||||
|
||||
public boolean refreshHTMLs()
|
||||
{
|
||||
return refreshHTMLs;
|
||||
}
|
||||
|
||||
public void setRefreshImages(boolean refreshImages)
|
||||
{
|
||||
this.refreshImages = refreshImages;
|
||||
}
|
||||
|
||||
public boolean refreshImages()
|
||||
{
|
||||
return refreshImages;
|
||||
}
|
||||
|
||||
public void setRefreshOthers(boolean refreshOthers)
|
||||
{
|
||||
this.refreshOthers = refreshOthers;
|
||||
}
|
||||
|
||||
public boolean refreshOthers()
|
||||
{
|
||||
return refreshOthers;
|
||||
}
|
||||
|
||||
public void setSaveRootDirectory(File saveRootDirectory)
|
||||
{
|
||||
this.saveRootDirectory = saveRootDirectory;
|
||||
}
|
||||
|
||||
public File getSaveRootDirectory()
|
||||
{
|
||||
return saveRootDirectory;
|
||||
}
|
||||
|
||||
public void setMailtoLogFile(File mailtoLogFile)
|
||||
{
|
||||
this.mailtoLogFile = mailtoLogFile;
|
||||
}
|
||||
|
||||
public File getMailtoLogFile()
|
||||
{
|
||||
return mailtoLogFile;
|
||||
}
|
||||
|
||||
public void setStartLocation(URL startLocation)
|
||||
{
|
||||
this.startLocation = startLocation;
|
||||
}
|
||||
|
||||
public URL getStartLocation()
|
||||
{
|
||||
return startLocation;
|
||||
}
|
||||
|
||||
public void setURLMatch(String urlMatch)
|
||||
{
|
||||
this.urlMatch = urlMatch;
|
||||
}
|
||||
|
||||
public String getURLMatch()
|
||||
{
|
||||
return urlMatch;
|
||||
}
|
||||
|
||||
public List getInterestingURLSubstrings()
|
||||
{
|
||||
return interestingURLSubstrings;
|
||||
}
|
||||
|
||||
public void setInterestingURLSubstrings(List interestingURLSubstrings)
|
||||
{
|
||||
this.interestingURLSubstrings = interestingURLSubstrings;
|
||||
}
|
||||
|
||||
public List getBoringURLSubstrings()
|
||||
{
|
||||
return boringURLSubstrings;
|
||||
}
|
||||
|
||||
public void setBoringURLSubstrings(List boringURLSubstrings)
|
||||
{
|
||||
this.boringURLSubstrings = boringURLSubstrings;
|
||||
}
|
||||
|
||||
public boolean isInteresting(URL u)
|
||||
{
|
||||
return matchURL(u, interestingURLSubstrings);
|
||||
}
|
||||
|
||||
public boolean isBoring(URL u)
|
||||
{
|
||||
return matchURL(u, boringURLSubstrings);
|
||||
}
|
||||
|
||||
private boolean matchURL(URL u, List substrings)
|
||||
{
|
||||
String str = u.toExternalForm();
|
||||
for(Iterator i = substrings.iterator(); i.hasNext(); )
|
||||
{
|
||||
String substr = (String) i.next();
|
||||
if(str.indexOf(substr) != -1)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
public void setDepthFirstSearch(boolean depthFirst)
|
||||
{
|
||||
this.depthFirst = depthFirst;
|
||||
}
|
||||
|
||||
public boolean isDepthFirstSearch()
|
||||
{
|
||||
return depthFirst;
|
||||
}
|
||||
|
||||
public void setMaxDepth(int maxDepth)
|
||||
{
|
||||
this.maxDepth = maxDepth;
|
||||
}
|
||||
|
||||
public int getMaxDepth()
|
||||
{
|
||||
return maxDepth;
|
||||
}
|
||||
|
||||
public void setUserAgent(String userAgent)
|
||||
{
|
||||
this.userAgent = userAgent;
|
||||
}
|
||||
|
||||
public String getUserAgent()
|
||||
{
|
||||
return userAgent;
|
||||
}
|
||||
|
||||
public void setBasicAuthUser(String basicAuthUser)
|
||||
{
|
||||
this.basicAuthUser = basicAuthUser;
|
||||
}
|
||||
|
||||
public String getBasicAuthUser()
|
||||
{
|
||||
return basicAuthUser;
|
||||
}
|
||||
|
||||
public void setBasicAuthPassword(String basicAuthPassword)
|
||||
{
|
||||
this.basicAuthPassword = basicAuthPassword;
|
||||
}
|
||||
|
||||
public String getBasicAuthPassword()
|
||||
{
|
||||
return basicAuthPassword;
|
||||
}
|
||||
|
||||
public void setSpiderThreads(int spiderThreads)
|
||||
{
|
||||
this.spiderThreads = spiderThreads;
|
||||
}
|
||||
|
||||
public int getSpiderThreads()
|
||||
{
|
||||
return spiderThreads;
|
||||
}
|
||||
|
||||
public void setCheckpointInterval(long interval)
|
||||
{
|
||||
this.checkpointInterval = interval;
|
||||
}
|
||||
|
||||
public long getCheckpointInterval()
|
||||
{
|
||||
return checkpointInterval;
|
||||
}
|
||||
|
||||
public String toString()
|
||||
{
|
||||
return "depthFirst:\t" + depthFirst
|
||||
+ "\nmaxDepth:\t" + maxDepth
|
||||
+ "\nhtmlExtensions:\t" + fromSet(htmlExtensions)
|
||||
+ "\nimageExtensions:\t" + fromSet(imageExtensions)
|
||||
+ "\nrefreshHTMLs:\t" + refreshHTMLs
|
||||
+ "\nrefreshImages:\t" + refreshImages
|
||||
+ "\nrefreshOthers:\t" + refreshOthers
|
||||
+ "\nsaveRootDirectory:\t" + saveRootDirectory
|
||||
+ "\nstartLocation:\t" + startLocation
|
||||
+ "\nurlMatch:\t" + urlMatch
|
||||
+ "\nuserAgent:\t" + userAgent
|
||||
+ "\nbasicAuthUser:\t" + basicAuthUser
|
||||
+ "\nbasicAuthPassword:\t" + "***"
|
||||
+ "\nspiderThreads:\t" + spiderThreads
|
||||
+ "\ncheckpointInterval:\t" + checkpointInterval;
|
||||
}
|
||||
|
||||
private Set parseSet(String str)
|
||||
{
|
||||
LechLogger.debug("parseSet(" + str + ")");
|
||||
HashSet result = new HashSet();
|
||||
StringTokenizer sTok = new StringTokenizer(str, ",");
|
||||
while(sTok.hasMoreTokens())
|
||||
{
|
||||
String tok = sTok.nextToken().trim();
|
||||
result.add(tok);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String fromSet(Set s)
|
||||
{
|
||||
StringBuffer sb = new StringBuffer();
|
||||
boolean first = true;
|
||||
for(Iterator i = s.iterator(); i.hasNext(); )
|
||||
{
|
||||
String str = (String) i.next();
|
||||
if(first)
|
||||
{
|
||||
first = false;
|
||||
}
|
||||
else
|
||||
{
|
||||
sb.append(",");
|
||||
}
|
||||
sb.append(str);
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
} // End class SpiderConfig
|
||||
|
|
@ -0,0 +1,138 @@
|
|||
/*
|
||||
* This is the MIT license, see also http://www.opensource.org/licenses/mit-license.html
|
||||
*
|
||||
* Copyright (c) 2001 Brian Pitcher
|
||||
* Copyright (c) 2004 Andrew Coleman
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
package weblech.spider;
|
||||
|
||||
import java.net.HttpURLConnection;
|
||||
import java.net.URL;
|
||||
import java.net.Authenticator;
|
||||
import java.io.*;
|
||||
|
||||
import weblech.ui.LechLogger;
|
||||
|
||||
public class URLGetter
|
||||
{
|
||||
private int failureCount = 0;
|
||||
|
||||
private final SpiderConfig config;
|
||||
|
||||
public URLGetter(SpiderConfig config)
|
||||
{
|
||||
LechLogger.debug("URLGetter()");
|
||||
this.config = config;
|
||||
|
||||
Authenticator.setDefault(new DumbAuthenticator(config.getBasicAuthUser(), config.getBasicAuthPassword()));
|
||||
}
|
||||
|
||||
public URLObject getURL(URLToDownload url)
|
||||
{
|
||||
LechLogger.debug("getURL(" + url + ")");
|
||||
|
||||
if(failureCount > 10)
|
||||
{
|
||||
LechLogger.warn("Lots of failures recently, waiting 5 seconds before attempting download");
|
||||
try { Thread.sleep(5 * 1000); } catch(InterruptedException e) { };
|
||||
failureCount = 0;
|
||||
}
|
||||
|
||||
URL requestedURL = url.getURL();
|
||||
URL referer = url.getReferer();
|
||||
|
||||
try
|
||||
{
|
||||
LechLogger.debug("Creating HTTP connection to " + requestedURL);
|
||||
HttpURLConnection conn = (HttpURLConnection) requestedURL.openConnection();
|
||||
if(referer != null)
|
||||
{
|
||||
LechLogger.debug("Setting Referer header to " + referer);
|
||||
conn.setRequestProperty("Referer", referer.toExternalForm());
|
||||
}
|
||||
|
||||
if(config.getUserAgent() != null)
|
||||
{
|
||||
LechLogger.debug("Setting User-Agent to " + config.getUserAgent());
|
||||
conn.setRequestProperty("User-Agent", config.getUserAgent());
|
||||
}
|
||||
|
||||
conn.setUseCaches(false);
|
||||
|
||||
LechLogger.debug("Opening URL");
|
||||
long startTime = System.currentTimeMillis();
|
||||
conn.connect();
|
||||
|
||||
String resp = conn.getResponseMessage();
|
||||
LechLogger.debug("Remote server response: " + resp);
|
||||
|
||||
String respStr = conn.getHeaderField(0);
|
||||
LechLogger.info("Server response: " + respStr);
|
||||
|
||||
for(int i = 1; ; i++)
|
||||
{
|
||||
String key = conn.getHeaderFieldKey(i);
|
||||
if(key == null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
String value = conn.getHeaderField(key);
|
||||
LechLogger.debug("Received header " + key + ": " + value);
|
||||
}
|
||||
|
||||
LechLogger.debug("Getting buffered input stream from remote connection");
|
||||
BufferedInputStream remoteBIS = new BufferedInputStream(conn.getInputStream());
|
||||
ByteArrayOutputStream baos = new ByteArrayOutputStream(10240);
|
||||
byte[] buf = new byte[1024];
|
||||
int bytesRead = 0;
|
||||
while(bytesRead >= 0)
|
||||
{
|
||||
baos.write(buf, 0, bytesRead);
|
||||
bytesRead = remoteBIS.read(buf);
|
||||
}
|
||||
|
||||
byte[] content = baos.toByteArray();
|
||||
long timeTaken = System.currentTimeMillis() - startTime;
|
||||
if(timeTaken < 100) timeTaken = 500;
|
||||
|
||||
int bytesPerSec = (int) ((double) content.length / ((double)timeTaken / 1000.0));
|
||||
LechLogger.info("Downloaded " + content.length + " bytes, " + bytesPerSec + " bytes/sec");
|
||||
if(content.length < conn.getContentLength())
|
||||
{
|
||||
LechLogger.warn("Didn't download full content for URL: " + url);
|
||||
failureCount++;
|
||||
return null;
|
||||
}
|
||||
return new URLObject(requestedURL, conn.getContentType(), content, config);
|
||||
}
|
||||
catch(FileNotFoundException fnfe) {
|
||||
LechLogger.warn("File not found: " + fnfe.getMessage());
|
||||
return null;
|
||||
}
|
||||
catch(IOException ioe)
|
||||
{
|
||||
LechLogger.warn("Caught IO Exception: " + ioe.getMessage(), ioe);
|
||||
failureCount++;
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,206 @@
|
|||
/*
|
||||
* This is the MIT license, see also http://www.opensource.org/licenses/mit-license.html
|
||||
*
|
||||
* Copyright (c) 2001 Brian Pitcher
|
||||
* Copyright (c) 2004 Andrew Coleman
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
package weblech.spider;
|
||||
|
||||
import java.io.*;
|
||||
import java.net.URL;
|
||||
import java.net.URLEncoder;
|
||||
|
||||
import weblech.ui.LechLogger;
|
||||
|
||||
public class URLObject
|
||||
{
|
||||
private final URL sourceURL;
|
||||
private final String contentType;
|
||||
private final byte[] content;
|
||||
|
||||
private final SpiderConfig config;
|
||||
|
||||
public URLObject(URL sourceURL, String contentType, byte[] content, SpiderConfig config)
|
||||
{
|
||||
this.sourceURL = sourceURL;
|
||||
this.contentType = contentType;
|
||||
this.content = content;
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
public URLObject(URL sourceURL, SpiderConfig config)
|
||||
{
|
||||
this.sourceURL = sourceURL;
|
||||
this.config = config;
|
||||
|
||||
String s = sourceURL.toExternalForm().toLowerCase();
|
||||
if(s.indexOf(".jpg") != -1)
|
||||
{
|
||||
contentType = "image/jpeg";
|
||||
}
|
||||
else if(s.indexOf(".gif") != -1)
|
||||
{
|
||||
contentType = "image/gif";
|
||||
}
|
||||
else
|
||||
{
|
||||
contentType = "text/html";
|
||||
}
|
||||
|
||||
if(existsOnDisk())
|
||||
{
|
||||
|
||||
File f = new File(convertToFileName());
|
||||
if(f.isDirectory())
|
||||
{
|
||||
f = new File(f, "index.html");
|
||||
}
|
||||
content = new byte[(int) f.length()];
|
||||
try
|
||||
{
|
||||
FileInputStream in = new FileInputStream(f);
|
||||
in.read(content);
|
||||
in.close();
|
||||
}
|
||||
catch(IOException ioe)
|
||||
{
|
||||
LechLogger.warn("IO Exception reading disk version of URL " + sourceURL, ioe);
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
content = new byte[0];
|
||||
}
|
||||
}
|
||||
|
||||
public String getContentType()
|
||||
{
|
||||
return contentType;
|
||||
}
|
||||
|
||||
public boolean isHTML()
|
||||
{
|
||||
return contentType.toLowerCase().startsWith("text/html");
|
||||
}
|
||||
|
||||
public boolean isXML()
|
||||
{
|
||||
return contentType.toLowerCase().startsWith("text/xml");
|
||||
}
|
||||
|
||||
public boolean isImage()
|
||||
{
|
||||
return contentType.startsWith("image/");
|
||||
}
|
||||
|
||||
public String getStringContent()
|
||||
{
|
||||
return new String(content);
|
||||
}
|
||||
|
||||
private String convertToFileName()
|
||||
{
|
||||
String url = sourceURL.toExternalForm();
|
||||
int httpIdx = url.indexOf("http://");
|
||||
if(httpIdx == 0)
|
||||
{
|
||||
url = url.substring(7);
|
||||
}
|
||||
// Check for at least one slash -- otherwise host name (e.g. sourceforge.net)
|
||||
if(url.indexOf("/") < 0)
|
||||
{
|
||||
url = url + "/";
|
||||
}
|
||||
// If trailing slash, add index.html as default
|
||||
if(url.endsWith("/"))
|
||||
{
|
||||
url = url + "index.html";
|
||||
}
|
||||
try {
|
||||
/* the old encode method is now deprecated, updated to the new API -- Coleman */
|
||||
url = textReplace("?", URLEncoder.encode("?","UTF-8"), url);
|
||||
url = textReplace("&", URLEncoder.encode("&","UTF-8"), url);
|
||||
}
|
||||
catch ( java.io.UnsupportedEncodingException exception ) {
|
||||
LechLogger.error ( exception.toString() );
|
||||
}
|
||||
return config.getSaveRootDirectory().getPath() + "/" + url;
|
||||
}
|
||||
|
||||
public boolean existsOnDisk()
|
||||
{
|
||||
File f = new File(convertToFileName());
|
||||
return (f.exists() && !f.isDirectory());
|
||||
}
|
||||
|
||||
public void writeToFile()
|
||||
{
|
||||
writeToFile(convertToFileName());
|
||||
}
|
||||
|
||||
public void writeToFile(String fileName)
|
||||
{
|
||||
LechLogger.debug("writeToFile(" + fileName + ")");
|
||||
try
|
||||
{
|
||||
File f = new File(fileName);
|
||||
f.getParentFile().mkdirs();
|
||||
FileOutputStream out = new FileOutputStream(fileName);
|
||||
out.write(content);
|
||||
out.flush();
|
||||
out.close();
|
||||
}
|
||||
catch(IOException ioe)
|
||||
{
|
||||
LechLogger.warn("IO Exception writing to " + fileName, ioe);
|
||||
}
|
||||
}
|
||||
|
||||
public String toString()
|
||||
{
|
||||
StringBuffer sb = new StringBuffer();
|
||||
sb.append("URLObject: ");
|
||||
sb.append(contentType);
|
||||
if(false)//isHTML() || isXML())
|
||||
{
|
||||
sb.append("\n");
|
||||
sb.append(getStringContent());
|
||||
}
|
||||
return sb.toString();
|
||||
}
|
||||
|
||||
private String textReplace(String find, String replace, String input)
|
||||
{
|
||||
int startPos = 0;
|
||||
while(true)
|
||||
{
|
||||
int textPos = input.indexOf(find, startPos);
|
||||
if(textPos < 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
input = input.substring(0, textPos) + replace + input.substring(textPos + find.length());
|
||||
startPos = textPos + replace.length();
|
||||
}
|
||||
return input;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,68 @@
|
|||
/*
|
||||
* This is the MIT license, see also http://www.opensource.org/licenses/mit-license.html
|
||||
*
|
||||
* Copyright (c) 2001 Brian Pitcher
|
||||
*
|
||||
* Permission is hereby granted, free of charge, to any person obtaining a
|
||||
* copy of this software and associated documentation files (the "Software"),
|
||||
* to deal in the Software without restriction, including without limitation
|
||||
* the rights to use, copy, modify, merge, publish, distribute, sublicense,
|
||||
* and/or sell copies of the Software, and to permit persons to whom the
|
||||
* Software is furnished to do so, subject to the following conditions:
|
||||
*
|
||||
* The above copyright notice and this permission notice shall be included in
|
||||
* all copies or substantial portions of the Software.
|
||||
*
|
||||
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
* SOFTWARE.
|
||||
*/
|
||||
|
||||
// $Header: /home/andrew/Projects/penguincoder/cvs/WebLech/weblech/spider/URLToDownload.java,v 1.1 2004/03/07 20:51:05 mercury Exp $
|
||||
|
||||
package weblech.spider;
|
||||
|
||||
import java.net.URL;
|
||||
|
||||
public class URLToDownload implements java.io.Serializable
|
||||
{
|
||||
private final URL url;
|
||||
private final URL referer;
|
||||
private final int depth;
|
||||
|
||||
public URLToDownload(URL url, int depth)
|
||||
{
|
||||
this(url, null, depth);
|
||||
}
|
||||
|
||||
public URLToDownload(URL url, URL referer, int depth)
|
||||
{
|
||||
this.url = url;
|
||||
this.referer = referer;
|
||||
this.depth = depth;
|
||||
}
|
||||
|
||||
public URL getURL()
|
||||
{
|
||||
return url;
|
||||
}
|
||||
|
||||
public URL getReferer()
|
||||
{
|
||||
return referer;
|
||||
}
|
||||
|
||||
public int getDepth()
|
||||
{
|
||||
return depth;
|
||||
}
|
||||
|
||||
public String toString()
|
||||
{
|
||||
return url + ", referer " + referer + ", depth " + depth;
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,138 @@
|
|||
/**
|
||||
* LechLogger.java: A Graphical Logger
|
||||
* The original weblech had a simple text interface and used an apache.org library for logging.
|
||||
* When i wanted to make this thing into a graphical appk, I realized the text logging
|
||||
* was going to have to go. It assumes you only want to log to one location (not a problem for
|
||||
* this application).
|
||||
*/
|
||||
package weblech.ui;
|
||||
|
||||
import javax.swing.JTextArea;
|
||||
import java.io.IOException;
|
||||
|
||||
public class LechLogger {
|
||||
|
||||
/**
|
||||
* The actual text area that will perform all output.
|
||||
*/
|
||||
private static JTextArea _loggerWindow;
|
||||
/**
|
||||
* These are flags for enabling different types of logging mechanisms.
|
||||
*/
|
||||
private static boolean error_enable, warn_enable, info_enable, debug_enable;
|
||||
|
||||
/**
|
||||
* Only need one initialization since this will be shared between many different
|
||||
* objects.
|
||||
*/
|
||||
static {
|
||||
_loggerWindow = null;
|
||||
error_enable = true;
|
||||
warn_enable = true;
|
||||
info_enable = true;
|
||||
debug_enable = true;
|
||||
}
|
||||
|
||||
/** Everybody wants to log, but you only need one logger! */
|
||||
public LechLogger() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets the textual component to perform the logging.
|
||||
*/
|
||||
public static void setTextArea ( JTextArea textarea ) {
|
||||
_loggerWindow = textarea;
|
||||
}
|
||||
|
||||
/**
|
||||
* A private method for actually writing the messages.
|
||||
* It is synchronized because the weblech spider is multi
|
||||
* threaded.
|
||||
*/
|
||||
private static synchronized void log ( String msg ) {
|
||||
if ( _loggerWindow == null ) {
|
||||
//System.out.println ( msg );
|
||||
return;
|
||||
}
|
||||
_loggerWindow.setEditable ( true );
|
||||
_loggerWindow.append ( msg );
|
||||
_loggerWindow.append ( "\n" );
|
||||
_loggerWindow.setEditable ( false );
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle error logging.
|
||||
*/
|
||||
public static void setErrorLogging() {
|
||||
error_enable = !error_enable;
|
||||
}
|
||||
|
||||
/**
|
||||
* Log an error message.
|
||||
*/
|
||||
public static void error ( String msg ) {
|
||||
if ( !error_enable ) return;
|
||||
log ( "*error>" + msg );
|
||||
}
|
||||
|
||||
/**
|
||||
* Log an error message and an exception.
|
||||
*/
|
||||
public static void error ( String msg, Exception exception ) {
|
||||
if ( !error_enable ) return;
|
||||
log ( "*error>" + msg + "\n" + exception.getMessage() );
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle informational messages.
|
||||
*/
|
||||
public static void setInformationalLogging() {
|
||||
info_enable = !info_enable;
|
||||
}
|
||||
|
||||
/**
|
||||
* Log an informational message.
|
||||
*/
|
||||
public static void info ( String msg ) {
|
||||
if ( !info_enable ) return;
|
||||
log ( "^info>" + msg );
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle warning messages.
|
||||
*/
|
||||
public static void setWarningLogging() {
|
||||
warn_enable = !warn_enable;
|
||||
}
|
||||
|
||||
/**
|
||||
* Log a warning message.
|
||||
*/
|
||||
public static void warn ( String msg ) {
|
||||
if ( !warn_enable ) return;
|
||||
log ( "-warn>" + msg );
|
||||
}
|
||||
|
||||
/**
|
||||
* Log a warning message with an exception.
|
||||
*/
|
||||
public static void warn ( String msg, IOException exception ) {
|
||||
if ( !warn_enable ) return;
|
||||
log ( "-warn>" + msg + "\n" + exception.getMessage() );
|
||||
}
|
||||
|
||||
/**
|
||||
* Toggle debug messages to be printed.
|
||||
*/
|
||||
public static void setDebugLogging() {
|
||||
debug_enable = !debug_enable;
|
||||
}
|
||||
|
||||
/**
|
||||
* Log a deubgging statement to the logging text area.
|
||||
*/
|
||||
public static void debug ( String msg ) {
|
||||
if ( !debug_enable ) return;
|
||||
log ( "@debug>" + msg );
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,251 @@
|
|||
/**
|
||||
* SpiderConfigPanel.java: A graphcial panel for configuring a SpiderConfig object.
|
||||
* This panel provides a way to change the more practical options of the WebLech Spider.
|
||||
* It supports saving and opening of SpiderConfigurations from a file. It does not use
|
||||
* any of the "interesting" or "boring" url features, or the email link save file.
|
||||
*/
|
||||
package weblech.ui;
|
||||
|
||||
import weblech.spider.Spider;
|
||||
import weblech.spider.SpiderConfig;
|
||||
|
||||
import javax.swing.JPanel;
|
||||
import javax.swing.JTextField;
|
||||
import javax.swing.JLabel;
|
||||
import javax.swing.JButton;
|
||||
import javax.swing.JComboBox;
|
||||
import javax.swing.JFileChooser;
|
||||
|
||||
import java.awt.GridLayout;
|
||||
import java.awt.FlowLayout;
|
||||
|
||||
import java.awt.event.ActionListener;
|
||||
import java.awt.event.ActionEvent;
|
||||
|
||||
import java.util.Properties;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.FileOutputStream;
|
||||
import java.io.ObjectInputStream;
|
||||
import java.io.ObjectOutputStream;
|
||||
|
||||
public class SpiderConfigPanel extends JPanel implements ActionListener {
|
||||
|
||||
/**
|
||||
* A list of all of the spiders that the GUI will have downloading. It is assumed
|
||||
* that the user knows how much bandwidth you have and really wants to try and get
|
||||
* several different sites at the same time.
|
||||
*/
|
||||
private ArrayList spiders;
|
||||
/**
|
||||
* Various text fields for the configuration options.
|
||||
*/
|
||||
private JTextField sitenamefield, dirfield, usernamefield, passwordfield, agentfield, depthfield, matchfield;
|
||||
/**
|
||||
* A selection box for the number of threads a new Spider should use, I am limiting
|
||||
* the number of threads to 4 for simplicity.
|
||||
*/
|
||||
private JComboBox threadbox;
|
||||
|
||||
public SpiderConfigPanel() {
|
||||
super ( new GridLayout ( 8, 1 ) );
|
||||
spiders = new ArrayList();
|
||||
|
||||
/* Panel for the directory to save all files */
|
||||
JPanel sitepanel = new JPanel();
|
||||
((FlowLayout) sitepanel.getLayout()).setAlignment ( FlowLayout.RIGHT );
|
||||
JLabel sitelabel = new JLabel ( "Output Directory:" );
|
||||
dirfield = new JTextField ( System.getProperty ( "user.home" ), 20 );
|
||||
sitepanel.add ( sitelabel );
|
||||
sitepanel.add ( dirfield );
|
||||
|
||||
/* Panel for the site to download */
|
||||
JPanel outputpanel = new JPanel();
|
||||
((FlowLayout) outputpanel.getLayout()).setAlignment ( FlowLayout.RIGHT );
|
||||
JLabel dirlabel = new JLabel ( "Download Site:" );
|
||||
sitenamefield = new JTextField ( "http://www.google.com/", 20 );
|
||||
outputpanel.add ( dirlabel );
|
||||
outputpanel.add ( sitenamefield );
|
||||
|
||||
/* Panel for the HTTP username */
|
||||
JPanel usernamepanel = new JPanel();
|
||||
((FlowLayout) usernamepanel.getLayout()).setAlignment ( FlowLayout.RIGHT );
|
||||
JLabel usernamelabel = new JLabel ( "Username:" );
|
||||
usernamefield = new JTextField ( "", 20 );
|
||||
usernamepanel.add ( usernamelabel );
|
||||
usernamepanel.add ( usernamefield );
|
||||
|
||||
/* Panel for the HTTP password */
|
||||
JPanel passpanel = new JPanel();
|
||||
((FlowLayout) passpanel.getLayout()).setAlignment ( FlowLayout.RIGHT );
|
||||
JLabel passwdlabel = new JLabel ( "Password:" );
|
||||
passwordfield = new JTextField ( "", 20 );
|
||||
passpanel.add ( passwdlabel );
|
||||
passpanel.add ( passwordfield );
|
||||
|
||||
/* Panel for the HTTP user agent */
|
||||
JPanel agentpanel = new JPanel();
|
||||
((FlowLayout) agentpanel.getLayout()).setAlignment ( FlowLayout.RIGHT );
|
||||
JLabel agentlabel = new JLabel ( "User Agent:" );
|
||||
agentfield = new JTextField ( "WebLech [Version C]", 20 );
|
||||
agentpanel.add ( agentlabel );
|
||||
agentpanel.add ( agentfield );
|
||||
|
||||
/* Panel for a simple string match downloading limiter (no match, no download) */
|
||||
JPanel matchpanel = new JPanel();
|
||||
((FlowLayout) matchpanel.getLayout()).setAlignment ( FlowLayout.RIGHT );
|
||||
JLabel matchlabel = new JLabel ( "Match String:" );
|
||||
matchfield = new JTextField ( "", 20 );
|
||||
matchpanel.add ( matchlabel );
|
||||
matchpanel.add ( matchfield );
|
||||
|
||||
/* Provides a panel for placing both the maximum depth and threads for this spider */
|
||||
JPanel detailpanel = new JPanel ( new GridLayout ( 1, 2 ) );
|
||||
|
||||
JPanel depthpanel = new JPanel();
|
||||
((FlowLayout) depthpanel.getLayout()).setAlignment ( FlowLayout.RIGHT );
|
||||
JLabel depthlabel = new JLabel ( "Max Depth:" );
|
||||
depthfield = new JTextField ( Integer.toString ( 0 ), 5 );
|
||||
depthpanel.add ( depthlabel );
|
||||
depthpanel.add ( depthfield );
|
||||
|
||||
JPanel threadpanel = new JPanel();
|
||||
JLabel threadlabel = new JLabel ( "Spider Threads:" );
|
||||
Integer[] threaditems = { new Integer ( "1" ), new Integer ( "2" ), new Integer ( "3" ), new Integer ( "4" ) };
|
||||
threadbox = new JComboBox ( threaditems );
|
||||
threadpanel.add ( threadlabel );
|
||||
threadpanel.add ( threadbox );
|
||||
|
||||
detailpanel.add ( depthpanel );
|
||||
detailpanel.add ( threadpanel );
|
||||
|
||||
/* Panel of buttons for various operations */
|
||||
JPanel buttonpanel = new JPanel();
|
||||
JButton save = new JButton ( "Save" );
|
||||
JButton spiderbutton = new JButton ( "Spider It" );
|
||||
JButton open = new JButton ( "Open" );
|
||||
JButton qbutton = new JButton ( "Quit" );
|
||||
buttonpanel.add ( save );
|
||||
buttonpanel.add ( spiderbutton );
|
||||
buttonpanel.add ( open );
|
||||
buttonpanel.add ( qbutton );
|
||||
|
||||
add ( sitepanel );
|
||||
add ( outputpanel );
|
||||
add ( usernamepanel );
|
||||
add ( passpanel );
|
||||
add ( agentpanel );
|
||||
add ( matchpanel );
|
||||
add ( detailpanel );
|
||||
add ( buttonpanel );
|
||||
|
||||
/* Configure the button actions */
|
||||
save.setActionCommand ( "save" );
|
||||
open.setActionCommand ( "open" );
|
||||
spiderbutton.setActionCommand ( "spider" );
|
||||
qbutton.setActionCommand ( "quit" );
|
||||
save.addActionListener ( this );
|
||||
open.addActionListener ( this );
|
||||
spiderbutton.addActionListener ( this );
|
||||
qbutton.addActionListener ( this );
|
||||
LechLogger.debug ( "Actions" );
|
||||
}
|
||||
|
||||
/**
|
||||
* This method will create a Properties object good for instantiating a new SpiderConfig
|
||||
* Object.
|
||||
*/
|
||||
private Properties createProperties() {
|
||||
Properties p = new Properties();
|
||||
p.setProperty ( "saveRootDirectory", dirfield.getText() );
|
||||
p.setProperty ( "startLocation", sitenamefield.getText() );
|
||||
p.setProperty ( "basicAuthUser", usernamefield.getText() );
|
||||
p.setProperty ( "basicAuthPassword", passwordfield.getText() );
|
||||
p.setProperty ( "urlMatch", matchfield.getText() );
|
||||
p.setProperty ( "spiderThreads", ((Integer) threadbox.getSelectedItem()).toString() );
|
||||
p.setProperty ( "maxDepth", depthfield.getText() );
|
||||
p.setProperty ( "userAgent", agentfield.getText() );
|
||||
p.setProperty ( "interestingURLs", "" );
|
||||
return p;
|
||||
}
|
||||
|
||||
/**
|
||||
* This method will extract all of the values from a SpiderConfig object that the GUI uses
|
||||
* and updates the panel to show the values in the object.
|
||||
*/
|
||||
private void setSpiderConfig ( SpiderConfig sc ) {
|
||||
dirfield.setText ( sc.getSaveRootDirectory().toString() );
|
||||
sitenamefield.setText ( sc.getStartLocation().toString() );
|
||||
usernamefield.setText ( sc.getBasicAuthUser() );
|
||||
passwordfield.setText ( sc.getBasicAuthPassword() );
|
||||
matchfield.setText ( sc.getURLMatch() );
|
||||
int t = sc.getSpiderThreads();
|
||||
if ( t < 1 || t > 4 ) {
|
||||
t = 1;
|
||||
sc.setSpiderThreads ( t );
|
||||
}
|
||||
threadbox.setSelectedIndex ( t - 1 );
|
||||
depthfield.setText ( Integer.toString ( sc.getMaxDepth() ) );
|
||||
agentfield.setText ( sc.getUserAgent() );
|
||||
}
|
||||
|
||||
/**
|
||||
* This method will coordinate all of the actions for the various buttons used.
|
||||
*/
|
||||
public void actionPerformed ( ActionEvent event ) {
|
||||
String cmd = event.getActionCommand();
|
||||
/* Download a new site */
|
||||
if ( cmd.equals ( "spider" ) ) {
|
||||
SpiderConfig c = new SpiderConfig ( createProperties() );
|
||||
Spider spider = new Spider ( c );
|
||||
/* But only if we are not already downloading the site */
|
||||
if ( spiders.contains ( spider ) ) {
|
||||
LechLogger.warn ( "Already have an instance of a Spider at " + c.getStartLocation() );
|
||||
return;
|
||||
}
|
||||
spiders.add ( spider );
|
||||
spider.start();
|
||||
}
|
||||
/* Save the current configuration to a file */
|
||||
else if ( cmd.equals ( "save" ) ) {
|
||||
JFileChooser f = new JFileChooser ( System.getProperty ( "user.home" ) );
|
||||
int r = f.showSaveDialog ( this );
|
||||
if ( r != JFileChooser.APPROVE_OPTION ) return;
|
||||
File outfile = f.getSelectedFile();
|
||||
try {
|
||||
ObjectOutputStream os = new ObjectOutputStream ( new FileOutputStream ( outfile ) );
|
||||
os.writeObject ( new SpiderConfig ( createProperties() ) );
|
||||
os.close();
|
||||
}
|
||||
catch ( Exception exception ) {
|
||||
LechLogger.error ( exception.toString() );
|
||||
}
|
||||
}
|
||||
/* Open a saved configuration from a file */
|
||||
else if ( cmd.equals ( "open" ) ) {
|
||||
JFileChooser f = new JFileChooser ( System.getProperty ( "user.home" ) );
|
||||
int r = f.showOpenDialog ( this );
|
||||
if ( r != JFileChooser.APPROVE_OPTION ) return;
|
||||
File infile = f.getSelectedFile();
|
||||
if ( !infile.canRead() ) {
|
||||
LechLogger.error ( "file " + f.toString() + " is not readable" );
|
||||
return;
|
||||
}
|
||||
try {
|
||||
ObjectInputStream os = new ObjectInputStream ( new FileInputStream ( infile ) );
|
||||
SpiderConfig sc = (SpiderConfig) os.readObject();
|
||||
os.close();
|
||||
setSpiderConfig ( sc );
|
||||
}
|
||||
catch ( Exception exception ) {
|
||||
LechLogger.error ( exception.toString() );
|
||||
}
|
||||
}
|
||||
/* Just quit */
|
||||
else if ( cmd.equals ( "quit" ) ) {
|
||||
System.exit ( 0 );
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
/**
|
||||
* Troll.java: A user interface to the weblech spider download utility.
|
||||
*/
|
||||
package weblech.ui;
|
||||
|
||||
/* I like to explicitly import all of my packages to remind me to KISS */
|
||||
import javax.swing.JFrame;
|
||||
import javax.swing.JPanel;
|
||||
import javax.swing.JTabbedPane;
|
||||
import javax.swing.JTextArea;
|
||||
import javax.swing.JScrollPane;
|
||||
import javax.swing.JMenuBar;
|
||||
import javax.swing.JMenu;
|
||||
import javax.swing.JMenuItem;
|
||||
import javax.swing.JCheckBoxMenuItem;
|
||||
|
||||
import java.awt.Dimension;
|
||||
|
||||
import java.awt.event.ActionListener;
|
||||
import java.awt.event.ActionEvent;
|
||||
|
||||
public class Troll extends JFrame implements ActionListener {
|
||||
|
||||
/**
|
||||
* This SpiderConfigPanel is a custom panel that provides many of the more
|
||||
* practical features of the weblech spider. It also controls the spiders
|
||||
* created by the user.
|
||||
*/
|
||||
private SpiderConfigPanel configpanel;
|
||||
/**
|
||||
* This is the area that all of the logging facilities will use. This makes debugging
|
||||
* in a system like Max OS X much simpler (:^)
|
||||
*/
|
||||
private static JTextArea logarea;
|
||||
|
||||
/* This just initializes the logging text box and readies it for recording events before
|
||||
* the rest of the object is even loaded.
|
||||
*/
|
||||
static {
|
||||
logarea = new JTextArea();
|
||||
LechLogger.setTextArea ( logarea );
|
||||
LechLogger.setDebugLogging();
|
||||
}
|
||||
|
||||
Troll() {
|
||||
super ( "Troll" );
|
||||
Dimension initialsize = new Dimension ( 400, 375 );
|
||||
setSize ( initialsize);
|
||||
|
||||
/* Create a menubar for controlling which aspects of the log you wish to see */
|
||||
JMenuBar menubar = new JMenuBar();
|
||||
JMenu logmenu = new JMenu ( "Log Options" );
|
||||
JCheckBoxMenuItem showdebug = new JCheckBoxMenuItem ( "Show Debug Messages", false );
|
||||
showdebug.setActionCommand ( "debug" );
|
||||
showdebug.addActionListener ( this );
|
||||
JCheckBoxMenuItem showinfo = new JCheckBoxMenuItem ( "Show Informational Messages", true );
|
||||
showinfo.setActionCommand ( "info" );
|
||||
showinfo.addActionListener ( this );
|
||||
JCheckBoxMenuItem showwarn = new JCheckBoxMenuItem ( "Show Warnings", true );
|
||||
showwarn.setActionCommand ( "warn" );
|
||||
showwarn.addActionListener ( this );
|
||||
JCheckBoxMenuItem showerror = new JCheckBoxMenuItem ( "Show Errors", true );
|
||||
showerror.setActionCommand ( "error" );
|
||||
showerror.addActionListener ( this );
|
||||
logmenu.add ( showdebug );
|
||||
logmenu.add ( showinfo );
|
||||
logmenu.add ( showwarn );
|
||||
logmenu.add ( showerror );
|
||||
menubar.add ( logmenu );
|
||||
|
||||
/* A simple tab interface between configuration and error checking */
|
||||
configpanel = new SpiderConfigPanel();
|
||||
JPanel logpanel = new JPanel();
|
||||
logpanel.add ( logarea );
|
||||
JScrollPane logscroller = new JScrollPane ( logpanel );
|
||||
JTabbedPane tabs = new JTabbedPane();
|
||||
tabs.addTab ( "Spider", configpanel );
|
||||
tabs.addTab ( "Log", logscroller );
|
||||
|
||||
/* Configure the JFrame to a usable state */
|
||||
setJMenuBar ( menubar );
|
||||
getContentPane().add ( tabs );
|
||||
setLocationRelativeTo ( null );
|
||||
setVisible ( true );
|
||||
setDefaultCloseOperation ( JFrame.EXIT_ON_CLOSE );
|
||||
}
|
||||
|
||||
/**
|
||||
* This method basically toggles all of the logging options.
|
||||
*/
|
||||
public void actionPerformed ( ActionEvent event ) {
|
||||
String cmd = event.getActionCommand();
|
||||
if ( cmd.equals ( "debug" ) ) {
|
||||
LechLogger.setDebugLogging();
|
||||
}
|
||||
else if ( cmd.equals ( "info" ) ) {
|
||||
LechLogger.setInformationalLogging();
|
||||
}
|
||||
else if ( cmd.equals ( "warn" ) ) {
|
||||
LechLogger.setDebugLogging();
|
||||
}
|
||||
else if ( cmd.equals ( "error" ) ) {
|
||||
LechLogger.setErrorLogging();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a new troll and go
|
||||
*/
|
||||
public static void main ( String[] args ) {
|
||||
Troll t = new Troll();
|
||||
}
|
||||
}
|
||||
Reference in New Issue