Name: rm29839 Date: 10/29/97


Using this code:
My application is designed to fetch index pages from
a (pre-verified) list of websites and write them to local
files. Here's an example of the configuration file,
sites.cfg:

bbcadpk1.icl.net 80 /
emap.admedia.net 80 /
www.gumby.iclnet.co.uk 80 /
www.manchesteronline.co.uk 80 /
www.marks-and-spencer.co.uk 80 /
www.marks-and-spencer.com 80 /
www.midscoop.co.uk 80 /
www.nature.com 80 /
www.ndb.iclnet.co.uk 80 /
www.nie.northcliffe.co.uk 80 /
www.octavian.co.uk 80 /

We start a GrabPage thread for each of the sites in the
file.

On using: java GrabIt sites.cfg

the first couple complete (almost) immediately. The third
causes all subsequent threads started to hang until
its lookup is finished. This causes problems with long lists
of websites, as the VM seems to run out of File Handles.

The third site's DNS is purposely broken, to reproduce a
a problem we recently had with someone's real DNS going down.

Here's the code:

import java.net.*;
import java.io.*;
import java.util.*;


class GrabPage extends Thread {
public String wcmethod;
public String wcurl;
public int wcport;
public String wcpage;

public void run() {
try {
        String fname,tname;

        System.out.println("GrabPage thread start for "+wcurl);

        fname="/tmp/"+wcurl+".out";

        PrintWriter pwcf = new PrintWriter (new BufferedOutputStream (new FileOutputStream(fname)));
        
        URL webcheck = new URL(wcmethod, wcurl, wcport, wcpage);
        URLConnection webc = webcheck.openConnection();
        webc.setRequestProperty("HTTP_HOST","http://"+wcurl);

        BufferedReader dis = new BufferedReader(new InputStreamReader(webc.getInputStream()));
        String inputLine;

        while ((inputLine = dis.readLine()) != null) {
                pwcf.println(inputLine);
        }
        dis.close();
        pwcf.close();
        System.out.println("GrabPage thread finish for "+wcurl);
        
    } catch (MalformedURLException me) {
        errout("/tmp/"+wcurl+".out","INVALID URL");
    } catch (UnknownHostException uh) {
        errout("/tmp/"+wcurl+".out","DNS");
    } catch (IOException ioe) {
        String thing;

        thing = ioe.toString();
        
        if (thing.indexOf("UnauthorizedHttpRequestException") >0) {
                errout("/tmp/"+wcurl+".out", "PASSWD");
        } else if (thing.indexOf("SocketException") >0) {
                errout("/tmp/"+wcurl+".out", "BADSOCKET");
        } else {
                System.out.println(ioe);
        }

    }
}

private void errout(String f, String mess) {
try {
        FileOutputStream wcf = new FileOutputStream(f);
        BufferedOutputStream bwcf = new BufferedOutputStream(wcf);
        PrintWriter pwcf = new PrintWriter(bwcf);
                
        pwcf.println(mess);

        pwcf.close();

        } catch (IOException e) {
                System.err.println(e);
    }
}


}



class GrabIt {
public static void main(String[] args) {
try {
        
         BufferedReader wcin = new BufferedReader (new InputStreamReader(new FileInputStream(args[0])));

        String line;
        
        for (;;) {
                line=wcin.readLine();
                                
                if (line == null) break;
                
                StringTokenizer lstr = new StringTokenizer(line, " ");

                String t;
                int startsec;
                GrabPage g;
                g = new GrabPage();

                g.wcmethod="http"; // Hardcode to http for now
                g.wcurl=lstr.nextToken(); // URL
                g.wcport=Integer.parseInt(lstr.nextToken()); // Port
                g.wcpage=lstr.nextToken(); // Page to grab
        
                g.start(); // Start the Thread
                
                // Delay for thread completion
                Thread.sleep(1000);

        }
        wcin.close();

    } catch (IOException e) {
                System.err.println(e);
    } catch (InterruptedException e) {
                System.err.println(e);
    }
}
}
(Review ID: 14638)
======================================================================