#! /usr/bin/perl

$home = $ENV{"HOME"};

require "URL.pl";

$url = "http://www.metaspy.com/spymagic/Spy?filter=false";
 
while(1){
 $status = &url_get($url, "&STDOUT");
 sleep(15);
}


sub url_get {

    local($url, $file) = @_;
    local($loseheader) = ($opt_h ? 0 : 1);
    local($debug) = ($opt_d ? 1 : 0);
    local($binary) = ($opt_b ? 1 : 0);
    local($dummy, $foo, $bar);


    ($protocol, $host, $port, $rest1, $rest2, $rest3) = &url'parse_url($url);

# Convert any characters in the string specified in hex by "%xx" to
# the correct character. Note we do this *after* parsing the URL!

    $rest1 =~ s/%(\w\w)/sprintf("%c", hex($1))/ge;

    if ($protocol eq "http") {

	return &url_get'http_get($host,$port,$rest1,$loseheader,$debug,$file);

    }

    die "Protocol $protocol not supported!\n";

}



package url_get;     # Everything after this is "private"


if ($] >= 5.0) {

    eval 'use Socket';

} else {

    eval 'require "sys/socket.ph"';

}


1;



sub http_get {

    local($host,$port,$request,$loseheader,$debug,$file) = @_;
    local($output) = "";
    local($redirect) = 0;
    local($location);


# Status code translation table. Key is HTTP status code (from
# http://info.cern.ch/hypertext/WWW/Protocols/HTTP/HTRESP.html),
# and value is status returned by url_get.


    %exit_status = (400,1,401,2,402,3,403,4,404,5,500,6,501,7,502,8,503,9);


    if ($file && !fileno(OUT)) { open(OUT, ">$file") || die "Error opening output file $file: $!\n"; }


    $ret = &url_get'open($host, $port);

    if (!defined($ret)) {

        if ($! && $! != "") {

            die "Error opening port $port on $host: $!\n";

        } else {

            die "Host not found: $host\n";

        }

    }

    #print CMD "GET $request HTTP/1.0\r\nAccept: */*\r\n\r\n";
    print CMD "GET $request HTTP/1.1\r\nUser-Agent: Mozilla/4.61 [en] (X11; U; Linux 2.2.16.artcontext i686)\r\nAccept: */*\r\nHost: $host\n\n";

    $_ = <CMD>;

    if (! $_) {

	die "Server unexpectedly closed connection - exiting.\n";

    }



# First, read the HTTP header


    if (m#^HTTP/([\.0-9]*) (\d\d\d) (.+)$#) {

	$http_version = $1;

        $status = $2;
        $reason = $3;

        if (! $debug && $status > 399) {

            warn "Error returned from server: $status $reason\n";

	    return $exit_status{$status};

        }

	if ($status >= 300) {

	    $redirect = 1;

	}

    } else {

	warn "Error - bad HTTP header: $_\n";

    }

    if ($debug) {

	warn "$_";

    }

    elsif (! $loseheader && ! $redirect) {

	if ($file) { print OUT $_; }

	else { $output .= $_; }

    }



# Next, read the MIME header



    while (<CMD>) {

        last if (/^\s*$/);

	if ($redirect && /^Location: (.*)$/) {

	    $location = $1;

	}

	if ($debug) {

	    warn "$_";

	}

	elsif (!$loseheader && !$redirect) {

	    if ($file) { print OUT $_; }

            else { $output .= $_; }

	}

	else {

            if (! /^[a-zA-Z\-]+: /) {

                warn "Bad MIME header line: $_";

            }

	}

    }

    if (! $_) {

	die "Server unexpectedly closed connection - exiting.\n";

    }

    if (! $debug && ! $loseheader && ! $redirect) {

	if ($file) { print OUT $_; }

        else { $output .= $_; }

    }



# Finally, read the rest



    while (<CMD>) {

	last if ($redirect && !$debug);

	if ($file) { 
	    #if($_ =~ /search.metacrawlwer.com/){
	    if($_ =~ /search\.metacrawler\.com/){
		  @parts = split(/general=/,$_);
		  @further = split("\"",$parts[1]);
		  @still = split("&",$further[0]);
		  $key = lc($still[0]);
		#print OUT $_; 
	        dbmopen(%LIST, "keywords",0644) || die "can't find\n";
		if($LIST{$key}){
		    $num = $LIST{$key}; 
		    $num++;
		    print($key . "  $num\n");
		    $LIST{$key} = $num;
		}else{
		    $LIST{$key} = 1;
		}
                dbmclose(%LIST);
		#print OUT $still[0] . "\n";  

	    }
	}
        else { 
			$output .= $_; 
	}

    }

    close(CMD);



# If we've been redirected to another location, get it there...



    if ($redirect && $location) {

	if (!$debug) {

	    warn "The item has been moved to URL: $location.\n";

	    warn "Attempting to obtain it from there...\n";

	}

	return &main'url_get($location, $file);

    }

    close(OUT) if ($file);

    return($output) unless ($file);

}



sub open {

    local($Host, $Port) = @_;

    local($destaddr, $destproc);



# Set the socket parameters. Note that we set the defaults to be the

# BSD values if we can't get them from the required files. Also note

# that, in the 4.0 version, the routines are in package ftp, since

# it does the "require sys/socket.ph" first.



    if ($] < 5.0) {

        (eval {$Inet = &ftp'AF_INET;}) || ($Inet=2);

        (eval {$Stream = &ftp'SOCK_STREAM;}) || ($Stream=1);

    } else {

        (eval {$Inet = &AF_INET;}) || ($Inet=2);

        (eval {$Stream = &SOCK_STREAM;}) || ($Stream=1);

    } 

#    warn "Inet = $Inet, Stream = $Stream\n";



    if ($Host =~ /^(\d+)+\.(\d+)\.(\d+)\.(\d+)$/) {

	$destaddr = pack('C4', $1, $2, $3, $4);

    } else {

	local(@temp) = gethostbyname($Host);

	unless (@temp) {

           $Error = "Can't get IP address of $Host";

           return undef;

        }

	$destaddr = $temp[4];

    }



    $Proto = (getprotobyname("tcp"))[2];

    $Sockaddr = 'S n a4 x8';

    $destproc = pack($Sockaddr, $Inet, $Port, $destaddr);

    if (socket(CMD, $Inet, $Stream, $Proto)) {

       if (connect(CMD, $destproc)) {



          ### This info will be used by future data connections ###

          $Cmdaddr = (unpack ($Sockaddr, getsockname(CMD)))[2];

          $Cmdname = pack($Sockaddr, $Inet, 0, $Cmdaddr);



          select((select(CMD), $| = 1)[$[]);



          return 1;

       }

    }



    close(CMD);

    return undef;

}

