#!/bin/bash

# Take a list of URLs in a file and fetch them
# Then extract all text with Links Browser
# Then zip them

# usage ./script.sh http://example.com/sitemap.html

# extract all URLs from the sitemap page (list of pages on the Web site)
links -dump $1 | egrep -o "http:.*\.aspx" >urls.txt

while read inputline
do
# read a URL out of the file
url="$(echo $inputline)"

# get the file
wget $url 

# extract the text from the file and save as .txt
# this line doesn't work because $url is not the filename, e.g., page.aspx
links -dump $url >$url.txt 

# delete the .aspx file
rm $url

# read from urls.txt
done < urls.txt

# zip all of the text files for export into plagiarism tool
zip website.zip *.txt