cat links.txt | while read i ; do
        wget --no-check-certificate -np -r -c --reject "index.html*" -e robots=off $i
done