diff options
author | Silvio Rhatto <rhatto@riseup.net> | 2013-08-25 20:01:03 -0300 |
---|---|---|
committer | Silvio Rhatto <rhatto@riseup.net> | 2013-08-25 20:01:03 -0300 |
commit | 32d0733a7ad32017649f0c0a5ac2b82e85af38db (patch) | |
tree | 723d1ec998c1f3f70de44d3d1c9c9feb7a826bf0 /lib/httracker/functions | |
parent | 63ee798f1f6c6a58165f328027cc3f427535c447 (diff) | |
download | httruta-32d0733a7ad32017649f0c0a5ac2b82e85af38db.tar.gz httruta-32d0733a7ad32017649f0c0a5ac2b82e85af38db.tar.bz2 |
Adding httracker_iterate
Diffstat (limited to 'lib/httracker/functions')
-rw-r--r-- | lib/httracker/functions | 28 |
1 files changed, 16 insertions, 12 deletions
diff --git a/lib/httracker/functions b/lib/httracker/functions index ea9a7fa..c7dff7f 100644 --- a/lib/httracker/functions +++ b/lib/httracker/functions @@ -41,27 +41,25 @@ function httracker_get { fi } -function httracker_get_single { -# grabs URLs from the network +function httracker_get_incremental { + # Creates target dir + year=`date +%Y` + month=`date +%m` + %day=`date +%d` + target=${MIRRORDIR}/${year}/${month} + sudo -u links mkdir -p ${target} + + # Grabs URLs from the network httrack --verbose \ --user links \ --depth=1 \ --purge-old=0 \ --index \ --cookies=1 \ - --list ${tmpfile} \ + --list ${URLS} \ --path ${target} \ } -function httracker_target_single { - # creates target dir - year=`date +%Y` - month=`date +%m` - %day=`date +%d` - target=${MIRRORDIR}/${year}/${month} - sudo -u links mkdir -p ${target} -} - function httracker_scuttle_config() { grep ${1} ${CONFIGFILE} | sed -e s/\[^\'\]\*\'// -e s/\'\.\*\$// } @@ -74,3 +72,9 @@ httracker_sqlquery() { --host=${dbhost} \ --execute="${1}" } + +function httracker_iterate { + for link in `cat $URLS | xargs`; do + httracker_get "$link" + done +} |