aboutsummaryrefslogtreecommitdiff
path: root/lib/httracker/functions
diff options
context:
space:
mode:
Diffstat (limited to 'lib/httracker/functions')
-rw-r--r--lib/httracker/functions28
1 files changed, 16 insertions, 12 deletions
diff --git a/lib/httracker/functions b/lib/httracker/functions
index ea9a7fa..c7dff7f 100644
--- a/lib/httracker/functions
+++ b/lib/httracker/functions
@@ -41,27 +41,25 @@ function httracker_get {
fi
}
-function httracker_get_single {
-# grabs URLs from the network
+function httracker_get_incremental {
+ # Creates target dir
+ year=`date +%Y`
+ month=`date +%m`
+ %day=`date +%d`
+ target=${MIRRORDIR}/${year}/${month}
+ sudo -u links mkdir -p ${target}
+
+ # Grabs URLs from the network
httrack --verbose \
--user links \
--depth=1 \
--purge-old=0 \
--index \
--cookies=1 \
- --list ${tmpfile} \
+ --list ${URLS} \
--path ${target} \
}
-function httracker_target_single {
- # creates target dir
- year=`date +%Y`
- month=`date +%m`
- %day=`date +%d`
- target=${MIRRORDIR}/${year}/${month}
- sudo -u links mkdir -p ${target}
-}
-
function httracker_scuttle_config() {
grep ${1} ${CONFIGFILE} | sed -e s/\[^\'\]\*\'// -e s/\'\.\*\$//
}
@@ -74,3 +72,9 @@ httracker_sqlquery() {
--host=${dbhost} \
--execute="${1}"
}
+
+function httracker_iterate {
+ for link in `cat $URLS | xargs`; do
+ httracker_get "$link"
+ done
+}