aboutsummaryrefslogtreecommitdiff
path: root/lib/httracker
diff options
context:
space:
mode:
authorSilvio Rhatto <rhatto@riseup.net>2013-08-25 22:04:26 -0300
committerSilvio Rhatto <rhatto@riseup.net>2013-08-25 22:04:26 -0300
commit51f40e313dc4c12b34546c4cc7e15131b691c70a (patch)
tree6f1067fbf3ec70f2cf7462bb6b2c26748e74373d /lib/httracker
parentafa9ae1f6a3f12e0bc7a30ee0a6c1c606e667511 (diff)
downloadhttruta-51f40e313dc4c12b34546c4cc7e15131b691c70a.tar.gz
httruta-51f40e313dc4c12b34546c4cc7e15131b691c70a.tar.bz2
Coding style and minor fixes
Diffstat (limited to 'lib/httracker')
-rw-r--r--lib/httracker/functions26
1 files changed, 15 insertions, 11 deletions
diff --git a/lib/httracker/functions b/lib/httracker/functions
index 1a4fa7f..0885206 100644
--- a/lib/httracker/functions
+++ b/lib/httracker/functions
@@ -3,6 +3,7 @@
# Misc httracker functions.
#
+# Set common httrack options
function httracker_opts {
OPTS=" --mirror \
--continue \
@@ -15,6 +16,7 @@ function httracker_opts {
--path ${target}"
}
+# Download URLs, mirror mode
function httracker_get {
# Options
local url="$1"
@@ -54,13 +56,14 @@ function httracker_get {
fi
}
+# Download URLs, incremental mode
function httracker_get_incremental {
- # Creates target dir
+ # Create target dir
year=`date +%Y`
month=`date +%m`
day=`date +%d`
target=${MIRRORDIR}/${year}/${month}
- sudo -u links mkdir -p ${target}
+ sudo -u ${USER} mkdir -p ${target}
# Basic options
httracker_opts
@@ -70,32 +73,33 @@ function httracker_get_incremental {
OPTS="--user $USER"
fi
- # Grabs URLs from the network
- httrack ${OPTS} \
- --user links \
- --list ${URLS} \
- --path ${target}
+ # Grab URLs from the network
+ httrack ${OPTS} --list ${URLS}
}
+# Get SemanticScuttle parameter
function httracker_scuttle_config() {
grep ${1} ${CONFIGFILE} | sed -e s/\[^\'\]\*\'// -e s/\'\.\*\$//
}
+# Query a mysql database
function httracker_sqlquery {
mysql --skip-column-names --batch \
- --user=${dbuser} \
- --password=${dbpass} \
- --database=${dbname} \
- --host=${dbhost} \
+ --user=${DBUSER} \
+ --password=${DBPASS} \
+ --database=${DBNAME} \
+ --host=${DBHOST} \
--execute="${1}"
}
+# Iterate over all URLs
function httracker_iterate {
for link in `cat $URLS | xargs`; do
httracker_get "$link"
done
}
+# Create basic folders
function httracker_setup_folders {
mkdir -p $MIRRORS $TMP