aboutsummaryrefslogtreecommitdiff
path: root/lib/httracker/functions
blob: a5144c90649aaf7a51d0acb4d2d0c5a04db7edc2 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
#!/bin/bash
#
# Misc httracker functions.
#

function httracker_get {
  local url="$1"
  local hash="`echo $1 | sha1sum | cut -d ' ' -f 1`"
  local target="$MIRRORS/$hash"

  mkdir -p $target

  # We already got this one
  if [ -f "$target/ok" ]; then
    return
  fi

  if [ "`whoami`" == "root" ]; then
    OPTS="--user $USER"
  else
    OPTS=""
  fi

  if [ "`whoami`" != "$USER" ] && [ "`whoami`" == "root" ]; then
    chown -R $USER.$GROUP $target/
  fi

  # Get each URL
  httrack               \
    --mirror            \
    --continue          \
    --depth=${DEPTH}    \
    --near              \
    --purge-old=0       \
    --index             \
    --cookies=1         \
    --path ${target}    \
    -r${LEVEL} ${OPTS} ${url}
    #-e%${EXT_LEVEL}    \
    #-m$FILESIZE        \
    #--verbose

  if [ "$?" == "0" ]; then
    # Mark as downloaded
    touch $target/ok
  else
    echo "Error fetching $url."
    rm -rf $target
  fi
}

function httracker_get_incremental {
  # Creates target dir
  year=`date +%Y`
  month=`date +%m`
  day=`date +%d`
  target=${MIRRORDIR}/${year}/${month}
  sudo -u links mkdir -p ${target}

  if [ "`whoami`" == "root" ]; then
    OPTS="--user $USER"
  else
    OPTS=""
  fi

  # Grabs URLs from the network
  httrack --verbose           \
          --mirror            \
          --continue          \
          --user links        \
          --depth=${DEPTH}    \
          --near              \
          --purge-old=0       \
          --index             \
          --cookies=1         \
          --list ${URLS}      \
          --path ${target} ${OPTS}

}

function httracker_scuttle_config() {
  grep ${1} ${CONFIGFILE} | sed -e s/\[^\'\]\*\'// -e s/\'\.\*\$//
}

function httracker_sqlquery {
  mysql --skip-column-names --batch \
        --user=${dbuser}          \
        --password=${dbpass}      \
        --database=${dbname}      \
        --host=${dbhost}          \
        --execute="${1}"
}

function httracker_iterate {
  for link in `cat $URLS | xargs`; do
    httracker_get "$link"
  done
}

function httracker_setup_folders {
  mkdir -p $MIRRORS $TMP

  if [ "`whoami`" == "root" ]; then
    chown -R $USER.$GROUP $TMP
  fi
}