From 896a68e38ce30c4a625555cb36393785a1141526 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Fri, 9 Nov 2018 21:00:51 -0200 Subject: Adds grab-site --- share/hoarder/grab-site/info | 1 + share/hoarder/grab-site/unix/linux/debian/install | 29 +++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 share/hoarder/grab-site/info create mode 100755 share/hoarder/grab-site/unix/linux/debian/install (limited to 'share') diff --git a/share/hoarder/grab-site/info b/share/hoarder/grab-site/info new file mode 100644 index 0000000..a245eed --- /dev/null +++ b/share/hoarder/grab-site/info @@ -0,0 +1 @@ +the archivist's web crawler: WARC output, dashboard for all crawls, dynamic ignore patterns diff --git a/share/hoarder/grab-site/unix/linux/debian/install b/share/hoarder/grab-site/unix/linux/debian/install new file mode 100755 index 0000000..be9c074 --- /dev/null +++ b/share/hoarder/grab-site/unix/linux/debian/install @@ -0,0 +1,29 @@ +#!/usr/bin/env sh +# +# Install grab-site +# + +# Parameters +SHARE="$1" +LIB="$2" + +# Include basic functions +. $LIB/trashman/functions || exit 1 +. $LIB/trashman/debian || exit 1 + +# Requirements +trashman_apt_install_packages git build-essential libssl-dev zlib1g-dev libbz2-dev libreadline-dev libsqlite3-dev libffi-dev libxml2-dev libxslt1-dev libre2-dev pkg-config + +# Setup pyenv and grab-site +mkdir -p $HOME/.local/bin $HOME/.local/share +cd $HOME/.local/bin +wget https://raw.githubusercontent.com/pyenv/pyenv-installer/master/bin/pyenv-installer +chmod +x pyenv-installer +./pyenv-installer +$HOME/.pyenv/bin/pyenv install 3.7.0 +$HOME/.pyenv/versions/3.7.0/bin/python -m venv $HOME/.local/share/gs-venv +$HOME/.local/share/gs-venv/bin/pip install --process-dependency-links --no-binary --upgrade git+https://github.com/ludios/grab-site + +echo "Add this to your ~/.bashrc or ~/.zshrc and then restart your shell (e.g. by opening a new terminal tab/window):" +echo "" +echo 'PATH="$PATH:'$HOME/.local/share/gs-venv/bin'"' -- cgit v1.2.3