aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSilvio Rhatto <rhatto@riseup.net>2019-07-08 14:04:43 -0300
committerSilvio Rhatto <rhatto@riseup.net>2019-07-08 14:04:43 -0300
commit7b6c8b16cedc678c6b35a7996ff7c3474e174d21 (patch)
treeb0d47109249deedce9dff4e8080fe7c5ce36daad
parente6bcfe8364968d3be47e97ee9f4a08a37caa7daf (diff)
downloadgrab-queue-7b6c8b16cedc678c6b35a7996ff7c3474e174d21.tar.gz
grab-queue-7b6c8b16cedc678c6b35a7996ff7c3474e174d21.tar.bz2
Fix basic logging
-rwxr-xr-xgrab-queue18
1 files changed, 12 insertions, 6 deletions
diff --git a/grab-queue b/grab-queue
index 19397d4..e2a7c7b 100755
--- a/grab-queue
+++ b/grab-queue
@@ -27,6 +27,7 @@ import argparse
import sys, os, subprocess, json
import yaml
from urllib.parse import urlencode
+from urllib.parse import urlparse
from hashlib import sha256
from tqdm import tqdm
@@ -64,13 +65,18 @@ class DownloadMultiple:
if self.debug:
print('Downloading ' + url + '...')
+ domain = urlparse(url).netloc
+
# Heuristics to check if dest is a folder
if local_filename[-1] == '/':
folder = local_filename
- log_sep = '/'
+ log_sep = '../logs/'
else:
folder = os.path.dirname(local_filename)
- log_sep = '.'
+ log_sep = ''
+
+ with open(log_sep + 'grab-queue.log', "a") as log:
+ log.write('Downloading ' + url + '...')
self.ensuredir(local_filename)
@@ -89,7 +95,7 @@ class DownloadMultiple:
stdout, stderr = await proc.communicate()
if stdout:
- output = open(local_filename + log_sep + 'stdout', 'w')
+ output = open(local_filename + log_sep + domain + '.stdout', 'w')
output.write(stdout.decode())
output.close()
@@ -97,18 +103,18 @@ class DownloadMultiple:
print(f'[stdout] {url} {stdout.decode()}')
if stderr:
- output = open(local_filename + log_sep + 'stderr', 'w')
+ output = open(local_filename + log_sep + domain + '.stderr', 'w')
output.write(stderr.decode())
output.close()
if self.debug:
print(f'[stderr] {url} {stderr.decode()}')
- output = open(local_filename + log_sep + 'returncode', 'w')
+ output = open(local_filename + log_sep + domain + '.returncode', 'w')
output.write(str(proc.returncode) + '\n')
output.close()
- output = open(local_filename + log_sep + 'date', 'w')
+ output = open(local_filename + log_sep + domain + '.date', 'w')
output.write(str(datetime.datetime.now()) + '\n')
output.close()