aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSilvio Rhatto <rhatto@riseup.net>2019-05-16 08:53:18 -0300
committerSilvio Rhatto <rhatto@riseup.net>2019-05-16 08:53:18 -0300
commit046e26a98759899f7275fbd22573ffe810304cef (patch)
tree8682820dafed8ea4ecb0334fb3e56b4a405fcff5
parentd94457aa7f882c7f38af6e271737679935d782ce (diff)
downloadckandumper-046e26a98759899f7275fbd22573ffe810304cef.tar.gz
ckandumper-046e26a98759899f7275fbd22573ffe810304cef.tar.bz2
Initial exit status tracking
-rwxr-xr-xckandumper31
1 files changed, 21 insertions, 10 deletions
diff --git a/ckandumper b/ckandumper
index 189459f..7362219 100755
--- a/ckandumper
+++ b/ckandumper
@@ -84,9 +84,14 @@ class DownloadMultiple:
print(f'[stderr] {url} {stderr.decode()}')
output = open(local_filename + '.returncode', 'w')
- output.write(str(proc.returncode))
+ output.write(str(proc.returncode) + '\n')
output.close()
+ if not str(proc.returncode) in self.stats['exitstatus']:
+ self.stats['exitstatus'][str(proc.returncode)] = []
+
+ self.stats['exitstatus'][str(proc.returncode)].append(url);
+
if self.debug:
print(f'[{cmd!r} exited with {proc.returncode}]')
@@ -109,12 +114,18 @@ class DownloadMultiple:
await asyncio.gather(*jobs)
def get(self, filepairs):
- self.bar = tqdm(total=len(filepairs)) if self.progress and len(filepairs) > 1 else False
- loop = asyncio.get_event_loop()
+ self.stats = { 'exitstatus': {} }
+ self.bar = tqdm(total=len(filepairs)) if self.progress and len(filepairs) > 1 else False
+ loop = asyncio.get_event_loop()
loop.set_debug(self.debug)
loop.run_until_complete(self.gather(filepairs))
+ if self.debug:
+ print(self.stats)
+
+ return self.stats
+
class CkanDumper:
"""Dumps CKAN data: metadata plus entire datasets"""
@@ -173,7 +184,7 @@ class CkanDumper:
# Groups
#
print(f'Downloading {self.url}{self.group_list}...')
- self.download.get([[self.url + self.group_list, group_list]])
+ status = self.download.get([[self.url + self.group_list, group_list]])
groups = self.load_json(group_list)
group_downloads = []
@@ -185,13 +196,13 @@ class CkanDumper:
group_downloads.append([self.url + self.group_show + urlencode({ 'id': group }, False, '', 'utf-8'), group_file])
- self.download.get(group_downloads)
+ status = self.download.get(group_downloads)
#
# Tags
#
print(f'Downloading {self.url}{self.tag_list}...')
- self.download.get([[self.url + self.tag_list, tag_list]])
+ status = self.download.get([[self.url + self.tag_list, tag_list]])
tags = self.load_json(tag_list)
tags_downloads = []
@@ -203,13 +214,13 @@ class CkanDumper:
tags_downloads.append([self.url + self.tag_show + urlencode({ 'id': tag }, False, '', 'utf-8'), tag_file])
- self.download.get(tags_downloads)
+ status = self.download.get(tags_downloads)
#
# Packages
#
print(f'Downloading {self.url}{self.package_list}...')
- self.download.get([[self.url + self.package_list, package_list]])
+ status = self.download.get([[self.url + self.package_list, package_list]])
packages = self.load_json(package_list)
packages_downloads = []
@@ -222,7 +233,7 @@ class CkanDumper:
packages_downloads.append([self.url + self.package_show + urlencode({ 'id': package }, False, '', 'utf-8'), package_file])
- self.download.get(packages_downloads)
+ status = self.download.get(packages_downloads)
#
# Package contents
@@ -251,7 +262,7 @@ class CkanDumper:
package_downloads.append([resource['url'], resource_file])
- self.download.get(package_downloads)
+ status = self.download.get(package_downloads)
if __name__ == "__main__":
# Parse CLI