diff options
author | Silvio Rhatto <rhatto@riseup.net> | 2019-05-16 08:53:18 -0300 |
---|---|---|
committer | Silvio Rhatto <rhatto@riseup.net> | 2019-05-16 08:53:18 -0300 |
commit | 046e26a98759899f7275fbd22573ffe810304cef (patch) | |
tree | 8682820dafed8ea4ecb0334fb3e56b4a405fcff5 | |
parent | d94457aa7f882c7f38af6e271737679935d782ce (diff) | |
download | ckandumper-046e26a98759899f7275fbd22573ffe810304cef.tar.gz ckandumper-046e26a98759899f7275fbd22573ffe810304cef.tar.bz2 |
Initial exit status tracking
-rwxr-xr-x | ckandumper | 31 |
1 files changed, 21 insertions, 10 deletions
@@ -84,9 +84,14 @@ class DownloadMultiple: print(f'[stderr] {url} {stderr.decode()}') output = open(local_filename + '.returncode', 'w') - output.write(str(proc.returncode)) + output.write(str(proc.returncode) + '\n') output.close() + if not str(proc.returncode) in self.stats['exitstatus']: + self.stats['exitstatus'][str(proc.returncode)] = [] + + self.stats['exitstatus'][str(proc.returncode)].append(url); + if self.debug: print(f'[{cmd!r} exited with {proc.returncode}]') @@ -109,12 +114,18 @@ class DownloadMultiple: await asyncio.gather(*jobs) def get(self, filepairs): - self.bar = tqdm(total=len(filepairs)) if self.progress and len(filepairs) > 1 else False - loop = asyncio.get_event_loop() + self.stats = { 'exitstatus': {} } + self.bar = tqdm(total=len(filepairs)) if self.progress and len(filepairs) > 1 else False + loop = asyncio.get_event_loop() loop.set_debug(self.debug) loop.run_until_complete(self.gather(filepairs)) + if self.debug: + print(self.stats) + + return self.stats + class CkanDumper: """Dumps CKAN data: metadata plus entire datasets""" @@ -173,7 +184,7 @@ class CkanDumper: # Groups # print(f'Downloading {self.url}{self.group_list}...') - self.download.get([[self.url + self.group_list, group_list]]) + status = self.download.get([[self.url + self.group_list, group_list]]) groups = self.load_json(group_list) group_downloads = [] @@ -185,13 +196,13 @@ class CkanDumper: group_downloads.append([self.url + self.group_show + urlencode({ 'id': group }, False, '', 'utf-8'), group_file]) - self.download.get(group_downloads) + status = self.download.get(group_downloads) # # Tags # print(f'Downloading {self.url}{self.tag_list}...') - self.download.get([[self.url + self.tag_list, tag_list]]) + status = self.download.get([[self.url + self.tag_list, tag_list]]) tags = self.load_json(tag_list) tags_downloads = [] @@ -203,13 +214,13 @@ class CkanDumper: tags_downloads.append([self.url + self.tag_show + urlencode({ 'id': tag }, False, '', 'utf-8'), tag_file]) - self.download.get(tags_downloads) + status = self.download.get(tags_downloads) # # Packages # print(f'Downloading {self.url}{self.package_list}...') - self.download.get([[self.url + self.package_list, package_list]]) + status = self.download.get([[self.url + self.package_list, package_list]]) packages = self.load_json(package_list) packages_downloads = [] @@ -222,7 +233,7 @@ class CkanDumper: packages_downloads.append([self.url + self.package_show + urlencode({ 'id': package }, False, '', 'utf-8'), package_file]) - self.download.get(packages_downloads) + status = self.download.get(packages_downloads) # # Package contents @@ -251,7 +262,7 @@ class CkanDumper: package_downloads.append([resource['url'], resource_file]) - self.download.get(package_downloads) + status = self.download.get(package_downloads) if __name__ == "__main__": # Parse CLI |