From 1831acd944f547ec63ffc1a8207d16eabb1a4b58 Mon Sep 17 00:00:00 2001 From: Silvio Rhatto Date: Thu, 16 May 2019 09:15:37 -0300 Subject: Dump global stats to a file --- ckandumper | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/ckandumper b/ckandumper index 5640dee..d9f179b 100755 --- a/ckandumper +++ b/ckandumper @@ -37,6 +37,7 @@ class DownloadMultiple: self.progress = progress self.debug = debug self.wget = wget + self.globalstats = { 'exitstatus': {} } def ensuredir(self, dest): """Ensures that the destination folder exists""" @@ -87,6 +88,11 @@ class DownloadMultiple: output.write(str(proc.returncode) + '\n') output.close() + if not str(proc.returncode) in self.globalstats['exitstatus']: + self.globalstats['exitstatus'][str(proc.returncode)] = [] + + self.globalstats['exitstatus'][str(proc.returncode)].append(url); + if not str(proc.returncode) in self.stats['exitstatus']: self.stats['exitstatus'][str(proc.returncode)] = [] @@ -178,6 +184,16 @@ class CkanDumper: print('') + def write_stats(self): + """Write global statistics to file""" + stats = open(self.dest + os.sep + 'ckandumper.stats.json', 'w') + stats.write(json.dumps(self.download.globalstats, indent=2) + '\n') + + def process_stats(self, stats): + """Process stats at each run""" + self.dump_stats(stats) + self.write_stats() + def dump(self): """Downloads all content listed in a CKAN repository""" package_list = self.dest + os.sep + 'package_list.json' @@ -201,7 +217,7 @@ class CkanDumper: group_downloads.append([self.url + self.group_show + urlencode({ 'id': group }, False, '', 'utf-8'), group_file]) stats = self.download.get(group_downloads) - self.dump_stats(stats) + self.process_stats(stats) # # Tags @@ -220,6 +236,7 @@ class CkanDumper: tags_downloads.append([self.url + self.tag_show + urlencode({ 'id': tag }, False, '', 'utf-8'), tag_file]) stats = self.download.get(tags_downloads) + self.process_stats(stats) # # Packages @@ -239,6 +256,7 @@ class CkanDumper: packages_downloads.append([self.url + self.package_show + urlencode({ 'id': package }, False, '', 'utf-8'), package_file]) stats = self.download.get(packages_downloads) + self.process_stats(stats) # # Package contents @@ -268,6 +286,7 @@ class CkanDumper: package_downloads.append([resource['url'], resource_file]) stats = self.download.get(package_downloads) + self.process_stats(stats) if __name__ == "__main__": # Parse CLI -- cgit v1.2.3