diff options
| author | Silvio Rhatto <rhatto@riseup.net> | 2019-05-16 14:15:21 -0300 |
|---|---|---|
| committer | Silvio Rhatto <rhatto@riseup.net> | 2019-05-16 14:15:21 -0300 |
| commit | 060975ed25259cf19d3a7f4ef288fc35c64148ed (patch) | |
| tree | a287f486fc30a4b8e590b17fe01a010d45fa04a4 | |
| parent | e239d30f713db3fecd44b7f96484dda03d29c85b (diff) | |
| download | ckandumper-060975ed25259cf19d3a7f4ef288fc35c64148ed.tar.gz ckandumper-060975ed25259cf19d3a7f4ef288fc35c64148ed.tar.bz2 | |
Adds --randomize
| -rwxr-xr-x | ckandumper | 27 |
1 files changed, 18 insertions, 9 deletions
@@ -162,6 +162,9 @@ class CkanDumper: if args.progress != None: self.progress = args.progress + if args.randomize != None: + self.randomize = args.randomize + if args.limit_rate != None: self.limit_rate = '--limit-rate=' + args.limit_rate else: @@ -296,22 +299,28 @@ class CkanDumper: package_downloads.append([resource['url'], resource_file]) + if self.randomize == True: + random.shuffle(package_downloads) + stats = self.download.get(package_downloads) self.process_stats(stats) if __name__ == "__main__": # Parse CLI parser = argparse.ArgumentParser(description='Dump CKAN metadata and datasets.') - parser.add_argument('url', nargs='+', help='CKAN instance URL') - parser.add_argument('dest', nargs='+', help='Destination folder') - parser.add_argument("--limit-rate", help="Limit the download speed to amount bytes per second, per download") - parser.add_argument("--limit-concurrent", help="Limit the total concurrent downloads") - parser.add_argument("--wget", help="Path of custom wget implementation") - parser.add_argument('--debug', dest='debug', action='store_true', help="Enable debug") - parser.add_argument('--no-debug', dest='debug', action='store_false', help="Disable debug") - parser.add_argument('--progress', dest='progress', action='store_true', help="Enable progress") - parser.add_argument('--no-progress', dest='progress', action='store_false', help="Disable progress") + parser.add_argument('url', nargs='+', help='CKAN instance URL') + parser.add_argument('dest', nargs='+', help='Destination folder') + parser.add_argument("--limit-rate", help="Limit the download speed to amount bytes per second, per download") + parser.add_argument("--limit-concurrent", help="Limit the total concurrent downloads") + parser.add_argument("--wget", help="Path of custom wget implementation") + parser.add_argument('--debug', dest='debug', action='store_true', help="Enable debug") + parser.add_argument('--no-debug', dest='debug', action='store_false', help="Disable debug") + parser.add_argument('--progress', dest='progress', action='store_true', help="Enable progress") + parser.add_argument('--no-progress', dest='progress', action='store_false', help="Disable progress") + parser.add_argument('--randomize', dest='randomize', action='store_true', help="Randomize the list of downloads to avoid consuming resources of the same remote endpoint") + parser.add_argument('--no-randomize', dest='randomize', action='store_false', help="Do not randomize the list of downloads") parser.set_defaults(debug=False) + parser.set_defaults(randomize=False) parser.set_defaults(progress=True) args = parser.parse_args() |