aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSilvio Rhatto <rhatto@riseup.net>2019-05-16 14:15:21 -0300
committerSilvio Rhatto <rhatto@riseup.net>2019-05-16 14:15:21 -0300
commit060975ed25259cf19d3a7f4ef288fc35c64148ed (patch)
treea287f486fc30a4b8e590b17fe01a010d45fa04a4
parente239d30f713db3fecd44b7f96484dda03d29c85b (diff)
downloadckandumper-060975ed25259cf19d3a7f4ef288fc35c64148ed.tar.gz
ckandumper-060975ed25259cf19d3a7f4ef288fc35c64148ed.tar.bz2
Adds --randomize
-rwxr-xr-xckandumper27
1 files changed, 18 insertions, 9 deletions
diff --git a/ckandumper b/ckandumper
index 7915c3b..f633215 100755
--- a/ckandumper
+++ b/ckandumper
@@ -162,6 +162,9 @@ class CkanDumper:
if args.progress != None:
self.progress = args.progress
+ if args.randomize != None:
+ self.randomize = args.randomize
+
if args.limit_rate != None:
self.limit_rate = '--limit-rate=' + args.limit_rate
else:
@@ -296,22 +299,28 @@ class CkanDumper:
package_downloads.append([resource['url'], resource_file])
+ if self.randomize == True:
+ random.shuffle(package_downloads)
+
stats = self.download.get(package_downloads)
self.process_stats(stats)
if __name__ == "__main__":
# Parse CLI
parser = argparse.ArgumentParser(description='Dump CKAN metadata and datasets.')
- parser.add_argument('url', nargs='+', help='CKAN instance URL')
- parser.add_argument('dest', nargs='+', help='Destination folder')
- parser.add_argument("--limit-rate", help="Limit the download speed to amount bytes per second, per download")
- parser.add_argument("--limit-concurrent", help="Limit the total concurrent downloads")
- parser.add_argument("--wget", help="Path of custom wget implementation")
- parser.add_argument('--debug', dest='debug', action='store_true', help="Enable debug")
- parser.add_argument('--no-debug', dest='debug', action='store_false', help="Disable debug")
- parser.add_argument('--progress', dest='progress', action='store_true', help="Enable progress")
- parser.add_argument('--no-progress', dest='progress', action='store_false', help="Disable progress")
+ parser.add_argument('url', nargs='+', help='CKAN instance URL')
+ parser.add_argument('dest', nargs='+', help='Destination folder')
+ parser.add_argument("--limit-rate", help="Limit the download speed to amount bytes per second, per download")
+ parser.add_argument("--limit-concurrent", help="Limit the total concurrent downloads")
+ parser.add_argument("--wget", help="Path of custom wget implementation")
+ parser.add_argument('--debug', dest='debug', action='store_true', help="Enable debug")
+ parser.add_argument('--no-debug', dest='debug', action='store_false', help="Disable debug")
+ parser.add_argument('--progress', dest='progress', action='store_true', help="Enable progress")
+ parser.add_argument('--no-progress', dest='progress', action='store_false', help="Disable progress")
+ parser.add_argument('--randomize', dest='randomize', action='store_true', help="Randomize the list of downloads to avoid consuming resources of the same remote endpoint")
+ parser.add_argument('--no-randomize', dest='randomize', action='store_false', help="Do not randomize the list of downloads")
parser.set_defaults(debug=False)
+ parser.set_defaults(randomize=False)
parser.set_defaults(progress=True)
args = parser.parse_args()