1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
|
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Dumps CKAN instance data: metadata plus entire datasets.
#
# Copyright (C) 2019 Silvio Rhatto <rhatto@riseup.net>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published
# by the Free Software Foundation, either version 3 of the License,
# or any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
# Dependencies
import argparse
import sys, os, subprocess, pycurl, json
from urllib.parse import urlencode
class CkanDumper:
"""Dumps CKAN data: metadata plus entire datasets"""
def __init__(self, args):
self.url = args.url[0]
self.dest = args.dest[0]
self.package_list = '/api/3/action/package_list'
self.package_show = '/api/3/action/package_show?'
self.group_list = '/api/3/action/group_list'
self.group_show = '/api/3/action/group_show?'
self.tag_list = '/api/3/action/tag_list'
self.tag_show = '/api/3/action/tag_show?'
if args.limit_rate != None:
self.limit_rate = '--limit-rate=' + args.limit_rate
def download(self, url, local_filename):
"""Downloads a file.
Using wget as it is more reliable
"""
subprocess.call('/usr/bin/wget ' + self.limit_rate + ' -c -O "' + local_filename + '" ' + url, shell=True)
def ensuredir(self, dest):
"""Ensures that the destination folder exists"""
if not os.path.exists(dest) and not os.path.isdir(dest):
os.makedirs(dest, 0o755);
elif os.path.exists(dest) and not os.path.isdir(dest):
raise ValueError('File exists and is not a folder:' + dest)
def load_json(self, file):
"""Loads a file with contents serialized as JSON"""
descriptor = open(file)
data = json.load(descriptor)
file.close()
def dump(self):
"""Downloads all content listed in a CKAN repository"""
self.ensuredir(self.dest)
# Move to dest folder
#os.chdir(self.dest)
package_list = self.dest + os.sep + 'package_list.json'
group_list = self.dest + os.sep + 'group_list.json'
tag_list = self.dest + os.sep + 'tag_list.json'
#
# Groups
#
self.download(self.url + self.group_list, group_list)
groups = self.load_json(group_list)
for group in groups['result']:
group_folder = self.dest + os.sep + 'groups' + os.sep + group
group_file = group_folder + os.sep + 'group.json'
self.ensuredir(group_folder)
print("Downloading " + self.url + self.group_show + 'id=' + group + '...')
self.download(self.url + self.group_show + urlencode({ 'id': group }, False, '', 'utf-8'), group_file)
#
# Tags
#
self.download(self.url + self.tag_list, tag_list)
tags = self.load_json(tag_list)
for tag in tags['result']:
tag_folder = self.dest + os.sep + 'tags' + os.sep + tag
tag_file = tag_folder + os.sep + 'tag.json'
self.ensuredir(tag_folder)
print("Downloading " + self.url + self.tag_show + 'id=' + tag + '...')
self.download(self.url + self.tag_show + urlencode({ 'id': tag }, False, '', 'utf-8'), tag_file)
#
# Packages
#
self.download(self.url + self.package_list, package_list)
packages = self.load_json(package_list)
for package in packages['result']:
package_folder = self.dest + os.sep + 'packages' + os.sep + package
package_file = package_folder + os.sep + 'package.json'
self.ensuredir(package_folder + os.sep + 'data')
print("Downloading " + self.url + self.package_show + 'id=' + package + '...')
self.download(self.url + self.package_show + urlencode({ 'id': package }, False, '', 'utf-8'), package_file)
contents = self.load_json(package_file)
for resource in contents['result']['resources']:
#if resource['name'] != None:
# name = resource['name']
#else
# name = resource['id']
name = resource['id']
if resource['format'] != None:
format = '.' + resource['format'].lower()
else:
format = ''
resource_file = package_folder + os.sep + 'data' + os.sep + name + format
self.download(resource['url'], resource_file)
# Run only once during development
#return
if __name__ == "__main__":
# Parse CLI
parser = argparse.ArgumentParser(description='Dump CKAN metadata and datasets.')
parser.add_argument('url', nargs='+', help='CKAN instance URL')
parser.add_argument('dest', nargs='+', help='Destination folder')
parser.add_argument("--limit-rate", help="Limit the download speed to amount bytes per second, per download")
args = parser.parse_args()
# Dispatch
ckan = CkanDumper(args)
ckan.dump()
|
