1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
#! /usr/bin/env python
# -*- coding: utf-8 -*-
# COPYRIGHT NOTICE STARTS HERE
# Copyright 2019 © Samsung Electronics Co., Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# COPYRIGHT NOTICE ENDS HERE
import argparse
import logging
import sys
import datetime
import timeit
import base
import docker_images
import git_repos
import http_files
import npm_packages
import pypi_packages
import rpm_packages
log = logging.getLogger(name=__name__)
def parse_args():
parser=argparse.ArgumentParser(description='Download data from lists')
list_group = parser.add_argument_group()
list_group.add_argument('--docker', action='append', nargs='+', default=[],
metavar=('list', 'dir-name'),
help='Docker type list. If second argument is specified '
'it is treated as directory where images will be saved '
'otherwise only pull operation is executed')
list_group.add_argument('--http', action='append', nargs=2, default=[],
metavar=('list', 'dir-name'),
help='Http type list and directory to save downloaded files')
list_group.add_argument('--npm', action='append', nargs=2, default=[],
metavar=('list', 'dir-name'),
help='npm type list and directory to save downloaded files')
list_group.add_argument('--rpm', action='append', nargs=2, default=[],
metavar=('list', 'dir-name'),
help='rpm type list and directory to save downloaded files')
list_group.add_argument('--git', action='append', nargs=2, default=[],
metavar=('list', 'dir-name'),
help='git repo type list and directory to save downloaded files')
list_group.add_argument('--pypi', action='append', nargs=2, default=[],
metavar=('list', 'dir-name'),
help='pypi packages type list and directory to save downloaded files')
parser.add_argument('--npm-registry', default='https://registry.npmjs.org',
help='npm registry to use (default: https://registry.npmjs.org)')
parser.add_argument('--check', '-c', action='store_true', default=False,
help='Check what is missing. No download.')
parser.add_argument('--debug', action='store_true', default=False,
help='Turn on debug output')
args = parser.parse_args()
for arg in ('docker', 'npm', 'http', 'rpm', 'git', 'pypi'):
if getattr(args, arg):
return args
parser.error('One of --docker, --npm, --http, --rpm, --git must be specified')
def run_cli():
args = parse_args()
console_handler = logging.StreamHandler(sys.stdout)
console_formatter = logging.Formatter('%(message)s')
console_handler.setFormatter(console_formatter)
now = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
log_file = 'download_data-{}.log'.format(now)
file_format = "%(asctime)s: %(filename)s: %(levelname)s: %(message)s"
if args.debug:
logging.basicConfig(level=logging.DEBUG, filename=log_file, format=file_format)
else:
logging.basicConfig(level=logging.INFO, filename=log_file, format=file_format)
root_logger = logging.getLogger()
root_logger.addHandler(console_handler)
list_with_errors = []
timer_start = timeit.default_timer()
for docker_list in args.docker:
log.info('Processing {}.'.format(docker_list[0]))
progress = None if args.check else base.init_progress('docker images')
save = False
if len(docker_list) > 1:
save = True
else:
docker_list.append(None)
try:
docker_images.download(docker_list[0], save,
docker_list[1], args.check, progress)
except RuntimeError:
list_with_errors.append(docker_list[0])
for http_list in args.http:
progress = None if args.check else base.init_progress('http files')
log.info('Processing {}.'.format(http_list[0]))
try:
http_files.download(http_list[0], http_list[1], args.check,
progress)
except RuntimeError:
list_with_errors.append(http_list[0])
for npm_list in args.npm:
progress = None if args.check else base.init_progress('npm packages')
log.info('Processing {}.'.format(npm_list[0]))
try:
npm_packages.download(npm_list[0], args.npm_registry, npm_list[1],
args.check, progress)
except RuntimeError:
list_with_errors.append(npm_list[0])
for rpm_list in args.rpm:
if args.check:
log.info('Check mode for rpm packages is not implemented')
break
log.info('Processing {}.'.format(rpm_list[0]))
try:
rpm_packages.download(rpm_list[0], rpm_list[1])
except RuntimeError:
list_with_errors.append(rpm_list[0])
for git_list in args.git:
if args.check:
log.info('Check mode for git repositories is not implemented')
break
progress = None if args.check else base.init_progress('git repositories')
log.info('Processing {}.'.format(git_list[0]))
try:
git_repos.download(git_list[0], git_list[1], progress)
except RuntimeError:
list_with_errors.append(git_list[0])
for pypi_list in args.pypi:
if args.check:
log.info('Check mode for pypi packages is not implemented')
break
progress = None if args.check else base.init_progress('pypi packages')
log.info('Processing {}.'.format(pypi_list[0]))
try:
pypi_packages.download(pypi_list[0], pypi_list[1], progress)
except RuntimeError:
list_with_errors.append(pypi_list[0])
e_time = datetime.timedelta(seconds=timeit.default_timer() - timer_start)
log.info(timeit.default_timer() - timer_start)
log.info('Execution ended. Total elapsed time {}'.format(e_time))
if list_with_errors:
log.error('Errors encountered while processing these lists:'
'\n{}'.format('\n'.join(list_with_errors)))
sys.exit(1)
if __name__ == '__main__':
run_cli()
|
