1 import math
2 import random
3 import string
4
5 from six import with_metaclass
6 from six.moves.urllib.parse import urljoin, urlparse, parse_qs
7 from textwrap import dedent
8 import re
9
10 import flask
11 import posixpath
12 from flask import url_for
13 from dateutil import parser as dt_parser
14 from netaddr import IPAddress, IPNetwork
15 from redis import StrictRedis
16 from sqlalchemy.types import TypeDecorator, VARCHAR
17 import json
18
19 from coprs import constants
20 from coprs import app
24 """ Generate a random string used as token to access the API
25 remotely.
26
27 :kwarg: size, the size of the token to generate, defaults to 30
28 chars.
29 :return: a string, the API token for the user.
30 """
31 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
32
33
34 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}"
35 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
36 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
37 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
42
45
47 if isinstance(attr, int):
48 for k, v in self.vals.items():
49 if v == attr:
50 return k
51 raise KeyError("num {0} is not mapped".format(attr))
52 else:
53 return self.vals[attr]
54
57 vals = {"nothing": 0, "request": 1, "approved": 2}
58
59 @classmethod
61 return [(n, k) for k, n in cls.vals.items() if n != without]
62
65 vals = {
66 "delete": 0,
67 "rename": 1,
68 "legal-flag": 2,
69 "createrepo": 3,
70 "update_comps": 4,
71 "gen_gpg_key": 5,
72 "rawhide_to_release": 6,
73 "fork": 7,
74 "update_module_md": 8,
75 "build_module": 9,
76 "cancel_build": 10,
77 }
78
81 vals = {"waiting": 0, "success": 1, "failure": 2}
82
83
84 -class RoleEnum(with_metaclass(EnumType, object)):
85 vals = {"user": 0, "admin": 1}
86
87
88 -class StatusEnum(with_metaclass(EnumType, object)):
89 vals = {
90 "failed": 0,
91 "succeeded": 1,
92 "canceled": 2,
93 "running": 3,
94 "pending": 4,
95 "skipped": 5,
96 "starting": 6,
97 "importing": 7,
98 "forked": 8,
99 "waiting": 9,
100 "unknown": 1000,
101 }
102
105 vals = {"pending": 0, "succeeded": 1, "failed": 2}
106
109 vals = {"unset": 0,
110 "link": 1,
111 "upload": 2,
112 "pypi": 5,
113 "rubygems": 6,
114 "scm": 8,
115 "custom": 9,
116 }
117
120 vals = {"unset": 0,
121
122 "unknown_error": 1,
123 "build_error": 2,
124 "srpm_import_failed": 3,
125 "srpm_download_failed": 4,
126 "srpm_query_failed": 5,
127 "import_timeout_exceeded": 6,
128 "git_clone_failed": 31,
129 "git_wrong_directory": 32,
130 "git_checkout_error": 33,
131 "srpm_build_error": 34,
132 }
133
136 """Represents an immutable structure as a json-encoded string.
137
138 Usage::
139
140 JSONEncodedDict(255)
141
142 """
143
144 impl = VARCHAR
145
147 if value is not None:
148 value = json.dumps(value)
149
150 return value
151
153 if value is not None:
154 value = json.loads(value)
155 return value
156
158
159 - def __init__(self, query, total_count, page=1,
160 per_page_override=None, urls_count_override=None,
161 additional_params=None):
162
163 self.query = query
164 self.total_count = total_count
165 self.page = page
166 self.per_page = per_page_override or constants.ITEMS_PER_PAGE
167 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT
168 self.additional_params = additional_params or dict()
169
170 self._sliced_query = None
171
172 - def page_slice(self, page):
173 return (self.per_page * (page - 1),
174 self.per_page * page)
175
176 @property
178 if not self._sliced_query:
179 self._sliced_query = self.query[slice(*self.page_slice(self.page))]
180 return self._sliced_query
181
182 @property
184 return int(math.ceil(self.total_count / float(self.per_page)))
185
187 if start:
188 if self.page - 1 > self.urls_count // 2:
189 return self.url_for_other_page(request, 1), 1
190 else:
191 if self.page < self.pages - self.urls_count // 2:
192 return self.url_for_other_page(request, self.pages), self.pages
193
194 return None
195
197 left_border = self.page - self.urls_count // 2
198 left_border = 1 if left_border < 1 else left_border
199 right_border = self.page + self.urls_count // 2
200 right_border = self.pages if right_border > self.pages else right_border
201
202 return [(self.url_for_other_page(request, i), i)
203 for i in range(left_border, right_border + 1)]
204
205 - def url_for_other_page(self, request, page):
206 args = request.view_args.copy()
207 args["page"] = page
208 args.update(self.additional_params)
209 return flask.url_for(request.endpoint, **args)
210
213 """
214 Get a git branch name from chroot. Follow the fedora naming standard.
215 """
216 os, version, arch = chroot.rsplit("-", 2)
217 if os == "fedora":
218 if version == "rawhide":
219 return "master"
220 os = "f"
221 elif os == "epel" and int(version) <= 6:
222 os = "el"
223 elif os == "mageia" and version == "cauldron":
224 os = "cauldron"
225 version = ""
226 elif os == "mageia":
227 os = "mga"
228 return "{}{}".format(os, version)
229
233 """
234 Pass in a standard style rpm fullname
235
236 Return a name, version, release, epoch, arch, e.g.::
237 foo-1.0-1.i386.rpm returns foo, 1.0, 1, i386
238 1:bar-9-123a.ia64.rpm returns bar, 9, 123a, 1, ia64
239 """
240
241 if filename[-4:] == '.rpm':
242 filename = filename[:-4]
243
244 archIndex = filename.rfind('.')
245 arch = filename[archIndex+1:]
246
247 relIndex = filename[:archIndex].rfind('-')
248 rel = filename[relIndex+1:archIndex]
249
250 verIndex = filename[:relIndex].rfind('-')
251 ver = filename[verIndex+1:relIndex]
252
253 epochIndex = filename.find(':')
254 if epochIndex == -1:
255 epoch = ''
256 else:
257 epoch = filename[:epochIndex]
258
259 name = filename[epochIndex + 1:verIndex]
260 return name, ver, rel, epoch, arch
261
264 """
265 Parse package name from possibly incomplete nvra string.
266 """
267
268 if pkg.count(".") >= 3 and pkg.count("-") >= 2:
269 return splitFilename(pkg)[0]
270
271
272 result = ""
273 pkg = pkg.replace(".rpm", "").replace(".src", "")
274
275 for delim in ["-", "."]:
276 if delim in pkg:
277 parts = pkg.split(delim)
278 for part in parts:
279 if any(map(lambda x: x.isdigit(), part)):
280 return result[:-1]
281
282 result += part + "-"
283
284 return result[:-1]
285
286 return pkg
287
311
314 """
315 Ensure that url either has http or https protocol according to the
316 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL"
317 """
318 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https":
319 return url.replace("http://", "https://")
320 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http":
321 return url.replace("https://", "http://")
322 else:
323 return url
324
327 """
328 Ensure that url either has http or https protocol according to the
329 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL"
330 """
331 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https":
332 return url.replace("http://", "https://")
333 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http":
334 return url.replace("https://", "http://")
335 else:
336 return url
337
340
342 """
343 Usage:
344
345 SQLAlchObject.to_dict() => returns a flat dict of the object
346 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object
347 and will include a flat dict of object foo inside of that
348 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns
349 a dict of the object, which will include dict of foo
350 (which will include dict of bar) and dict of spam.
351
352 Options can also contain two special values: __columns_only__
353 and __columns_except__
354
355 If present, the first makes only specified fields appear,
356 the second removes specified fields. Both of these fields
357 must be either strings (only works for one field) or lists
358 (for one and more fields).
359
360 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]},
361 "__columns_only__": "name"}) =>
362
363 The SQLAlchObject will only put its "name" into the resulting dict,
364 while "foo" all of its fields except "id".
365
366 Options can also specify whether to include foo_id when displaying
367 related foo object (__included_ids__, defaults to True).
368 This doesn"t apply when __columns_only__ is specified.
369 """
370
371 result = {}
372 if options is None:
373 options = {}
374 columns = self.serializable_attributes
375
376 if "__columns_only__" in options:
377 columns = options["__columns_only__"]
378 else:
379 columns = set(columns)
380 if "__columns_except__" in options:
381 columns_except = options["__columns_except__"]
382 if not isinstance(options["__columns_except__"], list):
383 columns_except = [options["__columns_except__"]]
384
385 columns -= set(columns_except)
386
387 if ("__included_ids__" in options and
388 options["__included_ids__"] is False):
389
390 related_objs_ids = [
391 r + "_id" for r, _ in options.items()
392 if not r.startswith("__")]
393
394 columns -= set(related_objs_ids)
395
396 columns = list(columns)
397
398 for column in columns:
399 result[column] = getattr(self, column)
400
401 for related, values in options.items():
402 if hasattr(self, related):
403 result[related] = getattr(self, related).to_dict(values)
404 return result
405
406 @property
408 return map(lambda x: x.name, self.__table__.columns)
409
413 self.host = config.get("REDIS_HOST", "127.0.0.1")
414 self.port = int(config.get("REDIS_PORT", "6379"))
415
417 return StrictRedis(host=self.host, port=self.port)
418
421 """
422 Creates connection to redis, now we use default instance at localhost, no config needed
423 """
424 return StrictRedis()
425
428 """
429 Converts datetime to unixtime
430 :param dt: DateTime instance
431 :rtype: float
432 """
433 return float(dt.strftime('%s'))
434
437 """
438 Converts datetime to unixtime from string
439 :param dt_string: datetime string
440 :rtype: str
441 """
442 return dt_to_unixtime(dt_parser.parse(dt_string))
443
446 """
447 Checks is ip is owned by the builders network
448 :param str ip: IPv4 address
449 :return bool: True
450 """
451 ip_addr = IPAddress(ip)
452 for subnet in app.config.get("BUILDER_IPS", ["127.0.0.1/24"]):
453 if ip_addr in IPNetwork(subnet):
454 return True
455
456 return False
457
460 if v is None:
461 return False
462 return v.lower() in ("yes", "true", "t", "1")
463
466 """
467 Examine given copr and generate proper URL for the `view`
468
469 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters,
470 and therefore you should *not* pass them manually.
471
472 Usage:
473 copr_url("coprs_ns.foo", copr)
474 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz)
475 """
476 if copr.is_a_group_project:
477 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs)
478 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
479
486
490
491
492 from sqlalchemy.engine.default import DefaultDialect
493 from sqlalchemy.sql.sqltypes import String, DateTime, NullType
494
495
496 PY3 = str is not bytes
497 text = str if PY3 else unicode
498 int_type = int if PY3 else (int, long)
499 str_type = str if PY3 else (str, unicode)
503 """Teach SA how to literalize various things."""
516 return process
517
528
531 """NOTE: This is entirely insecure. DO NOT execute the resulting strings."""
532 import sqlalchemy.orm
533 if isinstance(statement, sqlalchemy.orm.Query):
534 statement = statement.statement
535 return statement.compile(
536 dialect=LiteralDialect(),
537 compile_kwargs={'literal_binds': True},
538 ).string
539
542 app.update_template_context(context)
543 t = app.jinja_env.get_template(template_name)
544 rv = t.stream(context)
545 rv.enable_buffering(2)
546 return rv
547
555
558 """
559 Expands variables and sanitize repo url to be used for mock config
560 """
561 parsed_url = urlparse(repo_url)
562 if parsed_url.scheme == "copr":
563 user = parsed_url.netloc
564 prj = parsed_url.path.split("/")[1]
565 repo_url = "/".join([
566 flask.current_app.config["BACKEND_BASE_URL"],
567 "results", user, prj, chroot
568 ]) + "/"
569
570 repo_url = repo_url.replace("$chroot", chroot)
571 repo_url = repo_url.replace("$distname", chroot.rsplit("-", 2)[0])
572 return repo_url
573
576 """
577 :param repo: str repo from Copr/CoprChroot/Build/...
578 :param supported_keys list of supported optional parameters
579 :return: dict of optional parameters parsed from the repo URL
580 """
581 supported_keys = supported_keys or ["priority"]
582 if not repo.startswith("copr://"):
583 return {}
584
585 params = {}
586 qs = parse_qs(urlparse(repo).query)
587 for k, v in qs.items():
588 if k in supported_keys:
589
590
591 value = int(v[0]) if v[0].isnumeric() else v[0]
592 params[k] = value
593 return params
594
597 """ Return dict with proper build config contents """
598 chroot = None
599 for i in copr.copr_chroots:
600 if i.mock_chroot.name == chroot_id:
601 chroot = i
602 if not chroot:
603 return {}
604
605 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs
606
607 repos = [{
608 "id": "copr_base",
609 "url": copr.repo_url + "/{}/".format(chroot_id),
610 "name": "Copr repository",
611 }]
612
613 if not copr.auto_createrepo:
614 repos.append({
615 "id": "copr_base_devel",
616 "url": copr.repo_url + "/{}/devel/".format(chroot_id),
617 "name": "Copr buildroot",
618 })
619
620 def get_additional_repo_views(repos_list):
621 repos = []
622 for repo in repos_list:
623 params = parse_repo_params(repo)
624 repo_view = {
625 "id": generate_repo_name(repo),
626 "url": pre_process_repo_url(chroot_id, repo),
627 "name": "Additional repo " + generate_repo_name(repo),
628 }
629 repo_view.update(params)
630 repos.append(repo_view)
631 return repos
632
633 repos.extend(get_additional_repo_views(copr.repos_list))
634 repos.extend(get_additional_repo_views(chroot.repos_list))
635
636 return {
637 'project_id': copr.repo_id,
638 'additional_packages': packages.split(),
639 'repos': repos,
640 'chroot': chroot_id,
641 'use_bootstrap_container': copr.use_bootstrap_container,
642 'with_opts': chroot.with_opts.split(),
643 'without_opts': chroot.without_opts.split(),
644 }
645