1 import math
2 import random
3 import string
4 import html5_parser
5
6 from os.path import normpath
7 from six import with_metaclass
8 from six.moves.urllib.parse import urlparse, parse_qs, urlunparse, urlencode
9 import re
10
11 import flask
12 import posixpath
13 from flask import url_for
14 from dateutil import parser as dt_parser
15 from netaddr import IPAddress, IPNetwork
16 from redis import StrictRedis
17 from sqlalchemy.types import TypeDecorator, VARCHAR
18 import json
19
20 from copr_common.enums import EnumType
21 from copr_common.rpm import splitFilename
22 from coprs import constants
23 from coprs import app
27 """ Generate a random string used as token to access the API
28 remotely.
29
30 :kwarg: size, the size of the token to generate, defaults to 30
31 chars.
32 :return: a string, the API token for the user.
33 """
34 return ''.join(random.choice(string.ascii_lowercase) for x in range(size))
35
36
37 REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}"
38 CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
39 CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}"
40 PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}"
41
42
43 FINISHED_STATUSES = ["succeeded", "forked", "canceled", "skipped", "failed"]
48
51
52 vals = {"nothing": 0, "request": 1, "approved": 2}
53
54 @classmethod
56 return [(n, k) for k, n in cls.vals.items() if n != without]
57
60 vals = {"unset": 0,
61 "link": 1,
62 "upload": 2,
63 "pypi": 5,
64 "rubygems": 6,
65 "scm": 8,
66 "custom": 9,
67 }
68
71 """Represents an immutable structure as a json-encoded string.
72
73 Usage::
74
75 JSONEncodedDict(255)
76
77 """
78
79 impl = VARCHAR
80
82 if value is not None:
83 value = json.dumps(value)
84
85 return value
86
88 if value is not None:
89 value = json.loads(value)
90 return value
91
94 - def __init__(self, query, total_count, page=1,
95 per_page_override=None, urls_count_override=None,
96 additional_params=None):
97
98 self.query = query
99 self.total_count = total_count
100 self.page = page
101 self.per_page = per_page_override or constants.ITEMS_PER_PAGE
102 self.urls_count = urls_count_override or constants.PAGES_URLS_COUNT
103 self.additional_params = additional_params or dict()
104
105 self._sliced_query = None
106
107 - def page_slice(self, page):
108 return (self.per_page * (page - 1),
109 self.per_page * page)
110
111 @property
113 if not self._sliced_query:
114 self._sliced_query = self.query[slice(*self.page_slice(self.page))]
115 return self._sliced_query
116
117 @property
119 return int(math.ceil(self.total_count / float(self.per_page)))
120
122 if start:
123 if self.page - 1 > self.urls_count // 2:
124 return self.url_for_other_page(request, 1), 1
125 else:
126 if self.page < self.pages - self.urls_count // 2:
127 return self.url_for_other_page(request, self.pages), self.pages
128
129 return None
130
132 left_border = self.page - self.urls_count // 2
133 left_border = 1 if left_border < 1 else left_border
134 right_border = self.page + self.urls_count // 2
135 right_border = self.pages if right_border > self.pages else right_border
136
137 return [(self.url_for_other_page(request, i), i)
138 for i in range(left_border, right_border + 1)]
139
140 - def url_for_other_page(self, request, page):
141 args = request.view_args.copy()
142 args["page"] = page
143 args.update(self.additional_params)
144 return flask.url_for(request.endpoint, **args)
145
148 """
149 Get a git branch name from chroot. Follow the fedora naming standard.
150 """
151 os, version, arch = chroot.rsplit("-", 2)
152 if os == "fedora":
153 if version == "rawhide":
154 return "master"
155 os = "f"
156 elif os == "epel" and int(version) <= 6:
157 os = "el"
158 elif os == "mageia" and version == "cauldron":
159 os = "cauldron"
160 version = ""
161 elif os == "mageia":
162 os = "mga"
163 return "{}{}".format(os, version)
164
167 """
168 Parse package name from possibly incomplete nvra string.
169 """
170
171 if pkg.count(".") >= 3 and pkg.count("-") >= 2:
172 return splitFilename(pkg)[0]
173
174
175 result = ""
176 pkg = pkg.replace(".rpm", "").replace(".src", "")
177
178 for delim in ["-", "."]:
179 if delim in pkg:
180 parts = pkg.split(delim)
181 for part in parts:
182 if any(map(lambda x: x.isdigit(), part)):
183 return result[:-1]
184
185 result += part + "-"
186
187 return result[:-1]
188
189 return pkg
190
214
217 """
218 Ensure that url either has http or https protocol according to the
219 option in app config "ENFORCE_PROTOCOL_FOR_BACKEND_URL"
220 """
221 if app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "https":
222 return url.replace("http://", "https://")
223 elif app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] == "http":
224 return url.replace("https://", "http://")
225 else:
226 return url
227
230 """
231 Ensure that url either has http or https protocol according to the
232 option in app config "ENFORCE_PROTOCOL_FOR_FRONTEND_URL"
233 """
234 if app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "https":
235 return url.replace("http://", "https://")
236 elif app.config["ENFORCE_PROTOCOL_FOR_FRONTEND_URL"] == "http":
237 return url.replace("https://", "http://")
238 else:
239 return url
240
243
245 """
246 Usage:
247
248 SQLAlchObject.to_dict() => returns a flat dict of the object
249 SQLAlchObject.to_dict({"foo": {}}) => returns a dict of the object
250 and will include a flat dict of object foo inside of that
251 SQLAlchObject.to_dict({"foo": {"bar": {}}, "spam": {}}) => returns
252 a dict of the object, which will include dict of foo
253 (which will include dict of bar) and dict of spam.
254
255 Options can also contain two special values: __columns_only__
256 and __columns_except__
257
258 If present, the first makes only specified fields appear,
259 the second removes specified fields. Both of these fields
260 must be either strings (only works for one field) or lists
261 (for one and more fields).
262
263 SQLAlchObject.to_dict({"foo": {"__columns_except__": ["id"]},
264 "__columns_only__": "name"}) =>
265
266 The SQLAlchObject will only put its "name" into the resulting dict,
267 while "foo" all of its fields except "id".
268
269 Options can also specify whether to include foo_id when displaying
270 related foo object (__included_ids__, defaults to True).
271 This doesn"t apply when __columns_only__ is specified.
272 """
273
274 result = {}
275 if options is None:
276 options = {}
277 columns = self.serializable_attributes
278
279 if "__columns_only__" in options:
280 columns = options["__columns_only__"]
281 else:
282 columns = set(columns)
283 if "__columns_except__" in options:
284 columns_except = options["__columns_except__"]
285 if not isinstance(options["__columns_except__"], list):
286 columns_except = [options["__columns_except__"]]
287
288 columns -= set(columns_except)
289
290 if ("__included_ids__" in options and
291 options["__included_ids__"] is False):
292
293 related_objs_ids = [
294 r + "_id" for r, _ in options.items()
295 if not r.startswith("__")]
296
297 columns -= set(related_objs_ids)
298
299 columns = list(columns)
300
301 for column in columns:
302 result[column] = getattr(self, column)
303
304 for related, values in options.items():
305 if hasattr(self, related):
306 result[related] = getattr(self, related).to_dict(values)
307 return result
308
309 @property
312
316 self.host = config.get("REDIS_HOST", "127.0.0.1")
317 self.port = int(config.get("REDIS_PORT", "6379"))
318
320 return StrictRedis(host=self.host, port=self.port)
321
324 """
325 Creates connection to redis, now we use default instance at localhost, no config needed
326 """
327 return StrictRedis()
328
331 if v is None:
332 return False
333 return v.lower() in ("yes", "true", "t", "1")
334
337 """
338 Examine given copr and generate proper URL for the `view`
339
340 Values of `username/group_name` and `coprname` are automatically passed as the first two URL parameters,
341 and therefore you should *not* pass them manually.
342
343 Usage:
344 copr_url("coprs_ns.foo", copr)
345 copr_url("coprs_ns.foo", copr, arg1='bar', arg2='baz)
346 """
347 if copr.is_a_group_project:
348 return url_for(view, group_name=copr.group.name, coprname=copr.name, **kwargs)
349 return url_for(view, username=copr.user.name, coprname=copr.name, **kwargs)
350
353 """
354 For a given `owner` object, which may be either `models.User` or `models.Group`,
355 return an URL to its _profile_ page.
356 """
357
358
359 if hasattr(owner, "at_name"):
360 return url_for("groups_ns.list_projects_by_group", group_name=owner.name)
361 return url_for("coprs_ns.coprs_by_user", username=owner.username)
362
369
373
374
375 from sqlalchemy.engine.default import DefaultDialect
376 from sqlalchemy.sql.sqltypes import String, DateTime, NullType
377
378
379 PY3 = str is not bytes
380 text = str if PY3 else unicode
381 int_type = int if PY3 else (int, long)
382 str_type = str if PY3 else (str, unicode)
386 """Teach SA how to literalize various things."""
399 return process
400
411
414 """NOTE: This is entirely insecure. DO NOT execute the resulting strings.
415 This can be used for debuggin - it is not and should not be used in production
416 code.
417
418 It is useful if you want to debug an sqlalchemy query, i.e. copy the
419 resulting SQL query into psql console and try to tweak it so that it
420 actually works or works faster.
421 """
422 import sqlalchemy.orm
423 if isinstance(statement, sqlalchemy.orm.Query):
424 statement = statement.statement
425 return statement.compile(
426 dialect=LiteralDialect(),
427 compile_kwargs={'literal_binds': True},
428 ).string
429
432 app.update_template_context(context)
433 t = app.jinja_env.get_template(template_name)
434 rv = t.stream(context)
435 rv.enable_buffering(2)
436 return rv
437
445
448 """
449 Expands variables and sanitize repo url to be used for mock config
450 """
451 parsed_url = urlparse(repo_url)
452 query = parse_qs(parsed_url.query)
453
454 if parsed_url.scheme == "copr":
455 user = parsed_url.netloc
456 prj = parsed_url.path.split("/")[1]
457 repo_url = "/".join([
458 flask.current_app.config["BACKEND_BASE_URL"],
459 "results", user, prj, chroot
460 ]) + "/"
461
462 elif "priority" in query:
463 query.pop("priority")
464 query_string = urlencode(query, doseq=True)
465 parsed_url = parsed_url._replace(query=query_string)
466 repo_url = urlunparse(parsed_url)
467
468 repo_url = repo_url.replace("$chroot", chroot)
469 repo_url = repo_url.replace("$distname", chroot.rsplit("-", 2)[0])
470 return repo_url
471
474 """
475 :param repo: str repo from Copr/CoprChroot/Build/...
476 :param supported_keys list of supported optional parameters
477 :return: dict of optional parameters parsed from the repo URL
478 """
479 supported_keys = supported_keys or ["priority"]
480 params = {}
481 qs = parse_qs(urlparse(repo).query)
482 for k, v in qs.items():
483 if k in supported_keys:
484
485
486 value = int(v[0]) if v[0].isnumeric() else v[0]
487 params[k] = value
488 return params
489
492 """ Return dict with proper build config contents """
493 chroot = None
494 for i in copr.copr_chroots:
495 if i.mock_chroot.name == chroot_id:
496 chroot = i
497 if not chroot:
498 return {}
499
500 packages = "" if not chroot.buildroot_pkgs else chroot.buildroot_pkgs
501
502 repos = [{
503 "id": "copr_base",
504 "baseurl": copr.repo_url + "/{}/".format(chroot_id),
505 "name": "Copr repository",
506 }]
507
508 if not copr.auto_createrepo:
509 repos.append({
510 "id": "copr_base_devel",
511 "baseurl": copr.repo_url + "/{}/devel/".format(chroot_id),
512 "name": "Copr buildroot",
513 })
514
515 def get_additional_repo_views(repos_list):
516 repos = []
517 for repo in repos_list:
518 params = parse_repo_params(repo)
519 repo_view = {
520 "id": generate_repo_name(repo),
521 "baseurl": pre_process_repo_url(chroot_id, repo),
522 "name": "Additional repo " + generate_repo_name(repo),
523 }
524 repo_view.update(params)
525 repos.append(repo_view)
526 return repos
527
528 repos.extend(get_additional_repo_views(copr.repos_list))
529 repos.extend(get_additional_repo_views(chroot.repos_list))
530
531 return {
532 'project_id': copr.repo_id,
533 'additional_packages': packages.split(),
534 'repos': repos,
535 'chroot': chroot_id,
536 'use_bootstrap_container': copr.use_bootstrap_container,
537 'with_opts': chroot.with_opts.split(),
538 'without_opts': chroot.without_opts.split(),
539 }
540
548
551 if not url:
552 return None
553
554 return re.sub(r'(\.git)?/*$', '', url)
555
558 if not url:
559 return False
560
561 url = trim_git_url(url)
562 return urlparse(url)
563
567 if not subdir:
568 self.subdir = '.'
569 else:
570 self.subdir = normpath(subdir).strip('/')
571
573 if not path:
574 return False
575
576 changed = normpath(path).strip('/')
577 if changed == '.':
578 return False
579
580 if self.subdir == '.':
581 return True
582
583 return changed.startswith(self.subdir + '/')
584
587 parsed = html5_parser.parse(str(html_string))
588 elements = parsed.xpath(
589 "//section[contains(@class, 'commit_diff')]"
590 "//div[contains(@class, 'card-header')]"
591 "//a[contains(@class, 'font-weight-bold')]"
592 "/text()")
593
594 return set([str(x) for x in elements])
595
598 changes = set()
599 for line in text.split('\n'):
600 match = re.search(r'^(\+\+\+|---) [ab]/(.*)$', line)
601 if match:
602 changes.add(str(match.group(2)))
603 match = re.search(r'^diff --git a/(.*) b/(.*)$', line)
604 if match:
605 changes.add(str(match.group(1)))
606 changes.add(str(match.group(2)))
607 print(changes)
608
609 return changes
610