Package coprs :: Package logic :: Module builds_logic
[hide private]
[frames] | no frames]

Source Code for Module coprs.logic.builds_logic

   1  import tempfile 
   2  import shutil 
   3  import json 
   4  import os 
   5  import pprint 
   6  import time 
   7  import requests 
   8   
   9  from sqlalchemy.sql import text 
  10  from sqlalchemy.sql.expression import not_ 
  11  from sqlalchemy.orm import joinedload 
  12  from sqlalchemy import or_ 
  13  from sqlalchemy import and_ 
  14  from sqlalchemy import func, desc 
  15  from sqlalchemy.sql import false,true 
  16  from werkzeug.utils import secure_filename 
  17  from sqlalchemy import bindparam, Integer, String 
  18  from sqlalchemy.exc import IntegrityError 
  19   
  20  from copr_common.enums import FailTypeEnum, StatusEnum 
  21  from coprs import app 
  22  from coprs import db 
  23  from coprs import models 
  24  from coprs import helpers 
  25  from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT 
  26  from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException, \ 
  27                               UnrepeatableBuildException, RequestCannotBeExecuted, DuplicateException 
  28   
  29  from coprs.logic import coprs_logic 
  30  from coprs.logic import users_logic 
  31  from coprs.logic.actions_logic import ActionsLogic 
  32  from coprs.models import BuildChroot 
  33  from .coprs_logic import MockChrootsLogic 
  34  from coprs.logic.packages_logic import PackagesLogic 
  35   
  36  log = app.logger 
37 38 39 -class BuildsLogic(object):
40 @classmethod
41 - def get(cls, build_id):
42 return models.Build.query.filter(models.Build.id == build_id)
43 44 @classmethod
45 - def get_build_tasks(cls, status, background=None):
46 """ Returns tasks with given status. If background is specified then 47 returns normal jobs (false) or background jobs (true) 48 """ 49 result = models.BuildChroot.query.join(models.Build)\ 50 .filter(models.BuildChroot.status == status)\ 51 .order_by(models.Build.id.asc()) 52 if background is not None: 53 result = result.filter(models.Build.is_background == (true() if background else false())) 54 return result
55 56 @classmethod
57 - def get_srpm_build_tasks(cls, status, background=None):
58 """ Returns srpm build tasks with given status. If background is 59 specified then returns normal jobs (false) or background jobs (true) 60 """ 61 result = models.Build.query\ 62 .filter(models.Build.source_status == status)\ 63 .order_by(models.Build.id.asc()) 64 if background is not None: 65 result = result.filter(models.Build.is_background == (true() if background else false())) 66 return result
67 68 @classmethod
69 - def get_recent_tasks(cls, user=None, limit=100, period_days=2):
70 query_args = ( 71 models.BuildChroot.build_id, 72 func.max(models.BuildChroot.ended_on).label('max_ended_on'), 73 models.Build.submitted_on, 74 ) 75 group_by_args = ( 76 models.BuildChroot.build_id, 77 models.Build.submitted_on, 78 ) 79 80 81 if user: 82 query_args += (models.Build.user_id,) 83 group_by_args += (models.Build.user_id,) 84 85 subquery = (db.session.query(*query_args) 86 .join(models.Build) 87 .group_by(*group_by_args) 88 .having(func.count() == func.count(models.BuildChroot.ended_on)) 89 .having(models.Build.submitted_on > time.time() - 3600*24*period_days) 90 ) 91 if user: 92 subquery = subquery.having(models.Build.user_id == user.id) 93 94 subquery = subquery.order_by(desc('max_ended_on')).limit(limit).subquery() 95 96 query = models.Build.query.join(subquery, subquery.c.build_id == models.Build.id) 97 return list(query.all())
98 99 @classmethod
100 - def get_running_tasks_by_time(cls, start, end):
101 result = models.BuildChroot.query\ 102 .filter(models.BuildChroot.ended_on > start)\ 103 .filter(models.BuildChroot.started_on < end)\ 104 .order_by(models.BuildChroot.started_on.asc()) 105 106 return result
107 108 @classmethod
109 - def get_chroot_histogram(cls, start, end):
110 chroots = [] 111 chroot_query = BuildChroot.query\ 112 .filter(models.BuildChroot.started_on < end)\ 113 .filter(models.BuildChroot.ended_on > start)\ 114 .with_entities(BuildChroot.mock_chroot_id, 115 func.count(BuildChroot.mock_chroot_id))\ 116 .group_by(BuildChroot.mock_chroot_id)\ 117 .order_by(BuildChroot.mock_chroot_id) 118 119 for chroot in chroot_query: 120 chroots.append([chroot[0], chroot[1]]) 121 122 mock_chroots = coprs_logic.MockChrootsLogic.get_multiple() 123 for mock_chroot in mock_chroots: 124 for l in chroots: 125 if l[0] == mock_chroot.id: 126 l[0] = mock_chroot.name 127 128 return chroots
129 130 @classmethod
131 - def get_pending_jobs_bucket(cls, start, end):
132 query = text(""" 133 SELECT COUNT(*) as result 134 FROM build_chroot JOIN build on build.id = build_chroot.build_id 135 WHERE 136 build.submitted_on < :end 137 AND ( 138 build_chroot.started_on > :start 139 OR (build_chroot.started_on is NULL AND build_chroot.status = :status) 140 -- for currently pending builds we need to filter on status=pending because there might be 141 -- failed builds that have started_on=NULL 142 ) 143 AND NOT build.canceled 144 """) 145 146 res = db.engine.execute(query, start=start, end=end, status=StatusEnum("pending")) 147 return res.first().result
148 149 @classmethod
150 - def get_running_jobs_bucket(cls, start, end):
151 query = text(""" 152 SELECT COUNT(*) as result 153 FROM build_chroot 154 WHERE 155 started_on < :end 156 AND (ended_on > :start OR (ended_on is NULL AND status = :status)) 157 -- for currently running builds we need to filter on status=running because there might be failed 158 -- builds that have ended_on=NULL 159 """) 160 161 res = db.engine.execute(query, start=start, end=end, status=StatusEnum("running")) 162 return res.first().result
163 164 @classmethod
165 - def get_cached_graph_data(cls, params):
166 data = { 167 "pending": [], 168 "running": [], 169 } 170 result = models.BuildsStatistics.query\ 171 .filter(models.BuildsStatistics.stat_type == params["type"])\ 172 .filter(models.BuildsStatistics.time >= params["start"])\ 173 .filter(models.BuildsStatistics.time <= params["end"])\ 174 .order_by(models.BuildsStatistics.time) 175 176 for row in result: 177 data["pending"].append(row.pending) 178 data["running"].append(row.running) 179 180 return data
181 182 @classmethod
183 - def get_task_graph_data(cls, type):
184 data = [["pending"], ["running"], ["avg running"], ["time"]] 185 params = cls.get_graph_parameters(type) 186 cached_data = cls.get_cached_graph_data(params) 187 data[0].extend(cached_data["pending"]) 188 data[1].extend(cached_data["running"]) 189 190 for i in range(len(data[0]) - 1, params["steps"]): 191 step_start = params["start"] + i * params["step"] 192 step_end = step_start + params["step"] 193 pending = cls.get_pending_jobs_bucket(step_start, step_end) 194 running = cls.get_running_jobs_bucket(step_start, step_end) 195 data[0].append(pending) 196 data[1].append(running) 197 cls.cache_graph_data(type, time=step_start, pending=pending, running=running) 198 199 running_total = 0 200 for i in range(1, params["steps"] + 1): 201 running_total += data[1][i] 202 203 data[2].extend([running_total * 1.0 / params["steps"]] * (len(data[0]) - 1)) 204 205 for i in range(params["start"], params["end"], params["step"]): 206 data[3].append(time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(i))) 207 208 return data
209 210 @classmethod
211 - def get_small_graph_data(cls, type):
212 data = [[""]] 213 params = cls.get_graph_parameters(type) 214 cached_data = cls.get_cached_graph_data(params) 215 data[0].extend(cached_data["running"]) 216 217 for i in range(len(data[0]) - 1, params["steps"]): 218 step_start = params["start"] + i * params["step"] 219 step_end = step_start + params["step"] 220 running = cls.get_running_jobs_bucket(step_start, step_end) 221 data[0].append(running) 222 cls.cache_graph_data(type, time=step_start, running=running) 223 224 return data
225 226 @classmethod
227 - def cache_graph_data(cls, type, time, pending=0, running=0):
228 result = models.BuildsStatistics.query\ 229 .filter(models.BuildsStatistics.stat_type == type)\ 230 .filter(models.BuildsStatistics.time == time).first() 231 if result: 232 return 233 234 try: 235 cached_data = models.BuildsStatistics( 236 time = time, 237 stat_type = type, 238 running = running, 239 pending = pending 240 ) 241 db.session.add(cached_data) 242 db.session.commit() 243 except IntegrityError: # other process already calculated the graph data and cached it 244 db.session.rollback()
245 246 @classmethod
247 - def get_graph_parameters(cls, type):
248 if type is "10min": 249 # 24 hours with 10 minute intervals 250 step = 600 251 steps = 144 252 elif type is "30min": 253 # 24 hours with 30 minute intervals 254 step = 1800 255 steps = 48 256 elif type is "24h": 257 # 90 days with 24 hour intervals 258 step = 86400 259 steps = 90 260 261 end = int(time.time()) 262 end = end - (end % step) # align graph interval to a multiple of step 263 start = end - (steps * step) 264 265 return { 266 "type": type, 267 "step": step, 268 "steps": steps, 269 "start": start, 270 "end": end, 271 }
272 273 @classmethod
274 - def get_build_importing_queue(cls, background=None):
275 """ 276 Returns Builds which are waiting to be uploaded to dist git 277 """ 278 query = (models.Build.query 279 .filter(models.Build.canceled == false()) 280 .filter(models.Build.source_status == StatusEnum("importing")) 281 .order_by(models.Build.id.asc())) 282 if background is not None: 283 query = query.filter(models.Build.is_background == (true() if background else false())) 284 return query
285 286 @classmethod
287 - def get_pending_srpm_build_tasks(cls, background=None):
288 query = (models.Build.query 289 .filter(models.Build.canceled == false()) 290 .filter(models.Build.source_status == StatusEnum("pending")) 291 .order_by(models.Build.is_background.asc(), models.Build.id.asc())) 292 if background is not None: 293 query = query.filter(models.Build.is_background == (true() if background else false())) 294 return query
295 296 @classmethod
297 - def get_pending_build_tasks(cls, background=None):
298 query = (models.BuildChroot.query 299 .outerjoin(models.Build) 300 .outerjoin(models.CoprDir) 301 .outerjoin(models.Package, models.Package.id == models.Build.package_id) 302 .options(joinedload('build').joinedload('copr_dir'), 303 joinedload('build').joinedload('package')) 304 .filter(models.Build.canceled == false()) 305 .filter(or_( 306 models.BuildChroot.status == StatusEnum("pending"), 307 and_( 308 models.BuildChroot.status == StatusEnum("running"), 309 models.BuildChroot.started_on < int(time.time() - 1.1 * MAX_BUILD_TIMEOUT), 310 models.BuildChroot.ended_on.is_(None) 311 ) 312 )) 313 .order_by(models.Build.is_background.asc(), models.Build.id.asc())) 314 if background is not None: 315 query = query.filter(models.Build.is_background == (true() if background else false())) 316 return query
317 318 @classmethod
319 - def get_build_task(cls, task_id):
320 try: 321 build_id, chroot_name = task_id.split("-", 1) 322 except ValueError: 323 raise MalformedArgumentException("Invalid task_id {}".format(task_id)) 324 325 build_chroot = BuildChrootsLogic.get_by_build_id_and_name(build_id, chroot_name) 326 return build_chroot.join(models.Build).first()
327 328 @classmethod
329 - def get_srpm_build_task(cls, build_id):
330 return BuildsLogic.get_by_id(build_id).first()
331 332 @classmethod
333 - def get_multiple(cls):
334 return models.Build.query.order_by(models.Build.id.desc())
335 336 @classmethod
337 - def get_multiple_by_copr(cls, copr):
338 """ Get collection of builds in copr sorted by build_id descending 339 """ 340 return cls.get_multiple().filter(models.Build.copr == copr)
341 342 @classmethod
343 - def get_multiple_by_user(cls, user):
344 """ Get collection of builds in copr sorted by build_id descending 345 form the copr belonging to `user` 346 """ 347 return cls.get_multiple().join(models.Build.copr).filter( 348 models.Copr.user == user)
349 350 @classmethod
351 - def init_db(cls):
352 if db.engine.url.drivername == "sqlite": 353 return 354 355 status_to_order = """ 356 CREATE OR REPLACE FUNCTION status_to_order (x integer) 357 RETURNS integer AS $$ BEGIN 358 RETURN CASE WHEN x = 3 THEN 1 359 WHEN x = 6 THEN 2 360 WHEN x = 7 THEN 3 361 WHEN x = 4 THEN 4 362 WHEN x = 0 THEN 5 363 WHEN x = 1 THEN 6 364 WHEN x = 5 THEN 7 365 WHEN x = 2 THEN 8 366 WHEN x = 8 THEN 9 367 WHEN x = 9 THEN 10 368 ELSE x 369 END; END; 370 $$ LANGUAGE plpgsql; 371 """ 372 373 order_to_status = """ 374 CREATE OR REPLACE FUNCTION order_to_status (x integer) 375 RETURNS integer AS $$ BEGIN 376 RETURN CASE WHEN x = 1 THEN 3 377 WHEN x = 2 THEN 6 378 WHEN x = 3 THEN 7 379 WHEN x = 4 THEN 4 380 WHEN x = 5 THEN 0 381 WHEN x = 6 THEN 1 382 WHEN x = 7 THEN 5 383 WHEN x = 8 THEN 2 384 WHEN x = 9 THEN 8 385 WHEN x = 10 THEN 9 386 ELSE x 387 END; END; 388 $$ LANGUAGE plpgsql; 389 """ 390 391 db.engine.connect() 392 db.engine.execute(status_to_order) 393 db.engine.execute(order_to_status)
394 395 @classmethod
396 - def get_copr_builds_list(cls, copr, dirname=''):
397 query_select = """ 398 SELECT build.id, build.source_status, MAX(package.name) AS pkg_name, build.pkg_version, build.submitted_on, 399 MIN(statuses.started_on) AS started_on, MAX(statuses.ended_on) AS ended_on, order_to_status(MIN(statuses.st)) AS status, 400 build.canceled, MIN("group".name) AS group_name, MIN(copr.name) as copr_name, MIN("user".username) as user_name, build.copr_id 401 FROM build 402 LEFT OUTER JOIN package 403 ON build.package_id = package.id 404 LEFT OUTER JOIN (SELECT build_chroot.build_id, started_on, ended_on, status_to_order(status) AS st FROM build_chroot) AS statuses 405 ON statuses.build_id=build.id 406 LEFT OUTER JOIN copr 407 ON copr.id = build.copr_id 408 LEFT OUTER JOIN copr_dir 409 ON build.copr_dir_id = copr_dir.id 410 LEFT OUTER JOIN "user" 411 ON copr.user_id = "user".id 412 LEFT OUTER JOIN "group" 413 ON copr.group_id = "group".id 414 WHERE build.copr_id = :copr_id 415 AND (:dirname = '' OR :dirname = copr_dir.name) 416 GROUP BY 417 build.id 418 ORDER BY 419 build.id DESC; 420 """ 421 422 if db.engine.url.drivername == "sqlite": 423 def sqlite_status_to_order(x): 424 if x == 3: 425 return 1 426 elif x == 6: 427 return 2 428 elif x == 7: 429 return 3 430 elif x == 4: 431 return 4 432 elif x == 0: 433 return 5 434 elif x == 1: 435 return 6 436 elif x == 5: 437 return 7 438 elif x == 2: 439 return 8 440 elif x == 8: 441 return 9 442 elif x == 9: 443 return 10 444 return 1000
445 446 def sqlite_order_to_status(x): 447 if x == 1: 448 return 3 449 elif x == 2: 450 return 6 451 elif x == 3: 452 return 7 453 elif x == 4: 454 return 4 455 elif x == 5: 456 return 0 457 elif x == 6: 458 return 1 459 elif x == 7: 460 return 5 461 elif x == 8: 462 return 2 463 elif x == 9: 464 return 8 465 elif x == 10: 466 return 9 467 return 1000
468 469 conn = db.engine.connect() 470 conn.connection.create_function("status_to_order", 1, sqlite_status_to_order) 471 conn.connection.create_function("order_to_status", 1, sqlite_order_to_status) 472 statement = text(query_select) 473 statement.bindparams(bindparam("copr_id", Integer)) 474 statement.bindparams(bindparam("dirname", String)) 475 result = conn.execute(statement, {"copr_id": copr.id, "dirname": dirname}) 476 else: 477 statement = text(query_select) 478 statement.bindparams(bindparam("copr_id", Integer)) 479 statement.bindparams(bindparam("dirname", String)) 480 result = db.engine.execute(statement, {"copr_id": copr.id, "dirname": dirname}) 481 482 return result 483 484 @classmethod
485 - def join_group(cls, query):
486 return query.join(models.Copr).outerjoin(models.Group)
487 488 @classmethod
489 - def get_multiple_by_name(cls, username, coprname):
490 query = cls.get_multiple() 491 return (query.join(models.Build.copr) 492 .options(db.contains_eager(models.Build.copr)) 493 .join(models.Copr.user) 494 .filter(models.Copr.name == coprname) 495 .filter(models.User.username == username))
496 497 @classmethod
498 - def get_by_ids(cls, ids):
499 return models.Build.query.filter(models.Build.id.in_(ids))
500 501 @classmethod
502 - def get_by_id(cls, build_id):
503 return models.Build.query.filter(models.Build.id == build_id)
504 505 @classmethod
506 - def create_new_from_other_build(cls, user, copr, source_build, 507 chroot_names=None, **build_options):
508 skip_import = False 509 git_hashes = {} 510 511 if source_build.source_type == helpers.BuildSourceEnum('upload'): 512 if source_build.repeatable: 513 skip_import = True 514 for chroot in source_build.build_chroots: 515 git_hashes[chroot.name] = chroot.git_hash 516 else: 517 raise UnrepeatableBuildException("Build sources were not fully imported into CoprDistGit.") 518 519 build = cls.create_new(user, copr, source_build.source_type, source_build.source_json, chroot_names, 520 pkgs=source_build.pkgs, git_hashes=git_hashes, skip_import=skip_import, 521 srpm_url=source_build.srpm_url, copr_dirname=source_build.copr_dir.name, **build_options) 522 build.package_id = source_build.package_id 523 build.pkg_version = source_build.pkg_version 524 return build
525 526 @classmethod
527 - def create_new_from_url(cls, user, copr, url, chroot_names=None, 528 copr_dirname=None, **build_options):
529 """ 530 :type user: models.User 531 :type copr: models.Copr 532 533 :type chroot_names: List[str] 534 535 :rtype: models.Build 536 """ 537 source_type = helpers.BuildSourceEnum("link") 538 source_json = json.dumps({"url": url}) 539 srpm_url = None if url.endswith('.spec') else url 540 return cls.create_new(user, copr, source_type, source_json, chroot_names, 541 pkgs=url, srpm_url=srpm_url, copr_dirname=copr_dirname, **build_options)
542 543 @classmethod
544 - def create_new_from_scm(cls, user, copr, scm_type, clone_url, 545 committish='', subdirectory='', spec='', srpm_build_method='rpkg', 546 chroot_names=None, copr_dirname=None, **build_options):
547 """ 548 :type user: models.User 549 :type copr: models.Copr 550 551 :type chroot_names: List[str] 552 553 :rtype: models.Build 554 """ 555 source_type = helpers.BuildSourceEnum("scm") 556 source_json = json.dumps({"type": scm_type, 557 "clone_url": clone_url, 558 "committish": committish, 559 "subdirectory": subdirectory, 560 "spec": spec, 561 "srpm_build_method": srpm_build_method}) 562 return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
563 564 @classmethod
565 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, spec_template, 566 python_versions, chroot_names=None, copr_dirname=None, **build_options):
567 """ 568 :type user: models.User 569 :type copr: models.Copr 570 :type package_name: str 571 :type version: str 572 :type python_versions: List[str] 573 574 :type chroot_names: List[str] 575 576 :rtype: models.Build 577 """ 578 source_type = helpers.BuildSourceEnum("pypi") 579 source_json = json.dumps({"pypi_package_name": pypi_package_name, 580 "pypi_package_version": pypi_package_version, 581 "spec_template": spec_template, 582 "python_versions": python_versions}) 583 return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
584 585 @classmethod
586 - def create_new_from_rubygems(cls, user, copr, gem_name, chroot_names=None, 587 copr_dirname=None, **build_options):
588 """ 589 :type user: models.User 590 :type copr: models.Copr 591 :type gem_name: str 592 :type chroot_names: List[str] 593 :rtype: models.Build 594 """ 595 source_type = helpers.BuildSourceEnum("rubygems") 596 source_json = json.dumps({"gem_name": gem_name}) 597 return cls.create_new(user, copr, source_type, source_json, chroot_names, copr_dirname=copr_dirname, **build_options)
598 599 @classmethod
600 - def create_new_from_custom(cls, user, copr, script, script_chroot=None, script_builddeps=None, 601 script_resultdir=None, chroot_names=None, copr_dirname=None, **kwargs):
602 """ 603 :type user: models.User 604 :type copr: models.Copr 605 :type script: str 606 :type script_chroot: str 607 :type script_builddeps: str 608 :type script_resultdir: str 609 :type chroot_names: List[str] 610 :rtype: models.Build 611 """ 612 source_type = helpers.BuildSourceEnum("custom") 613 source_dict = { 614 'script': script, 615 'chroot': script_chroot, 616 'builddeps': script_builddeps, 617 'resultdir': script_resultdir, 618 } 619 620 return cls.create_new(user, copr, source_type, json.dumps(source_dict), 621 chroot_names, copr_dirname=copr_dirname, **kwargs)
622 623 @classmethod
624 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename, 625 chroot_names=None, copr_dirname=None, **build_options):
626 """ 627 :type user: models.User 628 :type copr: models.Copr 629 :param f_uploader(file_path): function which stores data at the given `file_path` 630 :return: 631 """ 632 tmp = tempfile.mkdtemp(dir=app.config["STORAGE_DIR"]) 633 tmp_name = os.path.basename(tmp) 634 filename = secure_filename(orig_filename) 635 file_path = os.path.join(tmp, filename) 636 f_uploader(file_path) 637 638 # make the pkg public 639 pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format( 640 baseurl=app.config["PUBLIC_COPR_BASE_URL"], 641 tmp_dir=tmp_name, 642 filename=filename) 643 644 # create json describing the build source 645 source_type = helpers.BuildSourceEnum("upload") 646 source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name}) 647 srpm_url = None if pkg_url.endswith('.spec') else pkg_url 648 649 try: 650 build = cls.create_new(user, copr, source_type, source_json, 651 chroot_names, pkgs=pkg_url, srpm_url=srpm_url, 652 copr_dirname=copr_dirname, **build_options) 653 except Exception: 654 shutil.rmtree(tmp) # todo: maybe we should delete in some cleanup procedure? 655 raise 656 657 return build
658 659 @classmethod
660 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None, pkgs="", 661 git_hashes=None, skip_import=False, background=False, batch=None, 662 srpm_url=None, copr_dirname=None, **build_options):
663 """ 664 :type user: models.User 665 :type copr: models.Copr 666 :type chroot_names: List[str] 667 :type source_type: int value from helpers.BuildSourceEnum 668 :type source_json: str in json format 669 :type pkgs: str 670 :type git_hashes: dict 671 :type skip_import: bool 672 :type background: bool 673 :type batch: models.Batch 674 :rtype: models.Build 675 """ 676 chroots = None 677 if chroot_names: 678 chroots = [] 679 for chroot in copr.active_chroots: 680 if chroot.name in chroot_names: 681 chroots.append(chroot) 682 683 build = cls.add( 684 user=user, 685 pkgs=pkgs, 686 copr=copr, 687 chroots=chroots, 688 source_type=source_type, 689 source_json=source_json, 690 enable_net=build_options.get("enable_net", copr.build_enable_net), 691 background=background, 692 git_hashes=git_hashes, 693 skip_import=skip_import, 694 batch=batch, 695 srpm_url=srpm_url, 696 copr_dirname=copr_dirname, 697 ) 698 699 if user.proven: 700 if "timeout" in build_options: 701 build.timeout = build_options["timeout"] 702 703 return build
704 705 @classmethod
706 - def add(cls, user, pkgs, copr, source_type=None, source_json=None, 707 repos=None, chroots=None, timeout=None, enable_net=True, 708 git_hashes=None, skip_import=False, background=False, batch=None, 709 srpm_url=None, copr_dirname=None):
710 711 if chroots is None: 712 chroots = [] 713 714 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action( 715 copr, "Can't build while there is an operation in progress: {action}") 716 users_logic.UsersLogic.raise_if_cant_build_in_copr( 717 user, copr, 718 "You don't have permissions to build in this copr.") 719 720 if not repos: 721 repos = copr.repos 722 723 # todo: eliminate pkgs and this check 724 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs): 725 raise MalformedArgumentException("Trying to create a build using src_pkg " 726 "with bad characters. Forgot to split?") 727 728 # just temporary to keep compatibility 729 if not source_type or not source_json: 730 source_type = helpers.BuildSourceEnum("link") 731 source_json = json.dumps({"url":pkgs}) 732 733 if skip_import and srpm_url: 734 chroot_status = StatusEnum("pending") 735 source_status = StatusEnum("succeeded") 736 else: 737 chroot_status = StatusEnum("waiting") 738 source_status = StatusEnum("pending") 739 740 copr_dir = None 741 if copr_dirname: 742 if not copr_dirname.startswith(copr.name+':') and copr_dirname != copr.name: 743 raise MalformedArgumentException("Copr dirname not starting with copr name.") 744 copr_dir = coprs_logic.CoprDirsLogic.get_or_create(copr, copr_dirname) 745 746 build = models.Build( 747 user=user, 748 pkgs=pkgs, 749 copr=copr, 750 repos=repos, 751 source_type=source_type, 752 source_json=source_json, 753 source_status=source_status, 754 submitted_on=int(time.time()), 755 enable_net=bool(enable_net), 756 is_background=bool(background), 757 batch=batch, 758 srpm_url=srpm_url, 759 copr_dir=copr_dir, 760 ) 761 762 if timeout: 763 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT 764 765 db.session.add(build) 766 767 for chroot in chroots: 768 # Chroots were explicitly set per-build. 769 git_hash = None 770 if git_hashes: 771 git_hash = git_hashes.get(chroot.name) 772 buildchroot = models.BuildChroot( 773 build=build, 774 status=chroot_status, 775 mock_chroot=chroot, 776 git_hash=git_hash, 777 ) 778 db.session.add(buildchroot) 779 780 return build
781 782 @classmethod
783 - def rebuild_package(cls, package, source_dict_update={}, copr_dir=None, update_callback=None, 784 scm_object_type=None, scm_object_id=None, 785 scm_object_url=None, submitted_by=None):
786 787 source_dict = package.source_json_dict 788 source_dict.update(source_dict_update) 789 source_json = json.dumps(source_dict) 790 791 if not copr_dir: 792 copr_dir = package.copr.main_dir 793 794 build = models.Build( 795 user=None, 796 pkgs=None, 797 package=package, 798 copr=package.copr, 799 repos=package.copr.repos, 800 source_status=StatusEnum("pending"), 801 source_type=package.source_type, 802 source_json=source_json, 803 submitted_on=int(time.time()), 804 enable_net=package.copr.build_enable_net, 805 timeout=DEFAULT_BUILD_TIMEOUT, 806 copr_dir=copr_dir, 807 update_callback=update_callback, 808 scm_object_type=scm_object_type, 809 scm_object_id=scm_object_id, 810 scm_object_url=scm_object_url, 811 submitted_by=submitted_by, 812 ) 813 db.session.add(build) 814 815 status = StatusEnum("waiting") 816 for chroot in package.chroots: 817 buildchroot = models.BuildChroot( 818 build=build, 819 status=status, 820 mock_chroot=chroot, 821 git_hash=None 822 ) 823 db.session.add(buildchroot) 824 825 cls.process_update_callback(build) 826 return build
827 828 829 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")} 830 831 @classmethod
832 - def get_buildchroots_by_build_id_and_branch(cls, build_id, branch):
833 """ 834 Returns a list of BuildChroots identified by build_id and dist-git 835 branch name. 836 """ 837 return ( 838 models.BuildChroot.query 839 .join(models.MockChroot) 840 .filter(models.BuildChroot.build_id==build_id) 841 .filter(models.MockChroot.distgit_branch_name==branch) 842 ).all()
843 844 845 @classmethod
846 - def delete_local_source(cls, build):
847 """ 848 Deletes the locally stored data for build purposes. This is typically 849 uploaded srpm file, uploaded spec file or webhook POST content. 850 """ 851 # is it hosted on the copr frontend? 852 data = json.loads(build.source_json) 853 if 'tmp' in data: 854 tmp = data["tmp"] 855 storage_path = app.config["STORAGE_DIR"] 856 try: 857 shutil.rmtree(os.path.join(storage_path, tmp)) 858 except: 859 pass
860 861 862 @classmethod
863 - def update_state_from_dict(cls, build, upd_dict):
864 """ 865 :param build: 866 :param upd_dict: 867 example: 868 { 869 "builds":[ 870 { 871 "id": 1, 872 "copr_id": 2, 873 "started_on": 1390866440 874 }, 875 { 876 "id": 2, 877 "copr_id": 1, 878 "status": 0, 879 "chroot": "fedora-18-x86_64", 880 "result_dir": "baz", 881 "ended_on": 1390866440 882 }] 883 } 884 """ 885 log.info("Updating build {} by: {}".format(build.id, upd_dict)) 886 887 # create the package if it doesn't exist 888 pkg_name = upd_dict.get('pkg_name', None) 889 if pkg_name: 890 if not PackagesLogic.get(build.copr_dir.id, pkg_name).first(): 891 try: 892 package = PackagesLogic.add( 893 build.copr.user, build.copr_dir, 894 pkg_name, build.source_type, build.source_json) 895 db.session.add(package) 896 db.session.commit() 897 except (IntegrityError, DuplicateException) as e: 898 app.logger.exception(e) 899 db.session.rollback() 900 return 901 build.package = PackagesLogic.get(build.copr_dir.id, pkg_name).first() 902 903 for attr in ["built_packages", "srpm_url", "pkg_version"]: 904 value = upd_dict.get(attr, None) 905 if value: 906 setattr(build, attr, value) 907 908 # update source build status 909 if str(upd_dict.get("task_id")) == str(build.task_id): 910 build.result_dir = upd_dict.get("result_dir", "") 911 912 new_status = upd_dict.get("status") 913 if new_status == StatusEnum("succeeded"): 914 new_status = StatusEnum("importing") 915 chroot_status=StatusEnum("waiting") 916 if not build.build_chroots: 917 # create the BuildChroots from Package setting, if not 918 # already set explicitly for concrete build 919 for chroot in build.package.chroots: 920 buildchroot = models.BuildChroot( 921 build=build, 922 status=chroot_status, 923 mock_chroot=chroot, 924 git_hash=None, 925 ) 926 db.session.add(buildchroot) 927 else: 928 for buildchroot in build.build_chroots: 929 buildchroot.status = chroot_status 930 db.session.add(buildchroot) 931 932 build.source_status = new_status 933 if new_status == StatusEnum("failed") or \ 934 new_status == StatusEnum("skipped"): 935 for ch in build.build_chroots: 936 ch.status = new_status 937 ch.ended_on = upd_dict.get("ended_on") or time.time() 938 db.session.add(ch) 939 940 if new_status == StatusEnum("failed"): 941 build.fail_type = FailTypeEnum("srpm_build_error") 942 943 cls.process_update_callback(build) 944 db.session.add(build) 945 return 946 947 if "chroot" in upd_dict: 948 # update respective chroot status 949 for build_chroot in build.build_chroots: 950 if build_chroot.name == upd_dict["chroot"]: 951 build_chroot.result_dir = upd_dict.get("result_dir", "") 952 953 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states: 954 build_chroot.status = upd_dict["status"] 955 956 if upd_dict.get("status") in BuildsLogic.terminal_states: 957 build_chroot.ended_on = upd_dict.get("ended_on") or time.time() 958 959 if upd_dict.get("status") == StatusEnum("starting"): 960 build_chroot.started_on = upd_dict.get("started_on") or time.time() 961 962 db.session.add(build_chroot) 963 964 # If the last package of a module was successfully built, 965 # then send an action to create module repodata on backend 966 if (build.module 967 and upd_dict.get("status") == StatusEnum("succeeded") 968 and all(b.status == StatusEnum("succeeded") for b in build.module.builds)): 969 ActionsLogic.send_build_module(build.copr, build.module) 970 971 cls.process_update_callback(build) 972 db.session.add(build)
973 974 @classmethod
975 - def process_update_callback(cls, build):
976 parsed_git_url = helpers.get_parsed_git_url(build.copr.scm_repo_url) 977 if not parsed_git_url: 978 return 979 980 if build.update_callback == 'pagure_flag_pull_request': 981 api_url = 'https://{0}/api/0/{1}/pull-request/{2}/flag'.format( 982 parsed_git_url.netloc, parsed_git_url.path, build.scm_object_id) 983 return cls.pagure_flag(build, api_url) 984 985 elif build.update_callback == 'pagure_flag_commit': 986 api_url = 'https://{0}/api/0/{1}/c/{2}/flag'.format( 987 parsed_git_url.netloc, parsed_git_url.path, build.scm_object_id) 988 return cls.pagure_flag(build, api_url)
989 990 @classmethod
991 - def pagure_flag(cls, build, api_url):
992 headers = { 993 'Authorization': 'token {}'.format(build.copr.scm_api_auth.get('api_key')) 994 } 995 996 if build.srpm_url: 997 progress = 50 998 else: 999 progress = 10 1000 1001 state_table = { 1002 'failed': ('failure', 0), 1003 'succeeded': ('success', 100), 1004 'canceled': ('canceled', 0), 1005 'running': ('pending', progress), 1006 'pending': ('pending', progress), 1007 'skipped': ('error', 0), 1008 'starting': ('pending', progress), 1009 'importing': ('pending', progress), 1010 'forked': ('error', 0), 1011 'waiting': ('pending', progress), 1012 'unknown': ('error', 0), 1013 } 1014 1015 build_url = os.path.join( 1016 app.config['PUBLIC_COPR_BASE_URL'], 1017 'coprs', build.copr.full_name.replace('@', 'g/'), 1018 'build', str(build.id) 1019 ) 1020 1021 data = { 1022 'username': 'Copr build', 1023 'comment': '#{}'.format(build.id), 1024 'url': build_url, 1025 'status': state_table[build.state][0], 1026 'percent': state_table[build.state][1], 1027 'uid': str(build.id), 1028 } 1029 1030 log.debug('Sending data to Pagure API: %s', pprint.pformat(data)) 1031 response = requests.post(api_url, data=data, headers=headers) 1032 log.debug('Pagure API response: %s', response.text)
1033 1034 @classmethod
1035 - def cancel_build(cls, user, build):
1036 if not user.can_build_in(build.copr): 1037 raise InsufficientRightsException( 1038 "You are not allowed to cancel this build.") 1039 if not build.cancelable: 1040 if build.status == StatusEnum("starting"): 1041 # this is not intuitive, that's why we provide more specific message 1042 err_msg = "Cannot cancel build {} in state 'starting'".format(build.id) 1043 else: 1044 err_msg = "Cannot cancel build {}".format(build.id) 1045 raise RequestCannotBeExecuted(err_msg) 1046 1047 if build.status == StatusEnum("running"): # otherwise the build is just in frontend 1048 ActionsLogic.send_cancel_build(build) 1049 1050 build.canceled = True 1051 cls.process_update_callback(build) 1052 1053 for chroot in build.build_chroots: 1054 chroot.status = 2 # canceled 1055 if chroot.ended_on is not None: 1056 chroot.ended_on = time.time()
1057 1058 @classmethod
1059 - def check_build_to_delete(cls, user, build):
1060 """ 1061 :type user: models.User 1062 :type build: models.Build 1063 """ 1064 if not user.can_edit(build.copr) or build.persistent: 1065 raise InsufficientRightsException( 1066 "You are not allowed to delete build `{}`.".format(build.id)) 1067 1068 if not build.finished: 1069 raise ActionInProgressException( 1070 "You can not delete build `{}` which is not finished.".format(build.id), 1071 "Unfinished build")
1072 1073 @classmethod
1074 - def delete_build(cls, user, build, send_delete_action=True):
1075 """ 1076 :type user: models.User 1077 :type build: models.Build 1078 """ 1079 cls.check_build_to_delete(user, build) 1080 1081 if send_delete_action: 1082 ActionsLogic.send_delete_build(build) 1083 1084 db.session.delete(build)
1085 1086 @classmethod
1087 - def delete_multiple_builds(cls, user, builds):
1088 """ 1089 :type user: models.User 1090 :type builds: list of models.Build 1091 """ 1092 to_delete = [] 1093 for build in builds: 1094 cls.check_build_to_delete(user, build) 1095 to_delete.append(build) 1096 1097 if to_delete: 1098 ActionsLogic.send_delete_multiple_builds(to_delete) 1099 1100 for build in to_delete: 1101 for build_chroot in build.build_chroots: 1102 db.session.delete(build_chroot) 1103 1104 db.session.delete(build)
1105 1106 @classmethod
1107 - def mark_as_failed(cls, build_id):
1108 """ 1109 Marks build as failed on all its non-finished chroots 1110 """ 1111 build = cls.get(build_id).one() 1112 chroots = filter(lambda x: x.status != StatusEnum("succeeded"), build.build_chroots) 1113 for chroot in chroots: 1114 chroot.status = StatusEnum("failed") 1115 if build.source_status != StatusEnum("succeeded"): 1116 build.source_status = StatusEnum("failed") 1117 cls.process_update_callback(build) 1118 return build
1119 1120 @classmethod
1121 - def last_modified(cls, copr):
1122 """ Get build datetime (as epoch) of last successful build 1123 1124 :arg copr: object of copr 1125 """ 1126 builds = cls.get_multiple_by_copr(copr) 1127 1128 last_build = ( 1129 builds.join(models.BuildChroot) 1130 .filter((models.BuildChroot.status == StatusEnum("succeeded")) 1131 | (models.BuildChroot.status == StatusEnum("skipped"))) 1132 .filter(models.BuildChroot.ended_on.isnot(None)) 1133 .order_by(models.BuildChroot.ended_on.desc()) 1134 ).first() 1135 if last_build: 1136 return last_build.ended_on 1137 else: 1138 return None
1139 1140 @classmethod
1141 - def filter_is_finished(cls, query, is_finished):
1142 # todo: check that ended_on is set correctly for all cases 1143 # e.g.: failed dist-git import, cancellation 1144 if is_finished: 1145 return query.join(models.BuildChroot).filter(models.BuildChroot.ended_on.isnot(None)) 1146 else: 1147 return query.join(models.BuildChroot).filter(models.BuildChroot.ended_on.is_(None))
1148 1149 @classmethod
1150 - def filter_by_group_name(cls, query, group_name):
1151 return query.filter(models.Group.name == group_name)
1152 1153 @classmethod
1154 - def filter_by_package_name(cls, query, package_name):
1155 return query.join(models.Package).filter(models.Package.name == package_name)
1156 1157 @classmethod
1158 - def clean_old_builds(cls):
1159 dirs = ( 1160 db.session.query( 1161 models.CoprDir.id, 1162 models.Package.id, 1163 models.Package.max_builds) 1164 .join(models.Build, models.Build.copr_dir_id==models.CoprDir.id) 1165 .join(models.Package) 1166 .filter(models.Package.max_builds > 0) 1167 .group_by( 1168 models.CoprDir.id, 1169 models.Package.max_builds, 1170 models.Package.id) 1171 .having(func.count(models.Build.id) > models.Package.max_builds) 1172 ) 1173 1174 for dir_id, package_id, limit in dirs.all(): 1175 delete_builds = ( 1176 models.Build.query.filter( 1177 models.Build.copr_dir_id==dir_id, 1178 models.Build.package_id==package_id) 1179 .order_by(desc(models.Build.id)) 1180 .offset(limit) 1181 .all() 1182 ) 1183 1184 for build in delete_builds: 1185 try: 1186 cls.delete_build(build.copr.user, build) 1187 except ActionInProgressException: 1188 # postpone this one to next day run 1189 log.error("Build(id={}) delete failed, unfinished action.".format(build.id))
1190 1191 @classmethod
1192 - def delete_orphaned_builds(cls):
1193 builds_to_delete = models.Build.query\ 1194 .join(models.Copr, models.Build.copr_id == models.Copr.id)\ 1195 .filter(models.Copr.deleted == True) 1196 1197 counter = 0 1198 for build in builds_to_delete: 1199 cls.delete_build(build.copr.user, build) 1200 counter += 1 1201 if counter >= 100: 1202 db.session.commit() 1203 counter = 0 1204 1205 if counter > 0: 1206 db.session.commit()
1207
1208 1209 -class BuildChrootsLogic(object):
1210 @classmethod
1211 - def get_by_build_id_and_name(cls, build_id, name):
1212 mc = MockChrootsLogic.get_from_name(name).one() 1213 1214 return ( 1215 BuildChroot.query 1216 .filter(BuildChroot.build_id == build_id) 1217 .filter(BuildChroot.mock_chroot_id == mc.id) 1218 )
1219 1220 @classmethod
1221 - def get_multiply(cls):
1222 query = ( 1223 models.BuildChroot.query 1224 .join(models.BuildChroot.build) 1225 .join(models.BuildChroot.mock_chroot) 1226 .join(models.Build.copr) 1227 .join(models.Copr.user) 1228 .outerjoin(models.Group) 1229 ) 1230 return query
1231 1232 @classmethod
1233 - def filter_by_build_id(cls, query, build_id):
1234 return query.filter(models.Build.id == build_id)
1235 1236 @classmethod
1237 - def filter_by_project_id(cls, query, project_id):
1238 return query.filter(models.Copr.id == project_id)
1239 1240 @classmethod
1241 - def filter_by_project_user_name(cls, query, username):
1242 return query.filter(models.User.username == username)
1243 1244 @classmethod
1245 - def filter_by_state(cls, query, state):
1246 return query.filter(models.BuildChroot.status == StatusEnum(state))
1247 1248 @classmethod
1249 - def filter_by_group_name(cls, query, group_name):
1250 return query.filter(models.Group.name == group_name)
1251
1252 1253 -class BuildsMonitorLogic(object):
1254 @classmethod
1255 - def get_monitor_data(cls, copr):
1256 query = """ 1257 SELECT 1258 package.id as package_id, 1259 package.name AS package_name, 1260 build.id AS build_id, 1261 build_chroot.status AS build_chroot_status, 1262 build.pkg_version AS build_pkg_version, 1263 mock_chroot.id AS mock_chroot_id, 1264 mock_chroot.os_release AS mock_chroot_os_release, 1265 mock_chroot.os_version AS mock_chroot_os_version, 1266 mock_chroot.arch AS mock_chroot_arch 1267 FROM package 1268 JOIN (SELECT 1269 MAX(build.id) AS max_build_id_for_chroot, 1270 build.package_id AS package_id, 1271 build_chroot.mock_chroot_id AS mock_chroot_id 1272 FROM build 1273 JOIN build_chroot 1274 ON build.id = build_chroot.build_id 1275 WHERE build.copr_id = {copr_id} 1276 AND build_chroot.status != 2 1277 GROUP BY build.package_id, 1278 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot 1279 ON package.id = max_build_ids_for_a_chroot.package_id 1280 JOIN build 1281 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot 1282 JOIN build_chroot 1283 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id 1284 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot 1285 JOIN mock_chroot 1286 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id 1287 JOIN copr_dir ON build.copr_dir_id=copr_dir.id WHERE copr_dir.main IS TRUE 1288 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC 1289 """.format(copr_id=copr.id) 1290 rows = db.session.execute(query) 1291 return rows
1292