1 import tempfile
2 import shutil
3 import json
4 import os
5 import pprint
6 import time
7 import flask
8 import sqlite3
9 from sqlalchemy.sql import text
10 from sqlalchemy import or_
11 from sqlalchemy import and_
12 from sqlalchemy.orm import joinedload
13 from sqlalchemy.orm.exc import NoResultFound
14 from sqlalchemy.sql import false,true
15 from werkzeug.utils import secure_filename
16 from sqlalchemy import desc,asc, bindparam, Integer
17 from collections import defaultdict
18
19 from coprs import app
20 from coprs import db
21 from coprs import exceptions
22 from coprs import models
23 from coprs import helpers
24 from coprs.constants import DEFAULT_BUILD_TIMEOUT, MAX_BUILD_TIMEOUT
25 from coprs.exceptions import MalformedArgumentException, ActionInProgressException, InsufficientRightsException, UnrepeatableBuildException
26 from coprs.helpers import StatusEnum
27
28 from coprs.logic import coprs_logic
29 from coprs.logic import users_logic
30 from coprs.logic.actions_logic import ActionsLogic
31 from coprs.models import BuildChroot,Build,Package,MockChroot
32 from .coprs_logic import MockChrootsLogic
33
34 log = app.logger
38 @classmethod
39 - def get(cls, build_id):
41
42 @classmethod
53
54 @classmethod
65
66 @classmethod
85
86 @classmethod
93
94 @classmethod
102
103 @classmethod
105 end = int(time.time())
106 start = end - 86399
107 step = 3600
108 tasks = cls.get_running_tasks_by_time(start, end)
109 steps = int(round((end - start) / step + 0.5))
110 current_step = 0
111
112 data = [[0] * (steps + 1)]
113 data[0][0] = ''
114 for t in tasks:
115 task = t.to_dict()
116 while task['started_on'] > start + step * (current_step + 1):
117 current_step += 1
118 data[0][current_step + 1] += 1
119 return data
120
121 @classmethod
133
134 @classmethod
143
144 @classmethod
160
161 @classmethod
170
171 @classmethod
174
175 @classmethod
178
179 @classmethod
184
185 @classmethod
192
193 @classmethod
195 if db.engine.url.drivername == "sqlite":
196 return
197
198 status_to_order = """
199 CREATE OR REPLACE FUNCTION status_to_order (x integer)
200 RETURNS integer AS $$ BEGIN
201 RETURN CASE WHEN x = 3 THEN 1
202 WHEN x = 6 THEN 2
203 WHEN x = 7 THEN 3
204 WHEN x = 4 THEN 4
205 WHEN x = 0 THEN 5
206 WHEN x = 1 THEN 6
207 WHEN x = 5 THEN 7
208 WHEN x = 2 THEN 8
209 WHEN x = 8 THEN 9
210 WHEN x = 9 THEN 10
211 ELSE x
212 END; END;
213 $$ LANGUAGE plpgsql;
214 """
215
216 order_to_status = """
217 CREATE OR REPLACE FUNCTION order_to_status (x integer)
218 RETURNS integer AS $$ BEGIN
219 RETURN CASE WHEN x = 1 THEN 3
220 WHEN x = 2 THEN 6
221 WHEN x = 3 THEN 7
222 WHEN x = 4 THEN 4
223 WHEN x = 5 THEN 0
224 WHEN x = 6 THEN 1
225 WHEN x = 7 THEN 5
226 WHEN x = 8 THEN 2
227 WHEN x = 9 THEN 8
228 WHEN x = 10 THEN 9
229 ELSE x
230 END; END;
231 $$ LANGUAGE plpgsql;
232 """
233
234 db.engine.connect()
235 db.engine.execute(status_to_order)
236 db.engine.execute(order_to_status)
237
238 @classmethod
240 query_select = """
241 SELECT build.id, build.source_status, MAX(package.name) AS pkg_name, build.pkg_version, build.submitted_on,
242 MIN(statuses.started_on) AS started_on, MAX(statuses.ended_on) AS ended_on, order_to_status(MIN(statuses.st)) AS status,
243 build.canceled, MIN("group".name) AS group_name, MIN(copr.name) as copr_name, MIN("user".username) as user_name
244 FROM build
245 LEFT OUTER JOIN package
246 ON build.package_id = package.id
247 LEFT OUTER JOIN (SELECT build_chroot.build_id, started_on, ended_on, status_to_order(status) AS st FROM build_chroot) AS statuses
248 ON statuses.build_id=build.id
249 LEFT OUTER JOIN copr
250 ON copr.id = build.copr_id
251 LEFT OUTER JOIN "user"
252 ON copr.user_id = "user".id
253 LEFT OUTER JOIN "group"
254 ON copr.group_id = "group".id
255 WHERE build.copr_id = :copr_id
256 GROUP BY
257 build.id;
258 """
259
260 if db.engine.url.drivername == "sqlite":
261 def sqlite_status_to_order(x):
262 if x == 3:
263 return 1
264 elif x == 6:
265 return 2
266 elif x == 7:
267 return 3
268 elif x == 4:
269 return 4
270 elif x == 0:
271 return 5
272 elif x == 1:
273 return 6
274 elif x == 5:
275 return 7
276 elif x == 2:
277 return 8
278 elif x == 8:
279 return 9
280 elif x == 9:
281 return 10
282 return 1000
283
284 def sqlite_order_to_status(x):
285 if x == 1:
286 return 3
287 elif x == 2:
288 return 6
289 elif x == 3:
290 return 7
291 elif x == 4:
292 return 4
293 elif x == 5:
294 return 0
295 elif x == 6:
296 return 1
297 elif x == 7:
298 return 5
299 elif x == 8:
300 return 2
301 elif x == 9:
302 return 8
303 elif x == 10:
304 return 9
305 return 1000
306
307 conn = db.engine.connect()
308 conn.connection.create_function("status_to_order", 1, sqlite_status_to_order)
309 conn.connection.create_function("order_to_status", 1, sqlite_order_to_status)
310 statement = text(query_select)
311 statement.bindparams(bindparam("copr_id", Integer))
312 result = conn.execute(statement, {"copr_id": copr.id})
313 else:
314 statement = text(query_select)
315 statement.bindparams(bindparam("copr_id", Integer))
316 result = db.engine.execute(statement, {"copr_id": copr.id})
317
318 return result
319
320 @classmethod
323
324 @classmethod
332
333 @classmethod
336
337 @classmethod
340
341 @classmethod
361
362 @classmethod
378
379 @classmethod
380 - def create_new_from_scm(cls, user, copr, scm_type, clone_url,
381 committish='', subdirectory='', spec='', srpm_build_method='rpkg',
382 chroot_names=None, **build_options):
383 """
384 :type user: models.User
385 :type copr: models.Copr
386
387 :type chroot_names: List[str]
388
389 :rtype: models.Build
390 """
391 source_type = helpers.BuildSourceEnum("scm")
392 source_json = json.dumps({"type": scm_type,
393 "clone_url": clone_url,
394 "committish": committish,
395 "subdirectory": subdirectory,
396 "spec": spec,
397 "srpm_build_method": srpm_build_method})
398 return cls.create_new(user, copr, source_type, source_json, chroot_names, **build_options)
399
400 @classmethod
401 - def create_new_from_pypi(cls, user, copr, pypi_package_name, pypi_package_version, python_versions,
402 chroot_names=None, **build_options):
419
420 @classmethod
433
434 @classmethod
435 - def create_new_from_custom(cls, user, copr,
436 script, script_chroot=None, script_builddeps=None,
437 script_resultdir=None, chroot_names=None, **kwargs):
438 """
439 :type user: models.User
440 :type copr: models.Copr
441 :type script: str
442 :type script_chroot: str
443 :type script_builddeps: str
444 :type script_resultdir: str
445 :type chroot_names: List[str]
446 :rtype: models.Build
447 """
448 source_type = helpers.BuildSourceEnum("custom")
449 source_dict = {
450 'script': script,
451 'chroot': script_chroot,
452 'builddeps': script_builddeps,
453 'resultdir': script_resultdir,
454 }
455
456 return cls.create_new(user, copr, source_type, json.dumps(source_dict),
457 chroot_names, **kwargs)
458
459 @classmethod
460 - def create_new_from_upload(cls, user, copr, f_uploader, orig_filename,
461 chroot_names=None, **build_options):
462 """
463 :type user: models.User
464 :type copr: models.Copr
465 :param f_uploader(file_path): function which stores data at the given `file_path`
466 :return:
467 """
468 tmp = tempfile.mkdtemp(dir=app.config["STORAGE_DIR"])
469 tmp_name = os.path.basename(tmp)
470 filename = secure_filename(orig_filename)
471 file_path = os.path.join(tmp, filename)
472 f_uploader(file_path)
473
474
475 pkg_url = "{baseurl}/tmp/{tmp_dir}/{filename}".format(
476 baseurl=app.config["PUBLIC_COPR_BASE_URL"],
477 tmp_dir=tmp_name,
478 filename=filename)
479
480
481 source_type = helpers.BuildSourceEnum("upload")
482 source_json = json.dumps({"url": pkg_url, "pkg": filename, "tmp": tmp_name})
483 srpm_url = None if pkg_url.endswith('.spec') else pkg_url
484
485 try:
486 build = cls.create_new(user, copr, source_type, source_json,
487 chroot_names, pkgs=pkg_url, srpm_url=srpm_url, **build_options)
488 except Exception:
489 shutil.rmtree(tmp)
490 raise
491
492 return build
493
494 @classmethod
495 - def create_new(cls, user, copr, source_type, source_json, chroot_names=None, pkgs="",
496 git_hashes=None, skip_import=False, background=False, batch=None,
497 srpm_url=None, **build_options):
498 """
499 :type user: models.User
500 :type copr: models.Copr
501 :type chroot_names: List[str]
502 :type source_type: int value from helpers.BuildSourceEnum
503 :type source_json: str in json format
504 :type pkgs: str
505 :type git_hashes: dict
506 :type skip_import: bool
507 :type background: bool
508 :type batch: models.Batch
509 :rtype: models.Build
510 """
511 if chroot_names is None:
512 chroots = [c for c in copr.active_chroots]
513 else:
514 chroots = []
515 for chroot in copr.active_chroots:
516 if chroot.name in chroot_names:
517 chroots.append(chroot)
518
519 build = cls.add(
520 user=user,
521 pkgs=pkgs,
522 copr=copr,
523 chroots=chroots,
524 source_type=source_type,
525 source_json=source_json,
526 enable_net=build_options.get("enable_net", copr.build_enable_net),
527 background=background,
528 git_hashes=git_hashes,
529 skip_import=skip_import,
530 batch=batch,
531 srpm_url=srpm_url,
532 )
533
534 if user.proven:
535 if "timeout" in build_options:
536 build.timeout = build_options["timeout"]
537
538 return build
539
540 @classmethod
541 - def add(cls, user, pkgs, copr, source_type=None, source_json=None,
542 repos=None, chroots=None, timeout=None, enable_net=True,
543 git_hashes=None, skip_import=False, background=False, batch=None,
544 srpm_url=None):
545
546 if chroots is None:
547 chroots = []
548
549 coprs_logic.CoprsLogic.raise_if_unfinished_blocking_action(
550 copr, "Can't build while there is an operation in progress: {action}")
551 users_logic.UsersLogic.raise_if_cant_build_in_copr(
552 user, copr,
553 "You don't have permissions to build in this copr.")
554
555 if not repos:
556 repos = copr.repos
557
558
559 if pkgs and (" " in pkgs or "\n" in pkgs or "\t" in pkgs or pkgs.strip() != pkgs):
560 raise exceptions.MalformedArgumentException("Trying to create a build using src_pkg "
561 "with bad characters. Forgot to split?")
562
563
564 if not source_type or not source_json:
565 source_type = helpers.BuildSourceEnum("link")
566 source_json = json.dumps({"url":pkgs})
567
568 if skip_import and srpm_url:
569 chroot_status = StatusEnum("pending")
570 source_status = StatusEnum("succeeded")
571 elif srpm_url:
572 chroot_status = StatusEnum("waiting")
573 source_status = StatusEnum("importing")
574 else:
575 chroot_status = StatusEnum("waiting")
576 source_status = StatusEnum("pending")
577
578 build = models.Build(
579 user=user,
580 pkgs=pkgs,
581 copr=copr,
582 repos=repos,
583 source_type=source_type,
584 source_json=source_json,
585 source_status=source_status,
586 submitted_on=int(time.time()),
587 enable_net=bool(enable_net),
588 is_background=bool(background),
589 batch=batch,
590 srpm_url=srpm_url,
591 )
592
593 if timeout:
594 build.timeout = timeout or DEFAULT_BUILD_TIMEOUT
595
596 db.session.add(build)
597
598
599
600 if not chroots:
601 chroots = copr.active_chroots
602
603 for chroot in chroots:
604 git_hash = None
605 if git_hashes:
606 git_hash = git_hashes.get(chroot.name)
607 buildchroot = models.BuildChroot(
608 build=build,
609 status=chroot_status,
610 mock_chroot=chroot,
611 git_hash=git_hash,
612 )
613 db.session.add(buildchroot)
614
615 return build
616
617 @classmethod
619 source_dict = package.source_json_dict
620 source_dict.update(source_dict_update)
621 source_json = json.dumps(source_dict)
622
623 build = models.Build(
624 user=None,
625 pkgs=None,
626 package_id=package.id,
627 copr=package.copr,
628 repos=package.copr.repos,
629 source_status=helpers.StatusEnum("pending"),
630 source_type=package.source_type,
631 source_json=source_json,
632 submitted_on=int(time.time()),
633 enable_net=package.copr.build_enable_net,
634 timeout=DEFAULT_BUILD_TIMEOUT
635 )
636
637 db.session.add(build)
638
639 chroots = package.copr.active_chroots
640
641 status = helpers.StatusEnum("waiting")
642
643 for chroot in chroots:
644 buildchroot = models.BuildChroot(
645 build=build,
646 status=status,
647 mock_chroot=chroot,
648 git_hash=None
649 )
650
651 db.session.add(buildchroot)
652
653 return build
654
655
656 terminal_states = {StatusEnum("failed"), StatusEnum("succeeded"), StatusEnum("canceled")}
657
658 @classmethod
670
671
672 @classmethod
674 """
675 Deletes the locally stored data for build purposes. This is typically
676 uploaded srpm file, uploaded spec file or webhook POST content.
677 """
678
679 data = json.loads(build.source_json)
680 if 'tmp' in data:
681 tmp = data["tmp"]
682 storage_path = app.config["STORAGE_DIR"]
683 try:
684 shutil.rmtree(os.path.join(storage_path, tmp))
685 except:
686 pass
687
688
689 @classmethod
691 """
692 :param build:
693 :param upd_dict:
694 example:
695 {
696 "builds":[
697 {
698 "id": 1,
699 "copr_id": 2,
700 "started_on": 139086644000
701 },
702 {
703 "id": 2,
704 "copr_id": 1,
705 "status": 0,
706 "chroot": "fedora-18-x86_64",
707 "result_dir": "baz",
708 "ended_on": 139086644000
709 }]
710 }
711 """
712 log.info("Updating build {} by: {}".format(build.id, upd_dict))
713
714
715 for attr in ["built_packages", "srpm_url"]:
716 value = upd_dict.get(attr, None)
717 if value:
718 setattr(build, attr, value)
719
720
721 if upd_dict.get("task_id") == build.task_id:
722 build.result_dir = upd_dict.get("result_dir", "")
723
724 if upd_dict.get("status") == StatusEnum("succeeded"):
725 new_status = StatusEnum("importing")
726 else:
727 new_status = upd_dict.get("status")
728
729 build.source_status = new_status
730 if new_status == StatusEnum("failed") or \
731 new_status == StatusEnum("skipped"):
732 for ch in build.build_chroots:
733 ch.status = new_status
734 ch.ended_on = upd_dict.get("ended_on") or time.time()
735 db.session.add(ch)
736
737 if new_status == StatusEnum("failed"):
738 build.fail_type = helpers.FailTypeEnum("srpm_build_error")
739
740 db.session.add(build)
741 return
742
743 if "chroot" in upd_dict:
744
745 for build_chroot in build.build_chroots:
746 if build_chroot.name == upd_dict["chroot"]:
747 build_chroot.result_dir = upd_dict.get("result_dir", "")
748
749 if "status" in upd_dict and build_chroot.status not in BuildsLogic.terminal_states:
750 build_chroot.status = upd_dict["status"]
751
752 if upd_dict.get("status") in BuildsLogic.terminal_states:
753 build_chroot.ended_on = upd_dict.get("ended_on") or time.time()
754
755 if upd_dict.get("status") == StatusEnum("starting"):
756 build_chroot.started_on = upd_dict.get("started_on") or time.time()
757
758 db.session.add(build_chroot)
759
760
761
762 if (build.module
763 and upd_dict.get("status") == StatusEnum("succeeded")
764 and all(b.status == StatusEnum("succeeded") for b in build.module.builds)):
765 ActionsLogic.send_build_module(build.copr, build.module)
766
767 db.session.add(build)
768
769 @classmethod
790
791 @classmethod
792 - def delete_build(cls, user, build, send_delete_action=True):
813
814 @classmethod
824
825 @classmethod
844
845 @classmethod
853
854 @classmethod
857
860 @classmethod
869
870 @classmethod
881
882 @classmethod
885
886 @classmethod
889
890 @classmethod
893
894 @classmethod
897
898 @classmethod
901
904 @classmethod
906 query = """
907 SELECT
908 package.id as package_id,
909 package.name AS package_name,
910 build.id AS build_id,
911 build_chroot.status AS build_chroot_status,
912 build.pkg_version AS build_pkg_version,
913 mock_chroot.id AS mock_chroot_id,
914 mock_chroot.os_release AS mock_chroot_os_release,
915 mock_chroot.os_version AS mock_chroot_os_version,
916 mock_chroot.arch AS mock_chroot_arch
917 FROM package
918 JOIN (SELECT
919 MAX(build.id) AS max_build_id_for_chroot,
920 build.package_id AS package_id,
921 build_chroot.mock_chroot_id AS mock_chroot_id
922 FROM build
923 JOIN build_chroot
924 ON build.id = build_chroot.build_id
925 WHERE build.copr_id = {copr_id}
926 AND build_chroot.status != 2
927 GROUP BY build.package_id,
928 build_chroot.mock_chroot_id) AS max_build_ids_for_a_chroot
929 ON package.id = max_build_ids_for_a_chroot.package_id
930 JOIN build
931 ON build.id = max_build_ids_for_a_chroot.max_build_id_for_chroot
932 JOIN build_chroot
933 ON build_chroot.mock_chroot_id = max_build_ids_for_a_chroot.mock_chroot_id
934 AND build_chroot.build_id = max_build_ids_for_a_chroot.max_build_id_for_chroot
935 JOIN mock_chroot
936 ON mock_chroot.id = max_build_ids_for_a_chroot.mock_chroot_id
937 ORDER BY package.name ASC, package.id ASC, mock_chroot.os_release ASC, mock_chroot.os_version ASC, mock_chroot.arch ASC
938 """.format(copr_id=copr.id)
939 rows = db.session.execute(query)
940 return rows
941