1
2
3 import json
4 import pprint
5 import zmq
6 import sys
7 import os
8 import logging
9 import requests
10 import re
11
12 sys.path.append(
13 os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
14 )
15
16 from coprs import db, app
17 from coprs.logic.coprs_logic import CoprsLogic
18 from coprs.logic.builds_logic import BuildsLogic
19 from coprs.logic.complex_logic import ComplexLogic
20 from coprs import helpers
21
22 try:
23 from urllib.parse import urlparse
24 except ImportError:
25 from urlparse import urlparse
26
27 SCM_SOURCE_TYPE = helpers.BuildSourceEnum("scm")
28
29 logging.basicConfig(
30 filename="{0}/build_on_pagure_commit.log".format(app.config.get("LOG_DIR")),
31 format='[%(asctime)s][%(levelname)6s]: %(message)s',
32 level=logging.DEBUG)
33
34 log = logging.getLogger(__name__)
35 log.addHandler(logging.StreamHandler(sys.stdout))
36
37 PAGURE_BASE_URL = "https://pagure.io/"
38 SRCFPO_BASE_URL = "https://src.fedoraproject.org/"
39
40 ENDPOINT = 'tcp://hub.fedoraproject.org:9940'
41
42 TOPICS = ['io.pagure.prod.pagure.git.receive', 'org.fedoraproject.prod.pagure.git.receive']
55
59
60 @classmethod
62 if db.engine.url.drivername == "sqlite":
63 placeholder = '?'
64 true = '1'
65 else:
66 placeholder = '%s'
67 true = 'true'
68
69 rows = db.engine.execute(
70 """
71 SELECT package.id AS package_id, package.source_json AS source_json, package.copr_id AS copr_id
72 FROM package
73 WHERE package.source_type = {0} AND
74 package.webhook_rebuild = {1} AND
75 package.source_json ILIKE {placeholder}
76 """.format(SCM_SOURCE_TYPE, true, placeholder=placeholder), '%'+clone_url+'%'
77 )
78 return [ScmPackage(row) for row in rows]
79
81 if not self.subdirectory or not raw_commit_text:
82 return True
83
84 for line in raw_commit_text.split('\n'):
85 match = re.search(r'^(\+\+\+|---) [ab]/(\w*)/?.*$', line)
86 if match and match.group(2).lower() == self.subdirectory.strip('./').lower():
87 return True
88
89 return False
90
93 log.debug("Setting up poller...")
94
95 ctx = zmq.Context()
96 s = ctx.socket(zmq.SUB)
97 s.connect(ENDPOINT)
98
99 for topic in TOPICS:
100 s.setsockopt_string(zmq.SUBSCRIBE, topic)
101
102 poller = zmq.Poller()
103 poller.register(s, zmq.POLLIN)
104
105 while True:
106 log.debug("Polling...")
107 evts = poller.poll(10000)
108 if not evts:
109 continue
110
111 log.debug("Receiving...")
112 _, msg_bytes = s.recv_multipart()
113 msg = msg_bytes.decode("utf-8")
114
115 log.debug("Parsing...")
116 data = json.loads(msg)
117
118 log.info("Got topic: {}".format(data['topic']))
119
120 project_url_path = data['msg']['repo']['url_path']
121 clone_url_path = data['msg']['repo']['fullname']
122 branch = data['msg']['branch']
123 start_commit = data['msg']['start_commit']
124 end_commit = data['msg']['end_commit']
125
126 if data['topic'] == 'org.fedoraproject.prod.pagure.git.receive':
127 base_url = SRCFPO_BASE_URL
128 else:
129 base_url = PAGURE_BASE_URL
130
131 clone_url = base_url + clone_url_path
132 log.info("\tclone_url = {}".format(clone_url))
133 log.info("\tbranch = {}".format(branch))
134
135 candidates = ScmPackage.get_candidates_for_rebuild(clone_url)
136 raw_commit_text = None
137 if candidates:
138
139
140 raw_commit_url = base_url + project_url_path + '/raw/' + end_commit
141 r = requests.get(raw_commit_url)
142 if r.status_code == requests.codes.ok:
143 raw_commit_text = r.text
144 else:
145 log.error("Bad http status {0} from url {1}".format(r.status_code, raw_commit_url))
146
147 for pkg in candidates:
148 log.info("Considering pkg id: {}, source_json: {}".format(pkg.pkg_id, pkg.source_json_dict))
149 if (pkg.clone_url == clone_url or pkg.clone_url == clone_url+'.git') \
150 and (not pkg.committish or branch.endswith(pkg.committish)) \
151 and pkg.is_dir_in_commit(raw_commit_text):
152 log.info("\t -> rebuilding.")
153 pkg.build(end_commit)
154 else:
155 log.info("\t -> skipping.")
156
157
158 if __name__ == '__main__':
159 while True:
160 try:
161 build_on_fedmsg_loop()
162 except KeyboardInterrupt:
163 sys.exit(1)
164 except:
165 log.exception('Error in fedmsg loop. Restarting it.')
166