copr-commits November 2014

copr-commits@lists.fedorahosted.org

2 participants
61 discussions

[copr] master: [backend] dispatcher: handle None value obtained from job_queue (a3fc2ff)
by vgologuz＠fedoraproject.org 28 Nov '14

28 Nov '14

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit a3fc2ff0b4639acfde435a8410f54037b8b669b8 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Nov 28 14:21:03 2014 +0100 [backend] dispatcher: handle None value obtained from job_queue >--------------------------------------------------------------- backend/backend/dispatcher.py | 2 ++ backend/tests/test_dispatcher.py | 12 ++++++++++++ 2 files changed, 14 insertions(+), 0 deletions(-) diff --git a/backend/backend/dispatcher.py b/backend/backend/dispatcher.py index 13fcb5d..ead03c9 100644 --- a/backend/backend/dispatcher.py +++ b/backend/backend/dispatcher.py @@ -446,6 +446,8 @@ class Worker(multiprocessing.Process): task = self.task_queue.dequeue() except TypeError: return + if not task: + return # import ipdb; ipdb.set_trace() job = BuildJob(task.data, self.opts) diff --git a/backend/tests/test_dispatcher.py b/backend/tests/test_dispatcher.py index 044500b..9f15282 100644 --- a/backend/tests/test_dispatcher.py +++ b/backend/tests/test_dispatcher.py @@ -617,6 +617,18 @@ class TestDispatcher(object): assert not self.worker.starting_build.called assert not self.worker.pkg_built_before.called + def test_obtain_job_dequeue_none_result(self, init_worker): + mc_tq = MagicMock() + self.worker.task_queue = mc_tq + self.worker.starting_build = MagicMock() + self.worker.pkg_built_before = MagicMock() + self.worker.pkg_built_before.return_value = False + + mc_tq.dequeue.return_value = None + assert self.worker.obtain_job() is None + assert not self.worker.starting_build.called + assert not self.worker.pkg_built_before.called + def test_obtain_job_on_starting_build(self, init_worker): mc_tq = MagicMock() self.worker.task_queue = mc_tq

1 0

[copr] master: [backend] split copr-be.py into more files (5c46b75)
by vgologuz＠fedoraproject.org 28 Nov '14

28 Nov '14

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 5c46b75de4667e6c0ea527ab9cbc97f50ea1c702 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Nov 28 14:04:52 2014 +0100 [backend] split copr-be.py into more files >--------------------------------------------------------------- backend/backend/daemons/__init__.py | 5 + backend/backend/daemons/backend.py | 182 +++++++++++++++++++ backend/backend/daemons/job_grab.py | 97 ++++++++++ backend/backend/daemons/log.py | 68 +++++++ backend/run/copr-be.py | 335 +---------------------------------- 5 files changed, 354 insertions(+), 333 deletions(-) diff --git a/backend/backend/daemons/__init__.py b/backend/backend/daemons/__init__.py new file mode 100644 index 0000000..d60155d --- /dev/null +++ b/backend/backend/daemons/__init__.py @@ -0,0 +1,5 @@ +# coding: utf-8 + +from .job_grab import CoprJobGrab +from .log import CoprLog +from .backend import CoprBackend, run_backend diff --git a/backend/backend/daemons/backend.py b/backend/backend/daemons/backend.py new file mode 100644 index 0000000..ff942c8 --- /dev/null +++ b/backend/backend/daemons/backend.py @@ -0,0 +1,182 @@ +# coding: utf-8 + +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import + +import grp +import multiprocessing +import os +import pwd +import signal +import sys +import time +from collections import defaultdict + +import lockfile +import daemon +from retask.queue import Queue +from retask import ConnectionError + +from ..exceptions import CoprBackendError +from ..dispatcher import Worker +from ..helpers import BackendConfigReader +from . import CoprJobGrab, CoprLog + + +class CoprBackend(object): + + """ + Core process - starts/stops/initializes workers + """ + + def __init__(self, config_file=None, ext_opts=None): + # read in config file + # put all the config items into a single self.opts bunch + + if not config_file: + raise CoprBackendError("Must specify config_file") + + self.config_file = config_file + self.ext_opts = ext_opts # to stow our cli options for read_conf() + self.workers_by_group_id = defaultdict(list) + self.max_worker_num_by_group_id = defaultdict(int) + + self.config_reader = BackendConfigReader(self.config_file, self.ext_opts) + self.opts = None + self.update_conf() + + self.lock = multiprocessing.Lock() + + self.task_queues = [] + try: + for group in self.opts.build_groups: + group_id = group["id"] + self.task_queues.append(Queue("copr-be-{0}".format(group_id))) + self.task_queues[group_id].connect() + except ConnectionError: + raise CoprBackendError( + "Could not connect to a task queue. Is Redis running?") + + # make sure there is nothing in our task queues + self.clean_task_queues() + + self.events = multiprocessing.Queue() + # event format is a dict {when:time, who:[worker|logger|job|main], + # what:str} + + # create logger + self._logger = CoprLog(self.opts, self.events) + self._logger.start() + + self.event("Starting up Job Grabber") + # create job grabber + self._jobgrab = CoprJobGrab(self.opts, self.events, self.lock) + self._jobgrab.start() + self.abort = False + + if not os.path.exists(self.opts.worker_logdir): + os.makedirs(self.opts.worker_logdir, mode=0o750) + + def event(self, what): + self.events.put({"when": time.time(), "who": "main", "what": what}) + + def update_conf(self): + self.opts = self.config_reader.read() + + def clean_task_queues(self): + try: + for queue in self.task_queues: + while queue.length: + queue.dequeue() + except ConnectionError: + raise CoprBackendError( + "Could not connect to a task queue. Is Redis running?") + + def run(self): + self.abort = False + while not self.abort: + # re-read config into opts + self.update_conf() + + for group in self.opts.build_groups: + group_id = group["id"] + self.event( + "# jobs in {0} queue: {1}" + .format(group["name"], self.task_queues[group_id].length) + ) + # this handles starting/growing the number of workers + if len(self.workers_by_group_id[group_id]) < group["max_workers"]: + self.event("Spinning up more workers") + for _ in range(group["max_workers"] - len(self.workers_by_group_id[group_id])): + self.max_worker_num_by_group_id[group_id] += 1 + w = Worker( + self.opts, self.events, + self.max_worker_num_by_group_id[group_id], + group_id, lock=self.lock + ) + + self.workers_by_group_id[group_id].append(w) + w.start() + self.event("Finished starting worker processes") + # FIXME - prune out workers + # if len(self.workers) > self.opts.num_workers: + # killnum = len(self.workers) - self.opts.num_workers + # for w in self.workers[:killnum]: + # insert a poison pill? Kill after something? I dunno. + # FIXME - if a worker bombs out - we need to check them + # and startup a new one if it happens + # check for dead workers and abort + preserved_workers = [] + for w in self.workers_by_group_id[group_id]: + if not w.is_alive(): + self.event("Worker {0} died unexpectedly".format(w.worker_num)) + if self.opts.exit_on_worker: + raise CoprBackendError( + "Worker died unexpectedly, exiting") + else: + w.terminate() # kill it with a fire + else: + preserved_workers.append(w) + self.workers_by_group_id[group_id] = preserved_workers + + time.sleep(self.opts.sleeptime) + + def terminate(self): + """ + Cleanup backend processes (just workers for now) + And also clean all task queues as they would survive copr restart + """ + + self.abort = True + for group in self.opts.build_groups: + group_id = group["id"] + for w in self.workers_by_group_id[group_id]: + self.workers_by_group_id[group_id].remove(w) + w.terminate() + self.clean_task_queues() + + +def run_backend(opts): + try: + context = daemon.DaemonContext( + pidfile=lockfile.FileLock(opts.pidfile), + gid=grp.getgrnam("copr").gr_gid, + uid=pwd.getpwnam("copr").pw_uid, + detach_process=opts.daemonize, + umask=0o22, + stderr=sys.stderr, + signal_map={ + signal.SIGTERM: "terminate", + signal.SIGHUP: "terminate", + }, + ) + with context: + cbe = CoprBackend(opts.config_file, ext_opts=opts) + cbe.run() + except (Exception, KeyboardInterrupt): + sys.stderr.write("Killing/Dying\n") + if "cbe" in locals(): + cbe.terminate() + raise diff --git a/backend/backend/daemons/job_grab.py b/backend/backend/daemons/job_grab.py new file mode 100644 index 0000000..c9accd4 --- /dev/null +++ b/backend/backend/daemons/job_grab.py @@ -0,0 +1,97 @@ +# coding: utf-8 + +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import + +import multiprocessing +import time +import setproctitle + +import requests +from retask.task import Task +from retask.queue import Queue + +from backend.actions import Action +from backend.frontend import FrontendClient + + +class CoprJobGrab(multiprocessing.Process): + + """ + Fetch jobs from the Frontend + - submit them to the jobs queue for workers + """ + + def __init__(self, opts, events, lock): + # base class initialization + multiprocessing.Process.__init__(self, name="jobgrab") + + self.opts = opts + self.events = events + self.task_queues = [] + for group in self.opts.build_groups: + self.task_queues.append(Queue("copr-be-{0}".format(group["id"]))) + self.task_queues[group["id"]].connect() + self.added_jobs = [] + self.lock = lock + + def event(self, what): + self.events.put({"when": time.time(), "who": "jobgrab", "what": what}) + + def load_tasks(self): + try: + r = requests.get( + "{0}/waiting/".format(self.opts.frontend_url), + auth=("user", self.opts.frontend_auth)) + r_json = r.json() + + except requests.RequestException as e: + self.event("Error retrieving jobs from {0}: {1}".format( + self.opts.frontend_url, e)) + return + + except ValueError as e: + self.event("Error getting JSON build list from FE {0}" + .format(e)) + return + + if "builds" in r_json and r_json["builds"]: + self.event("{0} jobs returned".format(len(r_json["builds"]))) + count = 0 + for task in r_json["builds"]: + if "task_id" in task and task["task_id"] not in self.added_jobs: + # this will ignore and throw away unconfigured architectures + # FIXME: don't do ^ + arch = task["chroot"].split("-")[2] + for group in self.opts.build_groups: + if arch in group["archs"]: + self.added_jobs.append(task["task_id"]) + task_obj = Task(task) + self.task_queues[group["id"]].enqueue(task_obj) + count += 1 + break + if count: + self.event("New jobs: %s" % count) + + if "actions" in r_json and r_json["actions"]: + self.event("{0} actions returned".format( + len(r_json["actions"]))) + + for action in r_json["actions"]: + ao = Action(self.events, action, self.lock, destdir=self.opts.destdir, + frontend_callback=FrontendClient(self.opts, self.events), + front_url=self.opts.frontend_base_url, + results_root_url=self.opts.results_baseurl) + ao.run() + + def run(self): + setproctitle.setproctitle("CoprJobGrab") + abort = False + try: + while not abort: + self.load_tasks() + time.sleep(self.opts.sleeptime) + except KeyboardInterrupt: + return diff --git a/backend/backend/daemons/log.py b/backend/backend/daemons/log.py new file mode 100644 index 0000000..f8a128c --- /dev/null +++ b/backend/backend/daemons/log.py @@ -0,0 +1,68 @@ +# coding: utf-8 + +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import + +import logging +import multiprocessing +import os +import sys +import time +import setproctitle + + +class CoprLog(multiprocessing.Process): + + """log mechanism where items from the events queue get recorded""" + + def __init__(self, opts, events): + + # base class initialization + multiprocessing.Process.__init__(self, name="logger") + + self.opts = opts + self.events = events + + logdir = os.path.dirname(self.opts.logfile) + if not os.path.exists(logdir): + os.makedirs(logdir, mode=0o750) + + def setup_log_handler(self): + sys.stderr.write("Running setup handler {} \n".format(self.opts)) + # setup a log file to write to + logging.basicConfig(filename=self.opts.logfile, level=logging.DEBUG) + + self.log({"when": time.time(), "who": self.__class__.__name__, "what": "Logger iniated"}) + + def log(self, event): + + when = time.strftime("%F %T", time.gmtime(event["when"])) + msg = "{0} : {1}: {2}".format(when, + event["who"], + event["what"].strip()) + try: + if self.opts.verbose: + sys.stderr.write("{0}\n".format(msg)) + sys.stderr.flush() + logging.debug(msg) + + except (IOError, OSError) as e: + + sys.stderr.write("Could not write to logfile {0} - {1}\n".format( + self.logfile, e)) + + # event format is a dict {when:time, who:[worker|logger|job|main], + # what:str} + def run(self): + setproctitle.setproctitle("CoprLog") + self.setup_log_handler() + abort = False + try: + while not abort: + e = self.events.get() + if "when" in e and "who" in e and "what" in e: + self.log(e) + except KeyboardInterrupt: + return diff --git a/backend/run/copr-be.py b/backend/run/copr-be.py index bdca6e8..61f62b8 100755 --- a/backend/run/copr-be.py +++ b/backend/run/copr-be.py @@ -5,323 +5,13 @@ from __future__ import unicode_literals from __future__ import division from __future__ import absolute_import -import ConfigParser -import grp -import lockfile -import logging -import multiprocessing import optparse import os -import pwd -import signal import sys -import time -from collections import defaultdict -import daemon -import requests -import setproctitle from bunch import Bunch -from retask.task import Task -from retask.queue import Queue -from retask import ConnectionError -from backend.exceptions import CoprBackendError -from backend.dispatcher import Worker -from backend.actions import Action -from backend.frontend import FrontendClient -from backend.helpers import BackendConfigReader - - -def _get_conf(cp, section, option, default, mode=None): - """ - To make returning items from config parser less irritating - - :param mode: convert obtained value, possible modes: - - None (default): do nothing - - "bool" or "boolean" - - "int" - - "float" - """ - - if cp.has_section(section) and cp.has_option(section, option): - if mode is None: - return cp.get(section, option) - elif mode in ["bool", "boolean"]: - return cp.getboolean(section, option) - elif mode == "int": - return cp.getint(section, option) - elif mode == "float": - return cp.getfloat(section, option) - return default - - -class CoprJobGrab(multiprocessing.Process): - - """ - Fetch jobs from the Frontend - - submit them to the jobs queue for workers - """ - - def __init__(self, opts, events, lock): - # base class initialization - multiprocessing.Process.__init__(self, name="jobgrab") - - self.opts = opts - self.events = events - self.task_queues = [] - for group in self.opts.build_groups: - self.task_queues.append(Queue("copr-be-{0}".format(group["id"]))) - self.task_queues[group["id"]].connect() - self.added_jobs = [] - self.lock = lock - - def event(self, what): - self.events.put({"when": time.time(), "who": "jobgrab", "what": what}) - - def load_tasks(self): - try: - r = requests.get( - "{0}/waiting/".format(self.opts.frontend_url), - auth=("user", self.opts.frontend_auth)) - r_json = r.json() - - except requests.RequestException as e: - self.event("Error retrieving jobs from {0}: {1}".format( - self.opts.frontend_url, e)) - return - - except ValueError as e: - self.event("Error getting JSON build list from FE {0}" - .format(e)) - return - - if "builds" in r_json and r_json["builds"]: - self.event("{0} jobs returned".format(len(r_json["builds"]))) - count = 0 - for task in r_json["builds"]: - if "task_id" in task and task["task_id"] not in self.added_jobs: - # this will ignore and throw away unconfigured architectures - # FIXME: don't do ^ - arch = task["chroot"].split("-")[2] - for group in self.opts.build_groups: - if arch in group["archs"]: - self.added_jobs.append(task["task_id"]) - task_obj = Task(task) - self.task_queues[group["id"]].enqueue(task_obj) - count += 1 - break - if count: - self.event("New jobs: %s" % count) - - if "actions" in r_json and r_json["actions"]: - self.event("{0} actions returned".format( - len(r_json["actions"]))) - - for action in r_json["actions"]: - ao = Action(self.events, action, self.lock, destdir=self.opts.destdir, - frontend_callback=FrontendClient(self.opts, self.events), - front_url=self.opts.frontend_base_url, - results_root_url=self.opts.results_baseurl) - ao.run() - - def run(self): - setproctitle.setproctitle("CoprJobGrab") - abort = False - try: - while not abort: - self.load_tasks() - time.sleep(self.opts.sleeptime) - except KeyboardInterrupt: - return - - -class CoprLog(multiprocessing.Process): - - """log mechanism where items from the events queue get recorded""" - - def __init__(self, opts, events): - - # base class initialization - multiprocessing.Process.__init__(self, name="logger") - - self.opts = opts - self.events = events - - logdir = os.path.dirname(self.opts.logfile) - if not os.path.exists(logdir): - os.makedirs(logdir, mode=0o750) - - def setup_log_handler(self): - sys.stderr.write("Running setup handler {} \n".format(self.opts)) - # setup a log file to write to - logging.basicConfig(filename=self.opts.logfile, level=logging.DEBUG) - - self.log({"when": time.time(), "who": self.__class__.__name__, "what": "Logger iniated"}) - - def log(self, event): - - when = time.strftime("%F %T", time.gmtime(event["when"])) - msg = "{0} : {1}: {2}".format(when, - event["who"], - event["what"].strip()) - try: - if self.opts.verbose: - sys.stderr.write("{0}\n".format(msg)) - sys.stderr.flush() - logging.debug(msg) - - except (IOError, OSError) as e: - - sys.stderr.write("Could not write to logfile {0} - {1}\n".format( - self.logfile, e)) - - # event format is a dict {when:time, who:[worker|logger|job|main], - # what:str} - def run(self): - setproctitle.setproctitle("CoprLog") - self.setup_log_handler() - abort = False - try: - while not abort: - e = self.events.get() - if "when" in e and "who" in e and "what" in e: - self.log(e) - except KeyboardInterrupt: - return - - -class CoprBackend(object): - - """ - Core process - starts/stops/initializes workers - """ - - def __init__(self, config_file=None, ext_opts=None): - # read in config file - # put all the config items into a single self.opts bunch - - if not config_file: - raise CoprBackendError("Must specify config_file") - - self.config_file = config_file - self.ext_opts = ext_opts # to stow our cli options for read_conf() - self.workers_by_group_id = defaultdict(list) - self.max_worker_num_by_group_id = defaultdict(int) - - self.config_reader = BackendConfigReader(self.config_file, self.ext_opts) - self.opts = None - self.update_conf() - - self.lock = multiprocessing.Lock() - - self.task_queues = [] - try: - for group in self.opts.build_groups: - group_id = group["id"] - self.task_queues.append(Queue("copr-be-{0}".format(group_id))) - self.task_queues[group_id].connect() - except ConnectionError: - raise CoprBackendError( - "Could not connect to a task queue. Is Redis running?") - - # make sure there is nothing in our task queues - self.clean_task_queues() - - self.events = multiprocessing.Queue() - # event format is a dict {when:time, who:[worker|logger|job|main], - # what:str} - - # create logger - self._logger = CoprLog(self.opts, self.events) - self._logger.start() - - self.event("Starting up Job Grabber") - # create job grabber - self._jobgrab = CoprJobGrab(self.opts, self.events, self.lock) - self._jobgrab.start() - self.abort = False - - if not os.path.exists(self.opts.worker_logdir): - os.makedirs(self.opts.worker_logdir, mode=0o750) - - def event(self, what): - self.events.put({"when": time.time(), "who": "main", "what": what}) - - def update_conf(self): - self.opts = self.config_reader.read() - - def clean_task_queues(self): - try: - for queue in self.task_queues: - while queue.length: - queue.dequeue() - except ConnectionError: - raise CoprBackendError( - "Could not connect to a task queue. Is Redis running?") - - def run(self): - self.abort = False - while not self.abort: - # re-read config into opts - self.update_conf() - - for group in self.opts.build_groups: - group_id = group["id"] - self.event( - "# jobs in {0} queue: {1}" - .format(group["name"], self.task_queues[group_id].length) - ) - # this handles starting/growing the number of workers - if len(self.workers_by_group_id[group_id]) < group["max_workers"]: - self.event("Spinning up more workers") - for _ in range(group["max_workers"] - len(self.workers_by_group_id[group_id])): - self.max_worker_num_by_group_id[group_id] += 1 - w = Worker( - self.opts, self.events, - self.max_worker_num_by_group_id[group_id], - group_id, lock=self.lock - ) - - self.workers_by_group_id[group_id].append(w) - w.start() - self.event("Finished starting worker processes") - # FIXME - prune out workers - # if len(self.workers) > self.opts.num_workers: - # killnum = len(self.workers) - self.opts.num_workers - # for w in self.workers[:killnum]: - # insert a poison pill? Kill after something? I dunno. - # FIXME - if a worker bombs out - we need to check them - # and startup a new one if it happens - # check for dead workers and abort - preserved_workers = [] - for w in self.workers_by_group_id[group_id]: - if not w.is_alive(): - self.event("Worker {0} died unexpectedly".format(w.worker_num)) - if self.opts.exit_on_worker: - raise CoprBackendError( - "Worker died unexpectedly, exiting") - else: - w.terminate() # kill it with a fire - else: - preserved_workers.append(w) - self.workers_by_group_id[group_id] = preserved_workers - - time.sleep(self.opts.sleeptime) - - def terminate(self): - """ - Cleanup backend processes (just workers for now) - And also clean all task queues as they would survive copr restart - """ - - self.abort = True - for group in self.opts.build_groups: - group_id = group["id"] - for w in self.workers_by_group_id[group_id]: - self.workers_by_group_id[group_id].remove(w) - w.terminate() - self.clean_task_queues() +from backend.daemons import run_backend def parse_args(args): @@ -356,28 +46,7 @@ def parse_args(args): def main(args): opts = parse_args(args) - - try: - context = daemon.DaemonContext( - pidfile=lockfile.FileLock(opts.pidfile), - gid=grp.getgrnam("copr").gr_gid, - uid=pwd.getpwnam("copr").pw_uid, - detach_process=opts.daemonize, - umask=0o22, - stderr=sys.stderr, - signal_map={ - signal.SIGTERM: "terminate", - signal.SIGHUP: "terminate", - }, - ) - with context: - cbe = CoprBackend(opts.config_file, ext_opts=opts) - cbe.run() - except (Exception, KeyboardInterrupt): - sys.stderr.write("Killing/Dying\n") - if "cbe" in locals(): - cbe.terminate() - raise + run_backend(opts) if __name__ == "__main__": try:

1 0

[copr] master: [backend] unittest for frontendclient (b37a5f8)
by vgologuz＠fedoraproject.org 28 Nov '14

28 Nov '14

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit b37a5f84b95c9aa69ac87fd4ba2eade4e480c1af Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Nov 28 13:47:01 2014 +0100 [backend] unittest for frontendclient >--------------------------------------------------------------- backend/backend/frontend.py | 28 +++----- backend/tests/test_frontend.py | 136 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+), 17 deletions(-) diff --git a/backend/backend/frontend.py b/backend/backend/frontend.py index c34857a..51551f4 100644 --- a/backend/backend/frontend.py +++ b/backend/backend/frontend.py @@ -1,5 +1,5 @@ import json -import requests +from requests import post, RequestException import time @@ -27,13 +27,12 @@ class FrontendClient(object): self.msg = None try: - response = requests.post(url, data=json.dumps(data), auth=auth, - headers=headers) - if response.status_code != 200: + response = post(url, data=json.dumps(data), auth=auth, headers=headers) + if response.status_code >= 400: self.msg = "Failed to submit to frontend: {0}: {1}".format( response.status_code, response.text) - raise requests.RequestException(self.msg) - except requests.RequestException as e: + raise RequestException(self.msg) + except RequestException as e: self.msg = "Post request failed: {0}".format(e) raise return response @@ -42,18 +41,13 @@ class FrontendClient(object): """ Make a request max_repeats-time to the frontend """ - repeats = 0 - while repeats <= max_repeats: + for i in range(max_repeats): try: - response = self._post_to_frontend(data, url_path) - break - except requests.RequestException: - - if repeats == max_repeats: - raise - repeats += 1 + return self._post_to_frontend(data, url_path) + except RequestException: time.sleep(5) - return response + else: + raise RequestException("Failed to post to frontend for {} times".format(max_repeats)) def update(self, data): """ @@ -70,5 +64,5 @@ class FrontendClient(object): data = {"build_id": build_id, "chroot": chroot_name} response = self._post_to_frontend_repeatedly(data, "starting_build") if "can_start" not in response.json(): - raise requests.RequestException("Bad respond from the frontend") + raise RequestException("Bad respond from the frontend") return response.json()["can_start"] diff --git a/backend/tests/test_frontend.py b/backend/tests/test_frontend.py new file mode 100644 index 0000000..c18d1f6 --- /dev/null +++ b/backend/tests/test_frontend.py @@ -0,0 +1,136 @@ +# coding: utf-8 +import copy + +from collections import defaultdict +import json +import multiprocessing +from pprint import pprint +from bunch import Bunch +from requests import RequestException +from backend.exceptions import BuilderError, BuilderTimeOutError, MockRemoteError, CoprSignError + +import tempfile +import shutil +import os + +import six +from backend.frontend import FrontendClient + +if six.PY3: + from unittest import mock + from unittest.mock import patch, MagicMock +else: + import mock + from mock import patch, MagicMock + +import pytest + + +(a)pytest.yield_fixture +def post_req(): + with mock.patch("backend.frontend.post") as obj: + yield obj + + +(a)pytest.yield_fixture +def mc_time(): + with mock.patch("backend.frontend.time") as obj: + yield obj + +class TestFrontendClient(object): + + def setup_method(self, method): + self.opts = Bunch( + frontend_url="http://example.com/", + frontend_auth="12345678", + ) + self.events = multiprocessing.Queue() + self.fc = FrontendClient(self.opts, self.events) + + self.data = { + "foo": "bar", + "bar": [1, 3, 5], + } + self.url_path = "sub_path" + + self.build_id = 12345 + self.chroot_name = "fedora-20-x86_64" + + @pytest.fixture + def mask_post_to_fe(self): + self.ptf = MagicMock() + self.fc._post_to_frontend = self.ptf + + def test_post_to_frontend(self, post_req): + post_req.return_value.status_code = 200 + self.fc._post_to_frontend(self.data, self.url_path) + + assert post_req.called + + def test_post_to_frontend_not_200(self, post_req): + post_req.return_value.status_code = 501 + with pytest.raises(RequestException): + self.fc._post_to_frontend(self.data, self.url_path) + + assert post_req.called + + def test_post_to_frontend_post_error(self, post_req): + post_req.side_effect = RequestException() + with pytest.raises(RequestException): + self.fc._post_to_frontend(self.data, self.url_path) + + assert post_req.called + + def test_post_to_frontend_repeated_first_try_ok(self, mask_post_to_fe, mc_time): + response = "ok\n" + self.ptf.return_value = response + + assert self.fc._post_to_frontend_repeatedly(self.data, self.url_path) == response + assert not mc_time.sleep.called + + def test_post_to_frontend_repeated_second_try_ok(self, mask_post_to_fe, mc_time): + response = "ok\n" + self.ptf.side_effect = [ + RequestException(), + response, + ] + + assert self.fc._post_to_frontend_repeatedly(self.data, self.url_path) == response + assert mc_time.sleep.called + + def test_post_to_frontend_repeated_all_attempts_failed(self, mask_post_to_fe, mc_time): + self.ptf.side_effect = RequestException() + + with pytest.raises(RequestException): + self.fc._post_to_frontend_repeatedly(self.data, self.url_path) + + assert mc_time.sleep.called + + def test_update(self): + ptfr = MagicMock() + self.fc._post_to_frontend_repeatedly = ptfr + self.fc.update(self.data) + assert ptfr.call_args == mock.call(self.data, "update") + + def test_starting_build(self): + ptfr = MagicMock() + self.fc._post_to_frontend_repeatedly = ptfr + for val in [True, False]: + ptfr.return_value.json.return_value = {"can_start": val} + + assert self.fc.starting_build(self.build_id, self.chroot_name) == val + + def test_starting_build_err(self): + ptfr = MagicMock() + self.fc._post_to_frontend_repeatedly = ptfr + + with pytest.raises(RequestException): + self.fc.starting_build(self.build_id, self.chroot_name) + + def test_starting_build_err_2(self): + ptfr = MagicMock() + self.fc._post_to_frontend_repeatedly = ptfr + ptfr.return_value.json.return_value = {} + + with pytest.raises(RequestException): + self.fc.starting_build(self.build_id, self.chroot_name)

1 0

[copr] master: [backend] renamed FrontendCallback to FrontendClient (1afb7bd)
by vgologuz＠fedoraproject.org 28 Nov '14

28 Nov '14

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 1afb7bd9830d976bdc373d7ba7e86ab2f307f6ec Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Nov 28 12:47:25 2014 +0100 [backend] renamed FrontendCallback to FrontendClient >--------------------------------------------------------------- backend/backend/dispatcher.py | 14 +++++++------- backend/backend/{callback.py => frontend.py} | 4 ++-- backend/run/copr-be.py | 4 ++-- backend/tests/test_dispatcher.py | 2 -- 4 files changed, 11 insertions(+), 13 deletions(-) diff --git a/backend/backend/dispatcher.py b/backend/backend/dispatcher.py index b17173a..13fcb5d 100644 --- a/backend/backend/dispatcher.py +++ b/backend/backend/dispatcher.py @@ -23,7 +23,7 @@ from .exceptions import MockRemoteError, CoprWorkerError, CoprWorkerSpawnFailErr from .job import BuildJob from .mockremote import MockRemote -from .callback import FrontendCallback +from .frontend import FrontendClient from .constants import BuildStatus ansible_playbook = "ansible-playbook" @@ -81,7 +81,7 @@ class Worker(multiprocessing.Process): self.lock = lock self.spawn_in_advance = self.opts.spawn_in_advance - self.frontend_callback = FrontendCallback(opts, events) + self.frontend_callback = FrontendClient(opts, events) self.callback = callback if not self.callback: @@ -573,8 +573,8 @@ class Worker(multiprocessing.Process): self.terminate_instance(vm_ip) vm_ip = None - # TODO: since spawn requires job object to create vm - # it's possible to have spawned VM with incorrect configuration - # disabling spawn in advance for now - # if self.spawn_in_advance: - # vm_ip = self.spawn_instance_with_check(job) + # TODO: since spawn requires job object to create vm + # it's possible to have spawned VM with incorrect configuration + # disabling spawn in advance for now + # if self.spawn_in_advance: + # vm_ip = self.spawn_instance_with_check(job) diff --git a/backend/backend/callback.py b/backend/backend/frontend.py similarity index 96% rename from backend/backend/callback.py rename to backend/backend/frontend.py index f76f5fd..c34857a 100644 --- a/backend/backend/callback.py +++ b/backend/backend/frontend.py @@ -3,13 +3,13 @@ import requests import time -class FrontendCallback(object): +class FrontendClient(object): """ Object to send data back to fronted """ def __init__(self, opts, events): - super(FrontendCallback, self).__init__() + super(FrontendClient, self).__init__() self.frontend_url = opts.frontend_url self.frontend_auth = opts.frontend_auth diff --git a/backend/run/copr-be.py b/backend/run/copr-be.py index fe1a65c..bdca6e8 100755 --- a/backend/run/copr-be.py +++ b/backend/run/copr-be.py @@ -29,7 +29,7 @@ from retask import ConnectionError from backend.exceptions import CoprBackendError from backend.dispatcher import Worker from backend.actions import Action -from backend.callback import FrontendCallback +from backend.frontend import FrontendClient from backend.helpers import BackendConfigReader @@ -120,7 +120,7 @@ class CoprJobGrab(multiprocessing.Process): for action in r_json["actions"]: ao = Action(self.events, action, self.lock, destdir=self.opts.destdir, - frontend_callback=FrontendCallback(self.opts, self.events), + frontend_callback=FrontendClient(self.opts, self.events), front_url=self.opts.frontend_base_url, results_root_url=self.opts.results_baseurl) ao.run() diff --git a/backend/tests/test_dispatcher.py b/backend/tests/test_dispatcher.py index 77bbb6d..044500b 100644 --- a/backend/tests/test_dispatcher.py +++ b/backend/tests/test_dispatcher.py @@ -678,8 +678,6 @@ class TestDispatcher(object): assert self.worker.obtain_job.called assert self.worker.terminate_instance.called - - @mock.patch("backend.dispatcher.time") def test_run_no_job(self, mc_time, init_worker): self.worker.init_fedmsg = MagicMock()

1 0

[copr] master: [backend] refactoring/unittest dispatcher (151e19d)
by vgologuz＠fedoraproject.org 28 Nov '14

28 Nov '14

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 151e19d510a4df5ad061d4c81dd150f9f16c16d1 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Thu Nov 27 09:23:00 2014 +0100 [backend] refactoring/unittest dispatcher >--------------------------------------------------------------- Diff suppressed because of size. To see it, use: git diff --patch-with-stat --no-color --find-copies-harder --ignore-space-at-eol ^151e19d510a4df5ad061d4c81dd150f9f16c16d1~1 151e19d510a4df5ad061d4c81dd150f9f16c16d1

1 0

[copr] master: [backend] PEP8 name convention (a6329a1)
by vgologuz＠fedoraproject.org 28 Nov '14

28 Nov '14

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit a6329a1e98cdce7babb680cc33b47f0bbce7f9f3 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Wed Nov 26 16:07:28 2014 +0100 [backend] PEP8 name convention >--------------------------------------------------------------- backend/backend/dispatcher.py | 6 +++--- 1 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/backend/dispatcher.py b/backend/backend/dispatcher.py index c267941..8beb8ab 100644 --- a/backend/backend/dispatcher.py +++ b/backend/backend/dispatcher.py @@ -429,7 +429,7 @@ class Worker(multiprocessing.Process): return True return False - def __spawn_with_check(self, job): + def spawn_instance_with_check(self, job): """ Wrapper around self.spawn_instance() with exception checking """ try: ip = self.spawn_instance(job) @@ -508,7 +508,7 @@ class Worker(multiprocessing.Process): if self.create and not self.ip: if not self.spawn_in_advance: - ip = self.__spawn_with_check(job) + ip = self.spawn_instance_with_check(job) # else we get ip from similar calling at the enf of this while-loop else: ip = self.ip @@ -611,4 +611,4 @@ class Worker(multiprocessing.Process): if self.create: self.terminate_instance(ip) if self.create and not self.ip and self.spawn_in_advance: - ip = self.__spawn_with_check(job) + ip = self.spawn_instance_with_check(job)

1 0

[copr] master: [backend] unittests for backend/sign (d3a9455)
by vgologuz＠fedoraproject.org 26 Nov '14

26 Nov '14

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit d3a945542024c4d2d8b8f2167ad32a616b5699f3 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Tue Nov 25 18:18:26 2014 +0100 [backend] unittests for backend/sign >--------------------------------------------------------------- backend/backend/sign.py | 9 +- backend/requirements.txt | 1 + backend/tests/test_sign.py | 296 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 303 insertions(+), 3 deletions(-) diff --git a/backend/backend/sign.py b/backend/backend/sign.py index 334beb9..21b6cf4 100755 --- a/backend/backend/sign.py +++ b/backend/backend/sign.py @@ -113,9 +113,12 @@ def sign_rpms_in_dir(username, projectname, path, opts, callback=None): rpm_list = [ os.path.join(path, filename) for filename in os.listdir(path) - if filename.endswith("rpm") + if filename.endswith(".rpm") ] + if not rpm_list: + return + try: get_pubkey(username, projectname) except CoprSignNoKeyError: @@ -135,8 +138,8 @@ def sign_rpms_in_dir(username, projectname, path, opts, callback=None): errors.append((rpm, e)) if errors: - raise MockRemoteError("Rpm sign failed, affected rpms: {}" - .format([err[0] for err in errors])) + raise CoprSignError("Rpm sign failed, affected rpms: {}" + .format([err[0] for err in errors])) def create_user_keys(username, projectname, opts): diff --git a/backend/requirements.txt b/backend/requirements.txt index 253297c..9576e53 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -5,3 +5,4 @@ redis retask python-daemon bunch +IPy diff --git a/backend/tests/test_sign.py b/backend/tests/test_sign.py new file mode 100644 index 0000000..7271c3c --- /dev/null +++ b/backend/tests/test_sign.py @@ -0,0 +1,296 @@ +import os + +from collections import defaultdict +import json +from pprint import pprint +from _pytest.capture import capsys +from bunch import Bunch +import pytest +import copy +import tempfile +import shutil + +import six +import time +from backend.exceptions import CoprSignError, CoprSignNoKeyError, CoprKeygenRequestError + +if six.PY3: + from unittest import mock + from unittest.mock import MagicMock +else: + import mock + from mock import MagicMock + +from backend.sign import get_pubkey, _sign_one, sign_rpms_in_dir, create_user_keys + + +STDOUT = "stdout" +STDERR = "stderr" + + +class TestSign(object): + + def setup_method(self, method): + self.username = "foo" + self.projectname = "bar" + + self.usermail = "foo_bar(a)copr.fedorahosted.org" + self.test_time = time.time() + self.tmp_dir_path = None + + self.opts = Bunch(keygen_host="example.com") + + def teardown_method(self, method): + if self.tmp_dir_path: + shutil.rmtree(self.tmp_dir_path) + + @pytest.fixture + def tmp_dir(self): + subdir = "test_createrepo_{}".format(time.time()) + self.tmp_dir_path = os.path.join(tempfile.gettempdir(), subdir) + os.mkdir(self.tmp_dir_path) + + @pytest.fixture + def tmp_files(self): + # ! require tmp_dir created before + self.file_names = ["foo.rpm", "bar.rpm", "bad", "morebadrpm"] + for name in self.file_names: + path = os.path.join(self.tmp_dir_path, name) + with open(path, "w") as handle: + handle.write("1") + + @mock.patch("backend.sign.Popen") + def test_get_pubkey(self, mc_popen): + mc_handle = MagicMock() + mc_handle.communicate.return_value = (STDOUT, STDERR) + mc_handle.returncode = 0 + mc_popen.return_value = mc_handle + + result = get_pubkey(self.username, self.projectname) + assert result == STDOUT + assert mc_popen.call_args[0][0] == ['sudo', '/bin/sign', '-u', self.usermail, '-p'] + + + @mock.patch("backend.sign.Popen") + def test_get_pubkey_error(self, mc_popen): + mc_popen.side_effect = IOError(STDERR) + + with pytest.raises(CoprSignError): + get_pubkey(self.username, self.projectname) + + + @mock.patch("backend.sign.Popen") + def test_get_pubkey_unknown_key(self, mc_popen): + mc_handle = MagicMock() + mc_handle.communicate.return_value = (STDOUT, "unknown key: foobar") + mc_handle.returncode = 1 + mc_popen.return_value = mc_handle + + with pytest.raises(CoprSignNoKeyError): + get_pubkey(self.username, self.projectname) + + @mock.patch("backend.sign.Popen") + def test_get_pubkey_unknown_error(self, mc_popen): + mc_handle = MagicMock() + mc_handle.communicate.return_value = (STDOUT, STDERR) + mc_handle.returncode = 1 + mc_popen.return_value = mc_handle + + with pytest.raises(CoprSignError): + get_pubkey(self.username, self.projectname) + + @mock.patch("backend.sign.Popen") + def test_get_pubkey_outfile(self, mc_popen, tmp_dir): + mc_handle = MagicMock() + mc_handle.communicate.return_value = (STDOUT, STDERR) + mc_handle.returncode = 0 + mc_popen.return_value = mc_handle + + outfile_path = os.path.join(self.tmp_dir_path, "out.pub") + assert not os.path.exists(outfile_path) + result = get_pubkey(self.username, self.projectname, outfile_path) + assert result == STDOUT + assert os.path.exists(outfile_path) + with open(outfile_path) as handle: + content = handle.read() + assert STDOUT == content + + @mock.patch("backend.sign.Popen") + def test_sign_one(self, mc_popen): + mc_handle = MagicMock() + mc_handle.communicate.return_value = (STDOUT, STDERR) + mc_handle.returncode = 0 + mc_popen.return_value = mc_handle + + fake_path = "/tmp/pkg.rpm" + result = _sign_one(fake_path, self.usermail) + assert STDOUT, STDERR == result + + expected_cmd = ['sudo', '/bin/sign', '-u', self.usermail, '-r', fake_path] + assert mc_popen.call_args[0][0] == expected_cmd + + @mock.patch("backend.sign.Popen") + def test_sign_one_popen_error(self, mc_popen): + mc_popen.side_effect = IOError() + + fake_path = "/tmp/pkg.rpm" + with pytest.raises(CoprSignError): + _sign_one(fake_path, self.usermail) + + mc_cb = MagicMock() + + with pytest.raises(CoprSignError): + _sign_one(fake_path, self.usermail, mc_cb) + assert isinstance(mc_cb.error.call_args[0][0], CoprSignError) + + @mock.patch("backend.sign.Popen") + def test_sign_one_cmd_erro(self, mc_popen): + mc_handle = MagicMock() + mc_handle.communicate.return_value = (STDOUT, STDERR) + mc_handle.returncode = 1 + mc_popen.return_value = mc_handle + + + fake_path = "/tmp/pkg.rpm" + with pytest.raises(CoprSignError): + _sign_one(fake_path, self.usermail) + + mc_cb = MagicMock() + + with pytest.raises(CoprSignError): + _sign_one(fake_path, self.usermail, mc_cb) + assert isinstance(mc_cb.error.call_args[0][0], CoprSignError) + + @mock.patch("backend.sign.request") + def test_create_user_keys(self, mc_request): + mc_request.return_value.status_code = 200 + create_user_keys(self.username, self.projectname, self.opts) + + assert mc_request.called + expected_call = mock.call( + url="http://example.com/gen_key", + data='{"name_real": "foo_bar", "name_email": "foo_bar(a)copr.fedorahosted.org"}', + method="post" + ) + assert mc_request.call_args == expected_call + + @mock.patch("backend.sign.request") + def test_create_user_keys_error_1(self, mc_request): + mc_request.side_effect = IOError() + with pytest.raises(CoprKeygenRequestError): + create_user_keys(self.username, self.projectname, self.opts) + + + @mock.patch("backend.sign.request") + def test_create_user_keys(self, mc_request): + for code in [400, 401, 404, 500, 599]: + mc_request.return_value.status_code = code + + with pytest.raises(CoprKeygenRequestError): + create_user_keys(self.username, self.projectname, self.opts) + + + @mock.patch("backend.sign._sign_one") + @mock.patch("backend.sign.create_user_keys") + @mock.patch("backend.sign.get_pubkey") + def test_sign_rpms_id_dir_nothing(self, mc_gp, mc_cuk, mc_so, + tmp_dir): + # empty target dir doesn't produce error + sign_rpms_in_dir(self.username, self.projectname, + self.tmp_dir_path, self.opts) + + assert not mc_gp.called + assert not mc_cuk.called + assert not mc_so.called + + @mock.patch("backend.sign._sign_one") + @mock.patch("backend.sign.create_user_keys") + @mock.patch("backend.sign.get_pubkey") + def test_sign_rpms_id_dir_ok(self, mc_gp, mc_cuk, mc_so, + tmp_dir, tmp_files): + mc_cb = MagicMock() + sign_rpms_in_dir(self.username, self.projectname, + self.tmp_dir_path, self.opts, callback=mc_cb) + + assert mc_gp.called + assert not mc_cuk.called + assert mc_so.called + + pathes = [call[0][0] for call in mc_so.call_args_list] + count = 0 + for name in self.file_names: + if name.endswith(".rpm"): + count += 1 + assert os.path.join(self.tmp_dir_path, name) in pathes + assert len(pathes) == count + + @mock.patch("backend.sign._sign_one") + @mock.patch("backend.sign.create_user_keys") + @mock.patch("backend.sign.get_pubkey") + def test_sign_rpms_id_dir_error_on_pubkey( + self, mc_gp, mc_cuk, mc_so, tmp_dir, tmp_files): + + mc_gp.side_effect = CoprSignError("foobar") + mc_cb = MagicMock() + with pytest.raises(CoprSignError): + sign_rpms_in_dir(self.username, self.projectname, + self.tmp_dir_path, self.opts, callback=mc_cb) + + assert mc_gp.called + assert not mc_cuk.called + assert not mc_so.called + + @mock.patch("backend.sign._sign_one") + @mock.patch("backend.sign.create_user_keys") + @mock.patch("backend.sign.get_pubkey") + def test_sign_rpms_id_dir_no_pub_key( + self, mc_gp, mc_cuk, mc_so, tmp_dir, tmp_files): + + mc_gp.side_effect = CoprSignNoKeyError("foobar") + mc_cb = MagicMock() + + sign_rpms_in_dir(self.username, self.projectname, + self.tmp_dir_path, self.opts, callback=mc_cb) + + assert mc_gp.called + assert mc_cuk.called + assert mc_so.called + + @mock.patch("backend.sign._sign_one") + @mock.patch("backend.sign.create_user_keys") + @mock.patch("backend.sign.get_pubkey") + def test_sign_rpms_id_dir_sign_error_one( + self, mc_gp, mc_cuk, mc_so, tmp_dir, tmp_files): + + mc_cb = MagicMock() + + mc_so.side_effect = [ + None, CoprSignError("foobar"), None + ] + with pytest.raises(CoprSignError): + sign_rpms_in_dir(self.username, self.projectname, + self.tmp_dir_path, self.opts, callback=mc_cb) + + assert mc_gp.called + assert not mc_cuk.called + + assert mc_so.called + + @mock.patch("backend.sign._sign_one") + @mock.patch("backend.sign.create_user_keys") + @mock.patch("backend.sign.get_pubkey") + def test_sign_rpms_id_dir_sign_error_all( + self, mc_gp, mc_cuk, mc_so, tmp_dir, tmp_files): + + mc_cb = MagicMock() + + mc_so.side_effect = CoprSignError("foobar") + with pytest.raises(CoprSignError): + sign_rpms_in_dir(self.username, self.projectname, + self.tmp_dir_path, self.opts, callback=mc_cb) + + assert mc_gp.called + assert not mc_cuk.called + + assert mc_so.called +

1 0

[copr] master: [backend] backend.sign: discover `keygen_host` from backend config file (853f547)
by vgologuz＠fedoraproject.org 26 Nov '14

26 Nov '14

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 853f547326eee205a11c0b567a1d81b285051f5a Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Tue Nov 25 19:13:50 2014 +0100 [backend] backend.sign: discover `keygen_host` from backend config file >--------------------------------------------------------------- backend/backend/helpers.py | 3 +++ backend/backend/mockremote/__init__.py | 3 ++- backend/backend/sign.py | 24 ++++++++++++++++-------- backend/conf/copr-be.conf.example | 6 +++++- backend/conf/copr-be.local.conf | 3 ++- 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py index 116bb54..a862db2 100644 --- a/backend/backend/helpers.py +++ b/backend/backend/helpers.py @@ -105,6 +105,9 @@ class BackendConfigReader(object): opts.do_sign = _get_conf( cp, "backend", "do_sign", False, mode="bool") + opts.keygen_host = _get_conf( + cp, "backend", "keygen_host", "copr-keygen.cloud.fedoraproject.org") + opts.build_user = _get_conf( cp, "backend", "build_user", DEF_BUILD_USER) diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py index d66c058..61c4233 100755 --- a/backend/backend/mockremote/__init__.py +++ b/backend/backend/mockremote/__init__.py @@ -211,7 +211,8 @@ class MockRemote(object): sign_rpms_in_dir(self.job.project_owner, self.job.project_name, get_target_dir(self.chroot_dir, self.pkg), - callback=self.callback) + opts=self.opts, + callback=self.callback,) except Exception as e: self.callback.error( "failed to sign packages " diff --git a/backend/backend/sign.py b/backend/backend/sign.py index 5049988..334beb9 100755 --- a/backend/backend/sign.py +++ b/backend/backend/sign.py @@ -19,10 +19,6 @@ from .exceptions import CoprSignError, CoprSignNoKeyError, \ SIGN_BINARY = "/bin/sign" DOMAIN = "fedorahosted.org" -# TODO: discover from config -# COPR_KEYGEN_URL = "http://127.0.0.1:3872/gen_key" -COPR_KEYGEN_URL = "http://209.132.184.124/gen_key" - def create_gpg_email(username, projectname): """ @@ -97,7 +93,7 @@ def _sign_one(path, email, callback=None): return stdout, stderr -def sign_rpms_in_dir(username, projectname, path, callback=None): +def sign_rpms_in_dir(username, projectname, path, opts, callback=None): """ Signs rpms using obs-signd. @@ -105,9 +101,11 @@ def sign_rpms_in_dir(username, projectname, path, callback=None): but we continue to try sign other pkgs. + :param username: copr username :param projectname: copr projectname :param path: directory with rpms to be signed + :param Bunch opts: backend config :param .mockremote.DefaultCallBack callback: object to log progress, two methods are utilised: ``log`` and ``error`` @@ -121,7 +119,7 @@ def sign_rpms_in_dir(username, projectname, path, callback=None): try: get_pubkey(username, projectname) except CoprSignNoKeyError: - create_user_keys(username, projectname) + create_user_keys(username, projectname, opts) errors = [] # tuples (rpm_filepath, exception) for rpm in rpm_list: @@ -141,13 +139,23 @@ def sign_rpms_in_dir(username, projectname, path, callback=None): .format([err[0] for err in errors])) -def create_user_keys(username, projectname): +def create_user_keys(username, projectname, opts): + """ + Generate a new key-pair at sign host + + :param username: + :param projectname: + :param opts: backend config + + :return: None + """ data = json.dumps({ "name_real": "{}_{}".format(username, projectname), "name_email": create_gpg_email(username, projectname) }) - query = dict(url=COPR_KEYGEN_URL, data=data, method="post") + keygen_url = "http://{}/gen_key".format(opts.keygen_host) + query = dict(url=keygen_url, data=data, method="post") try: response = request(**query) except Exception as e: diff --git a/backend/conf/copr-be.conf.example b/backend/conf/copr-be.conf.example index 69bcf7b..81b8333 100644 --- a/backend/conf/copr-be.conf.example +++ b/backend/conf/copr-be.conf.example @@ -94,7 +94,11 @@ worker_logdir=/var/log/copr/workers/ # enable package signing, require configured # signer host and correct /etc/sign.conf -#do_sign=false +# do_sign=false + +# host or ip of machine with copr-keygen +# usually the same as in /etc/sign.conf +# keygen_host=example.com # minimum age for builds to be pruned prune_days=14 diff --git a/backend/conf/copr-be.local.conf b/backend/conf/copr-be.local.conf index 76df89c..f59fb71 100644 --- a/backend/conf/copr-be.local.conf +++ b/backend/conf/copr-be.local.conf @@ -94,7 +94,8 @@ worker_logdir=/tmp/log/copr/workers/ # enable package signing, require configured # signer host and correct /etc/sign.conf -#do_sign=false +# do_sign=false +# keygen_host=example.com verbose=true

1 0

[copr] master: Fix mismatch between documentation and actual API in new build (c1d6550)
by Miroslav Suchý 26 Nov '14

26 Nov '14

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit c1d6550daa0ea98be895b7f2b6010a25653957b8 Author: Michael Simacek <msimacek(a)redhat.com> Date: Tue Nov 25 17:17:27 2014 +0100 Fix mismatch between documentation and actual API in new build Current implementation returns multiple ids, not just one >--------------------------------------------------------------- frontend/coprs_frontend/coprs/templates/api.html | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/frontend/coprs_frontend/coprs/templates/api.html b/frontend/coprs_frontend/coprs/templates/api.html index 2157aa0..a38c389 100644 --- a/frontend/coprs_frontend/coprs/templates/api.html +++ b/frontend/coprs_frontend/coprs/templates/api.html @@ -277,7 +277,7 @@ copr_url = {{ ('https://' + config['PUBLIC_COPR_HOSTNAME'])| fix_url_https_front { "output": "ok", "message": "Build was added to log4j.", - "id": 5 + "ids": [5] } </pre>

1 0

[copr] tag 'copr-backend-1.49-1' created
by vgologuz＠fedoraproject.org 25 Nov '14

25 Nov '14

Repository : http://git.fedorahosted.org/cgit/copr.git New tag : copr-backend-1.49-1 Referencing: 00dcaaf6d59c1e8028a3c41816498fbec1c4e48c

1 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

copr-commits November 2014