copr-devel December 2012

copr-devel@lists.stg.fedorahosted.org

4 participants
47 discussions

[copr] skvidal-backend: - make opts behave in face of cli-opts and on config reread - make exit be a bit quieter on ctrl-c (I hope) - prepare for a case where our builders are not dynamically created - obey pyflakes and pychecker suggestions (15066b2)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 15066b2d6c9c9a00994d2b7be1f13612466367a7 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 15:08:52 2012 -0500 - make opts behave in face of cli-opts and on config reread - make exit be a bit quieter on ctrl-c (I hope) - prepare for a case where our builders are not dynamically created - obey pyflakes and pychecker suggestions >--------------------------------------------------------------- backend/dispatcher.py | 19 +++++++++++-------- copr-be.py | 20 +++++++++++++------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index 434e671..f96bdb1 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -95,6 +95,7 @@ class Worker(multiprocessing.Process): self.opts = opts self.kill_received = False self.callback = callback + self.create = create if not self.callback: self.logfile = self.opts.worker_logdir + '/worker-%s.log' % self.worker_num self.callback = WorkerCallback(logfile = self.logfile) @@ -193,14 +194,15 @@ class Worker(multiprocessing.Process): job.jobfile = jobfile # spin up our build instance - try: - ip = self.spawn_instance() - if not ip: - raise errors.CoprWorkerError, "No IP found from creating instance" + if self.create: + try: + ip = self.spawn_instance() + if not ip: + raise errors.CoprWorkerError, "No IP found from creating instance" - except ansible.errors.AnsibleError, e: - self.callback.log('failure to setup instance: %s' % e) - raise + except ansible.errors.AnsibleError, e: + self.callback.log('failure to setup instance: %s' % e) + raise status = 1 job.started_on = time.time() @@ -243,5 +245,6 @@ class Worker(multiprocessing.Process): self.return_results(job) self.callback.log('worker finished build: %s' % ip) # clean up the instance - self.terminate_instance(ip) + if self.create: + self.terminate_instance(ip) diff --git a/copr-be.py b/copr-be.py index a3e372a..3be1828 100644 --- a/copr-be.py +++ b/copr-be.py @@ -20,7 +20,7 @@ def _get_conf(cp, section, option, default): class CoprBackend(object): - def __init__(self, config_file=None): + def __init__(self, config_file=None, ext_opts=None): # read in config file # put all the config items into a single self.opts bunch @@ -28,6 +28,7 @@ class CoprBackend(object): raise errors.CoprBackendError, "Must specify config_file" self.config_file = config_file + self._ext_opts = ext_opts # to stow our cli options for read_conf() self.opts = self.read_conf() logdir = os.path.dirname(self.opts.logfile) @@ -78,6 +79,9 @@ class CoprBackend(object): if not opts.jobsdir or not opts.destdir: raise errors.CoprBackendError, "Incomplete Config - must specify jobsdir and destdir in configuration" + if self._ext_opts: + for v in self._ext_opts: + setattr(opts, v, self.ext_opts.get(v)) return opts @@ -188,17 +192,19 @@ def parse_args(args): print "No config file found at: %s" % opts.config_file sys.exit(1) - return opts,args + ret_opts = Bunch() + for o in ('daemonize', 'exit_on_worker', 'pidfile', 'config_file'): + setattr(ret_opts, o, getattr(opts, o)) + + return ret_opts def main(args): - opts,args = parse_args(args) + opts = parse_args(args) try: - cbe = CoprBackend(opts.config_file) - cbe.opts.daemonize = opts.daemonize # just so we have it on hand - cbe.opts.exit_on_worker = opts.exit_on_worker + cbe = CoprBackend(opts.config_file, ext_opts=opts) if opts.daemonize: daemonize(opts.pidfile) cbe.run() @@ -207,7 +213,7 @@ def main(args): if 'cbe' in locals(): for w in cbe.workers: w.terminate() - raise + if __name__ == '__main__':

1 0

[copr] skvidal-backend: - check daemonize before daemonizing - output time in a more human readable format (4f0bd84)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 4f0bd842fd295465eefb1037f7458ff9b620faae Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 14:31:38 2012 -0500 - check daemonize before daemonizing - output time in a more human readable format >--------------------------------------------------------------- backend/dispatcher.py | 2 +- copr-be.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index e1a47dc..434e671 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -74,7 +74,7 @@ class WorkerCallback(object): if not self.logfile: return - now = time.time() + now = time.strftime('%F %T') try: open(self.logfile, 'a').write(str(now) + ':' + msg + '\n') except (IOError, OSError), e: diff --git a/copr-be.py b/copr-be.py index fc03430..a3e372a 100644 --- a/copr-be.py +++ b/copr-be.py @@ -82,7 +82,7 @@ class CoprBackend(object): def log(self, msg): - now = time.time() + now = time.strftime('%F %T') output = str(now) + ':' + msg if not self.opts.daemonize: print output @@ -199,7 +199,8 @@ def main(args): cbe = CoprBackend(opts.config_file) cbe.opts.daemonize = opts.daemonize # just so we have it on hand cbe.opts.exit_on_worker = opts.exit_on_worker - daemonize(opts.pidfile) + if opts.daemonize: + daemonize(opts.pidfile) cbe.run() except Exception, e: print 'Killing/Dying'

1 0

[copr] skvidal-backend: odd that pyflakes didn't catch a comma missing (c7f2dc7)
by Seth Vidal 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit c7f2dc73ab31b1f904c3bca2eb44c36738237e4e Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 14:25:23 2012 -0500 odd that pyflakes didn't catch a comma missing >--------------------------------------------------------------- copr-be.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/copr-be.py b/copr-be.py index 4fd48af..fc03430 100644 --- a/copr-be.py +++ b/copr-be.py @@ -176,7 +176,7 @@ def parse_args(args): parser = optparse.OptionParser('\ncopr-be [options]') parser.add_option('-c', '--config', default='/etc/copr-be.conf', dest='config_file', help="config file to use for copr-be run") - parser.add_option('-d' '--daemonize', default=False, dest='daemonize', + parser.add_option('-d','--daemonize', default=False, dest='daemonize', action='store_true', help="daemonize or not") parser.add_option('-p', '--pidfile', default='/var/run/copr-be.pid', dest='pidfile', help="pid file to use for copr-be if daemonized")

1 0

[copr] skvidal-backend: - add option parsing for config file, exit_on_worker, daemonize and pidfile - try out daemonize support (2840524)
by Seth Vidal 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 28405241f3b747ddae19f143ff486c86431cd20a Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 14:22:57 2012 -0500 - add option parsing for config file, exit_on_worker, daemonize and pidfile - try out daemonize support >--------------------------------------------------------------- copr-be.py | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 files changed, 75 insertions(+), 6 deletions(-) diff --git a/copr-be.py b/copr-be.py index 73ab70a..4fd48af 100644 --- a/copr-be.py +++ b/copr-be.py @@ -10,6 +10,7 @@ from backend.dispatcher import Worker from backend import errors from bunch import Bunch import ConfigParser +import optparse def _get_conf(cp, section, option, default): """to make returning items from config parser less irritating""" @@ -59,6 +60,8 @@ class CoprBackend(object): opts.terminate_playbook = _get_conf(cp,'backend','terminate_playbook', '/etc/copr/terminate_playbook.yml') opts.jobsdir = _get_conf(cp, 'backend', 'jobsdir', None) opts.destdir = _get_conf(cp, 'backend', 'destdir', None) + opts.daemonize = _get_conf(cp, 'backend', 'daemonize', True) + opts.exit_on_worker = _get_conf(cp, 'backend', 'exit_on_worker', False) opts.sleeptime = int(_get_conf(cp, 'backend', 'sleeptime', 10)) opts.num_workers = int(_get_conf(cp, 'backend', 'num_workers', 8)) opts.timeout = int(_get_conf(cp, 'builder', 'timeout', 1800)) @@ -80,8 +83,12 @@ class CoprBackend(object): def log(self, msg): now = time.time() + output = str(now) + ':' + msg + if not self.opts.daemonize: + print output + try: - open(self.logfile, 'a').write(str(now) + ':' + msg + '\n') + open(self.logfile, 'a').write(output + '\n') except (IOError, OSError), e: print >>sys.stderr, 'Could not write to logfile %s - %s' % (self.logfile, str(e)) @@ -120,17 +127,79 @@ class CoprBackend(object): # check for dead workers and abort for w in self.workers: if not w.is_alive(): - raise errors.CoprBackendError, "Worker died unexpectedly" + self.log('Worker %d died unexpectedly' % w.worker_num) + if self.opts.exit_on_worker: + raise errors.CoprBackendError, "Worker died unexpectedly, exiting" + time.sleep(self.opts.sleeptime) +# lifted from certmaster +def daemonize(pidfile=None): + """ + Daemonize this process with the UNIX double-fork trick. + Writes the new PID to the provided file name if not None. + """ -def main(args): - if len(args) < 1 or not os.path.exists(args[0]): - print 'Must pass in config file' + pid = os.fork() + if pid > 0: + sys.exit(0) + os.chdir("/") + os.setsid() + os.umask(077) + pid = os.fork() + + os.close(0) + os.close(1) + os.close(2) + + # The standard I/O file descriptors are redirected to /dev/null by default. + if (hasattr(os, "devnull")): + REDIRECT_TO = os.devnull + else: + REDIRECT_TO = "/dev/null" + + # based on http://code.activestate.com/recipes/278731/ + os.open(REDIRECT_TO, os.O_RDWR) # standard input (0) + + os.dup2(0, 1) # standard output (1) + os.dup2(0, 2) # standard error (2) + + + + if pid > 0: + if pidfile is not None: + open(pidfile, "w").write(str(pid)) + sys.exit(0) + +def parse_args(args): + parser = optparse.OptionParser('\ncopr-be [options]') + parser.add_option('-c', '--config', default='/etc/copr-be.conf', dest='config_file', + help="config file to use for copr-be run") + parser.add_option('-d' '--daemonize', default=False, dest='daemonize', + action='store_true', help="daemonize or not") + parser.add_option('-p', '--pidfile', default='/var/run/copr-be.pid', dest='pidfile', + help="pid file to use for copr-be if daemonized") + parser.add_option('-x', '--exit', default=False, dest='exit_on_worker', + action='store_true', help="exit on worker failure") + + opts, args = parser.parse_args(args) + if not os.path.exists(opts.config_file): + print "No config file found at: %s" % opts.config_file sys.exit(1) + + return opts,args + + + +def main(args): + opts,args = parse_args(args) + try: - cbe = CoprBackend(args[0]) + cbe = CoprBackend(opts.config_file) + cbe.opts.daemonize = opts.daemonize # just so we have it on hand + cbe.opts.exit_on_worker = opts.exit_on_worker + daemonize(opts.pidfile) cbe.run() except Exception, e: print 'Killing/Dying'

1 0

[copr] skvidal-backend: touch up the log outputs to make it easier to know what to do (8fc0c3e)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 8fc0c3e6a2eb3cd253ccdc75f4c2c05743e3590d Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 01:37:24 2012 -0500 touch up the log outputs to make it easier to know what to do >--------------------------------------------------------------- backend/dispatcher.py | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index eff1f27..e1a47dc 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -102,7 +102,7 @@ class Worker(multiprocessing.Process): if ip: self.callback.log('creating worker: %s' % ip) else: - self.callback.log('creating worker: with dynamic ip') + self.callback.log('creating worker: dynamic ip') def spawn_instance(self): """call the spawn playbook to startup/provision a building instance""" @@ -130,7 +130,9 @@ class Worker(multiprocessing.Process): # if we get here we're in trouble self.callback.log('No IP back from spawn_instance - dumping cache output') self.callback.log(str(play.SETUP_CACHE)) - self.callback.log(play.stats.summarize('localhost')) + self.callback.log(str(play.stats.summarize('localhost'))) + self.callback.log('Test spawn_instance playbook manually') + return None def terminate_instance(self,ip):

1 0

[copr] skvidal-backend: localhost as string to get back debug on heisenbug (c881da2)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit c881da21fc3f7fa723610919f627411b25e1a8e2 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 00:59:54 2012 -0500 localhost as string to get back debug on heisenbug >--------------------------------------------------------------- backend/dispatcher.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index 796a9b6..eff1f27 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -130,7 +130,7 @@ class Worker(multiprocessing.Process): # if we get here we're in trouble self.callback.log('No IP back from spawn_instance - dumping cache output') self.callback.log(str(play.SETUP_CACHE)) - self.callback.log(play.stats.summarize(localhost)) + self.callback.log(play.stats.summarize('localhost')) return None def terminate_instance(self,ip):

1 0

[copr] skvidal-backend: when we're dealing with no repos specified - make sure we don't treat '' as a repo (a1349ac)
by Seth Vidal 04 Dec '12

04 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit a1349acfffefce3ee6bd56b1eac12312e619ef1a Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 00:56:57 2012 -0500 when we're dealing with no repos specified - make sure we don't treat '' as a repo >--------------------------------------------------------------- backend/dispatcher.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index a9427aa..796a9b6 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -154,7 +154,7 @@ class Worker(multiprocessing.Process): build = d['builds'][0] jobdata = Bunch() jobdata.pkgs = build['pkgs'].split(' ') - jobdata.repos = build['repos'].split(' ') + jobdata.repos = [r for r in build['repos'].split(' ') if r.strip() ] jobdata.chroots = build['chroots'].split(' ') jobdata.memory_reqs = build['memory_reqs'] jobdata.timeout = build['timeout'] @@ -223,7 +223,7 @@ class Worker(multiprocessing.Process): # start the build - most importantly license checks. - self.callback.log('mockremote %s %s %s %s %s' % (ip, job.timeout, job.destdir, chroot, str(job.repos))) + self.callback.log('Calling mockremote with builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (ip, job.timeout, job.destdir, chroot, str(job.repos))) try: chrootlogfile = chroot_destdir + '/mockremote.log' mr = mockremote.MockRemote(builder=ip, timeout=job.timeout,

1 0

[copr] skvidal-backend: more debug output yay! (f568f12)
by Seth Vidal 04 Dec '12

04 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit f568f120f4dddf1fdb1818db9664d36efcded46b Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Tue Dec 4 16:55:14 2012 -0500 more debug output yay! >--------------------------------------------------------------- backend/dispatcher.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index 4afd9f9..a9427aa 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -130,6 +130,7 @@ class Worker(multiprocessing.Process): # if we get here we're in trouble self.callback.log('No IP back from spawn_instance - dumping cache output') self.callback.log(str(play.SETUP_CACHE)) + self.callback.log(play.stats.summarize(localhost)) return None def terminate_instance(self,ip):

1 0

[copr] skvidal-backend: dear world in the future - read your own code (16acf7c)
by skvidal＠fedorahosted.org 04 Dec '12

04 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 16acf7ce04ae4e242691c91aa8ca10fbb289ee2a Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Tue Dec 4 16:44:17 2012 -0500 dear world in the future - read your own code >--------------------------------------------------------------- backend/dispatcher.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index bf80e28..4afd9f9 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -228,7 +228,7 @@ class Worker(multiprocessing.Process): mr = mockremote.MockRemote(builder=ip, timeout=job.timeout, destdir=job.destdir, chroot=chroot, cont=True, recurse=True, repos=job.repos, - callback=mockremote.CLiLogCallBack(quiet=True,logfn=chrootlogfile)) + callback=mockremote.CliLogCallBack(quiet=True,logfn=chrootlogfile)) mr.build_pkgs(job.pkgs) except mockremote.MockRemoteError, e: # record and break

1 0

[copr] skvidal-backend: more logging in error conditions having to do with spawning instances (31ab388)
by skvidal＠fedorahosted.org 04 Dec '12

04 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 31ab388e827dbe9a279a8bb8eac76ff1acb60212 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Tue Dec 4 16:37:43 2012 -0500 more logging in error conditions having to do with spawning instances >--------------------------------------------------------------- backend/dispatcher.py | 9 ++++++++- copr-be.py | 1 + 2 files changed, 9 insertions(+), 1 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index 225b75d..bf80e28 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -99,7 +99,10 @@ class Worker(multiprocessing.Process): self.logfile = self.opts.worker_logdir + '/worker-%s.log' % self.worker_num self.callback = WorkerCallback(logfile = self.logfile) - self.callback.log('creating worker: %s' % ip) + if ip: + self.callback.log('creating worker: %s' % ip) + else: + self.callback.log('creating worker: with dynamic ip') def spawn_instance(self): """call the spawn playbook to startup/provision a building instance""" @@ -123,6 +126,10 @@ class Worker(multiprocessing.Process): if i =='localhost': continue return i + + # if we get here we're in trouble + self.callback.log('No IP back from spawn_instance - dumping cache output') + self.callback.log(str(play.SETUP_CACHE)) return None def terminate_instance(self,ip): diff --git a/copr-be.py b/copr-be.py index 2cc0fba..73ab70a 100644 --- a/copr-be.py +++ b/copr-be.py @@ -117,6 +117,7 @@ class CoprBackend(object): # #insert a poison pill? Kill after something? I dunno. # FIXME - if a worker bombs out - we need to check them # and startup a new one if it happens + # check for dead workers and abort for w in self.workers: if not w.is_alive(): raise errors.CoprBackendError, "Worker died unexpectedly"

1 0

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

copr-devel December 2012