copr-devel December 2012

copr-devel@lists.fedorahosted.org

4 participants
47 discussions

[copr] skvidal-backend: move the config reread out of the queuesize change - so we know it will happen in the loop even when no jobs are happening (00ccbef)
by Seth Vidal 06 Dec '12

06 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 00ccbefe32ed5a4f5827e4d7831a85f4752d597e Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Thu Dec 6 02:18:49 2012 -0500 move the config reread out of the queuesize change - so we know it will happen in the loop even when no jobs are happening >--------------------------------------------------------------- copr-be.py | 5 +++-- 1 files changed, 3 insertions(+), 2 deletions(-) diff --git a/copr-be.py b/copr-be.py index e5128d9..6564134 100644 --- a/copr-be.py +++ b/copr-be.py @@ -134,10 +134,11 @@ class CoprBackend(object): self.added_jobs.append(n) self.log('adding to work queue id %s' % n) + # re-read config into opts + self.opts = self.read_conf() + if self.jobs.qsize(): self.log("# jobs in queue: %s" % self.jobs.qsize()) - # re-read config into opts - self.opts = self.read_conf() # this handles starting/growing the number of workers if len(self.workers) < self.opts.num_workers: self.log("Spinning up more workers for jobs")

1 0

[copr] skvidal-backend: when we startup make sure our destdir exists when a worker dies unexpectedly - make sure we kill it and get rid of it (9da44f0)
by Seth Vidal 06 Dec '12

06 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 9da44f0b3c2417e53c9eadf9f44d17d3e5f272bf Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Thu Dec 6 02:16:25 2012 -0500 when we startup make sure our destdir exists when a worker dies unexpectedly - make sure we kill it and get rid of it >--------------------------------------------------------------- copr-be.py | 11 +++++++---- 1 files changed, 7 insertions(+), 4 deletions(-) diff --git a/copr-be.py b/copr-be.py index 27ceb03..e5128d9 100644 --- a/copr-be.py +++ b/copr-be.py @@ -37,6 +37,9 @@ class CoprBackend(object): if not os.path.exists(logdir): os.makedirs(logdir, mode=0750) + if not os.path.exists(self.opts.destdir): + os.makedirs(self.opts.destdir, mode=0755) + # setup a log file to write to self.logfile = self.opts.logfile self.log("Starting up new copr-be instance") @@ -106,8 +109,6 @@ class CoprBackend(object): except requests.RequestException, e: self.log('Error retrieving jobs from %s: %s' % (self.opts.frontend_url, e)) else: - - r_json = json.loads(r.content) # using old requests on el6 :( if 'builds' in r_json: self.log('%s jobs returned' % len(r_json['builds'])) @@ -131,7 +132,7 @@ class CoprBackend(object): if n not in self.added_jobs: self.jobs.put(f) self.added_jobs.append(n) - self.log('adding %s' % n) + self.log('adding to work queue id %s' % n) if self.jobs.qsize(): self.log("# jobs in queue: %s" % self.jobs.qsize()) @@ -159,7 +160,9 @@ class CoprBackend(object): self.log('Worker %d died unexpectedly' % w.worker_num) if self.opts.exit_on_worker: raise errors.CoprBackendError, "Worker died unexpectedly, exiting" - + else: + self.workers.remove(w) # it is not working anymore + w.terminate() # kill it with a fire time.sleep(self.opts.sleeptime)

1 0

[copr] skvidal-backend: older python-requests doesn't have attribute :( (02f9e18)
by Seth Vidal 06 Dec '12

06 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 02f9e18594e1a03d9bc127da601e45aebaca8b0c Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Thu Dec 6 02:00:11 2012 -0500 older python-requests doesn't have attribute :( >--------------------------------------------------------------- copr-be.py | 9 ++++++--- 1 files changed, 6 insertions(+), 3 deletions(-) diff --git a/copr-be.py b/copr-be.py index 8ab8470..27ceb03 100644 --- a/copr-be.py +++ b/copr-be.py @@ -106,10 +106,13 @@ class CoprBackend(object): except requests.RequestException, e: self.log('Error retrieving jobs from %s: %s' % (self.opts.frontend_url, e)) else: - if 'builds' in r.json: - self.log('%s jobs returned' % len(r.json['builds'])) + + + r_json = json.loads(r.content) # using old requests on el6 :( + if 'builds' in r_json: + self.log('%s jobs returned' % len(r_json['builds'])) count = 0 - for b in r.json['builds']: + for b in r_json['builds']: if 'id' in b: jobfile = self.opts.jobsdir + '/%s.json' % b['id'] if not os.path.exists(jobfile) and b['id'] not in self.added_jobs:

1 0

[copr] skvidal-backend: - saving the jobs out per-build - so don't expect a 'builds' dictionary - fetch the jobs from the front end (2eb8d67)
by Seth Vidal 06 Dec '12

06 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 2eb8d67a3a2ebbc509e8c25985d44b54287d8f62 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Thu Dec 6 01:52:44 2012 -0500 - saving the jobs out per-build - so don't expect a 'builds' dictionary - fetch the jobs from the front end >--------------------------------------------------------------- backend/dispatcher.py | 3 +-- copr-be.py | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index b9c1251..fc9a958 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -153,8 +153,7 @@ class Worker(multiprocessing.Process): def parse_job(self, jobfile): # read the json of the job in # break out what we need return a bunch of the info we need - d = json.load(open(jobfile)) - build = d['builds'][0] + build = json.load(open(jobfile)) jobdata = Bunch() jobdata.pkgs = build['pkgs'].split(' ') jobdata.repos = [r for r in build['repos'].split(' ') if r.strip() ] diff --git a/copr-be.py b/copr-be.py index bd26444..8ab8470 100644 --- a/copr-be.py +++ b/copr-be.py @@ -11,6 +11,8 @@ from backend import errors from bunch import Bunch import ConfigParser import optparse +import json +import requests def _get_conf(cp, section, option, default): """to make returning items from config parser less irritating""" @@ -97,10 +99,30 @@ class CoprBackend(object): print >>sys.stderr, 'Could not write to logfile %s - %s' % (self.logfile, str(e)) + def fetch_jobs(self): + self.log('fetching jobs') + try: + r = requests.get('%s/waiting_builds/' % self.opts.frontend_url) # auth stuff here? maybe/maybenot + except requests.RequestException, e: + self.log('Error retrieving jobs from %s: %s' % (self.opts.frontend_url, e)) + else: + if 'builds' in r.json: + self.log('%s jobs returned' % len(r.json['builds'])) + count = 0 + for b in r.json['builds']: + if 'id' in b: + jobfile = self.opts.jobsdir + '/%s.json' % b['id'] + if not os.path.exists(jobfile) and b['id'] not in self.added_jobs: + count += 1 + open(jobfile, 'w').write(json.dumps(b)) + self.log('Wrote job: %s' % b['id']) + self.log('New jobs: %s' % count) + def run(self): abort = False while not abort: + self.fetch_jobs() for f in sorted(glob.glob(self.opts.jobsdir + '/*.json')): n = os.path.basename(f).replace('.json', '') if n not in self.added_jobs:

1 0

[copr] skvidal-backend: make sure we're using whatever current umask is and we don't force into lesser levels (6d065fa)
by Seth Vidal 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 6d065fa3965f0974cdc57b6d9a54f4a7205f56bf Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 17:06:37 2012 -0500 make sure we're using whatever current umask is and we don't force into lesser levels >--------------------------------------------------------------- backend/dispatcher.py | 2 +- copr-be.py | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index 16e0825..b9c1251 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -216,7 +216,7 @@ class Worker(multiprocessing.Process): # setup our target dir locally if not os.path.exists(chroot_destdir): try: - os.makedirs(chroot_destdir, mode=0755) + os.makedirs(chroot_destdir) except (OSError, IOError), e: msg = "Could not make results dir for job: %s - %s" % (chroot_destdir, str(e)) self.callback.log(msg) diff --git a/copr-be.py b/copr-be.py index 9cd2bba..bd26444 100644 --- a/copr-be.py +++ b/copr-be.py @@ -145,12 +145,15 @@ def daemonize(pidfile=None): Writes the new PID to the provided file name if not None. """ + cur_umask = os.umask(077) + os.umask(cur_umask) + pid = os.fork() if pid > 0: sys.exit(0) os.chdir("/") os.setsid() - os.umask(077) + os.umask(cur_umask) pid = os.fork() os.close(0) @@ -223,6 +226,7 @@ if __name__ == '__main__': main(sys.argv[1:]) except Exception, e: print "ERROR: %s - %s" % (str(type(e)), str(e)) + # FIXME - maybe check on daemonize and do this as a syslog.syslog() call? sys.exit(1) except KeyboardInterrupt, e: print "\nUser cancelled, may need cleanup\n"

1 0

[copr] skvidal-backend: when we daemonize() we chdir() so if we don't abspath the config file we fall over and die :-/ (9f65c8f)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 9f65c8fd7a6cd0ce5931d61dce51b71ad5b3af0f Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 16:29:00 2012 -0500 when we daemonize() we chdir() so if we don't abspath the config file we fall over and die :-/ >--------------------------------------------------------------- copr-be.py | 3 ++- 1 files changed, 2 insertions(+), 1 deletions(-) diff --git a/copr-be.py b/copr-be.py index 9a5e3a4..9cd2bba 100644 --- a/copr-be.py +++ b/copr-be.py @@ -191,6 +191,7 @@ def parse_args(args): if not os.path.exists(opts.config_file): print "No config file found at: %s" % opts.config_file sys.exit(1) + opts.config_file = os.path.abspath(opts.config_file) ret_opts = Bunch() for o in ('daemonize', 'exit_on_worker', 'pidfile', 'config_file'): @@ -204,9 +205,9 @@ def main(args): opts = parse_args(args) try: + cbe = CoprBackend(opts.config_file, ext_opts=opts) if opts.daemonize: daemonize(opts.pidfile) - cbe = CoprBackend(opts.config_file, ext_opts=opts) cbe.run() except Exception, e: print 'Killing/Dying'

1 0

[copr] skvidal-backend: trying to figure out why the daemonized process dies (7ae63a9)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 7ae63a9178245688bb23ac7168fbe5546918187b Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 16:17:31 2012 -0500 trying to figure out why the daemonized process dies >--------------------------------------------------------------- copr-be.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/copr-be.py b/copr-be.py index 5b35905..9a5e3a4 100644 --- a/copr-be.py +++ b/copr-be.py @@ -204,9 +204,9 @@ def main(args): opts = parse_args(args) try: - cbe = CoprBackend(opts.config_file, ext_opts=opts) if opts.daemonize: daemonize(opts.pidfile) + cbe = CoprBackend(opts.config_file, ext_opts=opts) cbe.run() except Exception, e: print 'Killing/Dying'

1 0

[copr] skvidal-backend: - make pid file default to cwd unless specified - better log output for the building - fixme notes (2a047db)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 2a047db2d68b9439a6d63d01f1f31845f5a6a7a7 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 16:04:22 2012 -0500 - make pid file default to cwd unless specified - better log output for the building - fixme notes >--------------------------------------------------------------- backend/dispatcher.py | 7 ++++++- copr-be.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/backend/dispatcher.py b/backend/dispatcher.py index f96bdb1..16e0825 100644 --- a/backend/dispatcher.py +++ b/backend/dispatcher.py @@ -191,6 +191,10 @@ class Worker(multiprocessing.Process): # parse the job json into our info job = self.parse_job(jobfile) + # FIXME + # this is our best place to sanity check the job before starting + # up any longer process + job.jobfile = jobfile # spin up our build instance @@ -227,7 +231,7 @@ class Worker(multiprocessing.Process): # start the build - most importantly license checks. - self.callback.log('Calling mockremote with builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (ip, job.timeout, job.destdir, chroot, str(job.repos))) + self.callback.log('Starting build: builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (ip, job.timeout, job.destdir, chroot, str(job.repos))) try: chrootlogfile = chroot_destdir + '/mockremote.log' mr = mockremote.MockRemote(builder=ip, timeout=job.timeout, @@ -239,6 +243,7 @@ class Worker(multiprocessing.Process): # record and break self.callback.log('%s - %s' % (ip, e)) status = 0 # failure + self.callback.log('Finished build: builder=%r timeout=%r destdir=%r chroot=%r repos=%r' % (ip, job.timeout, job.destdir, chroot, str(job.repos))) job.ended_on = time.time() job.status = status diff --git a/copr-be.py b/copr-be.py index d98feea..5b35905 100644 --- a/copr-be.py +++ b/copr-be.py @@ -182,7 +182,7 @@ def parse_args(args): help="config file to use for copr-be run") parser.add_option('-d','--daemonize', default=False, dest='daemonize', action='store_true', help="daemonize or not") - parser.add_option('-p', '--pidfile', default='/var/run/copr-be.pid', dest='pidfile', + parser.add_option('-p', '--pidfile', default='copr-be.pid', dest='pidfile', help="pid file to use for copr-be if daemonized") parser.add_option('-x', '--exit', default=False, dest='exit_on_worker', action='store_true', help="exit on worker failure")

1 0

[copr] skvidal-backend: spaces in rsync command for name of copr :-/ (858a533)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 858a5331b1902764f7ee9a3838c7c1a2c8cfe10a Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 15:47:03 2012 -0500 spaces in rsync command for name of copr :-/ >--------------------------------------------------------------- backend/mockremote.py | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/backend/mockremote.py b/backend/mockremote.py index 7698de9..3d39f3d 100755 --- a/backend/mockremote.py +++ b/backend/mockremote.py @@ -329,6 +329,7 @@ class Builder(object): success = False rpd = self._get_remote_pkg_dir(pkg) + destdir = "'" + destdir.replace("'", "'\\''") + "'" # make spaces work w/our rsync command below :( # build rsync command line from the above remote_src = '%s@%s:%s' % (self.username, self.hostname, rpd) ssh_opts = "'ssh -o PasswordAuthentication=no -o StrictHostKeyChecking=no'"

1 0

[copr] skvidal-backend: still need something like a useful output when it does break (58c0441)
by skvidal＠fedorahosted.org 05 Dec '12

05 Dec '12

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : skvidal-backend >--------------------------------------------------------------- commit 58c0441ccecf4694cc45a82f43fcf372a9090528 Author: Seth Vidal <skvidal(a)fedoraproject.org> Date: Wed Dec 5 15:14:14 2012 -0500 still need something like a useful output when it does break >--------------------------------------------------------------- copr-be.py | 11 ++++++----- 1 files changed, 6 insertions(+), 5 deletions(-) diff --git a/copr-be.py b/copr-be.py index 3be1828..d98feea 100644 --- a/copr-be.py +++ b/copr-be.py @@ -28,7 +28,7 @@ class CoprBackend(object): raise errors.CoprBackendError, "Must specify config_file" self.config_file = config_file - self._ext_opts = ext_opts # to stow our cli options for read_conf() + self.ext_opts = ext_opts # to stow our cli options for read_conf() self.opts = self.read_conf() logdir = os.path.dirname(self.opts.logfile) @@ -79,8 +79,8 @@ class CoprBackend(object): if not opts.jobsdir or not opts.destdir: raise errors.CoprBackendError, "Incomplete Config - must specify jobsdir and destdir in configuration" - if self._ext_opts: - for v in self._ext_opts: + if self.ext_opts: + for v in self.ext_opts: setattr(opts, v, self.ext_opts.get(v)) return opts @@ -213,8 +213,9 @@ def main(args): if 'cbe' in locals(): for w in cbe.workers: w.terminate() - - + raise + except KeyboardInterrupt, e: + pass if __name__ == '__main__': try:

1 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

copr-devel December 2012