Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 00ccbefe32ed5a4f5827e4d7831a85f4752d597e
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Thu Dec 6 02:18:49 2012 -0500
move the config reread out of the queuesize change - so we know it will happen in the loop
even when no jobs are happening
>---------------------------------------------------------------
copr-be.py | 5 +++--
1 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index e5128d9..6564134 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -134,10 +134,11 @@ class CoprBackend(object):
self.added_jobs.append(n)
self.log('adding to work queue id %s' % n)
+ # re-read config into opts
+ self.opts = self.read_conf()
+
if self.jobs.qsize():
self.log("# jobs in queue: %s" % self.jobs.qsize())
- # re-read config into opts
- self.opts = self.read_conf()
# this handles starting/growing the number of workers
if len(self.workers) < self.opts.num_workers:
self.log("Spinning up more workers for jobs")
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 9da44f0b3c2417e53c9eadf9f44d17d3e5f272bf
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Thu Dec 6 02:16:25 2012 -0500
when we startup make sure our destdir exists
when a worker dies unexpectedly - make sure we kill it and get rid of it
>---------------------------------------------------------------
copr-be.py | 11 +++++++----
1 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 27ceb03..e5128d9 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -37,6 +37,9 @@ class CoprBackend(object):
if not os.path.exists(logdir):
os.makedirs(logdir, mode=0750)
+ if not os.path.exists(self.opts.destdir):
+ os.makedirs(self.opts.destdir, mode=0755)
+
# setup a log file to write to
self.logfile = self.opts.logfile
self.log("Starting up new copr-be instance")
@@ -106,8 +109,6 @@ class CoprBackend(object):
except requests.RequestException, e:
self.log('Error retrieving jobs from %s: %s' % (self.opts.frontend_url, e))
else:
-
-
r_json = json.loads(r.content) # using old requests on el6 :(
if 'builds' in r_json:
self.log('%s jobs returned' % len(r_json['builds']))
@@ -131,7 +132,7 @@ class CoprBackend(object):
if n not in self.added_jobs:
self.jobs.put(f)
self.added_jobs.append(n)
- self.log('adding %s' % n)
+ self.log('adding to work queue id %s' % n)
if self.jobs.qsize():
self.log("# jobs in queue: %s" % self.jobs.qsize())
@@ -159,7 +160,9 @@ class CoprBackend(object):
self.log('Worker %d died unexpectedly' % w.worker_num)
if self.opts.exit_on_worker:
raise errors.CoprBackendError, "Worker died unexpectedly, exiting"
-
+ else:
+ self.workers.remove(w) # it is not working anymore
+ w.terminate() # kill it with a fire
time.sleep(self.opts.sleeptime)
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 02f9e18594e1a03d9bc127da601e45aebaca8b0c
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Thu Dec 6 02:00:11 2012 -0500
older python-requests doesn't have attribute :(
>---------------------------------------------------------------
copr-be.py | 9 ++++++---
1 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 8ab8470..27ceb03 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -106,10 +106,13 @@ class CoprBackend(object):
except requests.RequestException, e:
self.log('Error retrieving jobs from %s: %s' % (self.opts.frontend_url, e))
else:
- if 'builds' in r.json:
- self.log('%s jobs returned' % len(r.json['builds']))
+
+
+ r_json = json.loads(r.content) # using old requests on el6 :(
+ if 'builds' in r_json:
+ self.log('%s jobs returned' % len(r_json['builds']))
count = 0
- for b in r.json['builds']:
+ for b in r_json['builds']:
if 'id' in b:
jobfile = self.opts.jobsdir + '/%s.json' % b['id']
if not os.path.exists(jobfile) and b['id'] not in self.added_jobs:
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 2eb8d67a3a2ebbc509e8c25985d44b54287d8f62
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Thu Dec 6 01:52:44 2012 -0500
- saving the jobs out per-build - so don't expect a 'builds' dictionary
- fetch the jobs from the front end
>---------------------------------------------------------------
backend/dispatcher.py | 3 +--
copr-be.py | 22 ++++++++++++++++++++++
2 files changed, 23 insertions(+), 2 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index b9c1251..fc9a958 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -153,8 +153,7 @@ class Worker(multiprocessing.Process):
def parse_job(self, jobfile):
# read the json of the job in
# break out what we need return a bunch of the info we need
- d = json.load(open(jobfile))
- build = d['builds'][0]
+ build = json.load(open(jobfile))
jobdata = Bunch()
jobdata.pkgs = build['pkgs'].split(' ')
jobdata.repos = [r for r in build['repos'].split(' ') if r.strip() ]
diff --git a/copr-be.py b/copr-be.py
index bd26444..8ab8470 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -11,6 +11,8 @@ from backend import errors
from bunch import Bunch
import ConfigParser
import optparse
+import json
+import requests
def _get_conf(cp, section, option, default):
"""to make returning items from config parser less irritating"""
@@ -97,10 +99,30 @@ class CoprBackend(object):
print >>sys.stderr, 'Could not write to logfile %s - %s' % (self.logfile, str(e))
+ def fetch_jobs(self):
+ self.log('fetching jobs')
+ try:
+ r = requests.get('%s/waiting_builds/' % self.opts.frontend_url) # auth stuff here? maybe/maybenot
+ except requests.RequestException, e:
+ self.log('Error retrieving jobs from %s: %s' % (self.opts.frontend_url, e))
+ else:
+ if 'builds' in r.json:
+ self.log('%s jobs returned' % len(r.json['builds']))
+ count = 0
+ for b in r.json['builds']:
+ if 'id' in b:
+ jobfile = self.opts.jobsdir + '/%s.json' % b['id']
+ if not os.path.exists(jobfile) and b['id'] not in self.added_jobs:
+ count += 1
+ open(jobfile, 'w').write(json.dumps(b))
+ self.log('Wrote job: %s' % b['id'])
+ self.log('New jobs: %s' % count)
+
def run(self):
abort = False
while not abort:
+ self.fetch_jobs()
for f in sorted(glob.glob(self.opts.jobsdir + '/*.json')):
n = os.path.basename(f).replace('.json', '')
if n not in self.added_jobs:
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 58c0441ccecf4694cc45a82f43fcf372a9090528
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Wed Dec 5 15:14:14 2012 -0500
still need something like a useful output when it does break
>---------------------------------------------------------------
copr-be.py | 11 ++++++-----
1 files changed, 6 insertions(+), 5 deletions(-)
diff --git a/copr-be.py b/copr-be.py
index 3be1828..d98feea 100644
--- a/copr-be.py
+++ b/copr-be.py
@@ -28,7 +28,7 @@ class CoprBackend(object):
raise errors.CoprBackendError, "Must specify config_file"
self.config_file = config_file
- self._ext_opts = ext_opts # to stow our cli options for read_conf()
+ self.ext_opts = ext_opts # to stow our cli options for read_conf()
self.opts = self.read_conf()
logdir = os.path.dirname(self.opts.logfile)
@@ -79,8 +79,8 @@ class CoprBackend(object):
if not opts.jobsdir or not opts.destdir:
raise errors.CoprBackendError, "Incomplete Config - must specify jobsdir and destdir in configuration"
- if self._ext_opts:
- for v in self._ext_opts:
+ if self.ext_opts:
+ for v in self.ext_opts:
setattr(opts, v, self.ext_opts.get(v))
return opts
@@ -213,8 +213,9 @@ def main(args):
if 'cbe' in locals():
for w in cbe.workers:
w.terminate()
-
-
+ raise
+ except KeyboardInterrupt, e:
+ pass
if __name__ == '__main__':
try: