Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit b336c6d22bc863b44410089d936504d2b2065c2b
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Sat Dec 8 13:04:44 2012 -0500
more todo's highlighting a bug
>---------------------------------------------------------------
TODO-backend | 2 ++
1 files changed, 2 insertions(+), 0 deletions(-)
diff --git a/TODO-backend b/TODO-backend
index 058330e..4cc5ca1 100644
--- a/TODO-backend
+++ b/TODO-backend
@@ -15,3 +15,5 @@
- work on a way to find and cancel a specific build that's happening other than just killing the instance
- determine if it is properly checking the timeout from a dead instance
- maybe dump out the PID of the worker that is running so we know which one to kill?
+- failure/success not being returned correctly. Should check for 'fail' in the directories and return based on
+ that. also anything lacking success is a failure.
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 04b2f891e06aaa4ea79188d641594af34d24fba7
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Dec 7 17:05:54 2012 -0500
add another TODO note
>---------------------------------------------------------------
TODO-backend | 1 +
1 files changed, 1 insertions(+), 0 deletions(-)
diff --git a/TODO-backend b/TODO-backend
index a978e13..058330e 100644
--- a/TODO-backend
+++ b/TODO-backend
@@ -14,3 +14,4 @@
- refactor mockremote/dispatcher.worker together?
- work on a way to find and cancel a specific build that's happening other than just killing the instance
- determine if it is properly checking the timeout from a dead instance
+- maybe dump out the PID of the worker that is running so we know which one to kill?
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 25b9a4f18ee342a0bbc797c796f27ec8756ee705
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Fri Dec 7 16:47:52 2012 -0500
remove/add todo's
>---------------------------------------------------------------
TODO-backend | 3 ++-
1 files changed, 2 insertions(+), 1 deletions(-)
diff --git a/TODO-backend b/TODO-backend
index e532fc9..a978e13 100644
--- a/TODO-backend
+++ b/TODO-backend
@@ -7,9 +7,10 @@
- LOADS of fixme and catching weird conditions
- make logging from mockremote more sane and coinsistent
- mock configs should be pushed to instances at creation time
- - should include enabled package_state pluging
- single url to repos, not mirrorlists
- consider making each worker return job to a completed queue so the primary
process can do other kinds of notification
- email notifications from backend?
- refactor mockremote/dispatcher.worker together?
+- work on a way to find and cancel a specific build that's happening other than just killing the instance
+- determine if it is properly checking the timeout from a dead instance
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit cc1f0ba6e55734f7f453a47321cbd778b6453de3
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Thu Dec 6 17:25:35 2012 -0500
add TODO list from my paper lists
>---------------------------------------------------------------
TODO-backend | 15 +++++++++++++++
1 files changed, 15 insertions(+), 0 deletions(-)
diff --git a/TODO-backend b/TODO-backend
new file mode 100644
index 0000000..e532fc9
--- /dev/null
+++ b/TODO-backend
@@ -0,0 +1,15 @@
+
+- change instance type by build request for more mem/procs/extend timeouts
+ - use extra-vars?
+ - need ansible 0.9?
+- auto-timer/cleanup script for old instances that may have been orphaned
+- prune out builders when we drop the number of them active
+- LOADS of fixme and catching weird conditions
+- make logging from mockremote more sane and coinsistent
+- mock configs should be pushed to instances at creation time
+ - should include enabled package_state pluging
+ - single url to repos, not mirrorlists
+- consider making each worker return job to a completed queue so the primary
+ process can do other kinds of notification
+- email notifications from backend?
+- refactor mockremote/dispatcher.worker together?
Repository : http://git.fedorahosted.org/cgit/copr.git
On branch : skvidal-backend
>---------------------------------------------------------------
commit 707d00179d06651528caa41cae30ca573afcd83d
Author: Seth Vidal <skvidal(a)fedoraproject.org>
Date: Thu Dec 6 15:16:16 2012 -0500
post status to frontend using python-requests
>---------------------------------------------------------------
backend/dispatcher.py | 61 +++++++++++++++++++++++++++++++++++++++---------
1 files changed, 49 insertions(+), 12 deletions(-)
diff --git a/backend/dispatcher.py b/backend/dispatcher.py
index db800cc..c113e8c 100644
--- a/backend/dispatcher.py
+++ b/backend/dispatcher.py
@@ -3,7 +3,6 @@
import os
import sys
-import shutil
import multiprocessing
import time
import Queue
@@ -15,13 +14,11 @@ import ansible
import ansible.playbook
import ansible.errors
from ansible import callbacks
+import requests
-
-#FIXME - this should be emitting to the per-worker log file
-# so we can know what is going on and where
class SilentPlaybookCallbacks(callbacks.PlaybookCallbacks):
''' playbook callbacks - quietly! '''
@@ -162,22 +159,60 @@ class Worker(multiprocessing.Process):
jobdata.timeout = build['timeout']
jobdata.destdir = self.opts.destdir + '/' + build['copr']['owner']['name'] + '/' + build['copr']['name'] + '/'
jobdata.build_id = build['id']
+ jobdata.results = self.opts.results_baseurl + '/' + build['copr']['owner']['name'] + '/' + build['copr']['name'] + '/'
jobdata.copr_id = build['copr']['id']
jobdata.user_id = build['user_id']
return jobdata
+ # maybe we move this to the callback?
+ def post_to_frontend(self, data):
+ """send data to frontend"""
+
+ headers = {'content-type': 'application/json'}
+ url='%s/update_builds/' % self.opts.frontend_url
+ auth=('user', self.opts.frontend_auth)
+
+ msg = None
+ try:
+ r = requests.post(url, data=json.dumps(data), auth=auth,
+ headers=headers)
+ if r.status_code != 200:
+ msg = 'Failed to submit to frontend: %s: %s' % (r.status_code, r.text)
+ except requests.RequestException, e:
+ msg = 'Post request failed: %s' % e
+
+ if msg:
+ self.callback.log(msg)
+ return False
+
+ return True
+
+ # maybe we move this to the callback?
+ def mark_started(self, job):
+
+ build = {'id':job.build_id,
+ 'started_on': job.started_on,
+ 'results': job.results,
+ }
+ data = {'builds':[build]}
+
+ if not self.post_to_frontend(data):
+ raise errors.CoprWorkerError, "Could not communicate to front end to submit status info"
+
+ # maybe we move this to the callback?
def return_results(self, job):
- """write out a completed json file to the results dir and submit the results to the frontend"""
self.callback.log('%s status %s. Took %s seconds' % (job.build_id, job.status, job.ended_on - job.started_on))
- jobfilename = os.path.basename(job.jobfile)
- shutil.move(job.jobfile, job.destdir + '/' + jobfilename)
- # reform json file with completed info (completed time, url, status, etc)
- # post it via requests to url with passcode
- # profit
+ build = {'id':job.build_id,
+ 'ended_on': job.ended_on,
+ 'status': job.status,
+ }
+ data = {'builds':[build]}
+
+ if not self.post_to_frontend(data):
+ raise errors.CoprWorkerError, "Could not communicate to front end to submit results"
- #FIXME - this should either return job status/results
- # into a queue or it should submit results directly to the frontend
+ os.unlink(job.jobfile)
def run(self):
# worker should startup and check if it can function
@@ -214,6 +249,8 @@ class Worker(multiprocessing.Process):
status = 1
job.started_on = time.time()
+ self.mark_started(job)
+
for chroot in job.chroots:
chroot_destdir = job.destdir + '/' + chroot