This patch adds support for building from git:// respositories. It is used like

koji call build 'git://<REPOSITORY>?<PACKAGE>#<TAG>' <TARGET>

The repository specified above must have the following layout:

<BASE_DIR>/<REPOSITORY> |- common/ | `- .git/ |- ... |- <PACKAGE> | `- .git/ |- ...

This means that every package is hold in an own git repository. There must be a 'common' repository which contains files required for building the srpm.

There is no explicit support for branches; the patch just checks out the given tag and builds from it. Perhaps it can be enhanced to check whether branch-point is some child of the tag...

TODO: the git and cvs handlers share some common, non trivial code. This should be generalized.

TODO: use a separate user account for the 'git' and 'make srpm' operations; currently they are executed under the uid 'kojid' is running (which is 'root). Hence, a '%(/sbin/killall5)' in the spec file can bring down the box.

Signed-off-by: Enrico Scholz enrico.scholz@informatik.tu-chemnitz.de ---

builder/kojid | 130 +++++++++++++++++++++++++++++++++++++++++++- cli/koji | 9 ++- koji.spec | 2 - koji/__init__.py | 11 ++++ www/kojiweb/taskinfo.chtml | 7 ++ www/kojiweb/tasks.chtml | 1 6 files changed, 153 insertions(+), 7 deletions(-)

diff --git a/builder/kojid b/builder/kojid index 5940954..bbb9774 100755 --- a/builder/kojid +++ b/builder/kojid @@ -1396,7 +1396,7 @@ class ChainBuildTask(BaseTaskHandler): subtasks = [] build_tasks = [] for src in build_level: - if src.startswith('cvs://'): + if src.startswith('cvs://') or src.startswith('git://'): task_id = session.host.subtask(method='build', arglist=[src, target, opts], parent=self.id) @@ -1437,8 +1437,10 @@ class BuildTask(BaseTaskHandler): raise koji.BuildError, "arch_override is only allowed for scratch builds" task_info = session.getTaskInfo(self.id) # only allow admins to perform non-scratch builds from srpm - if not src.startswith('cvs://') and not opts.get('scratch') \ - and not 'admin' in session.getUserPerms(task_info['owner']): + if not src.startswith('cvs://') and \ + not src.startswith('git://') and \ + not opts.get('scratch') and \ + not 'admin' in session.getUserPerms(task_info['owner']): raise koji.BuildError, "only admins may peform non-scratch builds from srpm" target_info = session.getBuildTarget(target) if not target_info: @@ -1496,6 +1498,8 @@ class BuildTask(BaseTaskHandler): if isinstance(src,str): if src.startswith('cvs://'): return self.getSRPMFromCVS(src) + if src.startswith('git://'): + return self.getSRPMFromGit(src) else: #assume this is a path under uploads return src @@ -1514,6 +1518,17 @@ class BuildTask(BaseTaskHandler): srpm = result['srpm'] return srpm

+ def getSRPMFromGit(self, url): + #TODO - allow different ways to get the srpm + task_id = session.host.subtask(method='buildSRPMFromGit', + arglist=[url], + label='srpm', + parent=self.id) + # wait for subtask to finish + result = self.wait(task_id)[task_id] + srpm = result['srpm'] + return srpm + def readSRPMHeader(self, srpm): #srpm arg should be a path relative to <BASEDIR>/work global options @@ -1767,6 +1782,115 @@ class TagBuildTask(BaseTaskHandler): exctype, value = sys.exc_info()[:2] session.host.tagNotification(False, tag_id, fromtag, build_id, user_id, ignore_success, "%s: %s" % (exctype, value)) raise e + + +class BuildSRPMFromGitTask(BaseTaskHandler): + + Methods = ['buildSRPMFromGit'] + _taskWeight = 0.75 + + def spec_sanity_checks(self, filename): + spec = open(filename).read() + for tag in ("Packager", "Distribution", "Vendor"): + if re.match("%s:" % tag, spec, re.M): + raise koji.BuildError, "%s is not allowed to be set in spec file" % tag + for tag in ("packager", "distribution", "vendor"): + if re.match("%%define\s+%s\s+" % tag, spec, re.M): + raise koji.BuildError, "%s is not allowed to be defined in spec file" % tag + + def handler(self,url): + if not url.startswith('git://'): + raise koji.BuildError("invalid git URL: %s" % url) + + # Hack it because it refuses to parse it properly otherwise + scheme, netloc, path, params, query, fragment = urlparse.urlparse('http'+url[3:]) + if not (netloc and path and fragment and query): + raise koji.BuildError("invalid git URL: %s" % url) + + # Steps: + # 1. GIT clone into tempdir + # 3. Run 'make srpm' + + gitdir = self.workdir + '/git' + koji.ensuredir(gitdir) + logfile = self.workdir + "/srpm.log" + uploadpath = self.getUploadDir() + sourcedir = '%s/%s' % (gitdir, query) + + cmd = ['git', 'clone', 'git://%s%s/%s' % (netloc, path, query), query] + if log_output(cmd[0], cmd, logfile, uploadpath, cwd=gitdir, logerror=1, append=1): + output = "(none)" + try: + output = open(logfile).read() + except IOError: + pass + raise koji.BuildError, "Error with clone 'git://%s%s/%s: %s" % (netloc, path, query, output) + + cmd = ['git', 'clone', 'git://%s%s/common' % (netloc, path)] + self.logger.debug("executing: %s" % cmd) + if log_output(cmd[0], cmd, logfile, uploadpath, cwd=gitdir, logerror=1, append=1): + output = "(none)" + try: + output = open(logfile).read() + except IOError: + pass + raise koji.BuildError, "Error with clone 'git://%s%s/common: %s" % (netloc, path, output) + + cmd = ['git', 'ls-files', '-t', '-s'] + log_output(cmd[0], cmd, logfile, uploadpath, cwd='%s/common' % gitdir, logerror=1, append=1) + + cmd = ['git', 'checkout', '-b', 'kojibuild', fragment] + self.logger.debug("executing: %s" % cmd) + if log_output(cmd[0], cmd, logfile, uploadpath, cwd=sourcedir, logerror=1, append=1): + output = "(none)" + try: + output = open(logfile).read() + except IOError: + pass + raise koji.BuildError, "Error with checking out tag '%s' from git://%s%s/%s: %s" % (fragment, netloc, path, query, output) + + cmd = ['git', 'ls-files', '-t', '-s'] + log_output(cmd[0], cmd, logfile, uploadpath, cwd=sourcedir, logerror=1, append=1) + + spec_files = glob.glob("%s/*.spec" % sourcedir) + if len(spec_files) == 0: + raise koji.BuildError("No spec file found") + elif len(spec_files) > 1: + raise koji.BuildError("Multiple spec files found: %s" % spec_files) + spec_file = spec_files[0] + + # Run spec file sanity checks. Any failures will throw a BuildError + self.spec_sanity_checks(spec_file) + + #build srpm + cmd = ['make', '-C', sourcedir, 'srpm', '_KOJI=1.2.2', '_KOJI_TAG=%s' % fragment] + self.logger.debug("executing: %s" % cmd) + if log_output(cmd[0], cmd, logfile, uploadpath, cwd=gitdir, logerror=1, append=1): + raise koji.BuildError, "Error building SRPM" + + srpms = glob.glob('%s/*.src.rpm' % sourcedir) + if len(srpms) == 0: + raise koji.BuildError, "No srpms found in %s" % sourcedir + elif len(srpms) > 1: + raise koji.BuildError, "Multiple srpms found in %s: %s" % (sourcedir, ", ".join(srpms)) + else: + srpm = srpms[0] + + # check srpm name + h = koji.get_rpm_header(srpm) + name = h[rpm.RPMTAG_NAME] + version = h[rpm.RPMTAG_VERSION] + release = h[rpm.RPMTAG_RELEASE] + srpm_name = "%(name)s-%(version)s-%(release)s.src.rpm" % locals() + if srpm_name != os.path.basename(srpm): + raise koji.BuildError, 'srpm name mismatch: %s != %s' % (srpm_name, os.path.basename(srpm)) + + #upload srpm and return + self.uploadFile(srpm) + return { + 'srpm' : "%s/%s" % (uploadpath, srpm_name), + 'log' : "%s/srpm.log" % uploadpath, + }

class BuildSRPMFromCVSTask(BaseTaskHandler):

diff --git a/cli/koji b/cli/koji index 718b6e5..4f05500 100755 --- a/cli/koji +++ b/cli/koji @@ -669,7 +669,7 @@ def handle_build(options, session, args): if build_opts.background: #relative to koji.PRIO_DEFAULT priority = 5 - if not source.startswith('cvs://'): + if not (source.startswith('cvs://') or source.startswith('git://')): # only allow admins to perform non-scratch builds from srpm if not opts['scratch'] and not session.hasPerm('admin'): parser.error(_("builds from srpm must use --scratch")) @@ -741,7 +741,7 @@ def handle_chain_build(options, session, args): if build_level: src_list.append(build_level) build_level = [] - elif src.startswith('cvs://'): + elif src.startswith('cvs://') or src.startswith('git://'): build_level.append(src) elif '/' not in src and len(src.split('-')) >= 3: # quick check that it looks like a N-V-R @@ -2432,8 +2432,13 @@ def _parseTaskParams(session, method, task_id): if method == 'buildFromCVS': lines.append("CVS URL: %s" % params[0]) lines.append("Build Target: %s" % params[1]) + elif method == 'buildFromGit': + lines.append("GIT URL: %s" % params[0]) + lines.append("Build Target: %s" % params[1]) elif method == 'buildSRPMFromCVS': lines.append("CVS URL: %s" % params[0]) + elif method == 'buildSRPMFromGit': + lines.append("GIT URL: %s" % params[0]) elif method == 'multiArchBuild': lines.append("SRPM: %s/work/%s" % (options.topdir, params[0])) lines.append("Build Target: %s" % params[1]) diff --git a/koji.spec b/koji.spec index f14bb6e..7d4d643 100644 --- a/koji.spec +++ b/koji.spec @@ -49,7 +49,7 @@ Requires(post): /sbin/service Requires(preun): /sbin/chkconfig Requires(preun): /sbin/service Requires(pre): /usr/sbin/useradd -Requires: cvs +Requires: cvs git-core make Requires: rpm-build Requires: redhat-rpm-config Requires: createrepo >= 0.4.10 diff --git a/koji/__init__.py b/koji/__init__.py index d808126..5789220 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1517,6 +1517,11 @@ def taskLabel(taskInfo): if source.startswith('cvs://'): source = source[source.rfind('/') + 1:] source = source.replace('#', ':') + elif source.startswith('git://'): + source = source[source.rfind('?') + 1:] + tmp = source[:source.find('#')] + source = source[source.find('#') + 1:] + source = '%s:%s' % (tmp, source[source.rfind('/') + 1:]) else: source = os.path.basename(source) extra = '%s, %s' % (target, source) @@ -1526,6 +1531,12 @@ def taskLabel(taskInfo): url = url[url.rfind('/') + 1:] url = url.replace('#', ':') extra = url + elif method == 'buildSRPMFromGit': + if taskInfo.has_key('request'): + url = taskInfo['request'][0] + url = url[url.rfind('/') + 1:] + url = url.replace('#', ':') + extra = url elif method == 'buildArch': if taskInfo.has_key('request'): srpm, tagID, arch = taskInfo['request'][:3] diff --git a/www/kojiweb/taskinfo.chtml b/www/kojiweb/taskinfo.chtml index 76df4bc..dd5cf72 100644 --- a/www/kojiweb/taskinfo.chtml +++ b/www/kojiweb/taskinfo.chtml @@ -67,6 +67,11 @@ <strong>Build Target:</strong> <a href="buildtargetinfo?name=$params[1]">$params[1]</a> #elif $task.method == 'buildSRPMFromCVS' <strong>CVS URL:</strong> $params[0] + #elif $task.method == 'buildFromGit' + <strong>GIT&nbsp;URL:</strong>&nbsp;$params[0]<br/> + <strong>Build Target:</strong> <a href="buildtargetinfo?name=$params[1]">$params[1]</a> + #elif $task.method == 'buildSRPMFromGit' + <strong>GIT URL:</strong> $params[0] #elif $task.method == 'multiArchBuild' <strong>SRPM:</strong> $params[0]<br/> <strong>Build Target:</strong> <a href="buildtargetinfo?name=$params[1]">$params[1]</a><br/> @@ -276,7 +281,7 @@ $cgi.escape($result.faultString.strip()) <a href="getfile?taskID=$task.id&name=$urllib.quote($filename)">$filename</a><br/> #end for #if $task.state not in ($koji.TASK_STATES.CLOSED, $koji.TASK_STATES.CANCELED, $koji.TASK_STATES.FAILED) and \ - $task.method in ('buildSRPMFromCVS', 'buildArch', 'createrepo') + $task.method in ('buildSRPMFromCVS', 'buildSRPMFromGit', 'buildArch', 'createrepo') <br/> <a href="watchlogs?taskID=$task.id">Watch logs</a> #end if diff --git a/www/kojiweb/tasks.chtml b/www/kojiweb/tasks.chtml index 70de06c..32ca6ad 100644 --- a/www/kojiweb/tasks.chtml +++ b/www/kojiweb/tasks.chtml @@ -104,6 +104,7 @@ All <option value="all" #if $method == 'all' then 'selected="selected"' else ''#>all</option> <option value="build" #if $method == 'build' then 'selected="selected"' else ''#>build</option> <option value="buildSRPMFromCVS" #if $method == 'buildSRPMFromCVS' then 'selected="selected"' else ''#>buildSRPMFromCVS</option> + <option value="buildSRPMFromGit" #if $method == 'buildSRPMFromGit' then 'selected="selected"' else ''#>buildSRPMFromGit</option> <option value="buildArch" #if $method == 'buildArch' then 'selected="selected"' else ''#>buildArch</option> <option value="buildNotification" #if $method == 'buildNotification' then 'selected="selected"' else ''#>buildNotification</option> <option value="tagBuild" #if $method == 'tagBuild' then 'selected="selected"' else ''#>tagBuild</option>