buildsys October 2013

buildsys@lists.fedoraproject.org

11 participants
12 discussions

[koji PATCH] allow setting build priority
by Dan Horák 09 Dec '13

09 Dec '13

When building for multiple tags it could be useful to set different priorities for the tasks, eg. rawhide builds will be lower prio than branched builds. --- util/koji-shadow | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/util/koji-shadow b/util/koji-shadow index 25492d5..8790cf8 100755 --- a/util/koji-shadow +++ b/util/koji-shadow @@ -140,6 +140,8 @@ def get_options(): help=_("tag sucessful builds into the tag we are building, default is to not tag")) parser.add_option("--arches", help=_("arches to use when creating tags")) + parser.add_option("--priority", type="int", default=5, + help=_("priority to set for submitted builds")) #parse once to get the config file (options, args) = parser.parse_args() @@ -1080,7 +1082,7 @@ class BuildTracker(object): #TODO ...and verify repo repo_id, event_id = session.getTaskResult(task_id) #kick off build - task_id = session.build(src, None, opts={'repo_id': repo_id}, priority=5 ) + task_id = session.build(src, None, opts={'repo_id': repo_id}, priority=options.priority ) return task_id def report(self): -- 1.8.1.4

2 1

[koji PATCH] fix format when printing rpms without buildroot
by Dan Horák 09 Dec '13

09 Dec '13

--- util/koji-shadow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/util/koji-shadow b/util/koji-shadow index 25492d5..da7b247 100755 --- a/util/koji-shadow +++ b/util/koji-shadow @@ -473,7 +473,7 @@ class TrackedBuild(object): if brs and bad: print "Warning: some rpms for %s lacked buildroots:" % self.nvr for rinfo in bad: - print " %(name)-%(version)-%(release).%(arch)" % rinfo + print " %(name)s-%(version)s-%(release)s.%(arch)s" % rinfo return brs.keys() def getDeps(self): -- 1.8.1.4

2 1

[koji PATCH] allow setting the timeout for builds from config file
by Dan Horák 09 Dec '13

09 Dec '13

--- builder/kojid | 5 +++-- builder/kojid.conf | 3 +++ koji/__init__.py | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/builder/kojid b/builder/kojid index f906d0b..18c1fa4 100755 --- a/builder/kojid +++ b/builder/kojid @@ -204,7 +204,7 @@ class BuildRoot(object): for k in ('repoid', 'tag_name'): if hasattr(self, k): opts[k] = getattr(self, k) - for k in ('mockdir', 'topdir', 'topurl', 'topurls', 'packager', 'vendor', 'distribution', 'mockhost', 'yum_proxy'): + for k in ('mockdir', 'topdir', 'topurl', 'topurls', 'packager', 'vendor', 'distribution', 'mockhost', 'yum_proxy', 'rpmbuild_timeout'): if hasattr(self.options, k): opts[k] = getattr(self.options, k) opts['buildroot_id'] = self.id @@ -3060,6 +3060,7 @@ def get_options(): 'maven_repo_ignore': '*.md5 *.sha1 maven-metadata*.xml _maven.repositories ' 'resolver-status.properties *.lastUpdated', 'failed_buildroot_lifetime' : 3600 * 4, + 'rpmbuild_timeout' : 3600 * 24, 'cert': '/etc/kojid/client.crt', 'ca': '/etc/kojid/clientca.crt', 'serverca': '/etc/kojid/serverca.crt'} @@ -3067,7 +3068,7 @@ def get_options(): for name, value in config.items('kojid'): if name in ['sleeptime', 'maxjobs', 'minspace', 'retry_interval', 'max_retries', 'offline_retry_interval', 'failed_buildroot_lifetime', - 'timeout',]: + 'timeout', 'rpmbuild_timeout',]: try: defaults[name] = int(value) except ValueError: diff --git a/builder/kojid.conf b/builder/kojid.conf index 1fa9ad4..05fe3b0 100644 --- a/builder/kojid.conf +++ b/builder/kojid.conf @@ -32,6 +32,9 @@ ; The _host string to use in mock ; mockhost=koji-linux-gnu +; Timeout for build duration (24 hours) +; rpmbuild_timeout=86400 + ; The URL for the xmlrpc server server=http://hub.example.com/kojihub diff --git a/koji/__init__.py b/koji/__init__.py index f42226b..8ab81a8 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -1266,7 +1266,7 @@ def genMockConfig(name, arch, managed=False, repoid=None, tag_name=None, **opts) # don't encourage network access from the chroot 'use_host_resolv': opts.get('use_host_resolv', False), # Don't let a build last more than 24 hours - 'rpmbuild_timeout': 86400 + 'rpmbuild_timeout': opts.get('rpmbuild_timeout', 86400) } # bind_opts are used to mount parts (or all of) /dev if needed. -- 1.7.7.6

2 1

[PATCH] builder: Upload recursively all files in all subdirs
by Mathieu Bridon 18 Nov '13

18 Nov '13

--- This will become necessary if we want to upload the results of a mash task, as I am currently working on. Should I have created a new uploadDirRecursively() method, instead of changing uploadDir()? builder/kojid | 33 ++++++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/builder/kojid b/builder/kojid index ca73cad..fbd4e73 100755 --- a/builder/kojid +++ b/builder/kojid @@ -371,15 +371,34 @@ class BuildRoot(object): append '.' + suffix to the filenames, so that successive uploads of the same directory won't overwrite each other, if the files have the same name but different contents.""" + def walker(arg, dirname, filenames): + # `arg` is completely ignored here + if dirname == dirpath: + uploadpath = rootuploadpath + + else: + dirrelpath = os.path.relpath(dirname, start=dirpath) + uploadpath = os.path.join(rootuploadpath, dirrelpath) + + for filename in filenames: + filepath = os.path.join(dirname, filename) + + if os.path.isdir(filepath): + # TODO: Should we do something about this? + continue + + if os.stat(filepath).st_size > 0: + if suffix: + filename = '%s.%s' % (filename, suffix) + + self.session.uploadWrapper(filepath, uploadpath, filename) + if not os.path.isdir(dirpath): return - uploadpath = self.getUploadPath() - for filename in os.listdir(dirpath): - filepath = os.path.join(dirpath, filename) - if os.stat(filepath).st_size > 0: - if suffix: - filename = '%s.%s' % (filename, suffix) - self.session.uploadWrapper(filepath, uploadpath, filename) + + rootuploadpath = self.getUploadPath() + + os.path.walk(dirpath, walker, None) def init(self): rv = self.mock(['--init']) -- 1.8.1.4

2 5

[PATCH] cli: Set the mtime from urlgrabber/pycurl, make users happy. BZ 872631.
by James Antill 06 Nov '13

06 Nov '13

--- cli/koji | 9 +++++++++ 1 files changed, 9 insertions(+), 0 deletions(-) diff --git a/cli/koji b/cli/koji index a870342..04d00bc 100755 --- a/cli/koji +++ b/cli/koji @@ -5726,6 +5726,15 @@ def anon_handle_download_build(options, session, args): os.write(out, buf) finally: os.close(out) + + # Set the mtime to the remote mtime, people like this. BZ 87263. + try: + import pycurl + mod_time = file.curl_obj.getinfo(pycurl.INFO_FILETIME) + os.utime(relpath, (mod_time, mod_time)) + except: + pass # Ignore errors here. + file.close() def anon_handle_wait_repo(options, session, args): -- 1.7.7.6

3 2

strange build errors
by Peter Bojtos 01 Nov '13

01 Nov '13

Dear List, I have a working koji setup with several clients. When I submit a few jobs, it works great. However when I submit lots of package (>100), some of the builds fail with the following error when mock is populating the buildroot (in root.log): DEBUG util.py:264: Installed: DEBUG util.py:264: bash.x86_64 0:4.1.2-15.el6_4 DEBUG util.py:264: bzip2.x86_64 0:1.0.5-7.el6_0 DEBUG util.py:264: coreutils.x86_64 0:8.4-19.el6_4.2 .... DEBUG util.py:264: xz.x86_64 0:4.999.9-0.5.beta.20091007git.slxrv3 DEBUG util.py:264: Dependency Installed: DEBUG util.py:264: audit-libs.x86_64 0:2.2-2.el6 DEBUG util.py:264: basesystem.noarch 0:10.0-4.el6 .... DEBUG util.py:264: xz-lzma-compat.x86_64 0:4.999.9-0.5.beta.20091007git.slxrv3 DEBUG util.py:264: zlib.x86_64 0:1.2.3-29.el6 DEBUG util.py:354: Child return code was: 0 DEBUG util.py:314: Executing command: /usr/bin/repoquery -c /var/lib/mock/dist-slxrv3-build-5544-1386/root//etc/yum.conf -a --qf '%{nevra} %{buildtime} %{size} %{pkgid} %{repoid}' > /var/lib/mock/dist-slxrv3-build-5544-1386/result/available_pkgs with env {'LANG': 'en_US.UTF-8', 'TERM': 'vt100', 'SHELL': '/bin/bash', 'HOSTNAME': 'mock', 'HOME': '/builddir', 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin'} DEBUG util.py:264: Traceback (most recent call last): DEBUG util.py:264: File "/usr/bin/repoquery", line 1510, in <module> DEBUG util.py:264: main(sys.argv) DEBUG util.py:264: File "/usr/bin/repoquery", line 1504, in main DEBUG util.py:264: repoq.runQuery(regexs) DEBUG util.py:264: File "/usr/bin/repoquery", line 982, in runQuery DEBUG util.py:264: pkgs = self.matchPkgs(items, plain_pkgs=plain_pkgs) DEBUG util.py:264: File "/usr/bin/repoquery", line 903, in matchPkgs DEBUG util.py:264: pkgs = self.returnPkgList(patterns=items) DEBUG util.py:264: File "/usr/bin/repoquery", line 856, in returnPkgList DEBUG util.py:264: pkgs = self.pkgSack.returnNewestByNameArch(**kwargs) DEBUG util.py:264: File "/usr/lib/python2.7/site-packages/yum/__init__.py", line 1013, in <lambda> DEBUG util.py:264: pkgSack = property(fget=lambda self: self._getSacks(), DEBUG util.py:264: File "/usr/lib/python2.7/site-packages/yum/__init__.py", line 779, in _getSacks DEBUG util.py:264: self.repos.populateSack(which=repos) DEBUG util.py:264: File "/usr/lib/python2.7/site-packages/yum/repos.py", line 309, in populateSack DEBUG util.py:264: self.doSetup() DEBUG util.py:264: File "/usr/lib/python2.7/site-packages/yum/repos.py", line 134, in doSetup DEBUG util.py:264: self.retrieveAllMD() DEBUG util.py:264: File "/usr/lib/python2.7/site-packages/yum/repos.py", line 92, in retrieveAllMD DEBUG util.py:264: downloading = repo._commonRetrieveDataMD_list(mdtypes) DEBUG util.py:264: File "/usr/lib/python2.7/site-packages/yum/yumRepo.py", line 1540, in _commonRetrieveDataMD_list DEBUG util.py:264: os.rename(local, local + '.old.tmp') DEBUG util.py:264: OSError: [Errno 2] No such file or directory DEBUG util.py:354: Child return code was: 1 DEBUG util.py:314: Executing command: ['/bin/umount', '-n', '-l', '/var/lib/mock/dist-slxrv3-build-5544-1386/root/dev/pts'] with env {'LANG': 'en_US.UTF-8', 'TERM': 'vt100', 'SHELL': '/bin/bash', 'HOSTNAME': 'mock', 'HOME': '/builddir', 'PATH': '/usr/bin:/bin:/usr/sbin:/sbin'} Is it because newRepo task is not finished but somehow creates the softlink to latest (and unfinished) koji repo? Have anyone seen the same error before? Thanks for your reply, Peter Bojtos

3 6

[PATCH 1/2] cli: Check pkg. to move-pkgs is in src tag before spawning task. BZ 527296.
by James Antill 29 Oct '13

29 Oct '13

--- cli/koji | 12 ++++++++++++ 1 files changed, 12 insertions(+), 0 deletions(-) diff --git a/cli/koji b/cli/koji index 04d00bc..b435f1a 100755 --- a/cli/koji +++ b/cli/koji @@ -5512,6 +5512,18 @@ def handle_move_build(opts, session, args): builds.append(build) for build in builds: + data = session.listTags(build=build['id']) + if not data: + print _("Package %s does not have any tags.") % arg + continue + data = [x['name'] for x in data] + if args[0] not in data: + msg = _("Package %s is not in %s, maybe you meant: %s") + arg = "%(package_name)s-%(version)s-%(release)s" % build + data = ",".join(data) + print msg % (arg, args[0], data) + continue + task_id = session.moveBuild(args[0], args[1], build['id'], options.force) tasks.append(task_id) print "Created task %s, moving %s" % (task_id, koji.buildLabel(build)) -- 1.7.7.6

1 1

[PATCH] cli: Simple text fix to make it more obvious what rpmdiff is doing. BZ 704129.
by James Antill 29 Oct '13

29 Oct '13

--- hub/kojihub.py | 4 ++-- 1 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hub/kojihub.py b/hub/kojihub.py index 6e256c4..9a4dafc 100644 --- a/hub/kojihub.py +++ b/hub/kojihub.py @@ -7162,8 +7162,8 @@ def rpmdiff(basepath, rpmlist): status = proc.wait() if os.WIFSIGNALED(status) or \ (os.WEXITSTATUS(status) != 0): - raise koji.BuildError, 'mismatch when analyzing %s, rpmdiff output was:\n%s' % \ - (os.path.basename(first_rpm), output) + raise koji.BuildError, 'mismatch when analyzing %s (from multiple arches), rpmdiff output was:\n 1. %s\n 2. %s\n%s' % \ + (os.path.basename(first_rpm), first_rpm, other_rpm, output) def importImageInternal(task_id, build_id, imgdata): """ -- 1.7.7.6

1 0

Need Koji Server Setup Help
by John.Florian＠dart.biz 28 Oct '13

28 Oct '13

Hi all, I'm a complete Koji newb and am attempting to get a Koji server up and running for internal use, but have run into a number of issues along the way, despite following https://fedoraproject.org/wiki/Koji/ServerHowTo and other similar pages found around the net. My setup is built atop Fedora 19, so I had to make several adjustments, such as: * access directives for Apache httpd 2.4 (bug filed: https://bugzilla.redhat.com/show_bug.cgi?id=1019323 and wiki likewise edited) * changed the ssl.cnf for the Koji CA to so that default_md=sha256 (instead of md5) under the [default_ca] section so that firefox wouldn't complain about the trustworthiness of the certificate I am able to use 'koji moshimoshi' both as kojiadmin and as myself from the Koji server as well as my workstation. I am also able to use the Koji-Web using http or https for some basic poking around. However, if I try the login link on the web page, firefox comes up with a message stating: The page isn't redirecting properly Firefox has detected that the server is redirecting the request for this address in a way that will never complete. This problem can sometimes be caused by disabling or refusing to accept cookies. >From what I can gather via /var/log/httpd/ssl_access_log and examination of /usr/share/koji-web/scripts/index.py's login function the page is trying to redirect the browser to force SSL (even if already using https when clicking on login) and somehow this results in what would be an infinite loop except that firefox halts the loop after a dozen or so cycles. I'm rather stumped as to where my investigation should be focused at this point however. Any advice? -- John Florian

4 11

Bootstrapping New Koji Server
by John.Florian＠dart.biz 22 Oct '13

22 Oct '13

First off, I apologize for the lengthy message, but I hope to eventually contribute to this project via code and/or docs. (I've already updated the Server Setup wiki page where I felt knowledgeable enough.) I'm loosely following the wiki page[1] for this process but may have myself confused about what some steps do and what overall is required. I want a Koji instance for building internally developed packages. I would rarely, if ever, want to rebuild a package already provided in Fedora. I've got lots of experience using rpmbuild and recently a little with mock, but am trying for a full fledged setup with Koji now to improve my workflow. For now, I am focusing on builds for F19 only, but will eventually add others to basically follow Fedora's EoL schedule. Thus far, I have: A basic setup working and can now login thanks to Mike's excellent help. A "f19" tag and a child "f19-build" tag for arches i386 and x86_64. Successfully imported the SRPMs for my locally developed packages only. I didn't import any binary RPMs as I wanted to see that Koji could do that as a test. I also didn't import anything of the Fedora-provided packages since I'm not interested in building those. A "f19" build target. A build group named "build" associated with my "f19-build" tag. I've tried doing a build of one of my SRPMs that I had imported and keep meeting failure. Examination of the root.log revealed only a few things, but none that sounded to worrisome: DEBUG util.py:266: There is no installed groups file. DEBUG util.py:266: Warning: Group build does not have any packages to install. When I got to the mock_output.log however, I saw this which I'm guessing is the reason for failure: ERROR: Could not find useradd in chroot, maybe the install failed? Seeing this I figured it was due to skipping the step for populating the "build" group with packages to be installed in the minimal buildroot. I skipped this because the instructions fail for me since I my FAS account isn't set up sufficiently to let me to auth against the real Fedora Koji instance, i.e., I don't have a signed user cert. Knowing that my current mock setup can manage this, I dug into its config and stumbled upon the "buildsys-build" package name and thus came up with: $ yum group info buildsys-build Loaded plugins: langpacks There is no installed groups file. Maybe run: yum groups mark convert Group: Buildsystem building group Group-Id: buildsys-build Mandatory Packages: bash bzip2 coreutils cpio diffutils fedora-release findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz This looks like a minimal buildroot to me, so I commenced with that list: $ koji add-group-pkg f19-build build bash bzip2 coreutils cpio diffutils fedora-release findutils gawk gcc gcc-c++ grep gzip info make patch redhat-rpm-config rpm-build sed shadow-utils tar unzip util-linux which xz I believe these are now "in place" correctly since: $ koji list-groups f19-build build [f19-build] bash: None, default [f19-build] bzip2: None, default [f19-build] coreutils: None, default [f19-build] cpio: None, default [f19-build] diffutils: None, default [f19-build] fedora-release: None, default [f19-build] findutils: None, default [f19-build] gawk: None, default [f19-build] gcc: None, default [f19-build] gcc-c++: None, default [f19-build] grep: None, default [f19-build] gzip: None, default [f19-build] info: None, default [f19-build] make: None, default [f19-build] patch: None, default [f19-build] redhat-rpm-config: None, default [f19-build] rpm-build: None, default [f19-build] sed: None, default [f19-build] shadow-utils: None, default [f19-build] tar: None, default [f19-build] unzip: None, default [f19-build] util-linux: None, default [f19-build] which: None, default [f19-build] xz: None, default [f19-build] However, when I resubmitted my build request, it failed and I see the same warnings/errors in the logs. I also see this in /var/log/kojid.log: 2013-10-22 16:48:57,582 [DEBUG] koji.TaskManager: bins: {'1:noarch': 1, '2:noarch': 1, '1:x86_64': 1, '1:i386': 1, '2:x86_64': 1, '2:i386': 1} 2013-10-22 16:49:12,598 [DEBUG] koji: Traceback (most recent call last): File "/usr/lib/python2.7/site-packages/koji/__init__.py", line 1895, in _callMethod return self._sendCall(handler, headers, request) File "/usr/lib/python2.7/site-packages/koji/__init__.py", line 1806, in _sendCall return self._sendOneCall(handler, headers, request) File "/usr/lib/python2.7/site-packages/koji/__init__.py", line 1825, in _sendOneCall cnx.send(request) File "/usr/lib64/python2.7/httplib.py", line 805, in send self.sock.sendall(data) File "/usr/lib/python2.7/site-packages/koji/ssl/SSLConnection.py", line 111, in sendall self.close() File "/usr/lib/python2.7/site-packages/koji/ssl/SSLConnection.py", line 82, in close self.shutdown() File "/usr/lib/python2.7/site-packages/koji/ssl/SSLConnection.py", line 53, in shutdown self.__dict__["conn"].shutdown() Error: [] 2013-10-22 16:49:12,599 [INFO] koji: Try #1 for call 1328 (listBuildroots) failed: [] I suspect I'm really close now, but have done something silly that's not obvious to me. Any pointers will be greatly appreciated. [1] https://fedoraproject.org/wiki/Koji/ServerBootstrap -- John Florian

3 2

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

2005

buildsys October 2013