buildsys September 2015

buildsys@lists.fedoraproject.org

8 participants
10 discussions

[PATCH] use createrepo_c instead of createrepo to make repos if it is available
by Dennis Gilmore 30 Oct '15

30 Oct '15

Signed-off-by: Dennis Gilmore <dennis(a)ausil.us> --- builder/kojid | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/builder/kojid b/builder/kojid index 3a92ec3..4e53fb5 100755 --- a/builder/kojid +++ b/builder/kojid @@ -4252,7 +4252,11 @@ class CreaterepoTask(BaseTaskHandler): def create_local_repo(self, rinfo, arch, pkglist, groupdata, oldrepo): koji.ensuredir(self.outdir) - cmd = ['/usr/bin/createrepo', '-vd', '-o', self.outdir] + if os.path.isfile('/usr/bin/createrepo_c'): + cmd = ['/usr/bin/createrepo_c'] + else: + cmd = ['/usr/bin/createrepo'] + cmd.extend(['-vd', '-o', self.outdir]) if pkglist is not None: cmd.extend(['-i', pkglist]) if os.path.isfile(groupdata): -- 2.4.0

2 3

[PATCH 1/2] Move config processing from CLI to koji.read_config().
by Daniel Mach 29 Oct '15

29 Oct '15

--- cli/koji | 85 ++------------------------------------------ koji/__init__.py | 106 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 109 insertions(+), 82 deletions(-) diff --git a/cli/koji b/cli/koji index 47d9691..66f4d99 100755 --- a/cli/koji +++ b/cli/koji @@ -186,88 +186,9 @@ def get_options(): list_commands() parser.error('Unknown command: %s' % args[0]) assert False - # load local config - defaults = { - 'server' : 'http://localhost/kojihub', - 'weburl' : 'http://localhost/koji', - 'topurl' : None, - 'pkgurl' : None, - 'topdir' : '/mnt/koji', - 'max_retries' : None, - 'retry_interval': None, - 'anon_retry' : None, - 'offline_retry' : None, - 'offline_retry_interval' : None, - 'keepalive' : True, - 'timeout' : None, - 'use_fast_upload': False, - 'poll_interval': 5, - 'krbservice': 'host', - 'cert': '~/.koji/client.crt', - 'ca': '~/.koji/clientca.crt', - 'serverca': '~/.koji/serverca.crt', - 'authtype': None - } - #note: later config files override earlier ones - configs = koji.config_directory_contents('/etc/koji.conf.d') - if os.access('/etc/koji.conf', os.F_OK): - configs.append('/etc/koji.conf') - if options.configFile: - fn = os.path.expanduser(options.configFile) - if os.path.isdir(fn): - contents = koji.config_directory_contents(fn) - if not contents: - parser.error("No config files found in directory: %s" % fn) - configs.extend(contents) - else: - if not os.access(fn, os.F_OK): - parser.error("No such file: %s" % fn) - configs.append(fn) - else: - user_config_dir = os.path.expanduser("~/.koji/config.d") - configs.extend(koji.config_directory_contents(user_config_dir)) - fn = os.path.expanduser("~/.koji/config") - if os.access(fn, os.F_OK): - configs.append(fn) - got_conf = False - for configFile in configs: - f = open(configFile) - config = ConfigParser.ConfigParser() - config.readfp(f) - f.close() - if config.has_section(options.profile): - got_conf = True - for name, value in config.items(options.profile): - #note the defaults dictionary also serves to indicate which - #options *can* be set via the config file. Such options should - #not have a default value set in the option parser. - if defaults.has_key(name): - if name in ('anon_retry', 'offline_retry', 'keepalive', 'use_fast_upload'): - defaults[name] = config.getboolean(options.profile, name) - elif name in ('max_retries', 'retry_interval', - 'offline_retry_interval', 'poll_interval', 'timeout'): - try: - defaults[name] = int(value) - except ValueError: - parser.error("value for %s config option must be a valid integer" % name) - assert False - else: - defaults[name] = value - if configs and not got_conf: - warn("Warning: no configuration for profile name: %s" % options.profile) - for name, value in defaults.iteritems(): - if getattr(options, name, None) is None: - setattr(options, name, value) - dir_opts = ('topdir', 'cert', 'ca', 'serverca') - for name in dir_opts: - # expand paths here, so we don't have to worry about it later - value = os.path.expanduser(getattr(options, name)) - setattr(options, name, value) - - #honor topdir - if options.topdir: - koji.BASEDIR = options.topdir - koji.pathinfo.topdir = options.topdir + + defaults = koji.read_config(options.profile, user_config=options.configFile) + options._update_loose(defaults.__dict__) #pkgurl is obsolete if options.pkgurl: diff --git a/koji/__init__.py b/koji/__init__.py index f45ff70..9951a28 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -28,6 +28,7 @@ except ImportError: sys.stderr.write("Warning: Could not install krbV module. Kerberos support will be disabled.\n") sys.stderr.flush() import base64 +import ConfigParser import datetime import errno from fnmatch import fnmatch @@ -35,6 +36,7 @@ import httplib import logging import logging.handlers from koji.util import md5_constructor +import optparse import os import os.path import pwd @@ -1457,6 +1459,110 @@ def config_directory_contents(dir_name): return configs +def _config_directory_contents(dir_name): + configs = [] + try: + conf_dir_contents = os.listdir(dir_name) + except OSError, exception: + if exception.errno != errno.ENOENT: + raise + else: + for name in sorted(conf_dir_contents): + if not name.endswith('.conf'): + continue + config_full_name = os.path.join(dir_name, name) + configs.append(config_full_name) + return configs + + +def read_config(profile_name, user_config=None): + config_defaults = { + 'server': 'http://localhost/kojihub', + 'weburl': 'http://localhost/koji', + 'topurl': None, + 'pkgurl': None, + 'topdir': '/mnt/koji', + 'max_retries': None, + 'retry_interval': None, + 'anon_retry': None, + 'offline_retry': None, + 'offline_retry_interval': None, + 'keepalive': True, + 'timeout': None, + 'use_fast_upload': False, + 'poll_interval': 5, + 'krbservice': 'host', + 'cert': '~/.koji/client.crt', + 'ca': '~/.koji/clientca.crt', + 'serverca': '~/.koji/serverca.crt', + 'authtype': None + } + + int_options = ['max_retries', 'retry_interval', 'offline_retry_interval', 'poll_interval', 'timeout'] + bool_options = ['anon_retry', 'offline_retry', 'keepalive', 'use_fast_upload'] + path_options = ['topdir', 'cert', 'ca', 'serverca'] + + result = config_defaults.copy() + for option in config_defaults: + if option in path_options: + result[option] = os.path.expanduser(result[option]) + + configs = [] + + # main config + configs.append("/etc/koji.conf") + + # conf.d + configs.extend(_config_directory_contents("/etc/koji.conf.d")) + + # user config + configs.append(os.path.expanduser("~/.koji/config")) + + # user conf.d + configs.extend(_config_directory_contents(os.path.expanduser("~/.koji/conf.d"))) + + # TODO: read configs via xdg.BaseDirectory.load_config_path("koji") + + # user config specified in runtime + if user_config is not None: + configs.append(user_config) + + # read configs in particular order, use the last value found + for config_path in configs: + if not os.access(config_path, os.F_OK): + continue + config = ConfigParser.SafeConfigParser() + config.readfp(open(config_path, "r")) + + if profile_name not in config.sections(): + continue + + # check for invalid options + invalid_options = [] + for option in config.options(profile_name): + if option not in result: + invalid_options.append(option) + + if invalid_options: + raise ValueError("Invalid options: %s" % ", ".join(invalid_options)) + + for option in config.options(profile_name): + if option in bool_options: + result[option] = config.getboolean(profile_name, option) + elif option in int_options: + result[option] = config.getint(profile_name, option) + else: + result[option] = config.get(profile_name, option) + if option in path_options: + result[option] = os.path.expanduser(result[option]) + + result["profile"] = profile_name + + # convert dict to optparse Values + options = optparse.Values(result) + return options + + class PathInfo(object): # ASCII numbers and upper- and lower-case letter for use in tmpdir() ASCII_CHARS = [chr(i) for i in range(48, 58) + range(65, 91) + range(97, 123)] -- 2.5.0

2 2

Frequent, intermittent failures of buildSRPMFromSCM since koji-1.10
by John Florian 29 Sep '15

29 Sep '15

About half of my builds fail with ... Traceback (most recent call last): File "/usr/lib/python2.7/site-packages/koji/daemon.py", line 1161, in runTask response = (handler.run(),) File "/usr/lib/python2.7/site-packages/koji/tasks.py", line 158, in run return koji.util.call_with_argcheck(self.handler, self.params, self.opts) File "/usr/lib/python2.7/site-packages/koji/util.py", line 154, in call_with_argcheck return func(*args, **kwargs) File "/usr/sbin/kojid", line 3817, in handler broot.init() File "/usr/sbin/kojid", line 460, in init rv = self.mock(['--init']) File "/usr/sbin/kojid", line 408, in mock incremental_upload(self.session, fname, fd, uploadpath, logger=self.logger) File "/usr/lib/python2.7/site-packages/koji/daemon.py", line 48, in incremental_upload fast_incremental_upload(session, fname, fd, path, retries, logger) File "/usr/lib/python2.7/site-packages/koji/daemon.py", line 87, in fast_incremental_upload result = session.rawUpload(contents, offset, path, fname, overwrite=True) File "/usr/lib/python2.7/site-packages/koji/__init__.py", line 1577, in __call__ return self.__func(self.__name,args,opts) File "/usr/lib/python2.7/site-packages/koji/__init__.py", line 1920, in _callMethod return self._sendCall(handler, headers, request) File "/usr/lib/python2.7/site-packages/koji/__init__.py", line 1831, in _sendCall return self._sendOneCall(handler, headers, request) File "/usr/lib/python2.7/site-packages/koji/__init__.py", line 1850, in _sendOneCall cnx.send(request) File "/usr/lib64/python2.7/httplib.py", line 820, in send self.sock.sendall(data) File "/usr/lib/python2.7/site-packages/koji/ssl/SSLConnection.py", line 111, in sendall self.close() File "/usr/lib/python2.7/site-packages/koji/ssl/SSLConnection.py", line 82, in close self.shutdown() File "/usr/lib/python2.7/site-packages/koji/ssl/SSLConnection.py", line 53, in shutdown self.__dict__["conn"].shutdown() Error: [] If I simply resubmit the same build request enough times the job will eventually succeed, but it's really annoying as is. This all started when I upgraded to koji-1.10. Is anyone else seeing this or did I perhaps screw something up with the upgrade? I filed a bug[0] a few days ago but have heard nothing there. [0] https://bugzilla.redhat.com/show_bug.cgi?id=1266245 -- John Florian

2 2

[PATCH 0/3] Added support for plugins on client
by Christos Triantafyllidis 17 Sep '15

17 Sep '15

This patch allows the usage of plugins on cli. Majority of the changes are just a re-ordering of existing code. Cheers, Christos Christos Triantafyllidis (3): Added support for plugins at client Added plugin configuration in client conf file Passing options variable to plugins cli/koji | 91 +++++++++++++++++++++++++++++++++++++---------------------- cli/koji.conf | 8 ++++++ 2 files changed, 65 insertions(+), 34 deletions(-) -- 2.4.3

1 3

mock-1.2.13 is available
by Miroslav Suchý 16 Sep '15

16 Sep '15

I just released new mock release (mock-1.2.13). It is bugfix release, but some bugfix may be interesting for you: * Fedora 23 configs are reverted back to use yum again. To be on pair with Koji * Lot of fixes for --new-chroot option * Mockchain can download SRPM from Dropbox * DNF does not install weak dependencies by default * When cleaning up chroots, mock now exclude mountpoints * When you build using DNF (rawhide) on systems, which does not have DNF (EL6, 7), mock will print warning, wait for confirmation, tell you how to suppress this warning next time. Nevertheless this warning is not fatal and Mock can continue using YUM. * Previously package_state plugin always used YUM, now it use DNF when chroot is configured to use DNF. * When file `/usr/bin/yum-deprecated` is present on your machine, then variable `config_opts['yum_command']` is set to this v alue by default. * Several others bugfixes https://bodhi.fedoraproject.org/updates/FEDORA-2015-16053 Mirek Suchy

1 0

[PATCH] PAM support for hub and BasicAuth for web
by Christos Triantafyllidis 16 Sep '15

16 Sep '15

Hello, The following patch is adding support for PAM authentication for the koji-hub and BasicAuth for the koji-web. This is useful for our internal use case as it allows us to login without the overhead of setting up either a CA or a kerberos realm for our users. The configuration is backwards compatible and hopefully similar to the other authntication methods. To active PAM support on hub you define the option: PAMService = koji in hub.conf. The value will be the name of the PAM service. Note the call to the PAM module is done via unpriviledged call thus the use of pam_unix won't be possible. Note that activating this option will have as result that username/password combinations from the DB will no longer be checked (similarly to when activating kerberos or SSL client auth). The BasicAuth for koji-web requires 2 changes: a) To enable WSGIPassAuthorization for /koji/login in httpd configuration. That passes the authorization variable from the apache to the application. b) Set the "BasicAuthRealm" option to the Basic Authentication Realm that will be presented to the user to login. Finally python-pam package has been added to the hub's dependencies. Cheers, Christos Christos Triantafyllidis (1): - Added PAM support for hub - Added BasicAuth support for web hub/hub.conf | 4 +++- hub/kojixmlrpc.py | 2 ++ koji.spec | 1 + koji/auth.py | 33 +++++++++++++++++++++++++-------- koji/server.py | 2 ++ www/conf/kojiweb.conf | 5 +++++ www/conf/web.conf | 3 +++ www/kojiweb/index.py | 18 +++++++++++++++++- www/kojiweb/wsgi_publisher.py | 9 +++++++-- 9 files changed, 65 insertions(+), 12 deletions(-) -- 2.4.3

1 3

[PATCH] add support for Image Factory generation of VMWare Fusion Vagrant boxes
by Ian McLeod 11 Sep '15

11 Sep '15

--- builder/kojid | 11 +++++++++-- cli/koji | 2 +- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/builder/kojid b/builder/kojid index c0759a8..4ea93f2 100755 --- a/builder/kojid +++ b/builder/kojid @@ -3063,7 +3063,7 @@ class BaseImageTask(OzImageTask): Some image formats require others to be processed first, which is why we have to do this. raw files in particular may not be kept. """ - supported = ('raw', 'raw-xz', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker', 'vagrant-virtualbox', 'vagrant-libvirt', 'vpc') + supported = ('raw', 'raw-xz', 'vmdk', 'qcow', 'qcow2', 'vdi', 'rhevm-ova', 'vsphere-ova', 'docker', 'vagrant-virtualbox', 'vagrant-libvirt', 'vagrant-vmware-fusion', 'vpc') for f in formats: if f not in supported: raise koji.ApplianceError('Invalid format: %s' % f) @@ -3098,6 +3098,7 @@ class BaseImageTask(OzImageTask): 'vsphere-ova': self._buildOVA, 'vagrant-virtualbox': self._buildOVA, 'vagrant-libvirt': self._buildOVA, + 'vagrant-vmware-fusion': self._buildOVA, 'docker': self._buildDocker } # add a handler to the logger so that we capture ImageFactory's logging @@ -3247,8 +3248,14 @@ class BaseImageTask(OzImageTask): if format == 'vagrant-libvirt': format = 'rhevm-ova' img_opts['rhevm_ova_format'] = 'vagrant-libvirt' + if format == 'vagrant-vmware-fusion': + format = 'vsphere-ova' + img_opts['vsphere_ova_format'] = 'vagrant-vmware-fusion' + # The initial disk image transform for VMWare Fusion/Workstation requires a "standard" VMDK + # not the stream oriented format used for VirtualBox or regular VMWare OVAs + img_opts['vsphere_vmdk_format'] = 'standard' targ = self._do_target_image(self.base_img.base_image.identifier, - format.replace('-ova', '')) + format.replace('-ova', ''), img_opts=img_opts) targ2 = self._do_target_image(targ.target_image.identifier, 'OVA', img_opts=img_opts) return {'image': targ2.target_image.data} diff --git a/cli/koji b/cli/koji index 47d9691..c74171c 100755 --- a/cli/koji +++ b/cli/koji @@ -5265,7 +5265,7 @@ def handle_image_build(options, session, args): """Create a disk image given an install tree""" formats = ('vmdk', 'qcow', 'qcow2', 'vdi', 'vpc', 'rhevm-ova', 'vsphere-ova', 'vagrant-virtualbox', 'vagrant-libvirt', - 'docker', 'raw-xz') + 'vagrant-vmware-fusion', 'docker', 'raw-xz') usage = _("usage: %prog image-build [options] <name> <version> " + "<target> <install-tree-url> <arch> [<arch>...]") usage += _("\n %prog image-build --config FILE") -- 2.1.0

1 0

koji and mergerepo_c
by Thomas 07 Sep '15

07 Sep '15

Hi Folks, Quick question, would you consider to use mergerepo_c with option --all [1] in koji ? I am happy to work on a patch if it would be accepted. Maybe, we can keep default behavior and enable with an option on the tag. My use case is to be able to ship different release built against fix external repo package that may not be latest version (Buildrequires:pkg = 1.0.0 while external repo has already 1.1.0) [1]: --all Include all packages with the same name and arch if version or release is different. If used --method argument is ignored! -- Thomas

2 1

Koji and DNF status?
by Miroslav Suchý 07 Sep '15

07 Sep '15

I would like to ask what is current status of migrating Koji to use DNF for building? I am mainly interested whether Koji will use DNF for building F23 packages, or you postpone it for F24? Because in such case I would revert fedora-23-*.cfg default configs in Mock to use Yum so Mock is on pair with Koji. -- Miroslav Suchy, RHCA Red Hat, Senior Software Engineer, #brno, #devexp, #fedora-buildsys

2 3

redhat-rpm-config and circular dependencies
by Colin Walters 02 Sep '15

02 Sep '15

Because Fedora is a self-hosted system, circular dependencies are a fact of life. Self-hosted compilers and the like will always exist. But I think the circular BR nature of redhat-rpm-config and macro packages is unnecessary self-inflicted pain. Currently, I am trying to backport (into a downstream distribution) the introduction of go-srpm-macros into redhat-rpm-config: http://pkgs.fedoraproject.org/cgit/redhat-rpm-config.git/commit/?id=ba49b89… Yet because redhat-rpm-config itself is pulled into the build root for go-srpm-macros, it introduces a circular build dependency. I could imagine external ways out of this (try dropping the BR of rrc for go-srpm-macros externally), but it would seem to me to be a lot saner just to include the macros themselves in redhat-rpm-config. Thoughts? (Is this the right list?)

3 2

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

2006

2005

buildsys September 2015