--- cli/koji | 137 ++++++++++++------------------------------------------- docs/profiles | 57 +++++++++++++++++++++++ koji/__init__.py | 130 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 215 insertions(+), 109 deletions(-) create mode 100644 docs/profiles

diff --git a/cli/koji b/cli/koji index f521d6d..29f26bb 100755 --- a/cli/koji +++ b/cli/koji @@ -186,88 +186,9 @@ def get_options(): list_commands() parser.error('Unknown command: %s' % args[0]) assert False - # load local config - defaults = { - 'server' : 'http://localhost/kojihub', - 'weburl' : 'http://localhost/koji', - 'topurl' : None, - 'pkgurl' : None, - 'topdir' : '/mnt/koji', - 'max_retries' : None, - 'retry_interval': None, - 'anon_retry' : None, - 'offline_retry' : None, - 'offline_retry_interval' : None, - 'keepalive' : True, - 'timeout' : None, - 'use_fast_upload': False, - 'poll_interval': 5, - 'krbservice': 'host', - 'cert': '~/.koji/client.crt', - 'ca': '~/.koji/clientca.crt', - 'serverca': '~/.koji/serverca.crt', - 'authtype': None - } - #note: later config files override earlier ones - configs = koji.config_directory_contents('/etc/koji.conf.d') - if os.access('/etc/koji.conf', os.F_OK): - configs.append('/etc/koji.conf') - if options.configFile: - fn = os.path.expanduser(options.configFile) - if os.path.isdir(fn): - contents = koji.config_directory_contents(fn) - if not contents: - parser.error("No config files found in directory: %s" % fn) - configs.extend(contents) - else: - if not os.access(fn, os.F_OK): - parser.error("No such file: %s" % fn) - configs.append(fn) - else: - user_config_dir = os.path.expanduser("~/.koji/config.d") - configs.extend(koji.config_directory_contents(user_config_dir)) - fn = os.path.expanduser("~/.koji/config") - if os.access(fn, os.F_OK): - configs.append(fn) - got_conf = False - for configFile in configs: - f = open(configFile) - config = ConfigParser.ConfigParser() - config.readfp(f) - f.close() - if config.has_section(options.profile): - got_conf = True - for name, value in config.items(options.profile): - #note the defaults dictionary also serves to indicate which - #options *can* be set via the config file. Such options should - #not have a default value set in the option parser. - if defaults.has_key(name): - if name in ('anon_retry', 'offline_retry', 'keepalive', 'use_fast_upload'): - defaults[name] = config.getboolean(options.profile, name) - elif name in ('max_retries', 'retry_interval', - 'offline_retry_interval', 'poll_interval', 'timeout'): - try: - defaults[name] = int(value) - except ValueError: - parser.error("value for %s config option must be a valid integer" % name) - assert False - else: - defaults[name] = value - if configs and not got_conf: - warn("Warning: no configuration for profile name: %s" % options.profile) - for name, value in defaults.iteritems(): - if getattr(options, name, None) is None: - setattr(options, name, value) - dir_opts = ('topdir', 'cert', 'ca', 'serverca') - for name in dir_opts: - # expand paths here, so we don't have to worry about it later - value = os.path.expanduser(getattr(options, name)) - setattr(options, name, value) - - #honor topdir - if options.topdir: - koji.BASEDIR = options.topdir - koji.pathinfo.topdir = options.topdir + + defaults = koji.read_config(options.profile, user_config=options.configFile) + options._update_loose(defaults.__dict__)

#pkgurl is obsolete if options.pkgurl: @@ -1503,11 +1424,11 @@ def linked_upload(localfile, path, name=None): try: if name is None: name = os.path.basename(localfile) - dest_dir = os.path.join(koji.pathinfo.work(), path) + dest_dir = os.path.join(koji_profile_module.pathinfo.work(), path) dst = os.path.join(dest_dir, name) koji.ensuredir(dest_dir) # fix uid/gid to keep httpd happy - st = os.stat(koji.pathinfo.work()) + st = os.stat(koji_profile_module.pathinfo.work()) os.chown(dest_dir, st.st_uid, st.st_gid) print "Linking rpm to: %s" % dst os.link(localfile, dst) @@ -2018,7 +1939,7 @@ def handle_prune_signed_copies(options, session, args): for sig in sigs: sigkey = sig['sigkey'] by_sig.setdefault(sigkey, []).append(rpminfo) - builddir = koji.pathinfo.build(binfo) + builddir = koji_profile_module.pathinfo.build(binfo) build_files = 0 build_space = 0 if not by_sig and options.debug: @@ -2028,7 +1949,7 @@ def handle_prune_signed_copies(options, session, args): archdirs = {} sigdirs = {} for rpminfo in rpms: - signedpath = "%s/%s" % (builddir, koji.pathinfo.signed(rpminfo, sigkey)) + signedpath = "%s/%s" % (builddir, koji_profile_module.pathinfo.signed(rpminfo, sigkey)) try: st = os.lstat(signedpath) except OSError: @@ -2285,8 +2206,8 @@ def handle_list_signed(options, session, args): binfo = session.getBuild(rinfo['build_id']) build_idx[binfo['id']] = binfo binfo['name'] = binfo['package_name'] - builddir = koji.pathinfo.build(binfo) - signedpath = "%s/%s" % (builddir, koji.pathinfo.signed(rinfo, sigkey)) + builddir = koji_profile_module.pathinfo.build(binfo) + signedpath = "%s/%s" % (builddir, koji_profile_module.pathinfo.signed(rinfo, sigkey)) if not os.path.exists(signedpath): if options.debug: print "No copy: %s" % signedpath @@ -2492,7 +2413,6 @@ def anon_handle_latest_build(options, session, args): if len(args) < 2: parser.error(_("A tag name and package name must be specified")) assert False - pathinfo = koji.PathInfo()

for pkg in args[1:]: if options.arch: @@ -2503,7 +2423,7 @@ def anon_handle_latest_build(options, session, args): for x in data: z = x.copy() x['name'] = builds_hash[x['build_id']]['package_name'] - x['path'] = os.path.join(pathinfo.build(x), pathinfo.rpm(z)) + x['path'] = os.path.join(koji_profile_module.pathinfo.build(x), koji_profile_module.pathinfo.rpm(z)) fmt = "%(path)s" else: fmt = "%(name)s-%(version)s-%(release)s.%(arch)s" @@ -2515,11 +2435,11 @@ def anon_handle_latest_build(options, session, args): if options.paths: if options.type == 'maven': for x in data: - x['path'] = pathinfo.mavenbuild(x) + x['path'] = koji_profile_module.pathinfo.mavenbuild(x) fmt = "%(path)-40s %(tag_name)-20s %(maven_group_id)-20s %(maven_artifact_id)-20s %(owner_name)s" else: for x in data: - x['path'] = pathinfo.build(x) + x['path'] = koji_profile_module.pathinfo.build(x) fmt = "%(path)-40s %(tag_name)-20s %(owner_name)s" else: if options.type == 'maven': @@ -2597,7 +2517,6 @@ def anon_handle_list_tagged(options, session, args): parser.error(_("Only one package name may be specified")) assert False activate_session(session) - pathinfo = koji.PathInfo() package = None if len(args) > 1: package = args[1] @@ -2630,14 +2549,14 @@ def anon_handle_list_tagged(options, session, args): build_idx = dict([(b['id'],b) for b in builds]) for rinfo in data: build = build_idx[rinfo['build_id']] - builddir = pathinfo.build(build) + builddir = koji_profile_module.pathinfo.build(build) if options.sigs: sigkey = rinfo['sigkey'] - signedpath = os.path.join(builddir, pathinfo.signed(rinfo, sigkey)) + signedpath = os.path.join(builddir, koji_profile_module.pathinfo.signed(rinfo, sigkey)) if os.path.exists(signedpath): rinfo['path'] = signedpath else: - rinfo['path'] = os.path.join(builddir, pathinfo.rpm(rinfo)) + rinfo['path'] = os.path.join(builddir, koji_profile_module.pathinfo.rpm(rinfo)) fmt = "%(path)s" data = [x for x in data if x.has_key('path')] else: @@ -2649,11 +2568,11 @@ def anon_handle_list_tagged(options, session, args): if options.paths: if options.type == 'maven': for x in data: - x['path'] = pathinfo.mavenbuild(x) + x['path'] = koji_profile_module.pathinfo.mavenbuild(x) fmt = "%(path)-40s %(tag_name)-20s %(maven_group_id)-20s %(maven_artifact_id)-20s %(owner_name)s" else: for x in data: - x['path'] = pathinfo.build(x) + x['path'] = koji_profile_module.pathinfo.build(x) fmt = "%(path)-40s %(tag_name)-20s %(owner_name)s" else: if options.type == 'maven': @@ -2721,7 +2640,6 @@ def anon_handle_list_untagged(options, session, args): opts = {} if package: opts['name'] = package - pathinfo = koji.PathInfo()

data = session.untaggedBuilds(**opts) if options.show_references: @@ -2741,7 +2659,7 @@ def anon_handle_list_untagged(options, session, args): #data = [x for x in data if not refs.has_key(x['id'])] if options.paths: for x in data: - x['path'] = pathinfo.build(x) + x['path'] = koji_profile_module.pathinfo.build(x) fmt = "%(path)s" else: fmt = "%(name)s-%(version)s-%(release)s" @@ -3053,9 +2971,9 @@ def anon_handle_rpminfo(options, session, args): print "External Repository: %(name)s [%(id)i]" % repo print "External Repository url: %(url)s" % repo else: - print "RPM Path: %s" % os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(info)) + print "RPM Path: %s" % os.path.join(koji_profile_module.pathinfo.build(buildinfo), koji_profile_module.pathinfo.rpm(info)) print "SRPM: %(epoch)s%(name)s-%(version)s-%(release)s [%(id)d]" % buildinfo - print "SRPM Path: %s" % os.path.join(koji.pathinfo.build(buildinfo), koji.pathinfo.rpm(buildinfo)) + print "SRPM Path: %s" % os.path.join(koji_profile_module.pathinfo.build(buildinfo), koji_profile_module.pathinfo.rpm(buildinfo)) print "Built: %s" % time.strftime('%a, %d %b %Y %H:%M:%S %Z', time.localtime(info['buildtime'])) print "Payload: %(payloadhash)s" %info print "Size: %(size)s" %info @@ -3127,23 +3045,23 @@ def anon_handle_buildinfo(options, session, args): if maven_archives: print "Maven archives:" for archive in maven_archives: - print os.path.join(koji.pathinfo.mavenbuild(info), koji.pathinfo.mavenfile(archive)) + print os.path.join(koji_profile_module.pathinfo.mavenbuild(info), koji_profile_module.pathinfo.mavenfile(archive)) win_archives = session.listArchives(buildID=info['id'], type='win') if win_archives: print "Windows archives:" for archive in win_archives: - print os.path.join(koji.pathinfo.winbuild(info), koji.pathinfo.winfile(archive)) + print os.path.join(koji_profile_module.pathinfo.winbuild(info), koji_profile_module.pathinfo.winfile(archive)) rpms = session.listRPMs(buildID=info['id']) image_info = session.getImageBuild(info['id']) img_archives = session.listArchives(buildID=info['id'], type='image') if img_archives: print 'Image archives:' for archive in img_archives: - print os.path.join(koji.pathinfo.imagebuild(info), archive['filename']) + print os.path.join(koji_profile_module.pathinfo.imagebuild(info), archive['filename']) if rpms: print "RPMs:" for rpm in rpms: - print os.path.join(koji.pathinfo.build(info), koji.pathinfo.rpm(rpm)) + print os.path.join(koji_profile_module.pathinfo.build(info), koji_profile_module.pathinfo.rpm(rpm)) if options.changelog: changelog = session.getChangelogEntries(info['id']) if changelog: @@ -4261,7 +4179,7 @@ def _printTaskInfo(session, task_id, level=0, recurse=True, verbose=True): files = session.listTaskOutput(task_id) logs = [filename for filename in files if filename.endswith('.log')] output = [filename for filename in files if not filename.endswith('.log')] - files_dir = '%s/%s' % (koji.pathinfo.work(), koji.pathinfo.taskrelpath(task_id)) + files_dir = '%s/%s' % (koji_profile_module.pathinfo.work(), koji_profile_module.pathinfo.taskrelpath(task_id))

owner = session.getUser(info['owner'])['name']

@@ -6124,9 +6042,8 @@ def anon_handle_download_build(options, session, args): if not suboptions.topurl: print "You must specify --topurl to download images" return 1 - pi = koji.PathInfo(topdir=suboptions.topurl) for archive in archives: - url = '%s/%s' % (pi.imagebuild(info), archive['filename']) + url = '%s/%s' % (pathinfo.imagebuild(info), archive['filename']) urls.append((url, archive['filename'])) else: # can't happen @@ -6628,7 +6545,9 @@ def activate_session(session):

if __name__ == "__main__": global options + global koji_profile_module options, command, args = get_options() + koji_profile_module = koji.get_profile_module(options.profile)

logger = logging.getLogger("koji") handler = logging.StreamHandler(sys.stderr) diff --git a/docs/profiles b/docs/profiles new file mode 100644 index 0000000..2f21723 --- /dev/null +++ b/docs/profiles @@ -0,0 +1,57 @@ +============= +Koji Profiles +============= +This document describes how to work with koji profiles. + + +Command Line Interface +====================== +Koji client allows connecting to multiple koji instances from CLI +by using profiles. The default profile is given by executable file name, +which is 'koji'. + +To change koji profile, you can: +* run koji with --profile=$profile_name argument +* change executable file name by symlinking $profile_name -> koji + + +Configuration Files +=================== +Configuration files are located in following locations: +* /etc/koji.conf +* /etc/koji.conf.d/*.conf +* ~/.koji/config.d/*.conf +* user-specified config +Koji reads them all, looking for [$profile_name] sections. + + +Using Koji Profiles in Python +============================= +Instead of using koji module directly, +get profile specific module by calling: +>>> mod = koji.get_profile_module($profile_name) + +This module is clone of koji module with additional +profile specific tweaks. + +Profile configuration is available via: +>>> mod.config + + +Example +------- +import koji + +fedora_koji = koji.get_profile_module("koji") +ppc_koji = koji.get_profile_module("ppc-koji") + +for i in (fedora_koji, ppc_koji): + print "PROFILE: %s" % i.config.profile + for key, value in sorted(i.config.__dict__.items()): + print " %s = %s" % (key, value) + print + + +TODO +==== +* consider using pyxdg for user config locations diff --git a/koji/__init__.py b/koji/__init__.py index c0d5a30..dae52fc 100644 --- a/koji/__init__.py +++ b/koji/__init__.py @@ -28,13 +28,16 @@ except ImportError: sys.stderr.write("Warning: Could not install krbV module. Kerberos support will be disabled.\n") sys.stderr.flush() import base64 +import ConfigParser import datetime import errno from fnmatch import fnmatch import httplib +import imp import logging import logging.handlers from koji.util import md5_constructor +import optparse import os import os.path import pwd @@ -1457,6 +1460,133 @@ def config_directory_contents(dir_name): return configs

+CONFIG_DEFAULTS = { + 'server': 'http://localhost/kojihub', + 'weburl': 'http://localhost/koji', + 'topurl': None, + 'pkgurl': None, + 'topdir': '/mnt/koji', + 'max_retries': None, + 'retry_interval': None, + 'anon_retry': None, + 'offline_retry': None, + 'offline_retry_interval': None, + 'keepalive': True, + 'timeout': None, + 'use_fast_upload': False, + 'poll_interval': 5, + 'krbservice': 'host', + 'cert': '~/.koji/client.crt', + 'ca': '~/.koji/clientca.crt', + 'serverca': '~/.koji/serverca.crt', + 'authtype': None +} + + +INT_OPTIONS = ['max_retries', 'retry_interval', 'offline_retry_interval', 'poll_interval', 'timeout'] +BOOL_OPTIONS = ['anon_retry', 'offline_retry', 'keepalive', 'use_fast_upload'] +PATH_OPTIONS = ['topdir', 'cert', 'ca', 'serverca'] + + +def config_directory_contents(dir_name): + configs = [] + try: + conf_dir_contents = os.listdir(dir_name) + except OSError, exception: + if exception.errno != errno.ENOENT: + raise + else: + for name in sorted(conf_dir_contents): + if not name.endswith('.conf'): + continue + config_full_name = os.path.join(dir_name, name) + configs.append(config_full_name) + return configs + + +def read_config(profile_name, user_config=None): + result = CONFIG_DEFAULTS.copy() + for option in CONFIG_DEFAULTS: + if option in PATH_OPTIONS: + result[option] = os.path.expanduser(result[option]) + + configs = [] + + # main config + configs.append("/etc/koji.conf") + + # conf.d + configs.extend(config_directory_contents("/etc/koji.conf.d")) + + # user config + configs.append(os.path.expanduser("~/.koji/config")) + + # user conf.d + configs.extend(config_directory_contents(os.path.expanduser("~/.koji/conf.d"))) + + # TODO: read configs via xdg.BaseDirectory.load_config_path("koji") + + # user config specified in runtime + if user_config is not None: + configs.append(user_config) + + # read configs in particular order, use the last value found + for config_path in configs: + if not os.access(config_path, os.F_OK): + continue + config = ConfigParser.SafeConfigParser() + config.readfp(open(config_path, "r")) + + if profile_name not in config.sections(): + continue + + # check for invalid options + invalid_options = [] + for option in config.options(profile_name): + if option not in result: + invalid_options.append(option) + + if invalid_options: + raise ValueError("Invalid options: %s" % ", ".join(invalid_options)) + + for option in config.options(profile_name): + if option in BOOL_OPTIONS: + result[option] = config.getboolean(profile_name, option) + elif option in INT_OPTIONS: + result[option] = config.getint(profile_name, option) + else: + result[option] = config.get(profile_name, option) + if option in PATH_OPTIONS: + result[option] = os.path.expanduser(result[option]) + + result["profile"] = profile_name + + # convert dict to optparse Values + options = optparse.Values(result) + return options + + +config = read_config("koji") + + +def get_profile_module(profile_name, config=None): + """ + Create module for a koji instance. + Override profile specific module attributes: + * BASEDIR + * config + * pathinfo + """ + if config is None: + config = read_config(profile_name) + mod = imp.new_module("_koji__%s" % profile_name) + mod.__dict__.update(globals()) + mod.config = config + mod.BASEDIR = config.topdir + mod.pathinfo = PathInfo(topdir=config.topdir) + return mod + + class PathInfo(object): # ASCII numbers and upper- and lower-case letter for use in tmpdir() ASCII_CHARS = [chr(i) for i in range(48, 58) + range(65, 91) + range(97, 123)]