copr-commits March 2015

copr-commits@lists.fedorahosted.org

2 participants
42 discussions

[copr] master: [backend] do logging from multiply processes through redis pubsub; some fixes to VM-management (9e8dec1)
by vgologuz＠fedoraproject.org 31 Mar '15

31 Mar '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 9e8dec1a8c637e21a50e62ba10f094ba3165cd9a Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Thu Mar 26 15:15:30 2015 +0100 [backend] do logging from multiply processes through redis pubsub; some fixes to VM-management >--------------------------------------------------------------- Diff suppressed because of size. To see it, use: git diff --patch-with-stat --no-color --find-copies-harder --ignore-space-at-eol ^9e8dec1a8c637e21a50e62ba10f094ba3165cd9a~1 9e8dec1a8c637e21a50e62ba10f094ba3165cd9a

1 0

[copr] master: [frontend] Bug 1206562 - Cannot delete Copr because it incorrectly thinks there are unfinished builds. Solution: `failed` but unfinished (ended_on is null) builds should be rescheduled. (cddbf44)
by vgologuz＠fedoraproject.org 30 Mar '15

30 Mar '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit cddbf44de7f8f05bf3fcedc58f7d1dde567948d3 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Mon Mar 30 14:23:59 2015 +0200 [frontend] Bug 1206562 - Cannot delete Copr because it incorrectly thinks there are unfinished builds. Solution: `failed` but unfinished (ended_on is null) builds should be rescheduled. >--------------------------------------------------------------- .../coprs_frontend/coprs/logic/builds_logic.py | 8 +++++--- 1 files changed, 5 insertions(+), 3 deletions(-) diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index 2712e48..b088ef9 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -59,7 +59,8 @@ class BuildsLogic(object): models.BuildChroot.status == helpers.StatusEnum("pending"), models.BuildChroot.status == helpers.StatusEnum("starting"), and_( - models.BuildChroot.status == helpers.StatusEnum("running"), + models.BuildChroot.status.in_([ + helpers.StatusEnum("running"), helpers.StatusEnum("failed")]), models.Build.started_on < int(time.time() - 1.1 * MAX_BUILD_TIMEOUT), models.Build.ended_on.is_(None) ) @@ -218,11 +219,12 @@ class BuildsLogic(object): """ if not user.can_build_in(build.copr): raise exceptions.InsufficientRightsException( - "You are not allowed to delete this build.") + "You are not allowed to delete build `{}`.".format(build.id)) if not build.deletable: + # from celery.contrib import rdb; rdb.set_trace() raise exceptions.ActionInProgressException( - "You can not delete build which is not finished.", + "You can not delete build `{}` which is not finished.".format(build.id), "Unfinished build") # Only failed, finished, succeeded get here.

1 0

[copr] master: [image_builder] initial release, due to OS bug, we cannot create snapshot after provision through API, need to do it manually in the WebUI. (daaaf9d)
by vgologuz＠fedoraproject.org 25 Mar '15

25 Mar '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit daaaf9d3adb5161fa78106065637479913f40d07 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Wed Mar 25 15:11:36 2015 +0100 [image_builder] initial release, due to OS bug, we cannot create snapshot after provision through API, need to do it manually in the WebUI. >--------------------------------------------------------------- backend/conf/playbooks/files/site-defaults.cfg | 156 ++++++++++++++++++++ backend/conf/playbooks/provision_builder_pb.yml | 16 ++ backend/conf/playbooks/provision_builder_tasks.yml | 62 ++++++++ backend/conf/playbooks/spawn_new_tasks.yml | 22 +++ builder_image/requirements.txt | 6 + builder_image/src/__init__.py | 1 + builder_image/src/config.py | 34 +++++ builder_image/src/copr_prepare_builder_image.py | 88 +++++++++++ 8 files changed, 385 insertions(+), 0 deletions(-) diff --git a/backend/conf/playbooks/files/site-defaults.cfg b/backend/conf/playbooks/files/site-defaults.cfg new file mode 100644 index 0000000..15ebd36 --- /dev/null +++ b/backend/conf/playbooks/files/site-defaults.cfg @@ -0,0 +1,156 @@ +# mock defaults +# vim:tw=0:ts=4:sw=4:et: +# +# This config file is for site-specific default values that apply across all +# configurations. Options specified in this config file can be overridden in +# the individual mock config files. +# +# The site-defaults.cfg delivered by default has NO options set. Only set +# options here if you want to override the defaults. +# +# Entries in this file follow the same format as other mock config files. +# config_opts['foo'] = bar + +############################################################################# +# +# Things that we recommend you set in site-defaults.cfg: +# +# config_opts['basedir'] = '/var/lib/mock/' +# config_opts['cache_topdir'] = '/var/cache/mock' +# Note: the path pointed to by basedir and cache_topdir must be owned +# by group 'mock' and must have mode: g+rws +# config_opts['rpmbuild_timeout'] = 0 +# config_opts['use_host_resolv'] = True + +# You can configure log format to pull from logging.ini formats of these names: +# config_opts['build_log_fmt_name'] = "unadorned" +# config_opts['root_log_fmt_name'] = "detailed" +# config_opts['state_log_fmt_name'] = "state" +# +# mock will normally set up a minimal chroot /dev. +# If you want to use a pre-configured /dev, disable this and use the bind-mount +# plugin to mount your special /dev +# config_opts['internal_dev_setup'] = True +# +# internal_setarch defaults to 'True' if the python 'ctypes' package is +# available. It is in the python std lib on >= python 2.5. On older versions, +# it is available as an addon. On systems w/o ctypes, it will default to 'False' +# config_opts['internal_setarch'] = False +# +# the cleanup_on_* options allow you to automatically clean and remove the +# mock build directory, but only take effect if --resultdir is used. +# config_opts provides fine-grained control. cmdline only has big hammer +# +# config_opts['cleanup_on_success'] = 1 +# config_opts['cleanup_on_failure'] = 1 + +# if you want mock to automatically run createrepo on the rpms in your +# resultdir. +# config_opts['createrepo_on_rpms'] = False +# config_opts['createrepo_command'] = '/usr/bin/createrepo -d -q -x *.src.rpm' + +############################################################################# +# +# plugin related. Below are the defaults. Change to suit your site +# policy. site-defaults.cfg is a good place to do this. +# +# NOTE: Some of the caching options can theoretically affect build +# reproducability. Change with care. +# +config_opts['plugin_conf']['package_state_enable'] = False +config_opts['plugin_conf']['ccache_enable'] = False +# config_opts['plugin_conf']['ccache_opts']['max_cache_size'] = '4G' +# config_opts['plugin_conf']['ccache_opts']['compress'] = None +# config_opts['plugin_conf']['ccache_opts']['dir'] = "%(cache_topdir)s/%(root)s/ccache/" +config_opts['plugin_conf']['yum_cache_enable'] = False +# config_opts['plugin_conf']['yum_cache_opts']['max_age_days'] = 30 +# config_opts['plugin_conf']['yum_cache_opts']['dir'] = "%(cache_topdir)s/%(root)s/yum_cache/" +config_opts['plugin_conf']['root_cache_enable'] = False +# config_opts['plugin_conf']['root_cache_opts']['max_age_days'] = 15 +# config_opts['plugin_conf']['root_cache_opts']['dir'] = "%(cache_topdir)s/%(root)s/root_cache/" +# config_opts['plugin_conf']['root_cache_opts']['compress_program'] = "pigz" +# config_opts['plugin_conf']['root_cache_opts']['extension'] = ".gz" +# config_opts['plugin_conf']['root_cache_opts']['exclude_dirs'] = ["./proc", "./sys", "./dev", +# "./tmp/ccache", "./var/cache/yum" ] +# +# bind mount plugin is enabled by default but has no configured directories to +# mount +config_opts['plugin_conf']['bind_mount_enable'] = False +# config_opts['plugin_conf']['bind_mount_opts']['dirs'].append(('/host/path', '/bind/mount/path/in/chroot/' )) +# +# config_opts['plugin_conf']['tmpfs_enable'] = False +# config_opts['plugin_conf']['tmpfs_opts']['required_ram_mb'] = 1024 +# config_opts['plugin_conf']['tmpfs_opts']['max_fs_size'] = '512m' + +############################################################################# +# +# environment for chroot +# +# config_opts['environment']['TERM'] = 'vt100' +# config_opts['environment']['SHELL'] = '/bin/bash' +# config_opts['environment']['HOME'] = '/builddir' +# config_opts['environment']['HOSTNAME'] = 'mock' +# config_opts['environment']['PATH'] = '/usr/bin:/bin:/usr/sbin:/sbin' +# config_opts['environment']['PROMPT_COMMAND'] = 'echo -n "<mock-chroot>"' +# config_opts['environment']['LANG'] = os.environ.setdefault('LANG', 'en_US.UTF-8') +# config_opts['environment']['TZ'] = os.environ.setdefault('TZ', 'EST5EDT') + +############################################################################# +# +# Things that you can change, but we dont recommend it: +# config_opts['chroothome'] = '/builddir' +# config_opts['clean'] = True + +############################################################################# +# +# Things that must be adjusted if SCM integration is used: +# +# config_opts['scm'] = True +# config_opts['scm_opts']['method'] = 'git' +# config_opts['scm_opts']['cvs_get'] = 'cvs -d /srv/cvs co SCM_BRN SCM_PKG' +# config_opts['scm_opts']['git_get'] = 'git clone SCM_BRN git://localhost/SCM_PKG.git SCM_PKG' +# config_opts['scm_opts']['svn_get'] = 'svn co file:///srv/svn/SCM_PKG/SCM_BRN SCM_PKG' +# config_opts['scm_opts']['spec'] = 'SCM_PKG.spec' +# config_opts['scm_opts']['ext_src_dir'] = '/dev/null' +# config_opts['scm_opts']['write_tar'] = True +# config_opts['scm_opts']['git_timestamps'] = True + +# These options are also recognized but usually defined in cmd line +# with --scm-option package=<pkg> --scm-option branch=<branch> +# config_opts['scm_opts']['package'] = 'mypkg' +# config_opts['scm_opts']['branch'] = 'master' + +############################################################################# +# +# Things that are best suited for individual chroot config files: +# +# MUST SET (in individual chroot cfg file): +# config_opts['root'] = 'name-of-yum-build-dir' +# config_opts['target_arch'] = 'i386' +# config_opts['yum.conf'] = '' +# config_opts['yum_common_opts'] = [] +# +# CAN SET, defaults usually work ok: +# config_opts['chroot_setup_cmd'] = 'install buildsys-build' +# config_opts['log_config_file'] = 'logging.ini' +# config_opts['more_buildreqs']['srpm_name-version-release'] = 'dependencies' +# config_opts['macros']['%Add_your_macro_name_here'] = "add macro value here" +# config_opts['files']['path/name/no/leading/slash'] = "put file contents here." +# config_opts['chrootuid'] = os.getuid() + +# If you change chrootgid, you must also change "mock" to the correct group +# name in this line of the mock PAM config: +# auth sufficient pam_succeed_if.so user ingroup mock use_uid quiet +# config_opts['chrootgid'] = grp.getgrnam("mock")[2] + +# config_opts['useradd'] = '/usr/sbin/useradd -m -u %(uid)s -g %(gid)s -d %(home)s -n %(user)s' # Fedora/RedHat +# +# Security related +# config_opts['no_root_shells'] = False +# +# Proxy settings (https_proxy, ftp_proxy, and no_proxy can also be set) +# config_opts['http_proxy'] = 'http://localhost:3128' + +config_opts['plugin_conf']['compress_logs_enable'] = True +config_opts['plugin_conf']['compress_logs_opts'] = {} +config_opts['plugin_conf']['compress_logs_opts']['command'] = "/usr/bin/gzip -9" diff --git a/backend/conf/playbooks/provision_builder_pb.yml b/backend/conf/playbooks/provision_builder_pb.yml new file mode 100644 index 0000000..f2f94bf --- /dev/null +++ b/backend/conf/playbooks/provision_builder_pb.yml @@ -0,0 +1,16 @@ +- hosts: localhost + gather_facts: False + tasks: + - local_action: add_host hostname={{ vm_ip }} groupname=builder_temp_group + +- hosts: builder_temp_group + user: fedora + sudo: true + gather_facts: True + #vars: + #- files: files/ + + tasks: + # - debug: msg="test" + - include: "provision_builder_tasks.yml" + diff --git a/backend/conf/playbooks/provision_builder_tasks.yml b/backend/conf/playbooks/provision_builder_tasks.yml new file mode 100644 index 0000000..7183692 --- /dev/null +++ b/backend/conf/playbooks/provision_builder_tasks.yml @@ -0,0 +1,62 @@ +- name: install pkgs + yum: state=present pkg={{ item }} + with_items: + - dnf + - mock + - mock-lvm + - createrepo + - yum-utils + - pyliblzma + - rsync + - openssh-clients + - rsync + - libselinux-python + - libsemanage-python + +- name: make sure newest rpm + yum: name={{ item }} state=latest + with_items: + - rpm + - glib2 + - ca-certificates + +- name: put updated mock configs into /etc/mock + copy: src=files/{{ item }} dest=/etc/mock + with_items: + - site-defaults.cfg + +- name: prepare caches + async: 3600 + command: mock -r {{ item }} --init + with_items: + - epel-5-i386 + - epel-5-x86_64 + - epel-6-i386 + - epel-6-x86_64 + - epel-7-x86_64 + - fedora-20-i386 + - fedora-20-x86_64 + - fedora-21-i386 + - fedora-21-x86_64 + - fedora-22-i386 + - fedora-22-x86_64 + - fedora-rawhide-i386 + - fedora-rawhide-x86_64 + +- name: mockbuilder user + user: name=mockbuilder groups=mock + +- name: mockbuilder .ssh + file: state=directory path=/home/mockbuilder/.ssh mode=0700 owner=mockbuilder group=mockbuilder + +- name: mockbuilder authorized_keys + authorized_key: user=mockbuilder key='{{ lookup('file', '/home/copr/provision/files/buildsys.pub') }}' + +- name: root authorized_keys + authorized_key: user=root key='{{ lookup('file', '/home/copr/provision/files/buildsys.pub') }}' + +- lineinfile: dest=/etc/mock/fedora-rawhide-x86_64.cfg line="config_opts['package_manager'] = 'dnf'" state=absent +- lineinfile: dest=/etc/mock/fedora-rawhide-i386.cfg line="config_opts['package_manager'] = 'dnf'" state=absent + +- lineinfile: dest=/etc/security/limits.conf line="* soft nofile 10240" insertafter=EOF +- lineinfile: dest=/etc/security/limits.conf line="* hard nofile 10240" insertafter=EOF diff --git a/backend/conf/playbooks/spawn_new_tasks.yml b/backend/conf/playbooks/spawn_new_tasks.yml new file mode 100644 index 0000000..802c0d8 --- /dev/null +++ b/backend/conf/playbooks/spawn_new_tasks.yml @@ -0,0 +1,22 @@ +- name: spin UP VM using nova_compute + sudo: False + local_action: + module: nova_compute + insecure: True + auth_url: "{{os_auth_url}}" + login_username: "admin" + login_password: "{{ADMIN_PASS}}" + login_tenant_name: "{{inventory_tenant}}" + name: "{{inventory_instance_name}}" + image_id: "{{ image }}" + wait_for: 300 + flavor_id: "{{ flavor_id }}" + security_groups: "{{security_group}}" + key_name: "{{ keypair }}" + nics: "{{ cloud_networks }}" + # floating_ip_pools: "{{ floating_ip_pools }}" + floating_ips: + - "{{public_ip}}" + register: nova_result + +- debug: msg="{{ nova_result.info }}" diff --git a/builder_image/requirements.txt b/builder_image/requirements.txt new file mode 100644 index 0000000..7f4b3d2 --- /dev/null +++ b/builder_image/requirements.txt @@ -0,0 +1,6 @@ +ipdb +ansible +python-novaclient +python-glanceclient +python-keystoneclient +# neutronclient diff --git a/builder_image/src/__init__.py b/builder_image/src/__init__.py new file mode 100644 index 0000000..57d631c --- /dev/null +++ b/builder_image/src/__init__.py @@ -0,0 +1 @@ +# coding: utf-8 diff --git a/builder_image/src/config.py b/builder_image/src/config.py new file mode 100644 index 0000000..d738fc0 --- /dev/null +++ b/builder_image/src/config.py @@ -0,0 +1,34 @@ +# coding: utf-8 + + +class Config(object): + OS_AUTH_URL = "https://fed-cloud09.cloud.fedoraproject.org:5000/v2.0" + + # With the addition of Keystone we have standardized on the term **tenant** + # as the entity that owns the resources. + # OS_TENANT_ID = "a3fa9697346046ef9e691a21527923a5" + # OS_TENANT_NAME = "copr" + OS_TENANT_ID = "566a072fb1694950998ad191fee3833b" + OS_TENANT_NAME = "coprdev" + + # In addition to the owning entity (tenant), openstack stores the entity + # performing the action as the **user**. + OS_USERNAME = "username" + OS_PASSWORD = "password" + + # If your configuration has multiple regions, we set that information here. + # OS_REGION_NAME is optional and only valid in certain environments. + OS_REGION_NAME = "RegionOne" + + flavor_name = "m1.small" + + vm_name = "builder_base_image" + image_name = "Fedora-Cloud-Base-20141203-21" + key_name = "fas" + + net_name_list = ["coprdev-net", "copr-net", "external"] + security_groups = ["default", "ssh-anywhere-coprdev"] + + provision_pb_path = "" + + floating_ips_pool = "external" diff --git a/builder_image/src/copr_prepare_builder_image.py b/builder_image/src/copr_prepare_builder_image.py new file mode 100644 index 0000000..adf1f2d --- /dev/null +++ b/builder_image/src/copr_prepare_builder_image.py @@ -0,0 +1,88 @@ +# coding: utf-8 + + +from ansible import errors, runner +from glanceclient import Client as GlanceClient +from keystoneclient import session +from keystoneclient.auth.identity import v2 as identity +# from neutronclient.neutron.client import Client as NeutronClient +from novaclient.v2.client import Client +import glanceclient.exc +import json +import novaclient.exceptions + +from config import Config + + +class ImageBuilder(object): + + """ + + :type config: Config + """ + def __init__(self, config): + self.config = config + self.vm_ip = None + + self.nova_result = None + self.float_ip = None + + def post_init(self): + self.nt = Client( + self.config.OS_USERNAME, + self.config.OS_PASSWORD, + self.config.OS_TENANT_NAME, + self.config.OS_AUTH_URL, + insecure=True, + ) + + def spawn_vm(self): + if len(self.nt.servers.findall(name=self.config.vm_name)) > 0: + raise RuntimeError("Server already exists") + + nics = [ + {"net-id": net.id} + for net in [ + self.nt.networks.find(human_id=net_name) + for net_name in self.config.net_name_list + ]] + self.nova_result = self.nt.servers.create( + name=self.config.vm_name, + flavor=self.nt.flavors.find(name=self.config.flavor_name), + image=self.nt.images.find(name=self.config.image_name), + key_name=self.config.key_name, + nics=nics, + security_groups=self.config.security_groups, + ) + + if getattr(self.config, "floating_ips_pool"): + self.float_ip = self.nt.floating_ips.create(pool=self.config.floating_ips_pool) + self.nova_result.add_floating_ip(self.float_ip) + + def run_provision(self): + pass + + def create_image_from_vm(self, image_name): + pass + + + def terminate_vm(self): + if self.nova_result: + self.nova_result.delete() + if self.float_ip: + self.float_ip.delete() + + +def main(): + ib = ImageBuilder(Config()) + try: + ib.post_init() + + + + finally: + ib.terminate_vm() + + +if __name__ == "__main__": + main()

1 0

[copr] master: [backend] tests cleanup (bf311c8)
by vgologuz＠fedoraproject.org 24 Mar '15

24 Mar '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit bf311c8767831f4453256ba07f0a4400b3489716 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Tue Mar 24 16:28:11 2015 +0100 [backend] tests cleanup >--------------------------------------------------------------- backend/backend/daemons/vm_master.py | 4 +++- backend/backend/mockremote/builder.py | 6 +++--- backend/backend/vm_manage/spawn.py | 2 +- backend/conf/copr-be.conf.example | 2 +- backend/copr-backend.spec | 9 +++++---- backend/run/copr_get_vm_info.py | 3 +++ backend/tests/mockremote/test_builder.py | 21 +++++++++++++++++---- backend/tests/run/test_copr_prune_results.py | 6 ++---- 8 files changed, 35 insertions(+), 18 deletions(-) diff --git a/backend/backend/daemons/vm_master.py b/backend/backend/daemons/vm_master.py index 0fa49f2..ad1fd70 100644 --- a/backend/backend/daemons/vm_master.py +++ b/backend/backend/daemons/vm_master.py @@ -84,6 +84,8 @@ class VmMaster(Process): self.vmm.start_vm_check(vmd.vm_name) def try_spawn_one(self, group): + # TODO: add setting "max_vm_in_ready_state", when this number reached, do not spawn more VMS, min value = 1 + max_vm_total = self.opts.build_groups[group]["max_vm_total"] active_vmd_list = self.vmm.get_vm_by_group_and_state_list( group, [VmStates.GOT_IP, VmStates.READY, VmStates.IN_USE, VmStates.CHECK_HEALTH]) @@ -92,7 +94,7 @@ class VmMaster(Process): total_vm_estimation = len(active_vmd_list) + self.vmm.spawner.children_number if total_vm_estimation >= max_vm_total: self.log("Skip spawn: max total vm reached for group {}: vm count: {}, spawn process: {}" - .format(group, len(active_vmd_list), self.opts.build_groups[group]["max_vm_total"])) + .format(group, len(active_vmd_list), self.vmm.spawner.children_number)) return last_vm_spawn_start = self.vmm.rc.hget(KEY_VM_POOL_INFO.format(group=group), "last_vm_spawn_start") if last_vm_spawn_start: diff --git a/backend/backend/mockremote/builder.py b/backend/backend/mockremote/builder.py index 064c7fb..a22f2a3 100644 --- a/backend/backend/mockremote/builder.py +++ b/backend/backend/mockremote/builder.py @@ -354,10 +354,10 @@ class Builder(object): def check(self): # do check of host try: + # requires name resolve facility socket.gethostbyname(self.hostname) - except socket.gaierror: - raise BuilderError("{0} could not be resolved".format( - self.hostname)) + except IOError: + raise BuilderError("{0} could not be resolved".format(self.hostname)) try: # check_for_ans_error(res, self.hostname) diff --git a/backend/backend/vm_manage/spawn.py b/backend/backend/vm_manage/spawn.py index 77658be..38a6e64 100644 --- a/backend/backend/vm_manage/spawn.py +++ b/backend/backend/vm_manage/spawn.py @@ -143,7 +143,7 @@ class Spawner(Executor): def start_spawn(self, group): self.recycle() - spawn_playbook = None + # spawn_playbook = None try: spawn_playbook = self.opts.build_groups[group]["spawn_playbook"] except KeyError: diff --git a/backend/conf/copr-be.conf.example b/backend/conf/copr-be.conf.example index 70e78ac..4a34206 100644 --- a/backend/conf/copr-be.conf.example +++ b/backend/conf/copr-be.conf.example @@ -41,7 +41,7 @@ group0_archs=i386,x86_64 group0_spawn_playbook=/srv/copr-work/provision/builderpb-PC.yml group0_terminate_playbook=/srv/copr-work/provision/terminatepb-PC.yml group0_max_workers=8 - +group0_max_vm_total=2 # That variables will be send to # terminate_playbook ansible script. diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec index 45e5fff..e6a2b5c 100644 --- a/backend/copr-backend.spec +++ b/backend/copr-backend.spec @@ -20,15 +20,14 @@ BuildArch: noarch BuildRequires: asciidoc BuildRequires: libxslt BuildRequires: util-linux +BuildRequires: yum-utils BuildRequires: python-setuptools BuildRequires: python-requests BuildRequires: python2-devel BuildRequires: python-copr BuildRequires: systemd BuildRequires: redis -%if 0%{?rhel} < 7 && 0%{?rhel} > 0 -BuildRequires: python-argparse -%endif + #for doc package BuildRequires: epydoc BuildRequires: graphviz @@ -48,6 +47,7 @@ BuildRequires: ansible >= 1.2 BuildRequires: python-IPy BuildRequires: python-paramiko BuildRequires: python-psutil +# BuildRequires: python-ipdb BuildRequires: wget Requires: obs-signd @@ -75,6 +75,7 @@ Requires: fedmsg Requires: gawk Requires: crontabs Requires: python-paramiko +# Requires: python-ipdb # Requires: logstash Requires(post): systemd @@ -166,7 +167,7 @@ cp -a conf/playbooks %{buildroot}%{_pkgdocdir}/ redis-server --port 7777 &> /dev/null & PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \ - -s --cov-report term-missing --cov ./backend --cov ./run ./tests/ + -s -v --cov-report term-missing --cov ./backend --cov ./run ./tests/ kill %1 diff --git a/backend/run/copr_get_vm_info.py b/backend/run/copr_get_vm_info.py index 3d80611..bf9f3e6 100644 --- a/backend/run/copr_get_vm_info.py +++ b/backend/run/copr_get_vm_info.py @@ -1,5 +1,8 @@ # coding: utf-8 +import sys +sys.path.append("/usr/share/copr/") + from backend.helpers import BackendConfigReader from backend.vm_manage.manager import VmManager diff --git a/backend/tests/mockremote/test_builder.py b/backend/tests/mockremote/test_builder.py index 8a414f9..f2da9b3 100644 --- a/backend/tests/mockremote/test_builder.py +++ b/backend/tests/mockremote/test_builder.py @@ -3,6 +3,7 @@ import copy from collections import defaultdict from pprint import pprint +import socket from bunch import Bunch from backend.exceptions import BuilderError, BuilderTimeOutError, AnsibleCallError, AnsibleResponseError, VmError @@ -35,6 +36,13 @@ from backend.mockremote.builder import Builder MODULE_REF = "backend.mockremote.builder" + +(a)pytest.yield_fixture +def mc_socket(): + with mock.patch("{}.socket".format(MODULE_REF)) as handle: + yield handle + + def noop(*args, **kwargs): pass @@ -301,7 +309,8 @@ class TestBuilder(object): results = {"contacted": {}, "dark": {}} assert {} == builder_module.get_ans_results(results, self.BUILDER_HOSTNAME) - def test_check_hostname_check(self): + def test_check_hostname_check(self, mc_socket): + mc_socket.gethostbyname.side_effect = socket.gaierror() builder = self.get_test_builder() for name in ["*", "256.0.0.1"]: with pytest.raises(BuilderError): @@ -309,8 +318,9 @@ class TestBuilder(object): builder.hostname = name builder.check() - def test_check_missing_required_binaries(self): + def test_check_missing_required_binaries(self, mc_socket): builder = self.get_test_builder() + self.stage = 0 def ans_run(): self.stage_ctx[self.stage]["conn"] = copy.deepcopy(builder.conn) @@ -326,11 +336,12 @@ class TestBuilder(object): builder.check() # import ipdb; ipdb.set_trace() + # pprint(self.stage_ctx) assert "/bin/rpm -q mock rsync" in self.stage_ctx[0]["conn"].module_args assert "does not have mock or rsync installed" in err.value.msg - def test_check_missing_mockchain_or_mock_config(self): + def test_check_missing_mockchain_or_mock_config(self, mc_socket): builder = self.get_test_builder() def ans_run(): @@ -346,13 +357,14 @@ class TestBuilder(object): with pytest.raises(BuilderError) as err: builder.check() + # pprint(self.stage_ctx) assert "/usr/bin/test -f /usr/bin/mockchain" in self.stage_ctx[1]["conn"].module_args # assert "/usr/bin/test -f /etc/mock/{}.cfg".format(self.BUILDER_CHROOT) in \ # self.stage_ctx[1]["conn"].module_args assert "missing mockchain binary" in err.value.msg - def test_check_missing_mock_config(self): + def test_check_missing_mock_config(self, mc_socket): builder = self.get_test_builder() ret_map = { @@ -370,6 +382,7 @@ class TestBuilder(object): with pytest.raises(BuilderError) as err: builder.check() + # pprint(self.stage_ctx) assert "/usr/bin/test -f /usr/bin/mockchain" in self.stage_ctx[1]["conn"].module_args assert "/usr/bin/test -f /etc/mock/{}.cfg".format(self.BUILDER_CHROOT) in \ self.stage_ctx[2]["conn"].module_args diff --git a/backend/tests/run/test_copr_prune_results.py b/backend/tests/run/test_copr_prune_results.py index 25c4b80..1a4be8c 100644 --- a/backend/tests/run/test_copr_prune_results.py +++ b/backend/tests/run/test_copr_prune_results.py @@ -78,7 +78,7 @@ class TestPruneResults(object): self.prj = "foox" self.chroots = ["fedora-20-i386", "fedora-20-x86_64"] self.opts = Bunch( - days=14, + prune_days=14, find_obsolete_script=os.path.abspath( os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, @@ -113,7 +113,7 @@ class TestPruneResults(object): def make_temp_dir(self): root_tmp_dir = tempfile.gettempdir() - subdir = "test_action_{}".format(self.test_time) + subdir = "test_prune_old_{}".format(self.test_time) self.tmp_dir_name = os.path.join(root_tmp_dir, subdir) os.mkdir(self.tmp_dir_name) @@ -300,7 +300,6 @@ class TestPruneResults(object): assert mc_cru.called - def test_prune_project_skip_when_acr_disabled(self, test_pruner, mc_cru, mc_gacs): self.pruner.prune_failed_builds = MagicMock() self.pruner.prune_obsolete_success_builds = MagicMock() @@ -318,7 +317,6 @@ class TestPruneResults(object): self.pruner.run() - def test_main(self, mc_pruner, mc_bcr): prune_main() assert mc_pruner.called

1 0

[copr] master: [backend] ensure that prune script running under the copr user; simpler `copr_find_obsolete_builds` (513441f)
by vgologuz＠fedoraproject.org 24 Mar '15

24 Mar '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 513441f93ca10e58cb61d262f0db57f0090171f0 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Tue Mar 24 14:09:34 2015 +0100 [backend] ensure that prune script running under the copr user; simpler `copr_find_obsolete_builds` >--------------------------------------------------------------- backend/run/copr_find_obsolete_builds.sh | 3 +-- backend/run/copr_prune_results.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/backend/run/copr_find_obsolete_builds.sh b/backend/run/copr_find_obsolete_builds.sh index dcd3e34..e46df34 100755 --- a/backend/run/copr_find_obsolete_builds.sh +++ b/backend/run/copr_find_obsolete_builds.sh @@ -6,7 +6,7 @@ DAYS=$2 echo "# looking for obsolete files in: " $CHROOT_PATH if [ -d $CHROOT_PATH ] then - pushd $CHROOT_PATH >/dev/null + cd $CHROOT_PATH # query latest (sucessfull) packages LATEST_PKGS=$(mktemp) @@ -33,5 +33,4 @@ then done rm -f $LATEST_PKGS $ERR_LOG - popd >/dev/null fi diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py index d9445fd..de96dc3 100644 --- a/backend/run/copr_prune_results.py +++ b/backend/run/copr_prune_results.py @@ -11,6 +11,7 @@ import sys import logging from subprocess import Popen, PIPE import time +import pwd log = logging.getLogger(__name__) @@ -156,8 +157,12 @@ def main(): pruner.run() if __name__ == "__main__": - logging.basicConfig( - filename="/var/log/copr/prune_old.log", - format='[%(asctime)s][%(levelname)6s]: %(message)s', - level=logging.INFO) - main() + if pwd.getpwuid(os.getuid())[0] != "copr": + print("This script should be executed under the `copr` user") + sys.exit(1) + else: + logging.basicConfig( + filename="/var/log/copr/prune_old.log", + format='[%(asctime)s][%(levelname)6s]: %(message)s', + level=logging.INFO) + main()

1 0

[copr] master: [backend] safer copr_prune_results script, unittests (e539a10)
by vgologuz＠fedoraproject.org 24 Mar '15

24 Mar '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit e539a10a7a34d3f4ff983704470d30306a132d07 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Tue Mar 24 13:42:09 2015 +0100 [backend] safer copr_prune_results script, unittests >--------------------------------------------------------------- backend/backend/createrepo.py | 2 + backend/run/copr_prune_results.py | 28 ++-- backend/tests/_resources/foox.tar.gz | Bin 17135362 -> 6549985 bytes backend/tests/run/test_copr_prune_results.py | 168 ++++++++++++++++++++++++-- 4 files changed, 171 insertions(+), 27 deletions(-) diff --git a/backend/backend/createrepo.py b/backend/backend/createrepo.py index 092fa6c..c20656f 100644 --- a/backend/backend/createrepo.py +++ b/backend/backend/createrepo.py @@ -47,6 +47,8 @@ def createrepo_unsafe(path, lock=None, dest_dir=None, base_url=None): cmd = Popen(comm, stdout=subprocess.PIPE, stderr=subprocess.PIPE) out, err = cmd.communicate() + # TODO: rewrite with raise CreateRepoException on returncode != 0 + # and return only stdout on success return cmd.returncode, out, err diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py index 9f77f81..d9445fd 100644 --- a/backend/run/copr_prune_results.py +++ b/backend/run/copr_prune_results.py @@ -58,20 +58,19 @@ class Pruner(object): log.info("Removing failed build: {}".format(build_path)) shutil.rmtree(build_path) - def prune_obsolete_success_builds(self, chroot_path, username, projectname): + def prune_obsolete_success_builds(self, chroot_path): """ Uses bash script which invokes repoquery to find obsolete build_dirs :param chroot_path: path to the chroot directory - :return: """ # import ipdb; ipdb.set_trace() cmd = map(str, [self.prune_sh_script, chroot_path, self.days]) handle = Popen(cmd, stdout=PIPE, stderr=PIPE) stdout, stderr = handle.communicate() if handle.returncode != 0: - log.error("Failed to prune old builds for copr {}/{}\n STDOUT: \n{}\n STDERR: \n{}\n" - .format(username, projectname, stdout.decode(), stderr.decode())) + log.error("Failed to prune old builds at: {} \n STDOUT: \n{}\n STDERR: \n{}\n" + .format(chroot_path, stdout.decode(), stderr.decode())) return # log.debug("find obsolete returned:\n {}\n stderr: \n {}".format(stdout, stderr)) for line in stdout.split("\n"): @@ -121,31 +120,32 @@ class Pruner(object): # run prune project sh for sub_dir_name in os.listdir(project_path): - if sub_dir_name == "repodata": - continue chroot_path = os.path.join(project_path, sub_dir_name) if not os.path.isdir(chroot_path): continue try: self.prune_failed_builds(chroot_path) - self.prune_obsolete_success_builds(chroot_path, username, projectname) + self.prune_obsolete_success_builds(chroot_path) except Exception as err: log.exception(err) - log.error("Error during prune copr {}/{}".format(username, projectname)) + log.error("Error during prune copr {}/{}:{}".format(username, projectname, sub_dir_name)) - log.debug("Prune done for {}/{}".format(username, projectname)) + log.debug("Prune done for {}/{}:{}".format(username, projectname, sub_dir_name)) # run createrepo try: - retcode, stdout, stderr = createrepo_unsafe(project_path) + retcode, stdout, stderr = createrepo_unsafe(chroot_path) if retcode != 0: log.error( - "Createrepo for copr {}/{}\n STDOUT: \n{}\n STDERR: \n{}\n" - .format(username, projectname, stdout.decode(), stderr.decode())) + "Failed to createrepo for copr {}/{}:{}\n STDOUT: \n{}\n STDERR: \n{}\n" + .format(username, projectname, sub_dir_name, stdout.decode(), stderr.decode())) + else: + log.info("Createrepo done for copr {}/{}:{}" + .format(username, projectname, sub_dir_name)) except Exception as exception: - log.exception("Createrepo for copr {}/{} failed with error: {}" - .format(username, projectname, exception)) + log.exception("Createrepo for copr {}/{}:{} failed with error: {}" + .format(username, projectname, sub_dir_name, exception)) log.info("Prune finished for copr {}/{}".format(username, projectname)) diff --git a/backend/tests/_resources/foox.tar.gz b/backend/tests/_resources/foox.tar.gz index aaebe8f..7b3c975 100644 Binary files a/backend/tests/_resources/foox.tar.gz and b/backend/tests/_resources/foox.tar.gz differ diff --git a/backend/tests/run/test_copr_prune_results.py b/backend/tests/run/test_copr_prune_results.py index 074e9dc..25c4b80 100644 --- a/backend/tests/run/test_copr_prune_results.py +++ b/backend/tests/run/test_copr_prune_results.py @@ -8,6 +8,7 @@ import tempfile import time from bunch import Bunch from subprocess import Popen, PIPE +from copr.client.exceptions import CoprException import pytest @@ -30,8 +31,28 @@ def mc_popen(): with mock.patch("{}.Popen".format(MODULE_REF)) as handle: yield handle +(a)pytest.yield_fixture +def mc_bcr(): + with mock.patch("{}.BackendConfigReader".format(MODULE_REF)) as handle: + yield handle + +(a)pytest.yield_fixture +def mc_cru(): + with mock.patch("{}.createrepo_unsafe".format(MODULE_REF)) as handle: + yield handle + +(a)pytest.yield_fixture +def mc_gacs(): + with mock.patch("{}.get_auto_createrepo_status".format(MODULE_REF)) as handle: + yield handle + +(a)pytest.yield_fixture +def mc_pruner(): + with mock.patch("{}.Pruner".format(MODULE_REF)) as handle: + yield handle from copr_prune_results import Pruner +from copr_prune_results import main as prune_main def assert_same_dirs(a, b): @@ -42,6 +63,7 @@ def assert_same_dirs(a, b): raise AssertionError("Directories {}, {} aren't equal, diff out: \n {} \n {}" .format(a, b, stdout, stderr)) + class TestPruneResults(object): def setup_method(self, method): @@ -54,22 +76,24 @@ class TestPruneResults(object): self.pkg_2 = "hello-2.8-1.fc20" self.pkg_2_obsolete = "hello-1.0.fc20" self.prj = "foox" - self.chroots = ["fedora-20-i386", "fedora-20-x86_64", - "fedora-21-i386", "fedora-21-x86_64"] + self.chroots = ["fedora-20-i386", "fedora-20-x86_64"] self.opts = Bunch( days=14, find_obsolete_script=os.path.abspath( os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, - "run", "copr_find_obsolete_builds.sh")) + "run", "copr_find_obsolete_builds.sh")), + + frontend_base_url="http://example.com", + destdir=self.tmp_dir_name ) self.log_file_path = os.path.join(self.tmp_dir_name, "_unittest.log") logging.basicConfig( - # filename=self.log_file_path, - stream=sys.stdout, + filename=self.log_file_path, + # stream=sys.stdout, format='[%(asctime)s][%(levelname)6s]: %(message)s', level=logging.DEBUG ) @@ -161,10 +185,7 @@ class TestPruneResults(object): os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1_obsolete, "success"), (time.time(), time.time())) - self.pruner.prune_obsolete_success_builds( - os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]), - self.username, self.coprname - ) + self.pruner.prune_obsolete_success_builds(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0])) assert_same_dirs( os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]), @@ -178,13 +199,134 @@ class TestPruneResults(object): mc_handle.communicate.return_value = ("foo", "bar") # doesn't touch FS if `find_obsolete_build` produce error return code - self.pruner.prune_obsolete_success_builds( - os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]), - self.username, self.coprname - ) + self.pruner.prune_obsolete_success_builds(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0])) assert mc_popen.called assert_same_dirs( os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]), os.path.join(self.expect_dir_name, self.prj, self.chroots[0]), ) + + def test_prune_project_ok(self, test_pruner, mc_cru, mc_gacs): + self.pruner.prune_failed_builds = MagicMock() + self.pruner.prune_obsolete_success_builds = MagicMock() + mc_cru.return_value = (0, "", "") + + self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj), + self.username, self.coprname) + + expected_path_set = set([ + os.path.join(self.tmp_dir_name, self.prj, chroot) + for chroot in self.chroots + ]) + assert set([ + call[0][0] for call in + self.pruner.prune_failed_builds.call_args_list + ]) == expected_path_set + assert set([ + call[0][0] for call in + self.pruner.prune_obsolete_success_builds.call_args_list + ]) == expected_path_set + assert set([ + call[0][0] for call in + mc_cru.call_args_list + ]) == expected_path_set + + + def test_prune_project_handle_gacs_error(self, test_pruner, mc_cru, mc_gacs): + self.pruner.prune_failed_builds = MagicMock() + self.pruner.prune_obsolete_success_builds = MagicMock() + + mc_gacs.side_effect = CoprException() + + self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj), + self.username, self.coprname) + + assert not self.pruner.prune_failed_builds.called + assert not self.pruner.prune_obsolete_success_builds.called + + def test_prune_project_handle_errors(self, test_pruner, mc_cru, mc_gacs): + self.pruner.prune_failed_builds = MagicMock() + self.pruner.prune_obsolete_success_builds = MagicMock() + mc_gacs.return_value = True + + + # 0. createrepo_unsafe failure + mc_cru.return_value = (1, "stdout", "stderr") + + self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj), + self.username, self.coprname) + + assert self.pruner.prune_failed_builds.called + assert self.pruner.prune_obsolete_success_builds.called + + self.pruner.prune_failed_builds.reset_mock() + self.pruner.prune_obsolete_success_builds.reset_mock() + + mc_cru.return_value = (0, "", "") + # 1. createrepo_unsafe raises error + mc_cru.side_effect = IOError() + + self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj), + self.username, self.coprname) + + assert self.pruner.prune_failed_builds.called + assert self.pruner.prune_obsolete_success_builds.called + + self.pruner.prune_failed_builds.reset_mock() + self.pruner.prune_obsolete_success_builds.reset_mock() + + mc_cru.side_effect = None + # 2. prune_obsolete_success_builds raises error + + self.pruner.prune_obsolete_success_builds.side_effect = IOError() + + self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj), + self.username, self.coprname) + + assert mc_cru.called + + self.pruner.prune_failed_builds.reset_mock() + self.pruner.prune_obsolete_success_builds.reset_mock() + mc_cru.reset_mock() + + # 3. prune_failed_builds raises error + + self.pruner.prune_obsolete_success_builds.side_effect = None + self.pruner.prune_failed_builds.side_effect = IOError() + + self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj), + self.username, self.coprname) + + assert mc_cru.called + + + def test_prune_project_skip_when_acr_disabled(self, test_pruner, mc_cru, mc_gacs): + self.pruner.prune_failed_builds = MagicMock() + self.pruner.prune_obsolete_success_builds = MagicMock() + + mc_gacs.return_value = False + + self.pruner.prune_project(os.path.join(self.tmp_dir_name, self.prj), + self.username, self.coprname) + + assert not self.pruner.prune_failed_builds.called + assert not self.pruner.prune_obsolete_success_builds.called + + def test_run(self, test_pruner): + self.pruner.prune_project = MagicMock() + + self.pruner.run() + + + def test_main(self, mc_pruner, mc_bcr): + prune_main() + assert mc_pruner.called + assert mc_pruner.return_value.run.called + assert mc_bcr.called + assert mc_bcr.call_args[0][0] == "/etc/copr/copr-be.conf" + + os.environ["BACKEND_CONFIG"] = "foobar" + prune_main() + assert mc_bcr.call_args[0][0] == "foobar" +

1 0

[copr] master: [backend] safer copr_prune_results script, [WIP] (10909c0)
by vgologuz＠fedoraproject.org 23 Mar '15

23 Mar '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 10909c0dadde9fa30c63e7bc5385df3181974292 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Tue Mar 24 02:00:08 2015 +0100 [backend] safer copr_prune_results script, [WIP] >--------------------------------------------------------------- backend/conf/crontab/copr-backend | 2 +- backend/run/copr_find_obsolete_builds.sh | 37 +++++ backend/run/copr_prune_old_builds.sh | 52 ------- backend/run/copr_prune_results.py | 193 +++++++++++++++++--------- backend/tests/_resources/foox.tar.gz | Bin 0 -> 17135362 bytes backend/tests/run/test_copr_prune_results.py | 190 +++++++++++++++++++++++++ 6 files changed, 355 insertions(+), 119 deletions(-) diff --git a/backend/conf/crontab/copr-backend b/backend/conf/crontab/copr-backend index 68ba1a4..de9f250 100644 --- a/backend/conf/crontab/copr-backend +++ b/backend/conf/crontab/copr-backend @@ -1,3 +1,3 @@ #!/usr/bin/sh -runuser -c "PYTHONUNBUFFERED=1 python /usr/bin/copr_prune_results.py" - copr >> /var/log/copr/prune_old.log 2>&1 +runuser -c "PYTHONUNBUFFERED=1 python /usr/bin/copr_prune_results.py" - copr diff --git a/backend/run/copr_find_obsolete_builds.sh b/backend/run/copr_find_obsolete_builds.sh new file mode 100755 index 0000000..dcd3e34 --- /dev/null +++ b/backend/run/copr_find_obsolete_builds.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +CHROOT_PATH=$1 +DAYS=$2 + +echo "# looking for obsolete files in: " $CHROOT_PATH +if [ -d $CHROOT_PATH ] +then + pushd $CHROOT_PATH >/dev/null + + # query latest (sucessfull) packages + LATEST_PKGS=$(mktemp) + ERR_LOG=$(mktemp) + # "yum clean metadata" does not work on this custom repos + rm -rf $(find /var/tmp -name query &> /dev/null) &> /dev/null + rm -rf /var/tmp/createrepo* &> /dev/null + rm -rf /var/cache/yum/* &> /dev/null + + repoquery --repofrompath=query,$CHROOT_PATH --repoid=query -a --location 2>$ERR_LOG \ + | cut -c8- > $LATEST_PKGS + + # Remove builds older then $DAYS days and which have newer builds available + for SUCCESS in $(find -name success -mtime +$DAYS); do + DIR=$(basename $(dirname $SUCCESS)) + echo "# checking dir: " $DIR + for PACKAGE in $(ls $DIR); do + grep $PACKAGE $LATEST_PKGS >/dev/null && continue 2 + done + # package was not found in $LATEST_PKGS + #rm -rf $DIR + #echo -n . + echo $DIR + done + + rm -f $LATEST_PKGS $ERR_LOG + popd >/dev/null +fi diff --git a/backend/run/copr_prune_old_builds.sh b/backend/run/copr_prune_old_builds.sh deleted file mode 100755 index 5b50c4c..0000000 --- a/backend/run/copr_prune_old_builds.sh +++ /dev/null @@ -1,52 +0,0 @@ -#!/bin/bash - -REPOPATH=$1 -DAYS=$2 - - -pushd $REPOPATH >/dev/null - - -for CHROOT in $(ls -d $REPOPATH/*/); do - pushd $CHROOT >/dev/null - - # remove old failed builds - for FAILED in $(find -name 'fail' -mtime +$DAYS); do - rm -rf $(dirname $FAILED) - echo -n . - done - - # query latest (sucessfull) packages - LATEST_PKGS=$(mktemp) - ERR_LOG=$(mktemp) - # "yum clean metadata" does not work on this custom repos - rm -rf $(find /var/tmp -name query &> /dev/null) &> /dev/null - rm -rf /var/tmp/createrepo* &> /dev/null - rm -rf /var/cache/yum/* &> /dev/null - - repoquery --repofrompath=query,$REPOPATH/$CHROOT --repoid=query -a --location 2>$ERR_LOG \ - | cut -c8- > $LATEST_PKGS - - # remove directory if it does not have repodata at all - if $(grep --quiet 'Cannot retrieve repository metadata' $ERR_LOG) || $(grep --quiet "Can't download or revert repomd.xml" $ERR_LOG) ; then - # don't remove when /devel presents - if [ ! -d $REPOPATH/$CHROOT/devel ]; then - rm -rf $REPOPATH/$CHROOT/ - fi - else - # Remove builds older then $DAYS days and which have newer builds available - for SUCCESS in $(find -name success -mtime +$DAYS); do - DIR=$(basename $(dirname $SUCCESS)) - for PACKAGE in $(ls $DIR); do - grep $PACKAGE $LATEST_PKGS >/dev/null && continue 2 - done - # package was not found in $LATEST_PKGS - rm -rf $DIR - echo -n . - done - fi - - rm -f $LATEST_PKGS $ERR_LOG - popd >/dev/null -done -popd >/dev/null diff --git a/backend/run/copr_prune_results.py b/backend/run/copr_prune_results.py index 70bc20c..9f77f81 100644 --- a/backend/run/copr_prune_results.py +++ b/backend/run/copr_prune_results.py @@ -6,9 +6,15 @@ from __future__ import division from __future__ import absolute_import import os +import shutil import sys import logging from subprocess import Popen, PIPE +import time + + +log = logging.getLogger(__name__) + from copr.client.exceptions import CoprException, CoprRequestException @@ -17,11 +23,9 @@ sys.path.append("/usr/share/copr/") from backend.helpers import BackendConfigReader, get_auto_createrepo_status from backend.createrepo import createrepo_unsafe -log = logging.getLogger(__name__) -log.addHandler(logging.NullHandler()) DEF_DAYS = 14 -DEF_PRUNE_SCRIPT = "/usr/bin/copr_prune_old_builds.sh" +DEF_FIND_OBSOLETE_SCRIPT = "/usr/bin/copr_find_obsolete_builds.sh" def list_subdir(path): @@ -29,74 +33,131 @@ def list_subdir(path): return dir_names, map(lambda x: os.path.join(path, x), dir_names) -def prune_project(opts, path, username, projectname): - log.debug("Going to prune {}/{}".format(username, projectname)) - # get ACR - try: - if not get_auto_createrepo_status(opts.frontend_base_url, username, projectname): - log.debug("Skipped {}/{} since auto createrepo option is disabled" - .format(username, projectname)) +class Pruner(object): + def __init__(self, opts): + self.opts = opts + self.days = getattr(self.opts, "prune_days", DEF_DAYS) + self.prune_sh_script = getattr(self.opts, "find_obsolete_script", DEF_FIND_OBSOLETE_SCRIPT) + + def prune_failed_builds(self, chroot_path): + """ + Deletes subdirs (project directories) which contains file `fail` + with mtime older then self.days + + :param chroot_path: path to the chroot directory + """ + for sub_dir_name in os.listdir(chroot_path): + build_path = os.path.join(chroot_path, sub_dir_name) + if not os.path.isdir(build_path): + # log.debug("Not a project directory, skipping: {}".format(build_path)) + continue + + fail_file_path = os.path.join(build_path, "fail") + if os.path.exists(fail_file_path) and not os.path.exists(os.path.join(build_path, "success")): + if time.time() - os.path.getmtime(fail_file_path) > self.days: + log.info("Removing failed build: {}".format(build_path)) + shutil.rmtree(build_path) + + def prune_obsolete_success_builds(self, chroot_path, username, projectname): + """ + Uses bash script which invokes repoquery to find obsolete build_dirs + + :param chroot_path: path to the chroot directory + :return: + """ + # import ipdb; ipdb.set_trace() + cmd = map(str, [self.prune_sh_script, chroot_path, self.days]) + handle = Popen(cmd, stdout=PIPE, stderr=PIPE) + stdout, stderr = handle.communicate() + if handle.returncode != 0: + log.error("Failed to prune old builds for copr {}/{}\n STDOUT: \n{}\n STDERR: \n{}\n" + .format(username, projectname, stdout.decode(), stderr.decode())) + return + # log.debug("find obsolete returned:\n {}\n stderr: \n {}".format(stdout, stderr)) + for line in stdout.split("\n"): + if not line.strip() or (len(line) > 0 and line[0] == "#"): + continue + to_delete = line.strip() + log.debug("Obsolete path, check for remove: {}".format(to_delete)) + + if to_delete in os.listdir(chroot_path): + to_delete_path = os.path.join(chroot_path, to_delete) + if os.path.isdir(to_delete_path): + log.info("Removing obsolete build: {}".format(to_delete_path)) + shutil.rmtree(to_delete_path) + + def run(self): + results_dir = self.opts.destdir + log.info("Pruning results dir: {} ".format(results_dir)) + user_dir_names, user_dirs = list_subdir(results_dir) + + log.info("Going to process total number: {} of user's directories".format(len(user_dir_names))) + log.info("Going to process user's directories: {}".format(user_dir_names)) + + counter = 0 + for username, subpath in zip(user_dir_names, user_dirs): + log.debug("For user `{}` exploring path: {}".format(username, subpath)) + for projectname, project_path in zip(*list_subdir(subpath)): + log.debug("Exploring project `{}` with path: {}".format(projectname, project_path)) + self.prune_project(project_path, username, projectname) + + counter += 1 + log.info("Pruned {}. projects".format(counter)) + + log.info("Pruning finished") + + def prune_project(self, project_path, username, projectname): + log.info("Going to prune {}/{}".format(username, projectname)) + # get ACR + try: + if not get_auto_createrepo_status(self.opts.frontend_base_url, username, projectname): + log.debug("Skipped {}/{} since auto createrepo option is disabled" + .format(username, projectname)) + return + except (CoprException, CoprRequestException) as exception: + log.debug("Failed to get project details for {}/{} with error: {}".format( + username, projectname, exception)) return - except (CoprException, CoprRequestException) as exception: - log.debug("Failed to get project details for {}/{} with error: {}".format( - username, projectname, exception)) - return - - # run prune project sh - days = getattr(opts, "prune_days", DEF_DAYS) - - cmd = map(str, [DEF_PRUNE_SCRIPT, path, days]) - - handle = Popen(cmd, stdout=PIPE, stderr=PIPE) - stdout, stderr = handle.communicate() - - if handle.returncode != 0: - print("Failed to prune old builds for copr {}/{}".format(username, projectname)) - print("STDOUT: \n{}".format(stdout.decode())) - print("STDERR: \n{}".format(stderr.decode())) - return - - # run createrepo - log.debug("Prune done for {}/{}".format(username, projectname)) - try: - retcode, stdout, stderr = createrepo_unsafe(path) - if retcode != 0: - print("Createrepo for {}/{} failed".format(username, projectname)) - print("STDOUT: \n{}".format(stdout.decode())) - print("STDERR: \n{}".format(stderr.decode())) - except Exception as exception: - print("Createrepo for {}/{} failed with error: {}" - .format(username, projectname, exception)) + + # run prune project sh + for sub_dir_name in os.listdir(project_path): + if sub_dir_name == "repodata": + continue + chroot_path = os.path.join(project_path, sub_dir_name) + if not os.path.isdir(chroot_path): + continue + + try: + self.prune_failed_builds(chroot_path) + self.prune_obsolete_success_builds(chroot_path, username, projectname) + except Exception as err: + log.exception(err) + log.error("Error during prune copr {}/{}".format(username, projectname)) + + log.debug("Prune done for {}/{}".format(username, projectname)) + # run createrepo + + try: + retcode, stdout, stderr = createrepo_unsafe(project_path) + if retcode != 0: + log.error( + "Createrepo for copr {}/{}\n STDOUT: \n{}\n STDERR: \n{}\n" + .format(username, projectname, stdout.decode(), stderr.decode())) + except Exception as exception: + log.exception("Createrepo for copr {}/{} failed with error: {}" + .format(username, projectname, exception)) + + log.info("Prune finished for copr {}/{}".format(username, projectname)) def main(): config_file = os.environ.get("BACKEND_CONFIG", "/etc/copr/copr-be.conf") - opts = BackendConfigReader(config_file).read() - - results_dir = opts.destdir - log.info("Pruning results dir: {} ".format(results_dir)) - user_dir_names, user_dirs = list_subdir(results_dir) - - print("Going to process total number: {} of user's directories".format(len(user_dir_names))) - log.info("Going to process user's directories: {}".format(user_dir_names)) - - counter = 0 - for username, subpath in zip(user_dir_names, user_dirs): - log.debug("For user `{}` exploring path: {}".format(username, subpath)) - for projectname, project_path in zip(*list_subdir(subpath)): - log.debug("Exploring project `{}` with path: {}".format(projectname, project_path)) - prune_project(opts, project_path, username, projectname) - - counter += 1 - print("Pruned {}. projects".format(counter)) - - print("Pruning finished") - + pruner = Pruner(BackendConfigReader(config_file).read()) + pruner.run() if __name__ == "__main__": - # logging.basicConfig( - # level=logging.DEBUG, - # format='[%(asctime)s] {%(pathname)s:%(lineno)d} %(levelname)s - %(message)s', - # datefmt='%H:%M:%S' - # ) + logging.basicConfig( + filename="/var/log/copr/prune_old.log", + format='[%(asctime)s][%(levelname)6s]: %(message)s', + level=logging.INFO) main() diff --git a/backend/tests/_resources/foox.tar.gz b/backend/tests/_resources/foox.tar.gz new file mode 100644 index 0000000..aaebe8f Binary files /dev/null and b/backend/tests/_resources/foox.tar.gz differ diff --git a/backend/tests/run/test_copr_prune_results.py b/backend/tests/run/test_copr_prune_results.py new file mode 100644 index 0000000..074e9dc --- /dev/null +++ b/backend/tests/run/test_copr_prune_results.py @@ -0,0 +1,190 @@ +# coding: utf-8 +import logging +import os +import sys +import shutil +import tarfile +import tempfile +import time +from bunch import Bunch +from subprocess import Popen, PIPE + +import pytest + +import six +if six.PY3: + from unittest import mock +else: + import mock + from mock import MagicMock + + + +sys.path.append("../../run") + + +MODULE_REF = "copr_prune_results" + +(a)pytest.yield_fixture +def mc_popen(): + with mock.patch("{}.Popen".format(MODULE_REF)) as handle: + yield handle + + +from copr_prune_results import Pruner + + +def assert_same_dirs(a, b): + proc = Popen(["diff", "-r", a, b], stdout=PIPE, stdin=PIPE) + stdout, stderr = proc.communicate() + + if proc.returncode != 0: + raise AssertionError("Directories {}, {} aren't equal, diff out: \n {} \n {}" + .format(a, b, stdout, stderr)) + +class TestPruneResults(object): + + def setup_method(self, method): + self.tmp_dir_name = None + self.test_time = time.time() + self.make_temp_dir() + + self.pkg_1 = "copr-backend-1.59-1.git.1.7ed2b8d.fc20" + self.pkg_1_obsolete = "copr-backend-1.40-1.fc20" + self.pkg_2 = "hello-2.8-1.fc20" + self.pkg_2_obsolete = "hello-1.0.fc20" + self.prj = "foox" + self.chroots = ["fedora-20-i386", "fedora-20-x86_64", + "fedora-21-i386", "fedora-21-x86_64"] + self.opts = Bunch( + days=14, + + find_obsolete_script=os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, + "run", "copr_find_obsolete_builds.sh")) + + + ) + + self.log_file_path = os.path.join(self.tmp_dir_name, "_unittest.log") + logging.basicConfig( + # filename=self.log_file_path, + stream=sys.stdout, + format='[%(asctime)s][%(levelname)6s]: %(message)s', + level=logging.DEBUG + ) + self.log = logging.getLogger(__name__) + self.log.info("setup method done") + + self.username = "vgologuz" + self.coprname = "foox" + + def teardown_method(self, method): + self.rm_tmp_dir() + + def rm_tmp_dir(self): + if self.tmp_dir_name: + shutil.rmtree(self.tmp_dir_name) + self.tmp_dir_name = None + + def make_temp_dir(self): + root_tmp_dir = tempfile.gettempdir() + subdir = "test_action_{}".format(self.test_time) + self.tmp_dir_name = os.path.join(root_tmp_dir, subdir) + + os.mkdir(self.tmp_dir_name) + self.expect_dir_name = os.path.join(self.tmp_dir_name, "_expect") + os.mkdir(self.expect_dir_name) + + self.unpack_resource("foox.tar.gz") + self.unpack_resource("foox.tar.gz", self.expect_dir_name) + + return self.tmp_dir_name + + def unpack_resource(self, resource_name, target=None): + if self.tmp_dir_name is None: + self.make_temp_dir() + + src_path = os.path.join(os.path.dirname(__file__), "..", + "_resources", resource_name) + + with tarfile.open(src_path, "r:gz") as tar_file: + tar_file.extractall(target or self.tmp_dir_name) + + + @pytest.fixture + def test_pruner(self): + self.pruner = Pruner(self.opts) + + return self.pruner + + def test_prune_old_failed(self, test_pruner): + """ + Delete if build with "fail" fail with mtime > 14 days + + """ + # TODO: what neither fail or success are present? + + shutil.move( + os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1, "success"), + os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1, "fail"),) + shutil.move( + os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_2, "success"), + os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_2, "fail"),) + shutil.move( + os.path.join(self.expect_dir_name, self.prj, self.chroots[0], self.pkg_2, "success"), + os.path.join(self.expect_dir_name, self.prj, self.chroots[0], self.pkg_2, "fail"),) + + os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1, "fail"), (0, 0)) + os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_2, "fail"), + (time.time(), time.time())) + + shutil.rmtree(os.path.join(self.expect_dir_name, self.prj, self.chroots[0], self.pkg_1)) + + self.pruner.prune_failed_builds(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0])) + + assert_same_dirs( + os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]), + os.path.join(self.expect_dir_name, self.prj, self.chroots[0]), + ) + + def test_prune_obsolete_builds(self, test_pruner): + + os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1, "success"), (0, 0)) + os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_2, "success"), (0, 0)) + + # old obsolete build folder should be removed + os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_2_obsolete, "success"), (0, 0)) + shutil.rmtree(os.path.join(self.expect_dir_name, self.prj, self.chroots[0], self.pkg_2_obsolete)) + + # recent obsolete build should remain + os.utime(os.path.join(self.tmp_dir_name, self.prj, self.chroots[0], self.pkg_1_obsolete, "success"), + (time.time(), time.time())) + + self.pruner.prune_obsolete_success_builds( + os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]), + self.username, self.coprname + ) + + assert_same_dirs( + os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]), + os.path.join(self.expect_dir_name, self.prj, self.chroots[0]), + ) + + def test_prune_obsolete_builds_handle_script_error(self, test_pruner, mc_popen): + mc_handle = MagicMock() + mc_popen.return_value = mc_handle + mc_handle.returncode = 1 + mc_handle.communicate.return_value = ("foo", "bar") + + # doesn't touch FS if `find_obsolete_build` produce error return code + self.pruner.prune_obsolete_success_builds( + os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]), + self.username, self.coprname + ) + assert mc_popen.called + + assert_same_dirs( + os.path.join(self.tmp_dir_name, self.prj, self.chroots[0]), + os.path.join(self.expect_dir_name, self.prj, self.chroots[0]), + )

1 0

[copr] master: [backend][frontend] Send for delete action only `src_pkg_name` instead of original URL. (7c4eba7)
by vgologuz＠fedoraproject.org 23 Mar '15

23 Mar '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 7c4eba7867e14cf9628ee319934010a05e284419 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Mon Mar 23 10:49:42 2015 +0100 [backend][frontend] Send for delete action only `src_pkg_name` instead of original URL. >--------------------------------------------------------------- backend/backend/actions.py | 8 +++--- backend/tests/test_action.py | 8 +++--- .../coprs_frontend/coprs/logic/builds_logic.py | 8 ++++- .../tests/test_logic/test_builds_logic.py | 26 ++++++++++++++++++++ .../test_views/test_coprs_ns/test_coprs_builds.py | 8 ++++-- 5 files changed, 45 insertions(+), 13 deletions(-) diff --git a/backend/backend/actions.py b/backend/backend/actions.py index c195100..1c178c8 100644 --- a/backend/backend/actions.py +++ b/backend/backend/actions.py @@ -106,12 +106,12 @@ class Action(object): projectname = ext_data["projectname"] chroots_requested = set(ext_data["chroots"]) - package_name = os.path.basename(ext_data["pkgs"]).replace(".src.rpm", "") - # TODO: replace with explicit pkg_name, send cleaned string into the action - if not package_name: - self.add_event("Bad package name {}, action data: {} ".format(package_name, ext_data)) + if not ext_data.get("src_pkg_name"): + self.add_event("Delete build action missing `src_pkg_name` field, check frontend version. Raw ext_data: {}" + .format(ext_data)) return + package_name = ext_data["src_pkg_name"] path = os.path.join(self.destdir, project) self.add_event("Deleting package {0}".format(package_name)) diff --git a/backend/tests/test_action.py b/backend/tests/test_action.py index 2ca6d47..08fb806 100644 --- a/backend/tests/test_action.py +++ b/backend/tests/test_action.py @@ -59,7 +59,7 @@ class TestAction(object): self.pkgs_stripped = ["foo"] self.ext_data_for_delete_build = json.dumps({ - "pkgs": " ".join(self.pkgs), + "src_pkg_name": self.pkgs_stripped[0], "username": "foo", "projectname": "bar", "chroots": ["fedora20", "epel7"] @@ -514,7 +514,7 @@ class TestAction(object): "id": 15, "old_value": "old_dir", "data": json.dumps({ - "pkgs": "rubygem-log4r-1.1.10-2.fc21.src.rpm", + "src_pkg_name": "rubygem-log4r-1.1.10-2.fc21", "username": "foo", "projectname": "bar", "chroots": ["fedora-20-x86_64", "fedora-21-x86_64"] @@ -583,7 +583,7 @@ class TestAction(object): "id": 15, "old_value": "old_dir", "data": json.dumps({ - "pkgs": "rubygem-log4r-1.1.10-2.fc21.src.rpm", + "src_pkg_name": "rubygem-log4r-1.1.10-2.fc21", "username": "foo", "projectname": "bar", "chroots": ["fedora-20-x86_64", "fedora-21-x86_64"] @@ -636,7 +636,7 @@ class TestAction(object): "id": 15, "old_value": "old_dir", "data": json.dumps({ - "pkgs": "http://example.com/", + "src_pkg_name": "", "username": "foo", "projectname": "bar", "chroots": ["fedora-20-x86_64", "fedora-21-x86_64"] diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index 0e839b3..2712e48 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -212,6 +212,10 @@ class BuildsLogic(object): @classmethod def delete_build(cls, user, build): + """ + :type user: models.User + :type build: models.Build + """ if not user.can_build_in(build.copr): raise exceptions.InsufficientRightsException( "You are not allowed to delete this build.") @@ -229,8 +233,8 @@ class BuildsLogic(object): if chroot.state not in ["skipped"] ] - if chroots_to_delete: - data_dict = {"pkgs": build.pkgs, + if chroots_to_delete and build.src_pkg_name: + data_dict = {"src_pkg_name": build.src_pkg_name, "username": build.copr.owner.name, "projectname": build.copr.name, "chroots": chroots_to_delete} diff --git a/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py b/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py index e8a375e..9423c36 100644 --- a/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py +++ b/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py @@ -120,6 +120,9 @@ class TestBuildsLogic(CoprsTestCase): def test_delete_build_basic( self, f_users, f_coprs, f_mock_chroots, f_builds, f_db): + + self.b1.pkgs = "http://example.com/copr-keygen-1.58-1.fc20.src.rpm" + self.db.session.add(self.b1) self.db.session.commit() expected_chroots_to_delete = set() @@ -134,11 +137,32 @@ class TestBuildsLogic(CoprsTestCase): action = ActionsLogic.get_many().one() delete_data = json.loads(action.data) assert "chroots" in delete_data + assert "copr-keygen-1.58-1.fc20" == delete_data["src_pkg_name"] assert expected_chroots_to_delete == set(delete_data["chroots"]) with pytest.raises(NoResultFound): BuildsLogic.get(self.b1.id).one() + def test_delete_build_bad_src_pkg( + self, f_users, f_coprs, f_mock_chroots, f_builds, f_db): + + self.b1.pkgs = "http://example.com/" + self.db.session.add(self.b1) + self.db.session.commit() + + expected_chroots_to_delete = set() + for bchroot in self.b1_bc: + expected_chroots_to_delete.add(bchroot.name) + + assert len(ActionsLogic.get_many().all()) == 0 + BuildsLogic.delete_build(self.u1, self.b1) + self.db.session.commit() + + assert len(ActionsLogic.get_many().all()) == 0 + + with pytest.raises(NoResultFound): + BuildsLogic.get(self.b1.id).one() + def test_delete_build_no_chroots_to_clean( self, f_users, f_coprs, f_mock_chroots, f_builds, f_db): @@ -159,6 +183,8 @@ class TestBuildsLogic(CoprsTestCase): for bchroot in self.b1_bc[1:-1]: bchroot.status = helpers.StatusEnum("skipped") + self.b1.pkgs = "http://example.com/copr-keygen-1.58-1.fc20.src.rpm" + self.db.session.add(self.b1) self.db.session.commit() assert len(ActionsLogic.get_many().all()) == 0 diff --git a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py index 393b00e..d8cbf08 100644 --- a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py +++ b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py @@ -104,9 +104,11 @@ class TestCoprDeleteBuild(CoprsTestCase): f_coprs, f_mock_chroots, f_builds, f_db): - self.db.session.add_all([self.u1, self.c1, self.b1]) - pkgs = "one two three" + pkgs = "http://example.com/one.src.rpm" self.b1.pkgs = pkgs + self.db.session.add_all([self.u1, self.c1, self.b1]) + self.db.session.commit() + r = self.test_client.post( "/coprs/{0}/{1}/delete_build/{2}/" .format(self.u1.name, self.c1.name, self.b1.id), @@ -120,7 +122,7 @@ class TestCoprDeleteBuild(CoprsTestCase): act = self.models.Action.query.first() assert act.object_type == "build" assert act.old_value == "user1/foocopr" - assert json.loads(act.data)["pkgs"] == pkgs + assert json.loads(act.data)["src_pkg_name"] == self.b1.src_pkg_name @TransactionDecorator("u2") def test_copr_build_non_submitter_cannot_delete_build(self, f_users,

1 0

[copr] master's head updated: Merge branch 'hotfix_1203753' (c9ea561)
by vgologuz＠fedoraproject.org 23 Mar '15

23 Mar '15

Repository : http://git.fedorahosted.org/cgit/copr.git Branch 'master' now includes: c8e4d8f cleanup 548bfc0 [backend][hotfix] 1203753 : don't process delete action if src_pkg is mallformed c7db8cb Automatic commit of package [copr-backend] release [1.59-1]. 7ed2b8d [backend][spec] start/stop redis server during package build tests 96cfc8a Automatic commit of package [copr-backend] release [1.60-1]. c9ea561 Merge branch 'hotfix_1203753'

1 0

[copr] master: Merge branch 'hotfix_1203753' (c9ea561)
by vgologuz＠fedoraproject.org 23 Mar '15

23 Mar '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit c9ea561667b71563e7f243829d9590c0731933fa Merge: c8e4d8f 96cfc8a Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Mon Mar 23 09:45:39 2015 +0100 Merge branch 'hotfix_1203753' >--------------------------------------------------------------- backend/backend/actions.py | 5 ++ backend/copr-backend.spec | 17 +++++- .../coprs/logic/__init__.py => backend/run/test.sh | 0 backend/tests/test_action.py | 55 ++++++++++++++++++++ rel-eng/packages/copr-backend | 2 +- 5 files changed, 75 insertions(+), 4 deletions(-) diff --git a/backend/backend/actions.py b/backend/backend/actions.py index 6b82ebe..c195100 100644 --- a/backend/backend/actions.py +++ b/backend/backend/actions.py @@ -107,6 +107,11 @@ class Action(object): chroots_requested = set(ext_data["chroots"]) package_name = os.path.basename(ext_data["pkgs"]).replace(".src.rpm", "") + # TODO: replace with explicit pkg_name, send cleaned string into the action + if not package_name: + self.add_event("Bad package name {}, action data: {} ".format(package_name, ext_data)) + return + path = os.path.join(self.destdir, project) self.add_event("Deleting package {0}".format(package_name)) diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec index 82cb38b..45e5fff 100644 --- a/backend/copr-backend.spec +++ b/backend/copr-backend.spec @@ -3,7 +3,7 @@ %endif Name: copr-backend -Version: 1.58 +Version: 1.60 Release: 1%{?dist} Summary: Backend for Copr @@ -25,6 +25,7 @@ BuildRequires: python-requests BuildRequires: python2-devel BuildRequires: python-copr BuildRequires: systemd +BuildRequires: redis %if 0%{?rhel} < 7 && 0%{?rhel} > 0 BuildRequires: python-argparse %endif @@ -162,9 +163,12 @@ cp -a conf/playbooks %{buildroot}%{_pkgdocdir}/ %check -#PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \ -# -s --cov-report term-missing --cov ./backend --cov ./run ./tests/ +redis-server --port 7777 &> /dev/null & +PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \ + -s --cov-report term-missing --cov ./backend --cov ./run ./tests/ + +kill %1 %pre getent group copr >/dev/null || groupadd -r copr @@ -223,6 +227,13 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %exclude %{_pkgdocdir}/playbooks %changelog +* Fri Mar 20 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.60-1 +- [backend][spec] start/stop redis server during package build tests + +* Fri Mar 20 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.59-1 +- [backend][hotfix] 1203753 : don't process delete action if src_pkg is + mallformed + * Mon Mar 02 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.58-1 - [rhbz:#1185959] - RFE: Present statistics about project popularity. A few more counters for downloads from backend's result diff --git a/frontend/coprs_frontend/coprs/logic/__init__.py b/backend/run/test.sh similarity index 100% copy from frontend/coprs_frontend/coprs/logic/__init__.py copy to backend/run/test.sh diff --git a/backend/tests/test_action.py b/backend/tests/test_action.py index 74a8e2b..2ca6d47 100644 --- a/backend/tests/test_action.py +++ b/backend/tests/test_action.py @@ -382,6 +382,8 @@ class TestAction(object): test_action.run() assert not os.path.exists(foo_pkg_dir) assert not os.path.exists(log_path) + assert os.path.exists(chroot_1_dir) + assert os.path.exists(chroot_2_dir) assert_what_from_queue(self.test_q, msg_list=[ "Action delete build", @@ -418,6 +420,10 @@ class TestAction(object): with pytest.raises(EmptyQueue): self.test_q.get_nowait() + assert os.path.exists(chroot_1_dir) + assert os.path.exists(chroot_2_dir) + + create_repo_expected_call = mock.call( username=u'foo', projectname=u'bar', @@ -531,6 +537,10 @@ class TestAction(object): assert not os.path.isdir(os.path.join(chroot_20_path, "rubygem-log4r-1.1.10-2.fc21")) assert not os.path.isdir(os.path.join(chroot_21_path, "rubygem-log4r-1.1.10-2.fc21")) + assert os.path.exists(chroot_20_path) + assert os.path.exists(chroot_21_path) + + @mock.patch("backend.actions.createrepo") def test_delete_two_chroots_two_remains(self, mc_createrepo_unsafe, mc_time): """ @@ -602,6 +612,51 @@ class TestAction(object): assert os.path.isdir(os.path.join(chroot_20_i386_path, "rubygem-log4r-1.1.10-2.fc21")) assert os.path.isdir(os.path.join(chroot_21_i386_path, "rubygem-log4r-1.1.10-2.fc21")) + @mock.patch("backend.actions.createrepo") + def test_delete_build_with_bad_pkg_name(self, mc_createrepo, mc_time): + """ + regression: https://bugzilla.redhat.com/show_bug.cgi?id=1203753 + + """ + mc_time.time.return_value = self.test_time + + resource_name = "1171796_doubled.tar.gz" + self.unpack_resource(resource_name) + + chroot_20_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-20-x86_64") + chroot_21_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-21-x86_64") + + mc_createrepo.return_value = 0, STDOUT, "" + mc_front_cb = MagicMock() + test_action = Action( + action={ + "action_type": ActionType.DELETE, + "object_type": "build", + "object_id": 15, + "id": 15, + "old_value": "old_dir", + "data": json.dumps({ + "pkgs": "http://example.com/", + "username": "foo", + "projectname": "bar", + "chroots": ["fedora-20-x86_64", "fedora-21-x86_64"] + }), + }, + events=self.test_q, lock=None, + frontend_callback=mc_front_cb, + destdir=self.tmp_dir_name, + front_url=None, + results_root_url=RESULTS_ROOT_URL + ) + + assert os.path.exists(chroot_20_path) + assert os.path.exists(chroot_21_path) + test_action.run() + # shouldn't touch chroot dirs + assert os.path.exists(chroot_20_path) + assert os.path.exists(chroot_21_path) + + @mock.patch("backend.actions.createrepo_unsafe") def test_delete_two_chroots_two_builds_stay_untouched(self, mc_createrepo_unsafe, mc_time): # TODO: prepare archive diff --git a/rel-eng/packages/copr-backend b/rel-eng/packages/copr-backend index d19bf95..0b75fea 100644 --- a/rel-eng/packages/copr-backend +++ b/rel-eng/packages/copr-backend @@ -1 +1 @@ -1.58-1 backend/ +1.60-1 backend/

1 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

copr-commits March 2015