copr-commits January 2015

copr-commits@lists.fedorahosted.org

2 participants
58 discussions

[copr] master: [backend] remove debug stuff from .spec (6b3b67a)
by vgologuz＠fedoraproject.org 30 Jan '15

30 Jan '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 6b3b67adf0c0346fb13fff8222bedc0bfc65c9a2 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Jan 30 18:19:52 2015 +0100 [backend] remove debug stuff from .spec >--------------------------------------------------------------- backend/copr-backend.spec | 4 ---- 1 files changed, 0 insertions(+), 4 deletions(-) diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec index a557bce..d7e869d 100644 --- a/backend/copr-backend.spec +++ b/backend/copr-backend.spec @@ -103,10 +103,6 @@ pushd documentation make %{?_smp_mflags} python popd -pushd /tmp -wget http://people.redhat.com/~vgologuz/permanent/hello_beaker_test_2-0.0.1-1.fc… -popd - %install install -d %{buildroot}%{_sharedstatedir}/copr

1 0

[copr] master: [backend] [rhbz:#1171796] copr sometimes doesn't delete build from repository (d20dc04)
by vgologuz＠fedoraproject.org 30 Jan '15

30 Jan '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit d20dc04e32784d938d471705ab65b71d2127864e Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Jan 30 14:00:07 2015 +0100 [backend] [rhbz:#1171796] copr sometimes doesn't delete build from repository >--------------------------------------------------------------- backend/backend/actions.py | 61 ++++--- backend/tests/_resources/1171796.tar.gz | Bin 0 -> 587365 bytes backend/tests/_resources/1171796_doubled.tar.gz | Bin 0 -> 1172212 bytes backend/tests/test_action.py | 217 ++++++++++++++++++----- frontend/run_tests.sh | 4 +- 5 files changed, 207 insertions(+), 75 deletions(-) diff --git a/backend/backend/actions.py b/backend/backend/actions.py index 4ae9c7c..f1d4557 100644 --- a/backend/backend/actions.py +++ b/backend/backend/actions.py @@ -39,6 +39,9 @@ class Action(object): self.front_url = front_url self.results_root_url = results_root_url + def __str__(self): + return "<Action: {}>".format(self.data) + def add_event(self, what): self.events.put({"when": time.time(), "who": "action", "what": what}) @@ -101,6 +104,7 @@ class Action(object): ext_data = json.loads(self.data["data"]) username = ext_data["username"] projectname = ext_data["projectname"] + chroots_requested = set(ext_data["chroots"]) packages = [os.path.basename(x).replace(".src.rpm", "") for x in ext_data["pkgs"].split()] @@ -111,34 +115,31 @@ class Action(object): self.add_event("Copr path {0}".format(path)) try: - chroot_list = os.listdir(path) + chroot_list = set(os.listdir(path)) except OSError: # already deleted - chroot_list = [] + chroot_list = set() + + chroots_to_do = chroot_list.intersection(chroots_requested) + if not chroots_to_do: + self.add_event("Nothing to delete for delete action: packages {}, {}" + .format(packages, ext_data)) + return - for chroot in chroot_list: + for chroot in chroots_to_do: self.add_event("In chroot {0}".format(chroot)) altered = False - # We need to delete the files only if they belong - # to the build. For example if my build fails and I send - # fixed pkg with the same version again, it succeeds and - # than I delete the failed, it would delete the succeeded - # files as well - that would be wrong. for pkg in packages: - if self.data["object_type"] == "build-succeeded" or ( - self.data["object_type"] == "build-failed" and - os.path.exists(os.path.join(path, chroot, pkg, "fail"))): - - pkg_path = os.path.join(path, chroot, pkg) - if os.path.isdir(pkg_path): - self.add_event("Removing build {0}".format(pkg_path)) - shutil.rmtree(pkg_path) - altered = True - else: - self.add_event( - "Package {0} dir not found in chroot {1}" - .format(pkg, chroot)) + pkg_path = os.path.join(path, chroot, pkg) + if os.path.isdir(pkg_path): + self.add_event("Removing build {0}".format(pkg_path)) + shutil.rmtree(pkg_path) + altered = True + else: + self.add_event( + "Package {0} dir not found in chroot {1}" + .format(pkg, chroot)) if altered: self.add_event("Running createrepo") @@ -154,13 +155,14 @@ class Action(object): self.add_event( "Error making local repo: {0}".format(err)) - log_path = os.path.join( - path, chroot, - 'build-{0}.log'.format(self.data['object_id'])) - - if os.path.isfile(log_path): - self.add_event("Removing log {0}".format(log_path)) - os.unlink(log_path) + logs_to_remove = [ + os.path.join(path, chroot, template.format(self.data['object_id'])) + for template in ['build-{}.log', 'build-{}.rsync.log'] + ] + for log_path in logs_to_remove: + if os.path.isfile(log_path): + self.add_event("Removing log {0}".format(log_path)) + os.remove(log_path) def run(self): """ Handle action (other then builds) - like rename or delete of project """ @@ -172,8 +174,7 @@ class Action(object): if action_type == ActionType.DELETE: if self.data["object_type"] == "copr": self.handle_delete_copr_project() - elif self.data["object_type"] in \ - ["build-succeeded", "build-skipped", "build-failed"]: + elif self.data["object_type"] == "build": self.handle_delete_build() result.result = ActionResult.SUCCESS diff --git a/backend/tests/_resources/1171796.tar.gz b/backend/tests/_resources/1171796.tar.gz new file mode 100644 index 0000000..6d3b522 Binary files /dev/null and b/backend/tests/_resources/1171796.tar.gz differ diff --git a/backend/tests/_resources/1171796_doubled.tar.gz b/backend/tests/_resources/1171796_doubled.tar.gz new file mode 100644 index 0000000..6cd04f3 Binary files /dev/null and b/backend/tests/_resources/1171796_doubled.tar.gz differ diff --git a/backend/tests/test_action.py b/backend/tests/test_action.py index 24e1367..3e7ce91 100644 --- a/backend/tests/test_action.py +++ b/backend/tests/test_action.py @@ -3,6 +3,7 @@ import json import tempfile import shutil import time +import tarfile import pytest import six @@ -60,7 +61,8 @@ class TestAction(object): self.ext_data_for_delete_build = json.dumps({ "pkgs": " ".join(self.pkgs), "username": "foo", - "projectname": "bar" + "projectname": "bar", + "chroots": ["fedora20", "epel7"] }) def teardown_method(self, method): @@ -83,6 +85,16 @@ class TestAction(object): return self.tmp_dir_name + def unpack_resource(self, resource_name): + if self.tmp_dir_name is None: + self.make_temp_dir() + + src_path = os.path.join(os.path.dirname(__file__), + "_resources", resource_name) + + with tarfile.open(src_path, "r:gz") as tfile: + tfile.extractall(os.path.join(self.tmp_dir_name, "old_dir")) + def test_action_event(self, mc_time): test_action = Action( events=self.test_q, @@ -308,40 +320,23 @@ class TestAction(object): tmp_dir = self.make_temp_dir() - for obj_type in ["build-succeeded", "build-skipped", "build-failed"]: - test_action = Action( - action={ - "action_type": ActionType.DELETE, - "object_type": obj_type, - "id": 7, - "old_value": "not-existing-project", - "data": self.ext_data_for_delete_build, - }, - events=self.test_q, lock=None, - frontend_callback=mc_front_cb, - destdir=tmp_dir, - front_url=None, - results_root_url=RESULTS_ROOT_URL - ) - with mock.patch("backend.actions.shutil") as mc_shutil: - test_action.run() - assert not mc_shutil.rmtree.called - - ev_1 = self.test_q.get_nowait() - assert "Action delete build" == ev_1["what"] - assert ev_1["who"] == "action" - - ev_2 = self.test_q.get_nowait() - assert "Packages to delete" in ev_2["what"] - assert " ".join(self.pkgs_stripped) in ev_2["what"] - assert ev_2["who"] == "action" - - ev_3 = self.test_q.get_nowait() - assert "Copr path" in ev_3["what"] - assert ev_3["who"] == "action" - - with pytest.raises(EmptyQueue): - self.test_q.get_nowait() + test_action = Action( + action={ + "action_type": ActionType.DELETE, + "object_type": "build", + "id": 7, + "old_value": "not-existing-project", + "data": self.ext_data_for_delete_build, + }, + events=self.test_q, lock=None, + frontend_callback=mc_front_cb, + destdir=tmp_dir, + front_url=None, + results_root_url=RESULTS_ROOT_URL + ) + with mock.patch("backend.actions.shutil") as mc_shutil: + test_action.run() + assert not mc_shutil.rmtree.called @mock.patch("backend.actions.createrepo") def test_delete_build_succeeded(self, mc_createrepo, mc_time): @@ -370,7 +365,7 @@ class TestAction(object): test_action = Action( action={ "action_type": ActionType.DELETE, - "object_type": "build-succeeded", + "object_type": "build", "id": 7, "old_value": "old_dir", "data": self.ext_data_for_delete_build, @@ -458,11 +453,11 @@ class TestAction(object): test_action = Action( action={ "action_type": ActionType.DELETE, - "object_type": "build-succeeded", + "object_type": "build", "id": 7, "old_value": "old_dir", "data": self.ext_data_for_delete_build, - "object_id": 42 + "object_id": 42, }, events=self.test_q, lock=None, frontend_callback=mc_front_cb, @@ -481,11 +476,147 @@ class TestAction(object): assert error_event_recorded + @mock.patch("backend.actions.createrepo") + def test_delete_two_chroots(self, mc_createrepo_unsafe, mc_time): + """ + Regression test, https://bugzilla.redhat.com/show_bug.cgi?id=1171796 + + """ + mc_createrepo_unsafe.return_value = 0, STDOUT, "" + + resource_name = "1171796.tar.gz" + self.unpack_resource(resource_name) + + chroot_20_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-20-x86_64") + chroot_21_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-21-x86_64") + + assert os.path.exists(os.path.join(chroot_20_path, "build-15.log")) + assert os.path.exists(os.path.join(chroot_21_path, "build-15.log")) + + assert os.path.exists(os.path.join(chroot_20_path, "build-15.rsync.log")) + assert os.path.exists(os.path.join(chroot_21_path, "build-15.rsync.log")) + + assert os.path.isdir(os.path.join(chroot_20_path, "rubygem-log4r-1.1.10-2.fc21")) + assert os.path.isdir(os.path.join(chroot_21_path, "rubygem-log4r-1.1.10-2.fc21")) + + mc_time.time.return_value = self.test_time + mc_front_cb = MagicMock() + + test_action = Action( + action={ + "action_type": ActionType.DELETE, + "object_type": "build", + "object_id": 15, + "id": 15, + "old_value": "old_dir", + "data": json.dumps({ + "pkgs": "rubygem-log4r-1.1.10-2.fc21.src.rpm", + "username": "foo", + "projectname": "bar", + "chroots": ["fedora-20-x86_64", "fedora-21-x86_64"] + }), + }, + events=self.test_q, lock=None, + frontend_callback=mc_front_cb, + destdir=self.tmp_dir_name, + front_url=None, + results_root_url=RESULTS_ROOT_URL + ) + test_action.run() + + assert not os.path.exists(os.path.join(chroot_20_path, "build-15.log")) + assert not os.path.exists(os.path.join(chroot_21_path, "build-15.log")) + + assert not os.path.exists(os.path.join(chroot_20_path, "build-15.rsync.log")) + assert not os.path.exists(os.path.join(chroot_21_path, "build-15.rsync.log")) + + assert not os.path.isdir(os.path.join(chroot_20_path, "rubygem-log4r-1.1.10-2.fc21")) + assert not os.path.isdir(os.path.join(chroot_21_path, "rubygem-log4r-1.1.10-2.fc21")) + + @mock.patch("backend.actions.createrepo") + def test_delete_two_chroots_two_remains(self, mc_createrepo_unsafe, mc_time): + """ + Regression test, https://bugzilla.redhat.com/show_bug.cgi?id=1171796 + extended: we also put two more chroots, which should be unaffected + """ + mc_createrepo_unsafe.return_value = 0, STDOUT, "" + + resource_name = "1171796_doubled.tar.gz" + self.unpack_resource(resource_name) + + chroot_20_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-20-x86_64") + chroot_21_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-21-x86_64") + chroot_20_i386_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-20-i386") + chroot_21_i386_path = os.path.join(self.tmp_dir_name, "old_dir", "fedora-21-i386") + + assert os.path.exists(os.path.join(chroot_20_path, "build-15.log")) + assert os.path.exists(os.path.join(chroot_21_path, "build-15.log")) + assert os.path.exists(os.path.join(chroot_20_i386_path, "build-15.log")) + assert os.path.exists(os.path.join(chroot_21_i386_path, "build-15.log")) + + assert os.path.exists(os.path.join(chroot_20_path, "build-15.rsync.log")) + assert os.path.exists(os.path.join(chroot_21_path, "build-15.rsync.log")) + assert os.path.exists(os.path.join(chroot_20_i386_path, "build-15.rsync.log")) + assert os.path.exists(os.path.join(chroot_21_i386_path, "build-15.rsync.log")) + + assert os.path.isdir(os.path.join(chroot_20_path, "rubygem-log4r-1.1.10-2.fc21")) + assert os.path.isdir(os.path.join(chroot_21_path, "rubygem-log4r-1.1.10-2.fc21")) + assert os.path.isdir(os.path.join(chroot_20_i386_path, "rubygem-log4r-1.1.10-2.fc21")) + assert os.path.isdir(os.path.join(chroot_21_i386_path, "rubygem-log4r-1.1.10-2.fc21")) + + mc_time.time.return_value = self.test_time + mc_front_cb = MagicMock() + + test_action = Action( + action={ + "action_type": ActionType.DELETE, + "object_type": "build", + "object_id": 15, + "id": 15, + "old_value": "old_dir", + "data": json.dumps({ + "pkgs": "rubygem-log4r-1.1.10-2.fc21.src.rpm", + "username": "foo", + "projectname": "bar", + "chroots": ["fedora-20-x86_64", "fedora-21-x86_64"] + }), + }, + events=self.test_q, lock=None, + frontend_callback=mc_front_cb, + destdir=self.tmp_dir_name, + front_url=None, + results_root_url=RESULTS_ROOT_URL + ) + test_action.run() + + assert not os.path.exists(os.path.join(chroot_20_path, "build-15.log")) + assert not os.path.exists(os.path.join(chroot_21_path, "build-15.log")) + assert os.path.exists(os.path.join(chroot_20_i386_path, "build-15.log")) + assert os.path.exists(os.path.join(chroot_21_i386_path, "build-15.log")) + + assert not os.path.exists(os.path.join(chroot_20_path, "build-15.rsync.log")) + assert not os.path.exists(os.path.join(chroot_21_path, "build-15.rsync.log")) + assert os.path.exists(os.path.join(chroot_20_i386_path, "build-15.rsync.log")) + assert os.path.exists(os.path.join(chroot_21_i386_path, "build-15.rsync.log")) + + assert not os.path.isdir(os.path.join(chroot_20_path, "rubygem-log4r-1.1.10-2.fc21")) + assert not os.path.isdir(os.path.join(chroot_21_path, "rubygem-log4r-1.1.10-2.fc21")) + assert os.path.isdir(os.path.join(chroot_20_i386_path, "rubygem-log4r-1.1.10-2.fc21")) + assert os.path.isdir(os.path.join(chroot_21_i386_path, "rubygem-log4r-1.1.10-2.fc21")) + + @mock.patch("backend.actions.createrepo_unsafe") + def test_delete_two_chroots_two_builds_stay_untouched(self, mc_createrepo_unsafe, mc_time): + # TODO: prepare archive + """ + Before: 2 builds of the same package-version, all using different chroot + """ + pass + @mock.patch("backend.actions.createrepo_unsafe") - def test_handle_createrepo_ok(self, mc_createrepo_unsfe, mc_time): + def test_handle_createrepo_ok(self, mc_createrepo_unsafe, mc_time): mc_front_cb = MagicMock() tmp_dir = self.make_temp_dir() - mc_createrepo_unsfe.return_value = 0, STDOUT, "" + mc_createrepo_unsafe.return_value = 0, STDOUT, "" action_data = json.dumps({ "chroots": ["epel-6-i386", "fedora-20-x86_64"], @@ -513,9 +644,9 @@ class TestAction(object): exp_call_1 = mock.call(lock=None, path=tmp_dir + u'/foo/bar/epel-6-i386') exp_call_2 = mock.call(lock=None, path=tmp_dir + u'/foo/bar/fedora-20-x86_64') - assert exp_call_1 in mc_createrepo_unsfe.call_args_list - assert exp_call_2 in mc_createrepo_unsfe.call_args_list - assert len(mc_createrepo_unsfe.call_args_list) == 2 + assert exp_call_1 in mc_createrepo_unsafe.call_args_list + assert exp_call_2 in mc_createrepo_unsafe.call_args_list + assert len(mc_createrepo_unsafe.call_args_list) == 2 @mock.patch("backend.actions.createrepo_unsafe") diff --git a/frontend/run_tests.sh b/frontend/run_tests.sh index 41e390e..2ecd9b9 100755 --- a/frontend/run_tests.sh +++ b/frontend/run_tests.sh @@ -1,6 +1,6 @@ #!/bin/sh cd coprs_frontend -COPR_CONFIG="$(pwd)/config/copr_unit_test.conf" python -m pytest tests \ - --cov-report term-missing --cov coprs $@ +COPR_CONFIG="$(pwd)/config/copr_unit_test.conf" python -m pytest tests -s \ + --cov-report term-missing --cov coprs $@

1 0

[copr] master: [frontend] [rhbz:#1171796] copr sometimes doesn't delete build from repository (ea42e4f)
by vgologuz＠fedoraproject.org 30 Jan '15

30 Jan '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit ea42e4f9b871e985db391ee288ac5c8bc98d8fc7 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Jan 30 14:53:07 2015 +0100 [frontend] [rhbz:#1171796] copr sometimes doesn't delete build from repository >--------------------------------------------------------------- .../coprs_frontend/coprs/logic/builds_logic.py | 41 +++++++----- .../tests/test_logic/test_builds_logic.py | 73 +++++++++++++++++++- .../test_views/test_coprs_ns/test_coprs_builds.py | 2 +- 3 files changed, 97 insertions(+), 19 deletions(-) diff --git a/frontend/coprs_frontend/coprs/logic/builds_logic.py b/frontend/coprs_frontend/coprs/logic/builds_logic.py index 5559722..d1cd9b8 100644 --- a/frontend/coprs_frontend/coprs/logic/builds_logic.py +++ b/frontend/coprs_frontend/coprs/logic/builds_logic.py @@ -212,23 +212,30 @@ class BuildsLogic(object): "You can not delete build which is not finished.", "Unfinished build") - # Only failed (and finished), succeeded, skipped and cancelled get here. - if build.state != "cancelled": # has nothing in backend to delete - object_type = "build-{0}".format(build.state) - data_dict = {"pkgs": build.pkgs, - "username": build.copr.owner.name, - "projectname": build.copr.name} - - action = models.Action( - action_type=helpers.ActionTypeEnum("delete"), - object_type=object_type, - object_id=build.id, - old_value="{0}/{1}".format(build.copr.owner.name, - build.copr.name), - data=json.dumps(data_dict), - created_on=int(time.time()) - ) - db.session.add(action) + # Only failed, finished, succeeded get here. + if build.state not in ["cancelled"]: # has nothing in backend to delete + # don't delete skipped chroots + chroots_to_delete = [ + chroot.name for chroot in build.build_chroots + if chroot.state not in ["skipped"] + ] + + if chroots_to_delete: + data_dict = {"pkgs": build.pkgs, + "username": build.copr.owner.name, + "projectname": build.copr.name, + "chroots": chroots_to_delete} + + action = models.Action( + action_type=helpers.ActionTypeEnum("delete"), + object_type="build", + object_id=build.id, + old_value="{0}/{1}".format(build.copr.owner.name, + build.copr.name), + data=json.dumps(data_dict), + created_on=int(time.time()) + ) + db.session.add(action) for build_chroot in build.build_chroots: db.session.delete(build_chroot) diff --git a/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py b/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py index c7dc07e..ef43cd0 100644 --- a/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py +++ b/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py @@ -1,10 +1,13 @@ # -*- encoding: utf-8 -*- +import json import pytest import time +from sqlalchemy.orm.exc import NoResultFound from coprs import helpers -from coprs.exceptions import ActionInProgressException +from coprs.exceptions import ActionInProgressException, InsufficientRightsException +from coprs.logic.actions_logic import ActionsLogic from coprs.logic.builds_logic import BuildsLogic from coprs.logic.builds_logic import BuildsMonitorLogic @@ -88,3 +91,71 @@ class TestBuildsLogic(CoprsTestCase): assert len(data) == 1 assert data[0] == self.b1_bc[0] + + def test_delete_build_exceptions( + self, f_users, f_coprs, f_mock_chroots, f_builds, f_db): + self.db.session.commit() + with pytest.raises(InsufficientRightsException): + BuildsLogic.delete_build(self.u1, self.b4) + + self.b1_bc[0].status = "running" + self.db.session.add(self.b1_bc[0]) + self.db.session.commit() + with pytest.raises(ActionInProgressException): + BuildsLogic.delete_build(self.u1, self.b1) + + def test_delete_build_basic( + self, f_users, f_coprs, f_mock_chroots, f_builds, f_db): + self.db.session.commit() + + expected_chroots_to_delete = set() + for bchroot in self.b1_bc: + expected_chroots_to_delete.add(bchroot.name) + + assert len(ActionsLogic.get_many().all()) == 0 + BuildsLogic.delete_build(self.u1, self.b1) + self.db.session.commit() + + assert len(ActionsLogic.get_many().all()) == 1 + action = ActionsLogic.get_many().one() + delete_data = json.loads(action.data) + assert "chroots" in delete_data + assert expected_chroots_to_delete == set(delete_data["chroots"]) + + with pytest.raises(NoResultFound): + BuildsLogic.get(self.b1.id).one() + + def test_delete_build_no_chroots_to_clean( + self, f_users, f_coprs, f_mock_chroots, f_builds, f_db): + + for bchroot in self.b1_bc: + bchroot.status = helpers.StatusEnum("skipped") + + self.db.session.commit() + assert len(ActionsLogic.get_many().all()) == 0 + BuildsLogic.delete_build(self.u1, self.b1) + self.db.session.commit() + assert len(ActionsLogic.get_many().all()) == 0 + + def test_delete_build_some_chroots( + self, f_users, f_coprs, f_mock_chroots, f_builds, f_db): + + expected_chroots_to_delete = set([self.b1_bc[0].name, + self.b1_bc[-1].name]) + for bchroot in self.b1_bc[1:-1]: + bchroot.status = helpers.StatusEnum("skipped") + + self.db.session.commit() + + assert len(ActionsLogic.get_many().all()) == 0 + BuildsLogic.delete_build(self.u1, self.b1) + self.db.session.commit() + + assert len(ActionsLogic.get_many().all()) == 1 + action = ActionsLogic.get_many().one() + delete_data = json.loads(action.data) + assert "chroots" in delete_data + assert expected_chroots_to_delete == set(delete_data["chroots"]) + + with pytest.raises(NoResultFound): + BuildsLogic.get(self.b1.id).one() diff --git a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py index 889f5e7..6a72158 100644 --- a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py +++ b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_builds.py @@ -118,7 +118,7 @@ class TestCoprDeleteBuild(CoprsTestCase): .first()) assert b is None act = self.models.Action.query.first() - assert act.object_type == "build-succeeded" + assert act.object_type == "build" assert act.old_value == "user1/foocopr" assert json.loads(act.data)["pkgs"] == pkgs

1 0

[copr] master: [backend] [rhbz:#1073333] Record consecutive builds fails to redis. Added script to produce warnings for nagios check from failures recorded to redis. (c66e6e2)
by vgologuz＠fedoraproject.org 30 Jan '15

30 Jan '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit c66e6e28fc58682b24b8a0bbbc6589f2762cac8f Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Jan 30 10:58:05 2015 +0100 [backend] [rhbz:#1073333] Record consecutive builds fails to redis. Added script to produce warnings for nagios check from failures recorded to redis. >--------------------------------------------------------------- backend/backend/constants.py | 4 ++ backend/backend/daemons/dispatcher.py | 8 ++++ backend/backend/helpers.py | 40 +++++++++++++++++- backend/run/check_consecutive_build_fails.py | 43 ++++++++++++++++++++ backend/tests/deamons/test_dispatcher.py | 2 + .../tests/test_logic/test_builds_logic.py | 6 ++- 6 files changed, 99 insertions(+), 4 deletions(-) diff --git a/backend/backend/constants.py b/backend/backend/constants.py index e6e01de..717ff27 100644 --- a/backend/backend/constants.py +++ b/backend/backend/constants.py @@ -14,6 +14,10 @@ DEF_MACROS = {} DEF_BUILDROOT_PKGS = "" +DEF_CONSECUTIVE_FAILURE_THRESHOLD = 10 +CONSECUTIVE_FAILURE_REDIS_KEY = "copr:sys:consecutive_build_fails" + + class BuildStatus(object): FAILURE = 0 SUCCEEDED = 1 diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py index 33facc0..2cfb086 100644 --- a/backend/backend/daemons/dispatcher.py +++ b/backend/backend/daemons/dispatcher.py @@ -18,6 +18,8 @@ from setproctitle import setproctitle from IPy import IP from retask.queue import Queue + + from ..mockremote.callback import CliLogCallBack from ..exceptions import MockRemoteError, CoprWorkerError, CoprWorkerSpawnFailError @@ -26,6 +28,7 @@ from ..job import BuildJob from ..mockremote import MockRemote from ..frontend import FrontendClient from ..constants import BuildStatus +from ..helpers import register_build_result ansible_playbook = "ansible-playbook" @@ -469,6 +472,8 @@ class Worker(multiprocessing.Process): raise CoprWorkerError( "No IP found from creating instance") except AnsibleError as e: + register_build_result(self.opts, failed=True) + self.callback.log("failure to setup instance: {0}".format(e)) raise @@ -600,10 +605,13 @@ class Worker(multiprocessing.Process): if self.opts.do_sign: mr.add_pubkey() + register_build_result(self.opts) + except MockRemoteError as e: # record and break self.callback.log("{0} - {1}".format(self.vm_ip, e)) status = BuildStatus.FAILURE + register_build_result(self.opts, failed=True) self.callback.log( "Finished build: id={0} builder={1} timeout={2} destdir={3}" diff --git a/backend/backend/helpers.py b/backend/backend/helpers.py index 62f7ad9..8554c8b 100644 --- a/backend/backend/helpers.py +++ b/backend/backend/helpers.py @@ -16,12 +16,20 @@ import datetime from copr.client import CoprClient -from backend.constants import DEF_BUILD_USER, DEF_BUILD_TIMEOUT +from backend.constants import DEF_BUILD_USER, DEF_BUILD_TIMEOUT, DEF_CONSECUTIVE_FAILURE_THRESHOLD, \ + CONSECUTIVE_FAILURE_REDIS_KEY from backend.exceptions import CoprBackendError +from redis import StrictRedis + +try: + import fedmsg +except ImportError: + # fedmsg is optional + fedmsg = None -class SortedOptParser(optparse.OptionParser): +class SortedOptParser(optparse.OptionParser): """Optparser which sorts the options by opt before outputting --help""" def format_help(self, formatter=None): @@ -146,8 +154,13 @@ class BackendConfigReader(object): cp, "backend", "sleeptime", 10, mode="int") opts.timeout = _get_conf( cp, "builder", "timeout", DEF_BUILD_TIMEOUT, mode="int") + opts.consecutive_failure_threshold = _get_conf( + cp, "builder", "consecutive_failure_threshold", + DEF_CONSECUTIVE_FAILURE_THRESHOLD, mode="int") opts.logfile = _get_conf( cp, "backend", "logfile", "/var/log/copr/backend.log") + opts.error_logfile = _get_conf( + cp, "backend", "error_logfile", "/var/log/copr/backend_error.log") opts.verbose = _get_conf( cp, "backend", "verbose", False, mode="bool") opts.worker_logdir = _get_conf( @@ -196,3 +209,26 @@ def log(lf, msg, quiet=None): "Could not write to logfile {0} - {1}\n".format(lf, str(e))) if not quiet: print(msg) + + +def register_build_result(opts=None, failed=False): + """ + Remember fails to redis. + Successful build resets counter to zero. + + :param opts: BackendConfig, when opts not provided default config location will be used + :param boolean failed: failure flag + :param str origin: name of component produced failure, default: `builder` + """ + if opts is None: + opts = BackendConfigReader().read() + + # TODO: add config options to specify redis host, port + conn = StrictRedis() # connecting to default local redis instance + + key = CONSECUTIVE_FAILURE_REDIS_KEY + if not failed: + conn.set(key, 0) + else: + conn.incr(key) + diff --git a/backend/run/check_consecutive_build_fails.py b/backend/run/check_consecutive_build_fails.py new file mode 100755 index 0000000..8e8e26a --- /dev/null +++ b/backend/run/check_consecutive_build_fails.py @@ -0,0 +1,43 @@ +#!/usr/bin/python -tt +# coding: utf-8 + +from __future__ import print_function +from __future__ import unicode_literals +from __future__ import division +from __future__ import absolute_import + +import sys + +from redis import StrictRedis + +sys.path.append("/usr/share/copr/") + + +from backend.helpers import BackendConfigReader +from backend.constants import CONSECUTIVE_FAILURE_REDIS_KEY + + +def main(): + opts = BackendConfigReader().read() + conn = StrictRedis() # connecting to default local redis instance + + key = CONSECUTIVE_FAILURE_REDIS_KEY + + value = int(conn.get(key) or 0) + if value > opts.consecutive_failure_threshold: + print("Critical") + sys.exit(2) + elif value > int(0.5 * opts.consecutive_failure_threshold): + print("Warning") + sys.exit(1) + else: + print("OK") + sys.exit(0) + + +if __name__ == "__main__": + try: + main() + except Exception as error: + print("UNKNOWN: {}".format(error)) + sys.exit(3) diff --git a/backend/tests/deamons/test_dispatcher.py b/backend/tests/deamons/test_dispatcher.py index b50f900..c56bbd4 100644 --- a/backend/tests/deamons/test_dispatcher.py +++ b/backend/tests/deamons/test_dispatcher.py @@ -92,6 +92,8 @@ class TestDispatcher(object): timeout=1800, destdir=self.tmp_dir_path, results_baseurl="/tmp", + + consecutive_failure_threshold=10, ) self.job = BuildJob(self.task, self.opts) diff --git a/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py b/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py index 1eeb17a..c7dc07e 100644 --- a/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py +++ b/frontend/coprs_frontend/tests/test_logic/test_builds_logic.py @@ -74,7 +74,7 @@ class TestBuildsLogic(CoprsTestCase): def test_build_queue_4(self, f_users, f_coprs, f_mock_chroots, f_builds, f_db): for build_chroots in [self.b1_bc, self.b2_bc]: for build_chroot in build_chroots: - build_chroot.status = 3 + build_chroot.status = 3 # running for build_chroots in [self.b3_bc, self.b4_bc]: for build_chroot in build_chroots: build_chroot.status = 0 @@ -85,4 +85,6 @@ class TestBuildsLogic(CoprsTestCase): self.db.session.commit() data = BuildsLogic.get_build_task_queue().all() - assert len(data) == 1 # + + assert len(data) == 1 + assert data[0] == self.b1_bc[0]

1 0

[copr] master: [frontend] correct url for pubkey in .repo (baf1b06)
by vgologuz＠fedoraproject.org 29 Jan '15

29 Jan '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit baf1b06286aea5fab7091bc016e87e2f956cb1fa Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Thu Jan 29 21:31:29 2015 +0100 [frontend] correct url for pubkey in .repo >--------------------------------------------------------------- .../coprs_frontend/coprs/templates/coprs/copr.repo | 2 +- .../coprs/views/coprs_ns/coprs_general.py | 4 +++- .../test_views/test_coprs_ns/test_coprs_general.py | 18 +++++++----------- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/frontend/coprs_frontend/coprs/templates/coprs/copr.repo b/frontend/coprs_frontend/coprs/templates/coprs/copr.repo index 15e5e7a..d316ae7 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/copr.repo +++ b/frontend/coprs_frontend/coprs/templates/coprs/copr.repo @@ -3,6 +3,6 @@ name=Copr repo for {{ copr.name }} owned by {{ copr.owner.name }} baseurl={{ url | fix_url_https_backend }} skip_if_unavailable=True gpgcheck=0 -gpgkey={{ url | fix_url_https_backend }}pubkey.gpg +gpgkey={{ pubkey_url | fix_url_https_backend }} enabled=1 diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py index 3f8f2c9..be91b69 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py @@ -1,5 +1,6 @@ import time import re +import urlparse import flask from flask import render_template @@ -545,8 +546,9 @@ def generate_repo_file(username, coprname, chroot, repofile): .format(username, coprname)) repo_url = generate_repo_url(mock_chroot, url) + pubkey_url = urlparse.urljoin(url, "pubkey.gpg") response = flask.make_response( - flask.render_template("coprs/copr.repo", copr=copr, url=repo_url)) + flask.render_template("coprs/copr.repo", copr=copr, url=repo_url, pubkey_url=pubkey_url)) response.mimetype = "text/plain" response.headers["Content-Disposition"] = \ diff --git a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py index f9d5069..8ecaacc 100644 --- a/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py +++ b/frontend/coprs_frontend/tests/test_views/test_coprs_ns/test_coprs_general.py @@ -713,17 +713,13 @@ class TestCoprRepoGeneration(CoprsTestCase): f_custom_builds, f_db): from coprs import app orig = app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] - try: - app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] = "https" - r = self.tc.get( - "/coprs/{0}/{1}/repo/fedora-18-x86_64/" - .format(self.u1.name, self.c1.name)) - - assert r.status_code == 200 - assert "baseurl=https://bar.baz" in r.data - except Exception as e: - app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] = orig - raise e + app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] = "https" + r = self.tc.get( + "/coprs/{0}/{1}/repo/fedora-18-x86_64/" + .format(self.u1.name, self.c1.name)) + + assert r.status_code == 200 + assert "baseurl=https://bar.baz" in r.data app.config["ENFORCE_PROTOCOL_FOR_BACKEND_URL"] = orig

1 0

[copr] master: correctly print job representation (c6e2cb9)
by Miroslav Suchý 26 Jan '15

26 Jan '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit c6e2cb9cf0d65a16a937d301b51f8cb19dae9f74 Author: Miroslav Suchý <msuchy(a)redhat.com> Date: Mon Jan 26 13:28:53 2015 +0100 correctly print job representation addressing: Process worker-builder: Traceback (most recent call last): File "/usr/lib64/python2.7/multiprocessing/process.py", line 258, in _bootstrap self.run() File "/usr/share/copr/backend/daemons/dispatcher.py", line 662, in run job = self.obtain_job() File "/usr/share/copr/backend/daemons/dispatcher.py", line 520, in obtain_job self.callback.log("obtained task: {}".format(job)) File "/usr/share/copr/backend/job.py", line 111, in __str__ return str(self.__unicode__()) File "/usr/share/copr/backend/job.py", line 115, in __unicode__ u"project: {project_name}, source pkg : {pkg} >".format(self.__dict__) KeyError: u'build_id' >--------------------------------------------------------------- backend/backend/job.py | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/backend/backend/job.py b/backend/backend/job.py index 6d07b26..232ec09 100644 --- a/backend/backend/job.py +++ b/backend/backend/job.py @@ -112,4 +112,4 @@ class BuildJob(object): def __unicode__(self): return u"BuildJob<id: {build_id}, owner: {project_owner}, " \ - u"project: {project_name}, source pkg : {pkg} >".format(self.__dict__) + u"project: {project_name}, source pkg : {pkg} >".format(**self.__dict__)

1 0

[copr] tag 'copr-backend-1.57-1' created
by vgologuz＠fedoraproject.org 23 Jan '15

23 Jan '15

Repository : http://git.fedorahosted.org/cgit/copr.git New tag : copr-backend-1.57-1 Referencing: cb59f6097870adbd4bbcfd84d6d96cc4c5f1803a

1 0

[copr] master: Automatic commit of package [copr-backend] release [1.57-1]. (ecdb0b4)
by vgologuz＠fedoraproject.org 23 Jan '15

23 Jan '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit ecdb0b4bbad9310ff6645dc3f563067156bb6bd7 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Jan 23 15:52:40 2015 +0100 Automatic commit of package [copr-backend] release [1.57-1]. >--------------------------------------------------------------- backend/copr-backend.spec | 26 +++++++++++++++++++++++++- rel-eng/packages/copr-backend | 2 +- 2 files changed, 26 insertions(+), 2 deletions(-) diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec index ac29eaa..a557bce 100644 --- a/backend/copr-backend.spec +++ b/backend/copr-backend.spec @@ -3,7 +3,7 @@ %endif Name: copr-backend -Version: 1.56 +Version: 1.57 Release: 1%{?dist} Summary: Backend for Copr @@ -211,6 +211,30 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %exclude %{_pkgdocdir}/playbooks %changelog +* Fri Jan 23 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.57-1 +- call correct Worker method on backend termination +- put gpg pubkey to the project results root dir (one level up from + the chroot dir) +- don't kill Worker from errors during job build +- [rhbz:#1169782] RFE - Show package "version-release" instead of + just "version" +- [rhbz:#1117446] add a build id tagfile into the package directory +- Updated unittests to reflect latest changes. +- builder: use only one log file for rsync per build +- dispatcher: run terminate_instance safely +- cleanup example config +- cleanup mockremote.builder +- Builder.download don't use Popen+PIPE.communicate with rsync, + output redirected to the files. +- disable networking only when required; python style exception + handling in mockremote*; removed run/copr_mockremote +- test build with disabled networking +- simplified mockremote.builder.Builder.check_for_ans_error; new + method mockremote.builder.Builder.run_ansible_with_check +- daemons.dispatched.Worker: don't fail when wrong group_id was + provided +- add vm_ip to worker process title (rhbz: 1182637) + * Wed Jan 14 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.56-1 - [backend] [.spec] fix %files section diff --git a/rel-eng/packages/copr-backend b/rel-eng/packages/copr-backend index ffdd123..34dea39 100644 --- a/rel-eng/packages/copr-backend +++ b/rel-eng/packages/copr-backend @@ -1 +1 @@ -1.56-1 backend/ +1.57-1 backend/

1 0

[copr] tag 'copr-frontend-1.52-1' created
by vgologuz＠fedoraproject.org 23 Jan '15

23 Jan '15

Repository : http://git.fedorahosted.org/cgit/copr.git New tag : copr-frontend-1.52-1 Referencing: 1e807827ba05aa8ff261c45a46015778ba297dc7

1 0

[copr] master: Automatic commit of package [copr-frontend] release [1.52-1]. (a137935)
by vgologuz＠fedoraproject.org 23 Jan '15

23 Jan '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit a1379358cbf41dbb09e27b661d9f2589bd38b005 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Jan 23 15:49:45 2015 +0100 Automatic commit of package [copr-frontend] release [1.52-1]. >--------------------------------------------------------------- frontend/copr-frontend.spec | 15 ++++++++++++++- rel-eng/packages/copr-frontend | 2 +- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/frontend/copr-frontend.spec b/frontend/copr-frontend.spec index 420972f..43e88ac 100644 --- a/frontend/copr-frontend.spec +++ b/frontend/copr-frontend.spec @@ -5,7 +5,7 @@ %endif Name: copr-frontend -Version: 1.51 +Version: 1.52 Release: 1%{?dist} Summary: Frontend for Copr @@ -168,6 +168,19 @@ service httpd condrestart %doc documentation/python-doc %changelog +* Fri Jan 23 2015 Valentin Gologuzov <vgologuz(a)redhat.com> 1.52-1 +- add url to gpg pubkey in .repo files +- [rhbz:#1183702] Interrupted builds aren't re-added to the + builder queue, and stuck forever in RUNNING state. +- [rhbz:#1133650] RFE: copr frontend on page of build details, + results section should show multiple links that link directly for every + chroot directory +- UI to control `enable_net` option, DB schema changes +- new command AddDebugUser for manage script +- [RHBZ:#1176364] Wrong value for the build timeout. +- [RHBZ:#1177179] Display the timezone with a format more similar to + ISO 8601 + * Mon Dec 15 2014 Valentin Gologuzov <vgologuz(a)redhat.com> 1.51-1 - bugfix: send correct chroots in on_auto_createrepo_change() - control auto_createrepo property of project through API diff --git a/rel-eng/packages/copr-frontend b/rel-eng/packages/copr-frontend index 3a85546..9d79b41 100644 --- a/rel-eng/packages/copr-frontend +++ b/rel-eng/packages/copr-frontend @@ -1 +1 @@ -1.51-1 frontend/ +1.52-1 frontend/

1 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

copr-commits January 2015