copr-commits July 2015

copr-commits@lists.fedorahosted.org

4 participants
267 discussions

[copr] master: [dist-git] copied systemd unit file from ansible (e225321)
by vgologuz＠fedoraproject.org 31 Jul '15

31 Jul '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit e22532174f0586eaccb46f8e484442f1844f7e26 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Jul 31 19:56:57 2015 +0200 [dist-git] copied systemd unit file from ansible >--------------------------------------------------------------- dist-git/copr-dist-git.service | 13 +++++++++++++ 1 files changed, 13 insertions(+), 0 deletions(-) diff --git a/dist-git/copr-dist-git.service b/dist-git/copr-dist-git.service index e69de29..7bce43b 100644 --- a/dist-git/copr-dist-git.service +++ b/dist-git/copr-dist-git.service @@ -0,0 +1,13 @@ +[Unit] +Description=copr aux service to import srpm into dist-git +Requires=dist-git.socket +After=dist-git.socket + + +[Service] +User=copr-service +Group=copr-service +ExecStart=/usr/bin/python2.7 /usr/share/copr/dist_git/dist_git_importer.py + +[Install] +WantedBy=multi-user.target

1 0

[copr] master: [dist-git] more test coverage (f6ec0a9)
by vgologuz＠fedoraproject.org 31 Jul '15

31 Jul '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit f6ec0a9cdf210a342d5786c0187fee1786a28529 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Jul 31 19:44:06 2015 +0200 [dist-git] more test coverage >--------------------------------------------------------------- backend/copr-backend.spec | 4 +- dist-git/copr-dist-git.spec | 20 ++- dist-git/dist_git/dist_git_importer.py | 29 ++-- dist-git/dist_git/helpers.py | 7 +- dist-git/tests/test_dist_git_importer.py | 236 +++++++++++++++++++++++++++++- 5 files changed, 267 insertions(+), 29 deletions(-) diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec index 62519f7..7f14487 100644 --- a/backend/copr-backend.spec +++ b/backend/copr-backend.spec @@ -185,8 +185,8 @@ cp -a conf/logstash/copr_backend.conf %{buildroot}%{_pkgdocdir}/examples/%{_sysc redis-server --port 7777 &> /dev/null & -#PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \ -# -s -v --cov-report term-missing --cov ./backend --cov ./run ./tests/ +PYTHONPATH=backend:run:$PYTHONPATH python -B -m pytest \ + -s -v --cov-report term-missing --cov ./backend --cov ./run ./tests/ kill %1 diff --git a/dist-git/copr-dist-git.spec b/dist-git/copr-dist-git.spec index d117676..652e3c0 100644 --- a/dist-git/copr-dist-git.spec +++ b/dist-git/copr-dist-git.spec @@ -14,13 +14,17 @@ Source0: %{name}-%{version}.tar.gz BuildArch: noarch -BuildRequires: systemd - -Requires: systemd -Requires: dist-git -Requires: python-bunch -Requires: python-requests -Requires: pyrpkg +BuildRequires: systemd +BuildRequires: dist-git +BuildRequires: python-bunch +BuildRequires: python-requests +BuildRequires: pyrpkg + +Requires: systemd +Requires: dist-git +Requires: python-bunch +Requires: python-requests +Requires: pyrpkg %description COPR is lightweight build system. It allows you to create new project in WebUI @@ -55,6 +59,8 @@ cp -a conf/logrotate %{buildroot}%{_sysconfdir}/logrotate.d/copr-dist-git touch %{buildroot}%{_var}/log/copr-dist-git/main.log %check +PYTHONPATH=.:$PYTHONPATH python -B -m pytest \ + -v --cov-report term-missing --cov ./dist_git ./tests/ %post # change context to be readable by cgit diff --git a/dist-git/dist_git/dist_git_importer.py b/dist-git/dist_git/dist_git_importer.py index f548863..88df6a6 100755 --- a/dist-git/dist_git/dist_git_importer.py +++ b/dist-git/dist_git/dist_git_importer.py @@ -109,6 +109,7 @@ class ImportTask(object): class DistGitImporter(object): def __init__(self, opts): + self.is_running = False self.opts = opts self.get_url = "{}/backend/importing/".format(self.opts.frontend_base_url) @@ -136,29 +137,25 @@ class DistGitImporter(object): @staticmethod def fetch_srpm(task, fetched_srpm_path): log.debug("download the package") - try: urlretrieve(task.package_url, fetched_srpm_path) except IOError: raise PackageDownloadException() - return fetched_srpm_path - @staticmethod - def my_upload(repo_dir, reponame, filename, filehash): + def my_upload(self, repo_dir, reponame, filename, filehash): """ This is a replacement function for uploading sources. Rpkg uses upload.cgi for uploading which doesn't make sense on the local machine. """ - lookaside = "/var/lib/dist-git/cache/lookaside/pkgs/" source = os.path.join(repo_dir, filename) - destination = os.path.join(lookaside, reponame, filename, filehash, filename) + destination = os.path.join(self.opts.lookaside_location, reponame, + filename, filehash, filename) if not os.path.exists(destination): os.makedirs(os.path.dirname(destination)) shutil.copyfile(source, destination) - @classmethod - def git_import_srpm(cls, task, filepath): + def git_import_srpm(self, task, filepath): """ Imports a source rpm file into local dist git. Repository name is in the Copr Style: user/project/package @@ -195,7 +192,7 @@ class DistGitImporter(object): # rpkg calls upload.cgi script on the dist git server # here, I just copy the source files manually with custom function # I also add one parameter "repo_dir" to that function with this hack - commands.lookasidecache.upload = types.MethodType(cls.my_upload, repo_dir) + commands.lookasidecache.upload = types.MethodType(self.my_upload, repo_dir) log.debug("clone the pkg repository into tmp directory") commands.clone(module, tmp, task.branch) @@ -219,10 +216,12 @@ class DistGitImporter(object): except rpkgError: log.exception("error during commit and push, ignored") - git_hash = commands.commithash + return commands.commithash + except Exception: + log.exception("Unexpected") + raise PackageImportException() finally: shutil.rmtree(tmp) - return git_hash @staticmethod def pkg_name_evr(srpm_path): @@ -310,7 +309,8 @@ class DistGitImporter(object): def run(self): log.info("DistGitImported initialized") - while True: + self.is_running = True + while self.is_running: mb_task = self.try_to_obtain_new_task() if mb_task is None: time.sleep(self.opts.sleep_time) @@ -347,10 +347,7 @@ def main(): log.info("Using configuration: \n" "{}".format(opts)) importer = DistGitImporter(opts) - try: - importer.run() - except KeyboardInterrupt: - return + importer.run() if __name__ == "__main__": diff --git a/dist-git/dist_git/helpers.py b/dist-git/dist_git/helpers.py index eb90f4a..3631840 100644 --- a/dist-git/dist_git/helpers.py +++ b/dist-git/dist_git/helpers.py @@ -77,6 +77,11 @@ class DistGitConfigReader(object): ) opts.cgit_pkg_list_location = _get_conf( - cp, "cgit_pkg_list_location", "log_dir", "/var/lib/copr-dist-git/cgit_pkg_list" + cp, "dist-git", "cgit_pkg_list_location", "/var/lib/copr-dist-git/cgit_pkg_list" ) + + opts.lookaside_location = _get_conf( + cp, "dist-git", "lookaside_location", "/var/lib/dist-git/cache/lookaside/pkgs/" + ) + return opts diff --git a/dist-git/tests/test_dist_git_importer.py b/dist-git/tests/test_dist_git_importer.py index 4cc54b4..7c003d9 100644 --- a/dist-git/tests/test_dist_git_importer.py +++ b/dist-git/tests/test_dist_git_importer.py @@ -9,6 +9,7 @@ import tempfile import shutil import time from bunch import Bunch +from pyrpkg import rpkgError import pytest import six @@ -20,14 +21,35 @@ else: import mock from mock import MagicMock -from dist_git.dist_git_importer import DistGitImporter, SourceType, ImportTask +from dist_git.dist_git_importer import DistGitImporter, SourceType, ImportTask, PackageDownloadException, \ + PackageImportException, PackageQueryException, main MODULE_REF = 'dist_git.dist_git_importer' @pytest.yield_fixture +def mc_dgcr(): + with mock.patch("{}.DistGitConfigReader".format(MODULE_REF)) as handle: + yield handle + +(a)pytest.yield_fixture +def mc_dgi(): + with mock.patch("{}.DistGitImporter".format(MODULE_REF)) as handle: + yield handle + +(a)pytest.yield_fixture +def mc_time(): + with mock.patch("{}.time".format(MODULE_REF)) as handle: + yield handle + +(a)pytest.yield_fixture def mc_popen(): - with mock.patch(MODULE_REF) as handle: + with mock.patch("{}.Popen".format(MODULE_REF)) as handle: + yield handle + +(a)pytest.yield_fixture +def mc_call(): + with mock.patch("{}.call".format(MODULE_REF)) as handle: yield handle @@ -42,8 +64,18 @@ def mc_post(): with mock.patch("{}.post".format(MODULE_REF)) as handle: yield handle +(a)pytest.yield_fixture +def mc_urlretrieve(): + with mock.patch("{}.urlretrieve".format(MODULE_REF)) as handle: + yield handle -if False: +(a)pytest.yield_fixture +def mc_pyrpkg_commands(): + with mock.patch("{}.Commands".format(MODULE_REF)) as handle: + yield handle + + +if True: logging.basicConfig( level=logging.DEBUG, format='[%(asctime)s][%(levelname)s][%(name)s][%(module)s:%(lineno)d] %(message)s', @@ -53,17 +85,27 @@ if False: class TestDistGitImporter(object): def setup_method(self, method): + self.tmp_dir_name = self.make_temp_dir() + self.lookaside_location = os.path.join(self.tmp_dir_name, "lookaside") self.opts = Bunch({ "frontend_base_url": "http://front", "frontend_auth": "secure_password", + "lookaside_location": self.lookaside_location, + + "cgit_pkg_list_location": self.tmp_dir_name, + "sleep_time": 10, + "log_dir": self.tmp_dir_name }) self.dgi = DistGitImporter(self.opts) self.USER_NAME = "foo" self.PROJECT_NAME = "bar" + self.PACKAGE_NAME = "bar_app" + self.PACKAGE_VERSION = "2:0.01-1.fc20" self.BRANCH = "f22" + self.FILE_HASH = "1234abc" self.task_data_1 = { "task_id": 123, "user": self.USER_NAME, @@ -86,6 +128,24 @@ class TestDistGitImporter(object): self.task_1 = ImportTask.from_dict(self.task_data_1, self.opts) self.task_2 = ImportTask.from_dict(self.task_data_2, self.opts) + self.fetched_srpm_path = "/tmp/none/" + + + def teardown_method(self, method): + self.rm_tmp_dir() + + def rm_tmp_dir(self): + if self.tmp_dir_name: + shutil.rmtree(self.tmp_dir_name) + self.tmp_dir_name = None + + def make_temp_dir(self): + root_tmp_dir = tempfile.gettempdir() + subdir = "test_{}_{}".format(MODULE_REF, time.time()) + self.tmp_dir_name = os.path.join(root_tmp_dir, subdir) + os.mkdir(self.tmp_dir_name) + return self.tmp_dir_name + def test_try_to_obtain_new_task_empty(self, mc_get): mc_get.return_value.json.return_value = {"builds": []} assert self.dgi.try_to_obtain_new_task() is None @@ -116,3 +176,173 @@ class TestDistGitImporter(object): task_data["source_type"] = 999999 mc_get.return_value.json.return_value = {"builds": [task_data]} assert self.dgi.try_to_obtain_new_task() is None + + def test_fetch_srpm_on(self, mc_urlretrieve): + self.dgi.fetch_srpm(self.task_1, self.fetched_srpm_path) + assert mc_urlretrieve.called + + mc_urlretrieve.side_effect = IOError + with pytest.raises(PackageDownloadException): + self.dgi.fetch_srpm(self.task_1, self.fetched_srpm_path) + + def test_my_upload(self): + filename = "source" + source_path = os.path.join(self.tmp_dir_name, filename) + with open(source_path, "w") as handle: + handle.write("1") + + reponame = self.PROJECT_NAME + target = "/".join([ + self.lookaside_location, reponame, filename, self.FILE_HASH, filename + ]) + assert not os.path.exists(target) + self.dgi.my_upload(self.tmp_dir_name, reponame, filename, self.FILE_HASH) + assert os.path.exists(target) + + def test_git_import_srpm(self, mc_pyrpkg_commands): + # stupid test, just for the coverage + mc_cmd = MagicMock() + mc_pyrpkg_commands.return_value = mc_cmd + mc_cmd.commithash = self.FILE_HASH + + filename = "source" + source_path = os.path.join(self.tmp_dir_name, filename) + + self.task_1.package_name = self.PACKAGE_NAME + assert self.dgi.git_import_srpm(self.task_1, source_path) == self.FILE_HASH + + # check exception handling + for err in [IOError, OSError, ValueError]: + mc_cmd.import_srpm.side_effect = err + with pytest.raises(PackageImportException): + self.dgi.git_import_srpm(self.task_1, source_path) + mc_cmd.import_srpm.side_effect = None + + mc_cmd.push.side_effect = rpkgError + assert self.dgi.git_import_srpm(self.task_1, source_path) == self.FILE_HASH + + for err in [IOError, OSError, ValueError]: + mc_pyrpkg_commands.side_effect = err + with pytest.raises(PackageImportException): + self.dgi.git_import_srpm(self.task_1, source_path) + + def test_pkg_name_evr(self, mc_popen): + mc_comm = MagicMock() + mc_popen.return_value.communicate = mc_comm + + test_plan = [ + (("(none)", "0.1", "1.fc20"), "0.1-1.fc20"), + (("2", "0.1", "1.fc20"), "2:0.1-1.fc20") + ] + for (e, v, r), expected in test_plan: + mc_comm.return_value = ("foo {} {} {}".format(e, v, r), None) + assert self.dgi.pkg_name_evr("/dev/null") == ("foo", expected) + + def test_pkg_name_evr_error_handling(self, mc_popen): + mc_comm = MagicMock() + mc_popen.return_value.communicate = mc_comm + + test_plan = [ + (("(none)", "0.1", "1.fc20"), "0.1-1.fc20"), + (("2", "0.1", "1.fc20"), "2:0.1-1.fc20") + ] + + mc_popen.side_effect = OSError + with pytest.raises(PackageQueryException): + self.dgi.pkg_name_evr("/dev/null") + + mc_popen.side_effect = None + for (e, v, r), expected in test_plan: + mc_comm.return_value = ("foo {} {} {}".format(e, v, r), "err msg") + with pytest.raises(PackageQueryException): + self.dgi.pkg_name_evr("/dev/null") + + for (e, v, r), expected in test_plan: + mc_comm.return_value = ("", None) + with pytest.raises(PackageQueryException): + self.dgi.pkg_name_evr("/dev/null") + + def test_before_git_import(self, mc_call): + # dummy test, just for coverage + self.dgi.before_git_import(self.task_1) + assert mc_call.called + + def test_after_git_import(self, mc_call): + # dummy test, just for coverage + self.dgi.after_git_import() + assert mc_call.called + + def test_past_back(self, mc_post): + dd = {"foo": "bar"} + self.dgi.post_back(dd) + assert mc_post.called + + def test_past_back_safe(self, mc_post): + dd = {"foo": "bar"} + self.dgi.post_back_safe(dd) + assert mc_post.called + mc_post.reset_mock() + assert not mc_post.called + + mc_post.side_effect = IOError + self.dgi.post_back_safe(dd) + assert mc_post.called + + def test_do_import(self): + internal_methods = [ + "fetch_srpm", + "pkg_name_evr", + "before_git_import", + "git_import_srpm", + "after_git_import", + "post_back" + ] + mc_methods = {name: MagicMock() for name in internal_methods} + for name, mc in mc_methods.items(): + setattr(self.dgi, name, mc) + + mc_methods["pkg_name_evr"].return_value = self.PACKAGE_NAME, self.PACKAGE_VERSION + mc_methods["git_import_srpm"].return_value = self.FILE_HASH + + self.dgi.do_import(self.task_1) + assert self.task_1.package_name == self.PACKAGE_NAME + assert self.task_1.package_version == self.PACKAGE_VERSION + assert self.task_1.git_hash == self.FILE_HASH + + for name in internal_methods: + mc = mc_methods[name] + for err in (IOError, PackageImportException, PackageDownloadException, PackageQueryException): + mc.side_effect = err + self.dgi.do_import(self.task_1) + + mc.side_effect = None + + def test_run(self, mc_time): + self.dgi.try_to_obtain_new_task = MagicMock() + self.dgi.do_import = MagicMock() + + def stop_run(*args, **kwargs): + self.dgi.is_running = False + + mc_time.sleep.side_effect = stop_run + + self.dgi.try_to_obtain_new_task.return_value = None + self.dgi.run() + assert not self.dgi.do_import.called + + self.dgi.try_to_obtain_new_task.return_value = self.task_1 + self.dgi.do_import.side_effect = stop_run + self.dgi.run() + assert self.dgi.do_import.call_args == mock.call(self.task_1) + + def test_main(self, mc_dgi, mc_dgcr): + # dummy test, just for coverage + mc_dgcr.return_value.read.return_value = self.opts + main() + + assert mc_dgi.called + + mc_dgcr.return_value.read.side_effect = IOError() + with pytest.raises(SystemExit): + main() +

1 0

[copr] master: [dist-git] added initial tests; renamed folder with sources to use underscore instead of dash (dd0ab58)
by vgologuz＠fedoraproject.org 31 Jul '15

31 Jul '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit dd0ab580430c28b95cbf1b4dc3474c557bc6b7c3 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Jul 31 17:17:25 2015 +0200 [dist-git] added initial tests; renamed folder with sources to use underscore instead of dash >--------------------------------------------------------------- dist-git/copr-dist-git.spec | 4 +- dist-git/{dist-git => dist_git}/__init__.py | 0 .../{dist-git => dist_git}/dist_git_importer.py | 74 +++++------- dist-git/{dist-git => dist_git}/helpers.py | 0 dist-git/run_tests.sh | 1 + dist-git/tests/test_dist_git_importer.py | 118 ++++++++++++++++++++ 6 files changed, 151 insertions(+), 46 deletions(-) diff --git a/dist-git/copr-dist-git.spec b/dist-git/copr-dist-git.spec index 09aace1..d117676 100644 --- a/dist-git/copr-dist-git.spec +++ b/dist-git/copr-dist-git.spec @@ -38,14 +38,14 @@ This package contains Copr services for Dist Git server %install -install -d %{buildroot}%{_datadir}/copr/dist-git +install -d %{buildroot}%{_datadir}/copr/dist_git install -d %{buildroot}%{_sysconfdir}/copr install -d %{buildroot}%{_sysconfdir}/logrotate.d/ install -d %{buildroot}%{_unitdir} install -d %{buildroot}%{_var}/log/copr-dist-git install -d %{buildroot}%{_sharedstatedir}/copr-dist-git -cp -a dist-git/* %{buildroot}%{_datadir}/copr/dist-git +cp -a dist_git/* %{buildroot}%{_datadir}/copr/dist_git cp -a conf/copr-dist-git.conf.example %{buildroot}%{_sysconfdir}/copr/copr-dist-git.conf cp -a copr-dist-git.service %{buildroot}%{_unitdir}/ diff --git a/dist-git/dist-git/__init__.py b/dist-git/dist_git/__init__.py similarity index 100% rename from dist-git/dist-git/__init__.py rename to dist-git/dist_git/__init__.py diff --git a/dist-git/dist-git/dist_git_importer.py b/dist-git/dist_git/dist_git_importer.py similarity index 86% rename from dist-git/dist-git/dist_git_importer.py rename to dist-git/dist_git/dist_git_importer.py index 058ae64..f548863 100755 --- a/dist-git/dist-git/dist_git_importer.py +++ b/dist-git/dist_git/dist_git_importer.py @@ -4,14 +4,13 @@ import os import json import time import types -import urllib +from urllib import urlretrieve import shutil import tempfile import logging -import subprocess +from subprocess import PIPE, Popen -from requests import get -from requests import post +from requests import get, post # pyrpkg uses os.getlogin(). It requires tty which is unavailable when we run this script as a daemon # very dirty solution for now @@ -30,17 +29,6 @@ from helpers import DistGitConfigReader log = logging.getLogger(__name__) -# Example usage: -# -# user = "asamalik" -# project = "project-for-dist-git" -# pkg_name = "devtoolset-3" -# branch = "f20" -# filepath = "/tmp/rh-php56-php-5.6.5-5.el7.src.rpm" -# -# git_hash = import_srpm(user, project, pkg_name, branch, filepath) - - class PackageImportException(Exception): pass @@ -53,22 +41,6 @@ class PackageQueryException(Exception): pass -def _my_upload(repo_dir, reponame, filename, filehash): - """ - This is a replacement function for uploading sources. - Rpkg uses upload.cgi for uploading which doesn't make sense - on the local machine. - """ - lookaside = "/var/lib/dist-git/cache/lookaside/pkgs/" - source = os.path.join(repo_dir, filename) - destination = os.path.join(lookaside, reponame, filename, filehash, filename) - if not os.path.exists(destination): - os.makedirs(os.path.dirname(destination)) - shutil.copyfile(source, destination) - - - - class SourceType: SRPM_LINK = 1 SRPM_UPLOAD = 2 @@ -166,13 +138,27 @@ class DistGitImporter(object): log.debug("download the package") try: - urllib.urlretrieve(task.package_url, fetched_srpm_path) + urlretrieve(task.package_url, fetched_srpm_path) except IOError: raise PackageDownloadException() return fetched_srpm_path @staticmethod - def git_import_srpm(task, filepath): + def my_upload(repo_dir, reponame, filename, filehash): + """ + This is a replacement function for uploading sources. + Rpkg uses upload.cgi for uploading which doesn't make sense + on the local machine. + """ + lookaside = "/var/lib/dist-git/cache/lookaside/pkgs/" + source = os.path.join(repo_dir, filename) + destination = os.path.join(lookaside, reponame, filename, filehash, filename) + if not os.path.exists(destination): + os.makedirs(os.path.dirname(destination)) + shutil.copyfile(source, destination) + + @classmethod + def git_import_srpm(cls, task, filepath): """ Imports a source rpm file into local dist git. Repository name is in the Copr Style: user/project/package @@ -180,12 +166,12 @@ class DistGitImporter(object): :type task: ImportTask """ - log.debug("importing it and delete the srpm") + log.debug("importing srpm into the dist-git") # I need to use git via SSH because of gitolite as it manages # permissions with it's hook that relies on gitolite console # which is a default shell on SSH - git_bas_eurl = "ssh://copr-dist-git@localhost/%(module)s" + git_base_url = "ssh://copr-dist-git@localhost/%(module)s" tmp = tempfile.mkdtemp() try: repo_dir = os.path.join(tmp, task.package_name) @@ -195,7 +181,7 @@ class DistGitImporter(object): lookaside="", lookasidehash="md5", lookaside_cgi="", - gitbaseurl=git_bas_eurl, + gitbaseurl=git_base_url, anongiturl="", branchre="", kojiconfig="", @@ -209,7 +195,7 @@ class DistGitImporter(object): # rpkg calls upload.cgi script on the dist git server # here, I just copy the source files manually with custom function # I also add one parameter "repo_dir" to that function with this hack - commands.lookasidecache.upload = types.MethodType(_my_upload, repo_dir) + commands.lookasidecache.upload = types.MethodType(cls.my_upload, repo_dir) log.debug("clone the pkg repository into tmp directory") commands.clone(module, tmp, task.branch) @@ -231,22 +217,22 @@ class DistGitImporter(object): commands.push() log.debug("commit and push done") except rpkgError: - pass + log.exception("error during commit and push, ignored") + git_hash = commands.commithash finally: shutil.rmtree(tmp) return git_hash @staticmethod - def pkg_name_evr(pkg): + def pkg_name_evr(srpm_path): """ Queries a package for its name and evr (epoch:version-release) """ log.debug("Verifying packagage, getting name and version.") - cmd = ['rpm', '-qp', '--nosignature', '--qf', '%{NAME} %{EPOCH} %{VERSION} %{RELEASE}', pkg] + cmd = ['rpm', '-qp', '--nosignature', '--qf', '%{NAME} %{EPOCH} %{VERSION} %{RELEASE}', srpm_path] try: - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + proc = Popen(cmd, stdout=PIPE, stderr=PIPE) output, error = proc.communicate() except OSError as e: raise PackageQueryException(e) @@ -280,14 +266,14 @@ class DistGitImporter(object): """ Could raise error related to networkd connection """ - post(self.upload_url, auth=self.auth, data=json.dumps(data_dict), headers=self.headers) + return post(self.upload_url, auth=self.auth, data=json.dumps(data_dict), headers=self.headers) def post_back_safe(self, data_dict): """ Ignores any error """ try: - self.post_back(data_dict) + return self.post_back(data_dict) except Exception: log.exception("Failed to post back to frontend : {}".format(data_dict)) diff --git a/dist-git/dist-git/helpers.py b/dist-git/dist_git/helpers.py similarity index 100% rename from dist-git/dist-git/helpers.py rename to dist-git/dist_git/helpers.py diff --git a/dist-git/run_tests.sh b/dist-git/run_tests.sh new file mode 100755 index 0000000..83bc02f --- /dev/null +++ b/dist-git/run_tests.sh @@ -0,0 +1 @@ +PYTHONPATH=./src: python2 -B -m pytest --cov-report term-missing --cov ./dist_git tests $@ diff --git a/dist-git/tests/test_dist_git_importer.py b/dist-git/tests/test_dist_git_importer.py new file mode 100644 index 0000000..4cc54b4 --- /dev/null +++ b/dist-git/tests/test_dist_git_importer.py @@ -0,0 +1,118 @@ +# coding: utf-8 +import json +import logging + +import os +import copy +import tarfile +import tempfile +import shutil +import time +from bunch import Bunch +import pytest + +import six + +if six.PY3: + from unittest import mock + from unittest.mock import MagicMock +else: + import mock + from mock import MagicMock + +from dist_git.dist_git_importer import DistGitImporter, SourceType, ImportTask + +MODULE_REF = 'dist_git.dist_git_importer' + + +(a)pytest.yield_fixture +def mc_popen(): + with mock.patch(MODULE_REF) as handle: + yield handle + + +(a)pytest.yield_fixture +def mc_get(): + with mock.patch("{}.get".format(MODULE_REF)) as handle: + yield handle + + +(a)pytest.yield_fixture +def mc_post(): + with mock.patch("{}.post".format(MODULE_REF)) as handle: + yield handle + + +if False: + logging.basicConfig( + level=logging.DEBUG, + format='[%(asctime)s][%(levelname)s][%(name)s][%(module)s:%(lineno)d] %(message)s', + datefmt='%H:%M:%S' + ) + + +class TestDistGitImporter(object): + def setup_method(self, method): + self.opts = Bunch({ + "frontend_base_url": "http://front", + "frontend_auth": "secure_password", + + }) + + self.dgi = DistGitImporter(self.opts) + + self.USER_NAME = "foo" + self.PROJECT_NAME = "bar" + self.BRANCH = "f22" + self.task_data_1 = { + "task_id": 123, + "user": self.USER_NAME, + "project": self.PROJECT_NAME, + + "branch": self.BRANCH, + "source_type": SourceType.SRPM_LINK, + "source_json": json.dumps({"url": "http://example.com/pkg.src.rpm"}) + } + self.task_data_2 = { + "task_id": 124, + "user": self.USER_NAME, + "project": self.PROJECT_NAME, + + "branch": self.BRANCH, + "source_type": SourceType.SRPM_UPLOAD, + "source_json": json.dumps({"tmp": "tmp_2", "pkg": "pkg_2.src.rpm"}) + } + + self.task_1 = ImportTask.from_dict(self.task_data_1, self.opts) + self.task_2 = ImportTask.from_dict(self.task_data_2, self.opts) + + def test_try_to_obtain_new_task_empty(self, mc_get): + mc_get.return_value.json.return_value = {"builds": []} + assert self.dgi.try_to_obtain_new_task() is None + + def test_try_to_obtain_handle_error(self, mc_get): + for err in [IOError, OSError, ValueError]: + mc_get.side_effect = err + assert self.dgi.try_to_obtain_new_task() is None + + def test_try_to_obtain_ok(self, mc_get): + mc_get.return_value.json.return_value = {"builds": [self.task_data_1, self.task_data_2]} + task = self.dgi.try_to_obtain_new_task() + assert task.task_id == self.task_data_1["task_id"] + assert task.user == self.USER_NAME + assert task.branch == self.BRANCH + assert task.package_url == "http://example.com/pkg.src.rpm" + + def test_try_to_obtain_ok_2(self, mc_get): + mc_get.return_value.json.return_value = {"builds": [self.task_data_2, self.task_data_1]} + task = self.dgi.try_to_obtain_new_task() + assert task.task_id == self.task_data_2["task_id"] + assert task.user == self.USER_NAME + assert task.branch == self.BRANCH + assert task.package_url == "http://front/tmp/tmp_2/pkg_2.src.rpm" + + def test_try_to_obtain_new_task_unknown_source_type(self, mc_get): + task_data = copy.deepcopy(self.task_data_1) + task_data["source_type"] = 999999 + mc_get.return_value.json.return_value = {"builds": [task_data]} + assert self.dgi.try_to_obtain_new_task() is None

1 0

[copr] master: Do not use email in contact automatically (790a1ad)
by asamalik＠fedoraproject.org 31 Jul '15

31 Jul '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 790a1adecc4be0eadd51ac62819529c067431e39 Author: Adam Samalik <asamalik(a)redhat.com> Date: Fri Jul 31 09:56:59 2015 +0200 Do not use email in contact automatically This is a new function that might get unnoticed. In that case, user's email would be exposed to public without their knowing. So let's make it opt-in instead. >--------------------------------------------------------------- frontend/coprs_frontend/coprs/forms.py | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/frontend/coprs_frontend/coprs/forms.py b/frontend/coprs_frontend/coprs/forms.py index 7a50562..9cbd29c 100644 --- a/frontend/coprs_frontend/coprs/forms.py +++ b/frontend/coprs_frontend/coprs/forms.py @@ -167,7 +167,6 @@ class CoprFormFactory(object): contact = wtforms.StringField( "Contact", - default=flask.g.user.mail, validators=[ wtforms.validators.Optional(), EmailOrURL()])

1 0

[copr] master: update copr form info (c3c8b62)
by asamalik＠fedoraproject.org 31 Jul '15

31 Jul '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit c3c8b62379545a0032ac54de9f714174ee515bf0 Author: Adam Samalik <asamalik(a)redhat.com> Date: Fri Jul 31 09:42:41 2015 +0200 update copr form info >--------------------------------------------------------------- .../coprs_frontend/coprs/templates/_helpers.html | 13 ++++++++++++- .../coprs/templates/coprs/_coprs_forms.html | 20 ++++++-------------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/frontend/coprs_frontend/coprs/templates/_helpers.html b/frontend/coprs_frontend/coprs/templates/_helpers.html index d420411..f8573fd 100644 --- a/frontend/coprs_frontend/coprs/templates/_helpers.html +++ b/frontend/coprs_frontend/coprs/templates/_helpers.html @@ -1,4 +1,4 @@ -{% macro render_field(field, label=None, class='') %} +{% macro render_field(field, label=None, class='', info=None) %} {% if not kwargs['hidden'] %} {% if field.errors %} <div class="form-group has-error"> @@ -10,11 +10,22 @@ </label> <div class="col-sm-10"> {{ field(class="form-control", **kwargs)|safe }} + <ul class="list-unstyled"> + {% if info %} + <li class="help-block"> + <small class="text-muted pficon pficon-info"> + {{info|safe}} + </small> + </li> + {% endif %} {% if field.errors %} {% for error in field.errors %} + <li class="help-block"> <span class="help-block">{{ error }}</span> + </li> {% endfor %} {% endif %} + </ul> </div> </div> {% else %} diff --git a/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html b/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html index 974bc31..56f393c 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/_coprs_forms.html @@ -22,21 +22,13 @@ {{ render_field(form.name, hidden = True) }} {{ render_field(form.name, label='Project Name', disabled = True) }} {% endif %} - {{ render_field(form.description, rows=5, cols=50, placeholder='Optional - describe your project briefly.') }} - {{ render_field(form.instructions, rows=5, cols=50, placeholder='Optional - describe how your project can be installed. Where users can report bugs and issues. Link to wiki is good as well.') }} - {{ render_field(form.homepage, label='Homepage', placeholder='Optional - project homepage') }} - {{ render_field(form.contact, label='Contact', placeholder='Optional - email address or contact url') }} + {{ render_field(form.description, rows=5, cols=50, placeholder='Optional - describe your project briefly.', + info='You can use <a href="http://daringfireball.net/projects/markdown/syntax">markdown syntax</a>, inline HTMLis forbidden.') }} + {{ render_field(form.instructions, rows=5, cols=50, placeholder='Optional - describe how your project can be installed. Where users can report bugs and issues. Link to wiki is good as well.', info='You can use <a href="http://daringfireball.net/projects/markdown/syntax">markdown syntax</a>, inline HTMLis forbidden.') }} - <div class="form-group"> - <label class="col-sm-2 control-label" for="textInput-markup"> - </label> - <div class="col-sm-10"> - <small class="text-muted pficon pficon-info"> - You can use <a href="http://daringfireball.net/projects/markdown/syntax">markdown syntax</a>, inline HTML is forbidden. - </small> - </div> - </div> + {{ render_field(form.homepage, label='Homepage', placeholder='Optional - project homepage') }} + {{ render_field(form.contact, label='Contact', placeholder='Optional - email address or contact url', info="Use e-mail address or a link to your issue tracker. This information will be shown publicly.") }} {% if form._mock_chroots_error %} <div class="form-group has-error"> @@ -59,7 +51,7 @@ </div> {% endfor %}{% endfor %} {% if form._mock_chroots_error %} - <span class=" help-block"> {{ form._mock_chroots_error }}</span> + <span class="help-block pficon pficon-warning-triangle-o"> {{ form._mock_chroots_error }}</span> {% endif %} </div> </div>

1 0

[copr] master: [dist-git] refactoring importer (3b504f7)
by vgologuz＠fedoraproject.org 30 Jul '15

30 Jul '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 3b504f78423aced5d1c9cade99e5384771ecb281 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Thu Jul 30 17:46:38 2015 +0200 [dist-git] refactoring importer >--------------------------------------------------------------- dist-git/dist-git/dist_git_importer.py | 408 ++++++++++++++++++-------------- dist-git/dist-git/helpers.py | 4 + 2 files changed, 233 insertions(+), 179 deletions(-) diff --git a/dist-git/dist-git/dist_git_importer.py b/dist-git/dist-git/dist_git_importer.py index a39528b..058ae64 100755 --- a/dist-git/dist-git/dist_git_importer.py +++ b/dist-git/dist-git/dist_git_importer.py @@ -67,98 +67,75 @@ def _my_upload(repo_dir, reponame, filename, filehash): shutil.copyfile(source, destination) -def import_srpm(user, project, pkg, branch, filepath): - """ - Imports a source rpm file into local dist git. - Repository name is in the Copr Style: user/project/package - filepath is a srpm file locally downloaded somewhere - """ - # I need to use git via SSH because of gitolite as it manages - # permissions with it's hook that relies on gitolite console - # which is a default shell on SSH - gitbaseurl = "ssh://copr-dist-git@localhost/%(module)s" - tmp = tempfile.mkdtemp() - try: - repo_dir = os.path.join(tmp, pkg) - log.debug("repo_dir: {}".format(repo_dir)) - # use rpkg for importing the source rpm - commands = Commands(path=repo_dir, - lookaside="", - lookasidehash="md5", - lookaside_cgi="", - gitbaseurl=gitbaseurl, - anongiturl="", - branchre="", - kojiconfig="", - build_client="") - - # rpkg gets module_name as a basename of git url - # we use module_name as "username/projectname/packagename" - # basename is not working here - so I'm setting it manually - module = "{}/{}/{}".format(user, project, pkg) - commands.module_name = module - # rpkg calls upload.cgi script on the dist git server - # here, I just copy the source files manually with custom function - # I also add one parameter "repo_dir" to that function with this hack - commands.lookasidecache.upload = types.MethodType(_my_upload, repo_dir) - - log.debug("clone the pkg repository into tmp directory") - commands.clone(module, tmp, branch) - - log.debug("import the source rpm into git and save filenames of sources") - try: - uploadfiles = commands.import_srpm(filepath) - except Exception: - log.exception("Failed to import the source rpm: {}".format(filepath)) - raise PackageImportException - log.info("save the source files into lookaside cache") - commands.upload(uploadfiles, replace=True) - log.debug("git push") - message = "Import of {}".format(os.path.basename(filepath)) - try: - commands.commit(message) - commands.push() - log.debug("commit and push done") - except rpkgError: - pass - git_hash = commands.commithash - finally: - shutil.rmtree(tmp) - return git_hash - - -def pkg_name_evr(pkg): - """ - Queries a package for its name and evr (epoch:version-release) - """ - log.debug("Verifying packagage, getting name and version.") - cmd = ['rpm', '-qp', '--nosignature', '--qf', '%{NAME} %{EPOCH} %{VERSION} %{RELEASE}', pkg] - try: - proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - output, error = proc.communicate() - except OSError as e: - raise PackageQueryException(e) - if error: - raise PackageQueryException('Error querying srpm: %s' % error) +class SourceType: + SRPM_LINK = 1 + SRPM_UPLOAD = 2 - try: - name, epoch, version, release = output.split(" ") - except ValueError as e: - raise PackageQueryException(e) - # Epoch is an integer or '(none)' if not set - if epoch.isdigit(): - evr = "{}:{}-{}".format(epoch, version, release) - else: - evr = "{}-{}".format(version, release) +class ImportTask(object): + def __init__(self): + + self.task_id = None + self.user = None + self.project = None + self.branch = None + + self.source_type = None + self.source_json = None + self.source_data = None + + self.package_name = None + self.package_version = None + self.git_hash = None + + self.package_url = None + + @property + def reponame(self): + if any(x is None for x in [self.user, self.project, self.package_name]): + return None + else: + return "{}/{}/{}".format(self.user, self.project, self.package_name) + + @staticmethod + def from_dict(dict_data, opts): + task = ImportTask() - return name, evr + task.task_id = dict_data["task_id"] + task.user = dict_data["user"] + task.project = dict_data["project"] + task.branch = dict_data["branch"] + task.source_type = dict_data["source_type"] + task.source_json = dict_data["source_json"] + task.source_data = json.loads(dict_data["source_json"]) -class DistGitImporter(): + if task.source_type == SourceType.SRPM_LINK: + task.package_url = json.loads(task.source_json)["url"] + + elif task.source_type == SourceType.SRPM_UPLOAD: + json_tmp = task.source_data["tmp"] + json_pkg = task.source_data["pkg"] + task.package_url = "{}/tmp/{}/{}".format(opts.frontend_base_url, json_tmp, json_pkg) + + else: + raise PackageImportException("Got unknown source type: {}".format(task.source_type)) + + return task + + def get_dict_for_frontend(self): + return { + "task_id": self.task_id, + "pkg_name": self.package_name, + "pkg_version": self.package_version, + "repo_name": self.reponame, + "git_hash": self.git_hash + } + + +class DistGitImporter(object): def __init__(self, opts): self.opts = opts @@ -167,104 +144,137 @@ class DistGitImporter(): self.auth = ("user", self.opts.frontend_auth) self.headers = {"content-type": "application/json"} - def run(self): + self.tmp_root = None - tmp = tempfile.mkdtemp() - log.info("DistGitImported initialized") + def try_to_obtain_new_task(self): + log.debug("1. Try to get task data") try: - while True: - log.info("1. Try to get task data") - try: - # get the data - r = get(self.get_url) - - # take the first task - builds_list = r.json()["builds"] - if len(builds_list) == 0: - log.debug("No new tasks to process") - time.sleep(30) - continue - task = builds_list[0] - - # extract data from it - task_id = task["task_id"] - user = task["user"] - project = task["project"] - #package = task["package"] - branch = task["branch"] - source_type = task["source_type"] - source_json = task["source_json"] - - if source_type == 1: # SRPM link - package_url = json.loads(source_json)["url"] - elif source_type == 2: # SRPM upload - json_tmp = json.loads(source_json)["tmp"] - json_pkg = json.loads(source_json)["pkg"] - package_url = "{}/tmp/{}/{}".format( - self.opts.frontend_base_url, json_tmp, json_pkg) - else: - log.error("Got unknown source type: {}".format(source_type)) - raise Exception("Got unknown source type: {}".format(source_type)) - - except KeyboardInterrupt: - sys.exit(0) - - except Exception: - log.exception("Failed acquire new packages for import") - time.sleep(30) - continue - - log.info("2. Importing the package: {}".format(package_url)) - try: - log.debug("download the package") - fetched_srpm_path = os.path.join(tmp, os.path.basename(package_url)) - try: - urllib.urlretrieve(package_url, fetched_srpm_path) - except IOError: - raise PackageDownloadException - - # todo: check that obtained file is a REAL srpm - # todo query package name & version and ise real name instead of task["package"] - # if fetched file is not a proper srpm set error state - name, version = pkg_name_evr(fetched_srpm_path) - - reponame = "{}/{}/{}".format(user, project, name) - log.debug("make sure repos exist: {}".format(reponame)) - call(["/usr/share/dist-git/git_package.sh", reponame]) - call(["/usr/share/dist-git/git_branch.sh", branch, reponame]) - - log.debug("import it and delete the srpm") - git_hash = import_srpm(user, project, name, branch, fetched_srpm_path) - - log.debug("send a response - success") - - # refresh cgit - call(["/usr/share/dist-git/cgit_pkg_list.sh", self.opts.cgit_pkg_list_location]) - - # send a response - success - data = {"task_id": task_id, - "pkg_name": name, - "pkg_version": version, - "repo_name": reponame, - "git_hash": git_hash} - self.post_back(data) - - except (PackageImportException, PackageDownloadException, PackageQueryException): - log.info("send a response - failure during import of: {}".format(package_url)) - self.post_back_safe({"task_id": task_id, "error": "error"}) - - except Exception: - log.exception("Unexpected error during package import") - self.post_back_safe({"task_id": task_id, "error": "error"}) - - finally: - try: - os.remove(fetched_srpm_path) - except Exception: - log.exception("Failed to remove fetched srpm") + # get the data + r = get(self.get_url) + # take the first task + builds_list = r.json()["builds"] + if len(builds_list) == 0: + log.debug("No new tasks to process") + return + return ImportTask.from_dict(builds_list[0], self.opts) + except Exception: + log.exception("Failed acquire new packages for import") + return + + @staticmethod + def fetch_srpm(task, fetched_srpm_path): + log.debug("download the package") + + try: + urllib.urlretrieve(task.package_url, fetched_srpm_path) + except IOError: + raise PackageDownloadException() + return fetched_srpm_path + + @staticmethod + def git_import_srpm(task, filepath): + """ + Imports a source rpm file into local dist git. + Repository name is in the Copr Style: user/project/package + filepath is a srpm file locally downloaded somewhere + + :type task: ImportTask + """ + log.debug("importing it and delete the srpm") + # I need to use git via SSH because of gitolite as it manages + # permissions with it's hook that relies on gitolite console + # which is a default shell on SSH + git_bas_eurl = "ssh://copr-dist-git@localhost/%(module)s" + tmp = tempfile.mkdtemp() + try: + repo_dir = os.path.join(tmp, task.package_name) + log.debug("repo_dir: {}".format(repo_dir)) + # use rpkg for importing the source rpm + commands = Commands(path=repo_dir, + lookaside="", + lookasidehash="md5", + lookaside_cgi="", + gitbaseurl=git_bas_eurl, + anongiturl="", + branchre="", + kojiconfig="", + build_client="") + + # rpkg gets module_name as a basename of git url + # we use module_name as "username/projectname/package_name" + # basename is not working here - so I'm setting it manually + module = "{}/{}/{}".format(task.user, task.project, task.package_name) + commands.module_name = module + # rpkg calls upload.cgi script on the dist git server + # here, I just copy the source files manually with custom function + # I also add one parameter "repo_dir" to that function with this hack + commands.lookasidecache.upload = types.MethodType(_my_upload, repo_dir) + + log.debug("clone the pkg repository into tmp directory") + commands.clone(module, tmp, task.branch) + + log.debug("import the source rpm into git and save filenames of sources") + try: + upload_files = commands.import_srpm(filepath) + except Exception: + log.exception("Failed to import the source rpm: {}".format(filepath)) + raise PackageImportException() + + log.info("save the source files into lookaside cache") + commands.upload(upload_files, replace=True) + + log.debug("git push") + message = "Import of {}".format(os.path.basename(filepath)) + try: + commands.commit(message) + commands.push() + log.debug("commit and push done") + except rpkgError: + pass + git_hash = commands.commithash finally: shutil.rmtree(tmp) + return git_hash + + @staticmethod + def pkg_name_evr(pkg): + """ + Queries a package for its name and evr (epoch:version-release) + """ + log.debug("Verifying packagage, getting name and version.") + cmd = ['rpm', '-qp', '--nosignature', '--qf', '%{NAME} %{EPOCH} %{VERSION} %{RELEASE}', pkg] + try: + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, error = proc.communicate() + except OSError as e: + raise PackageQueryException(e) + if error: + raise PackageQueryException('Error querying srpm: %s' % error) + + try: + name, epoch, version, release = output.split(" ") + except ValueError as e: + raise PackageQueryException(e) + + # Epoch is an integer or '(none)' if not set + if epoch.isdigit(): + evr = "{}:{}-{}".format(epoch, version, release) + else: + evr = "{}-{}".format(version, release) + + return name, evr + + def after_git_import(self): + log.debug("refreshing cgit listing") + call(["/usr/share/dist-git/cgit_pkg_list.sh", self.opts.cgit_pkg_list_location]) + + @staticmethod + def before_git_import(task): + log.debug("make sure repos exist: {}".format(task.reponame)) + call(["/usr/share/dist-git/git_package.sh", task.reponame]) + call(["/usr/share/dist-git/git_branch.sh", task.branch, task.reponame]) def post_back(self, data_dict): """ @@ -281,6 +291,46 @@ class DistGitImporter(): except Exception: log.exception("Failed to post back to frontend : {}".format(data_dict)) + def do_import(self, task): + """ + :type task: ImportTask + """ + log.debug("2. Importing the package: {}".format(task.package_url)) + tmp_root = tempfile.mkdtemp() + fetched_srpm_path = os.path.join(tmp_root, "package.src.rpm") + + try: + self.fetch_srpm(task, fetched_srpm_path) + task.package_name, task.package_version = self.pkg_name_evr(fetched_srpm_path) + + self.before_git_import(task) + task.git_hash = self.git_import_srpm(task, fetched_srpm_path) + self.after_git_import() + + log.debug("sending a response - success") + self.post_back(task.get_dict_for_frontend()) + + except (PackageImportException, PackageDownloadException, PackageQueryException): + log.info("send a response - failure during import of: {}".format(task.package_url)) + self.post_back_safe({"task_id": task.task_id, "error": "error"}) + + except Exception: + log.exception("Unexpected error during package import") + self.post_back_safe({"task_id": task.task_id, "error": "error"}) + + finally: + shutil.rmtree(tmp_root, ignore_errors=True) + + def run(self): + log.info("DistGitImported initialized") + + while True: + mb_task = self.try_to_obtain_new_task() + if mb_task is None: + time.sleep(self.opts.sleep_time) + else: + self.do_import(mb_task) + def main(): config_file = None @@ -291,7 +341,7 @@ def main(): config_reader = DistGitConfigReader(config_file) try: opts = config_reader.read() - except Exception as e: + except Exception: print("Failed to read config file, used file location: `{}`" .format(config_file)) # sys.exit(1) diff --git a/dist-git/dist-git/helpers.py b/dist-git/dist-git/helpers.py index bb7233d..eb90f4a 100644 --- a/dist-git/dist-git/helpers.py +++ b/dist-git/dist-git/helpers.py @@ -72,6 +72,10 @@ class DistGitConfigReader(object): cp, "dist-git", "log_dir", "/var/log/copr-dist-git" ) + opts.sleep_time = _get_conf( + cp, "dist-git", "sleep_time", 15, mode="int" + ) + opts.cgit_pkg_list_location = _get_conf( cp, "cgit_pkg_list_location", "log_dir", "/var/lib/copr-dist-git/cgit_pkg_list" )

1 0

[copr] master: [frontend] add importing queue (838d31d)
by asamalik＠fedoraproject.org 30 Jul '15

30 Jul '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 838d31deee083bd7de11cf166e53ffd89245912b Author: Adam Samalik <asamalik(a)redhat.com> Date: Thu Jul 30 16:55:22 2015 +0200 [frontend] add importing queue >--------------------------------------------------------------- .../coprs_frontend/coprs/templates/coprs/show.html | 4 ++++ .../coprs_frontend/coprs/templates/status.html | 3 +++ .../status/{waiting.html => importing.html} | 10 +++------- .../coprs/views/coprs_ns/coprs_general.py | 6 ++++++ .../coprs/views/status_ns/status_general.py | 9 +++++++++ 5 files changed, 25 insertions(+), 7 deletions(-) diff --git a/frontend/coprs_frontend/coprs/templates/coprs/show.html b/frontend/coprs_frontend/coprs/templates/coprs/show.html index fc8596e..350173a 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/show.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/show.html @@ -61,6 +61,10 @@ <h3 class="panel-title"> Task Queue </h3> </div> <div class="list-group"> + <a href="{{url_for('status_ns.importing')}}" class="list-group-item"> + <span class="badge">{{importing_tasks}}</span> + Importing + </a> <a href="{{url_for('status_ns.waiting')}}" class="list-group-item"> <span class="badge">{{waiting_tasks}}</span> Waiting diff --git a/frontend/coprs_frontend/coprs/templates/status.html b/frontend/coprs_frontend/coprs/templates/status.html index cabb9f1..c477e4b 100644 --- a/frontend/coprs_frontend/coprs/templates/status.html +++ b/frontend/coprs_frontend/coprs/templates/status.html @@ -14,6 +14,9 @@ </ol> <h1> Task queue </h1> <ul class="nav nav-tabs"> + <li class="{% block importing_selected %}{% endblock %}"> + <a href="{{ url_for('status_ns.importing') }}">Importing</a> + </li> <li class="{% block waiting_selected %}{% endblock %}"> <a href="{{ url_for('status_ns.waiting') }}">Waiting</a> </li> diff --git a/frontend/coprs_frontend/coprs/templates/status/waiting.html b/frontend/coprs_frontend/coprs/templates/status/importing.html similarity index 84% copy from frontend/coprs_frontend/coprs/templates/status/waiting.html copy to frontend/coprs_frontend/coprs/templates/status/importing.html index e1d34e5..c69c2bc 100644 --- a/frontend/coprs_frontend/coprs/templates/status/waiting.html +++ b/frontend/coprs_frontend/coprs/templates/status/importing.html @@ -1,8 +1,8 @@ {% extends "status.html" %} -{% block waiting_selected %}active{% endblock %} +{% block importing_selected %}active{% endblock %} {%block status_breadcrumb %} <li class="active"> - Waiting + Importing </li> {%endblock%} {% block status_body %} @@ -13,7 +13,7 @@ {%else%} {{number}} tasks {%endif%} - are waiting + are being imported </h2> <table class="table table-striped table-bordered"> @@ -30,11 +30,7 @@ {% for task in tasks %} <tr> <td> - {% if task.state == "starting" %} - <span class="build-starting">starting</span> - {% else %} {{ task.build.submitted_on | time_ago }} - {% endif %} </td> <td><a href="{{ url_for('coprs_ns.copr_detail', username = task.build.copr.owner.name, coprname = task.build.copr.name) }}">{{task.build.copr.owner.name}}/{{task.build.copr.name}}</a></td> <td><a href="{{ url_for('coprs_ns.copr_build', username = task.build.copr.owner.name, coprname = task.build.copr.name, build_id = task.build_id) }}">{{task.build_id}}</a></td> diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py index 8d8e906..26dcf5e 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py @@ -44,12 +44,15 @@ def coprs_show(page=1): waiting_tasks = len(list(builds_logic.BuildsLogic.get_build_task_queue())) running_tasks = len(list(builds_logic.BuildsLogic .get_build_tasks(helpers.StatusEnum("running")))) + importing_tasks = len(list(builds_logic.BuildsLogic + .get_build_tasks(helpers.StatusEnum("importing")))) return flask.render_template("coprs/show/all.html", coprs=coprs, paginator=paginator, waiting_tasks=waiting_tasks, running_tasks=running_tasks, + importing_tasks=importing_tasks, users_builds=users_builds) @@ -73,6 +76,8 @@ def coprs_by_owner(username=None, page=1): waiting_tasks = len(list(builds_logic.BuildsLogic.get_build_task_queue())) running_tasks = len(list(builds_logic.BuildsLogic .get_build_tasks(helpers.StatusEnum("running")))) + importing_tasks = len(list(builds_logic.BuildsLogic + .get_build_tasks(helpers.StatusEnum("importing")))) return flask.render_template("coprs/show/user.html", user=user, @@ -80,6 +85,7 @@ def coprs_by_owner(username=None, page=1): paginator=paginator, waiting_tasks=waiting_tasks, running_tasks=running_tasks, + importing_tasks=importing_tasks, users_builds=users_builds) diff --git a/frontend/coprs_frontend/coprs/views/status_ns/status_general.py b/frontend/coprs_frontend/coprs/views/status_ns/status_general.py index add0359..58301d0 100644 --- a/frontend/coprs_frontend/coprs/views/status_ns/status_general.py +++ b/frontend/coprs_frontend/coprs/views/status_ns/status_general.py @@ -21,3 +21,12 @@ def running(): return flask.render_template("status/running.html", number=len(list(tasks)), tasks=tasks) + + +(a)status_ns.route("/importing/") +def importing(): + tasks = builds_logic.BuildsLogic.get_build_tasks( + helpers.StatusEnum("importing")).limit(200) + return flask.render_template("status/importing.html", + number=len(list(tasks)), + tasks=tasks)

1 0

[copr] master: [dist-git] mark build as failed for any error during import (6024b6e)
by vgologuz＠fedoraproject.org 30 Jul '15

30 Jul '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 6024b6eeb80c0ec924d11c4409f779ada525f5b4 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Thu Jul 30 14:57:37 2015 +0200 [dist-git] mark build as failed for any error during import >--------------------------------------------------------------- dist-git/dist-git/dist_git_importer.py | 39 +++++++++++++++++++++---------- 1 files changed, 26 insertions(+), 13 deletions(-) diff --git a/dist-git/dist-git/dist_git_importer.py b/dist-git/dist-git/dist_git_importer.py index 2503f79..a39528b 100755 --- a/dist-git/dist-git/dist_git_importer.py +++ b/dist-git/dist-git/dist_git_importer.py @@ -162,11 +162,12 @@ class DistGitImporter(): def __init__(self, opts): self.opts = opts + self.get_url = "{}/backend/importing/".format(self.opts.frontend_base_url) + self.upload_url = "{}/backend/import-completed/".format(self.opts.frontend_base_url) + self.auth = ("user", self.opts.frontend_auth) + self.headers = {"content-type": "application/json"} + def run(self): - get_url = "{}/backend/importing/".format(self.opts.frontend_base_url) - upload_url = "{}/backend/import-completed/".format(self.opts.frontend_base_url) - auth = ("user", self.opts.frontend_auth) - headers = {"content-type": "application/json"} tmp = tempfile.mkdtemp() log.info("DistGitImported initialized") @@ -175,7 +176,7 @@ class DistGitImporter(): log.info("1. Try to get task data") try: # get the data - r = get(get_url) + r = get(self.get_url) # take the first task builds_list = r.json()["builds"] @@ -246,18 +247,15 @@ class DistGitImporter(): "pkg_version": version, "repo_name": reponame, "git_hash": git_hash} - post(upload_url, auth=auth, data=json.dumps(data), headers=headers) + self.post_back(data) - except (PackageImportException, PackageDownloadException): + except (PackageImportException, PackageDownloadException, PackageQueryException): log.info("send a response - failure during import of: {}".format(package_url)) - data = {"task_id": task_id, "error": "error"} - try: - post(upload_url, auth=auth, data=json.dumps(data), headers=headers) - except Exception: - log.exception("Failed to inform frontend about failed import: {}".format(data)) + self.post_back_safe({"task_id": task_id, "error": "error"}) except Exception: - log.exception("error during package import") + log.exception("Unexpected error during package import") + self.post_back_safe({"task_id": task_id, "error": "error"}) finally: try: @@ -268,6 +266,21 @@ class DistGitImporter(): finally: shutil.rmtree(tmp) + def post_back(self, data_dict): + """ + Could raise error related to networkd connection + """ + post(self.upload_url, auth=self.auth, data=json.dumps(data_dict), headers=self.headers) + + def post_back_safe(self, data_dict): + """ + Ignores any error + """ + try: + self.post_back(data_dict) + except Exception: + log.exception("Failed to post back to frontend : {}".format(data_dict)) + def main(): config_file = None

1 0

[copr] master: [backend] fix BuildJob.results_dir; eliminated MockRemote.pkg_dest_path (b0063eb)
by vgologuz＠fedoraproject.org 30 Jul '15

30 Jul '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit b0063eb1bd0aabaaf2112bd8f9034c2f1f0ac64b Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Thu Jul 30 12:38:59 2015 +0200 [backend] fix BuildJob.results_dir; eliminated MockRemote.pkg_dest_path >--------------------------------------------------------------- backend/backend/daemons/dispatcher.py | 2 ++ backend/backend/job.py | 6 +++--- backend/backend/mockremote/__init__.py | 11 +++-------- backend/tests/mockremote/test_mockremote.py | 8 ++++---- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/backend/backend/daemons/dispatcher.py b/backend/backend/daemons/dispatcher.py index 58076c7..70953e8 100644 --- a/backend/backend/daemons/dispatcher.py +++ b/backend/backend/daemons/dispatcher.py @@ -342,6 +342,8 @@ class Worker(multiprocessing.Process): def copy_mock_logs(self, job): if not os.path.isdir(job.results_dir): + self.log.info("Job results dir doesn't exists, couldn't copy main log; path: {}" + .format(job.results_dir)) return log_names = [(job.chroot_log_name, "mockchain.log.gz"), diff --git a/backend/backend/job.py b/backend/backend/job.py index 3a62678..a30c468 100644 --- a/backend/backend/job.py +++ b/backend/backend/job.py @@ -87,7 +87,7 @@ class BuildJob(object): @property def results_dir(self): - return os.path.join(self.chroot_dir, self.package_name) + return os.path.join(self.chroot_dir, self.target_dir_name) @property def target_dir_name(self): @@ -153,5 +153,5 @@ class BuildJob(object): return str(self.__unicode__()) def __unicode__(self): - return u"BuildJob<id: {build_id}, owner: {project_owner}, " \ - u"project: {project_name}, git branch: {git_branch}, git_hash: {git_hash} >".format(**self.__dict__) + return u"BuildJob<id: {build_id}, owner: {project_owner}, project: {project_name}, " \ + u"git branch: {git_branch}, git_hash: {git_hash}, status: {status} >".format(**self.__dict__) diff --git a/backend/backend/mockremote/__init__.py b/backend/backend/mockremote/__init__.py index 0b2e0a4..c96190d 100755 --- a/backend/backend/mockremote/__init__.py +++ b/backend/backend/mockremote/__init__.py @@ -172,11 +172,6 @@ class MockRemote(object): def pkg(self): return self.job.pkg - @property - def pkg_dest_path(self): - return os.path.normpath("{}/{}/{}".format( - self.job.destdir, self.job.chroot, self.job.target_dir_name)) - def add_pubkey(self): """ Adds pubkey.gpg with public key to ``chroot_dir`` @@ -266,7 +261,7 @@ class MockRemote(object): self.do_createrepo() def prepare_build_dir(self): - p_path = self.pkg_dest_path + p_path = self.job.results_dir # if it's marked as fail, nuke the failure and try to rebuild it if os.path.exists(os.path.join(p_path, "fail")): os.unlink(os.path.join(p_path, "fail")) @@ -316,7 +311,7 @@ class MockRemote(object): raise MockRemoteError("Error occurred during build {}: {}" .format(self.job, error)) finally: - self.builder.download(self.pkg_dest_path) + self.builder.download(self.job.results_dir) # self.add_log_symlinks() # todo: add config option, need this for nginx self.log.info("End Build: {0}".format(self.job)) @@ -329,7 +324,7 @@ class MockRemote(object): into the directory with downloaded files. """ - info_file_path = os.path.join(self.pkg_dest_path, "build.info") + info_file_path = os.path.join(self.job.results_dir, "build.info") self.log.info("marking build dir with build_id, ") try: with open(info_file_path, 'w') as info_file: diff --git a/backend/tests/mockremote/test_mockremote.py b/backend/tests/mockremote/test_mockremote.py index d1b17a8..d9986b8 100644 --- a/backend/tests/mockremote/test_mockremote.py +++ b/backend/tests/mockremote/test_mockremote.py @@ -176,7 +176,7 @@ class TestMockRemote(object): assert self.mr.do_createrepo.called def test_prepare_build_dir_erase_fail_file(self, f_mock_remote): - target_dir = self.mr.pkg_dest_path + target_dir = self.mr.job.results_dir os.makedirs(target_dir) fail_path = os.path.join(target_dir, "fail") with open(fail_path, "w") as handle: @@ -187,7 +187,7 @@ class TestMockRemote(object): assert os.path.exists(fail_path) is False def test_prepare_build_dir_erase_success_file(self, f_mock_remote): - target_dir = self.mr.pkg_dest_path + target_dir = self.mr.job.results_dir os.makedirs(target_dir) fail_path = os.path.join(target_dir, "success") with open(fail_path, "w") as handle: @@ -200,7 +200,7 @@ class TestMockRemote(object): def test_prepare_build_dir_creates_dirs(self, f_mock_remote): self.mr.prepare_build_dir() - assert os.path.exists(self.mr.pkg_dest_path) + assert os.path.exists(self.mr.job.results_dir) def test_build_pkg_and_process_results(self, f_mock_remote): self.mr.on_success_build = MagicMock() @@ -246,7 +246,7 @@ class TestMockRemote(object): def test_mark_dir_with_build_id(self, f_mock_remote): # TODO: create real test - target_dir = self.mr.pkg_dest_path + target_dir = self.mr.job.results_dir os.makedirs(target_dir) info_file_path = os.path.join(target_dir, "build.info")

1 0

[copr] master: [dist-git] don't break on the post failure to frontend (b91dafc)
by vgologuz＠fedoraproject.org 30 Jul '15

30 Jul '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit b91dafc383c89687a2983981eebe9f9d27517e47 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Thu Jul 30 12:18:37 2015 +0200 [dist-git] don't break on the post failure to frontend >--------------------------------------------------------------- dist-git/dist-git/dist_git_importer.py | 5 ++++- 1 files changed, 4 insertions(+), 1 deletions(-) diff --git a/dist-git/dist-git/dist_git_importer.py b/dist-git/dist-git/dist_git_importer.py index ef75d4d..2503f79 100755 --- a/dist-git/dist-git/dist_git_importer.py +++ b/dist-git/dist-git/dist_git_importer.py @@ -251,7 +251,10 @@ class DistGitImporter(): except (PackageImportException, PackageDownloadException): log.info("send a response - failure during import of: {}".format(package_url)) data = {"task_id": task_id, "error": "error"} - post(upload_url, auth=auth, data=json.dumps(data), headers=headers) + try: + post(upload_url, auth=auth, data=json.dumps(data), headers=headers) + except Exception: + log.exception("Failed to inform frontend about failed import: {}".format(data)) except Exception: log.exception("error during package import")

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

copr-commits July 2015