copr-commits February 2015

copr-commits@lists.fedorahosted.org

3 participants
40 discussions

[copr] master: [frontend][backend] [rhbz:#1185959] - RFE: Present statistics about project popularity. A few more counters for downloads from backend's result directory. (69f6553)
by vgologuz＠fedoraproject.org 25 Feb '15

25 Feb '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 69f65530119bbc6577dd298c71f2ef164aa95f7a Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Wed Feb 25 18:17:49 2015 +0100 [frontend][backend] [rhbz:#1185959] - RFE: Present statistics about project popularity. A few more counters for downloads from backend's result directory. >--------------------------------------------------------------- backend/conf/logstash/copr_backend.conf | 60 ++++++++++ backend/conf/logstash/lighttpd.pattern | 1 + backend/copr-backend.spec | 12 ++ frontend/conf/logstash.conf | 37 ++++--- frontend/copr-frontend.spec | 5 + frontend/coprs_frontend/coprs/__init__.py | 10 +- frontend/coprs_frontend/coprs/helpers.py | 43 +++++++- frontend/coprs_frontend/coprs/logic/stat_logic.py | 44 ++++++-- frontend/coprs_frontend/coprs/rmodels.py | 117 ++++++++++++++++++++ .../coprs/templates/coprs/detail/overview.html | 10 ++- .../coprs/views/coprs_ns/coprs_general.py | 21 +++- frontend/coprs_frontend/coprs/views/misc.py | 2 +- .../coprs/views/stats_ns/stats_receiver.py | 21 +--- frontend/coprs_frontend/tests/test_rmodels.py | 53 +++++++++ 14 files changed, 387 insertions(+), 49 deletions(-) diff --git a/backend/conf/logstash/copr_backend.conf b/backend/conf/logstash/copr_backend.conf new file mode 100644 index 0000000..e6a96e2 --- /dev/null +++ b/backend/conf/logstash/copr_backend.conf @@ -0,0 +1,60 @@ +input { + #file { + # path => "/var/log/copr/backend.log" + # type => "copr.backend.main" + #} + file { + path => "/var/log/lighttpd/access.log" + type => "lighttpd-access" + } +} + +filter { + mutate { + add_tag => [ "backend" ] + } + if [type] == 'lighttpd-access' { + grok { + patterns_dir => "/usr/share/logstash/patterns" + pattern => "%{LIGHTTPD}" + } + date { + match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ] + } + if "repodata/repomd.xml" in [request] and "devel/repodata/repomd.xml" not in [request] { + mutate { add_tag => "repomdxml" } + mutate { add_tag => "publish_stat" } + grok { + match => ["request", "/results/%{USERNAME:copr_user}/%{USERNAME:copr_project_name}/%{USERNAME:copr_chroot}/repodata/repomd.xml"] + } + } + if [request] =~ "rpm$" { + mutate { add_tag => "rpm" } + mutate { add_tag => "publish_stat" } + grok { + match => ["request", "/results/%{USERNAME:copr_user}/%{USERNAME:copr_project_name}/%{USERNAME:copr_chroot}/%{USERNAME:copr_build_dir}/%{USERNAME:copr_rpm}"] + } + } + } +} + +output { + if "publish_stat" in [tags] { + http { + url => "http://copr-fe-dev.cloud.fedoraproject.org/stats_rcv/from_logstash" + format => "json" + http_method => "post" + } + } + + file { + path => "/var/log/logstash/all.log" + codec => "rubydebug" + } + + file { + path => "/tmp/logstashall.log" + codec => "rubydebug" + } + +} diff --git a/backend/conf/logstash/lighttpd.pattern b/backend/conf/logstash/lighttpd.pattern new file mode 100644 index 0000000..7dce2a0 --- /dev/null +++ b/backend/conf/logstash/lighttpd.pattern @@ -0,0 +1 @@ +LIGHTTPD %{IPORHOST:clientip} %{IPORHOST:httphost} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{URIPATHPARAM:request}(?: HTTP/%{NUMBER:httpversion})|-)" %{NUMBER:response} (?:%{NUMBER:bytes}|-) "(?:%{URI:referrer}|-)" %{QS:agent} diff --git a/backend/copr-backend.spec b/backend/copr-backend.spec index ec3dc85..bb39b63 100644 --- a/backend/copr-backend.spec +++ b/backend/copr-backend.spec @@ -72,6 +72,7 @@ Requires: fedmsg Requires: gawk Requires: crontabs Requires: python-paramiko +Requires: logstash Requires(post): systemd Requires(preun): systemd @@ -145,6 +146,13 @@ touch %{buildroot}%{_var}/run/copr-backend/copr-be.pid install -m 0644 copr-backend.service %{buildroot}/%{_unitdir}/ install -m 0644 conf/copr.sudoers.d %{buildroot}%{_sysconfdir}/sudoers.d/copr + +install -d %{buildroot}%{_sysconfdir}/logstash.d +cp -a conf/logstash/copr_backend.conf %{buildroot}%{_sysconfdir}/logstash.d/copr_backend.conf +install -d %{buildroot}%{_datadir}/logstash/patterns/ +cp -a conf/logstash/lighttpd.pattern %{buildroot}%{_datadir}/logstash/patterns/lighttpd.pattern + + #doc cp -a documentation/python-doc %{buildroot}%{_pkgdocdir}/ cp -a conf/playbooks %{buildroot}%{_pkgdocdir}/ @@ -163,6 +171,7 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %post %systemd_post copr-backend.service +%systemd_post logstash.service %preun %systemd_preun copr-backend.service @@ -197,6 +206,9 @@ useradd -r -g copr -G lighttpd -s /bin/bash -c "COPR user" copr %{_bindir}/* %config(noreplace) %{_sysconfdir}/cron.daily/copr-backend +%config(noreplace) %{_sysconfdir}/logstash.d/copr_backend.conf +%{_datadir}/logstash/patterns/lighttpd.pattern + %config(noreplace) %attr(0600, root, root) %{_sysconfdir}/sudoers.d/copr diff --git a/frontend/conf/logstash.conf b/frontend/conf/logstash.conf index 82aa01d..36cb8cd 100644 --- a/frontend/conf/logstash.conf +++ b/frontend/conf/logstash.conf @@ -10,32 +10,39 @@ input { } filter { - grok { - type => "httpd-access" - pattern => "%{COMBINEDAPACHELOG}" + mutate { add_tag => "frontend" } + if [type] == "httpd-access" { + if "POST /stats_rcv/from_logstash HTTP" in [message] { + drop{} + } + grok { + type => "httpd-access" + pattern => "%{COMBINEDAPACHELOG}" + } + + if [request] =~ ".repo$" { + mutate { add_tag => "repo_dl" } + grok { + match => ["request", "/coprs/%{USERNAME:copr_user}/%{USERNAME:copr_project_name}/repo/%{USERNAME:copr_name_release}/%{USERNAME:copr_repo_file}"] + } + } + } } output { if [type] == "httpd-access" { - if ".repo" in [request] and "stats_rcv" not in [request] { + if "repo_dl" in [tags] { http { url => "http://127.0.0.1/stats_rcv/from_logstash" format => "json" http_method => "post" - message => "foobar" } } - #file { - # path => "/var/log/logstash/web.log" - # codec => rubydebug {} - # - #} } - # file { - # path => "/var/log/logstash/all.log" - # codec => rubydebug {} - # } - + file { + path => "/var/log/logstash/all.log" + codec => rubydebug {} + } } diff --git a/frontend/copr-frontend.spec b/frontend/copr-frontend.spec index bf83869..4439e30 100644 --- a/frontend/copr-frontend.spec +++ b/frontend/copr-frontend.spec @@ -59,6 +59,9 @@ Requires: python-mock Requires: python-decorator Requires: yum Requires: logstash +Requires: redis +Requires: python-redis +Requires: python-dateutil %if 0%{?rhel} < 7 && 0%{?rhel} > 0 BuildRequires: python-argparse %endif @@ -72,6 +75,8 @@ BuildRequires: python-flask-whooshee BuildRequires: python-pylibravatar BuildRequires: python-flask-wtf BuildRequires: python-netaddr +BuildRequires: python-redis +BuildRequires: python-dateutil BuildRequires: pytest BuildRequires: yum BuildRequires: python-flexmock diff --git a/frontend/coprs_frontend/coprs/__init__.py b/frontend/coprs_frontend/coprs/__init__.py index 492dec3..e86bb64 100644 --- a/frontend/coprs_frontend/coprs/__init__.py +++ b/frontend/coprs_frontend/coprs/__init__.py @@ -3,9 +3,9 @@ from __future__ import with_statement import os import flask -from flask.ext.sqlalchemy import SQLAlchemy -from flask.ext.openid import OpenID -from flask.ext.whooshee import Whooshee +from flask_sqlalchemy import SQLAlchemy +from flask_openid import OpenID +from flask_whooshee import Whooshee app = flask.Flask(__name__) @@ -25,12 +25,16 @@ oid = OpenID(app, app.config["OPENID_STORE"], safe_roots=[]) db = SQLAlchemy(app) whooshee = Whooshee(app) + import coprs.filters import coprs.log from coprs.log import setup_log import coprs.models import coprs.whoosheers +from coprs.helpers import RedisConnectionProvider +rcp = RedisConnectionProvider(config=app.config) + from coprs.views import admin_ns from coprs.views.admin_ns import admin_general from coprs.views import api_ns diff --git a/frontend/coprs_frontend/coprs/helpers.py b/frontend/coprs_frontend/coprs/helpers.py index f4f8efa..811a481 100644 --- a/frontend/coprs_frontend/coprs/helpers.py +++ b/frontend/coprs_frontend/coprs/helpers.py @@ -4,6 +4,10 @@ import string import urlparse import flask +from dateutil import parser as dt_parser + +from redis import StrictRedis + from coprs import constants from coprs import app @@ -21,7 +25,10 @@ def generate_api_token(size=30): return ''.join(random.choice(string.ascii_lowercase) for x in range(size)) -REPO_DL_STAT_FMT = "{user}@{copr}:{name_release}" +REPO_DL_STAT_FMT = "repo_dl_stat::{copr_user}@{copr_project_name}:{copr_name_release}" +CHROOT_REPO_MD_DL_STAT_FMT = "chroot_repo_metadata_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" +CHROOT_RPMS_DL_STAT_FMT = "chroot_rpms_dl_stat:hset::{copr_user}@{copr_project_name}:{copr_chroot}" +PROJECT_RPMS_DL_STAT_FMT = "project_rpms_dl_stat:hset::{copr_user}@{copr_project_name}" class CounterStatType(object): @@ -268,3 +275,37 @@ class Serializer(object): @property def serializable_attributes(self): return map(lambda x: x.name, self.__table__.columns) + + +class RedisConnectionProvider(object): + def __init__(self, config): + self.host = config.get("redis_host", "127.0.0.1") + self.port = int(config.get("redis_port", "6379")) + + def get_connection(self): + return StrictRedis(host=self.host, port=self.port) + + +def get_redis_connection(): + """ + Creates connection to redis, now we use default instance at localhost, no config needed + """ + return StrictRedis() + + +def dt_to_unixtime(dt): + """ + Converts datetime to unixtime + :param dt: DateTime instance + :rtype: float + """ + return float(dt.strftime('%s')) + + +def string_dt_to_unixtime(dt_string): + """ + Converts datetime to unixtime from string + :param dt_string: datetime string + :rtype: str + """ + return dt_to_unixtime(dt_parser.parse(dt_string)) diff --git a/frontend/coprs_frontend/coprs/logic/stat_logic.py b/frontend/coprs_frontend/coprs/logic/stat_logic.py index c1b2e0f..a2e4062 100644 --- a/frontend/coprs_frontend/coprs/logic/stat_logic.py +++ b/frontend/coprs_frontend/coprs/logic/stat_logic.py @@ -3,6 +3,8 @@ import json import os import pprint import time + + from sqlalchemy import or_ from sqlalchemy import and_ @@ -13,11 +15,11 @@ from coprs import db from coprs import exceptions from coprs.models import CounterStat from coprs import helpers -from coprs.helpers import REPO_DL_STAT_FMT +from coprs.helpers import REPO_DL_STAT_FMT, CHROOT_REPO_MD_DL_STAT_FMT, dt_to_unixtime, string_dt_to_unixtime, \ + CHROOT_RPMS_DL_STAT_FMT, PROJECT_RPMS_DL_STAT_FMT from coprs import signals from coprs.helpers import CounterStatType - - +from coprs.rmodels import TimedStatEvents class CounterStatLogic(object): @@ -65,9 +67,9 @@ class CounterStatLogic(object): chroot_by_stat_name = {} for chroot in copr.active_chroots: kwargs = { - "user": copr.owner.name, - "copr": copr.name, - "name_release": chroot.name_release + "copr_user": copr.owner.name, + "copr_project_name": copr.name, + "copr_name_release": chroot.name_release } chroot_by_stat_name[REPO_DL_STAT_FMT.format(**kwargs)] = chroot.name_release @@ -83,6 +85,30 @@ class CounterStatLogic(object): return repo_dl_stats - - - +def handle_logstash(rc, ls_data): + """ + :param rc: connection to redis + :type rc: StrictRedis + + :param ls_data: log stash record + :type ls_data: dict + """ + dt_unixtime = string_dt_to_unixtime(ls_data["@timestamp"]) + app.logger.debug("got ls_data: {}".format(ls_data)) + + if "tags" in ls_data: + tags = set(ls_data["tags"]) + if "frontend" in tags and "repo_dl": + name = REPO_DL_STAT_FMT.format(**ls_data) + CounterStatLogic.incr(name=name, counter_type=CounterStatType.REPO_DL) + db.session.commit() + + if "backend" in tags and "repomdxml" in tags: + key = CHROOT_REPO_MD_DL_STAT_FMT.format(**ls_data) + TimedStatEvents.add_event(rc, key, timestamp=dt_unixtime) + + if "backend" in tags and "rpm" in tags: + key_chroot = CHROOT_RPMS_DL_STAT_FMT.format(**ls_data) + key_project = PROJECT_RPMS_DL_STAT_FMT.format(**ls_data) + TimedStatEvents.add_event(rc, key_chroot, timestamp=dt_unixtime) + TimedStatEvents.add_event(rc, key_project, timestamp=dt_unixtime) diff --git a/frontend/coprs_frontend/coprs/rmodels.py b/frontend/coprs_frontend/coprs/rmodels.py new file mode 100644 index 0000000..43734ae --- /dev/null +++ b/frontend/coprs_frontend/coprs/rmodels.py @@ -0,0 +1,117 @@ +# coding: utf-8 + +""" Models to redis entities """ +import time +from math import ceil +from datetime import datetime, timedelta +from redis import StrictRedis + + +class GenericRedisModel(object): + _KEY_BASE = "copr:generic" + + @classmethod + def _get_key(cls, name, prefix=None): + if prefix: + return "{}:{}:{}".format(prefix, cls._KEY_BASE, name) + else: + return "{}:{}".format(cls._KEY_BASE, name) + + +class TimedStatEvents(GenericRedisModel): + """ + Wraps hset structure, where: + **key** - name of event, fix prefix specifying events type + **member** - bucket representing one day + **score** - events count + """ + _KEY_BASE = "copr:tse" + + @staticmethod + def timestamp_to_day(ut): + """ + :param ut: unix timestamp + :type ut: float + :return: name for the day bucket + """ + td = timedelta(days=1).total_seconds() + return int(ceil(ut / td)) + + @classmethod + def gen_days_interval(cls, min_ts, max_ts): + """ + Generate list of days bucket names which contains + all events between `min_ts` and `max_ts` + :param min_ts: min unix timestamp + :param max_ts: max unix timestamp + :rtype: list + """ + start_ut = cls.timestamp_to_day(min_ts) + end_ut = cls.timestamp_to_day(max_ts) + + return range(start_ut, end_ut + 1) + + @classmethod + def add_event(cls, rconnect, name, timestamp, count=1, prefix=None): + """ + Stoted new event to redist + :param rconnect: Connection to a redis + :type rconnect: StrictRedis + :param name: statistics name + :param timestamp: timestamp of event + :param count: number of events, default=1 + :param prefix: prefix for statistics, default is None + """ + count = int(count) + ut_day = cls.timestamp_to_day(timestamp) + + key = cls._get_key(name, prefix) + + rconnect.hincrby(key, ut_day, count) + + @classmethod + def get_count(cls, rconnect, name, day_min=None, prefix=None, day_max=None): + """ + Count total event occurency between day_min and day_max + :param rconnect: Connection to a redis + :type rconnect: StrictRedis + :param name: statistics name + :param day_min: default: seven days ago + :param day_max: default: tomorrow + :param prefix: prefix for statistics, default is None + + :rtype: int + """ + key = cls._get_key(name, prefix) + if day_min is None: + day_min = time.time() - timedelta(days=7).total_seconds() + + if day_max is None: + day_max = time.time() + timedelta(days=1).total_seconds() + + interval = cls.gen_days_interval(day_min, day_max) + if len(interval) == 0: + return 0 + + res = rconnect.hmget(key, interval) + return sum(int(amount) for amount in res if amount is not None) + + + @classmethod + def trim_before(cls, rconnect, name, threshold_timestamp, + prefix=None): + """ + Removes all records occured before `threshold_timestamp` + :param rconnect: StrictRedis + :param name: statistics name + :param threshold_timestamp: int + :param prefix: prefix for statistics, default is None + """ + + key = cls._get_key(name, prefix) + + threshold_day = cls.timestamp_to_day(threshold_timestamp) + 1 + all_members = rconnect.hgetall(key) + to_del = [mb for mb in all_members.keys() if int(mb) < threshold_day] + + rconnect.hdel(key, *to_del) diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html index 5c9629a..96c76e6 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html @@ -35,7 +35,15 @@ {{ repo.arch_list|join(", ") }} </td> <td> - <center> {{ repo.dl_stat }} </center> + <center> {{ repo.dl_stat }} </center> <br /> +  + # of repo dl last week<br /> + {% for arch in repo.arch_list %} + + {{arch}}: {{ repo.rpm_dl_stat[arch] }} <br /> + + {% endfor %} +  </td> <td class="rightmost"> <a href="{{ diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py index de05b68..2eb7312 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py @@ -12,18 +12,20 @@ from itertools import groupby from coprs import app from coprs import db +from coprs import rcp from coprs import exceptions from coprs import forms from coprs import helpers from coprs import models from coprs.logic.stat_logic import CounterStatLogic +from coprs.rmodels import TimedStatEvents from coprs.views.misc import login_required, page_not_found from coprs.views.coprs_ns import coprs_ns from coprs.logic import builds_logic, coprs_logic, actions_logic -from coprs.helpers import parse_package_name, generate_repo_url +from coprs.helpers import parse_package_name, generate_repo_url, CHROOT_RPMS_DL_STAT_FMT, CHROOT_REPO_MD_DL_STAT_FMT @coprs_ns.route("/", defaults={"page": 1}) @@ -189,16 +191,31 @@ def copr_detail(username, coprname): repos_info = {} for chroot in copr.active_chroots: + chroot_rpms_dl_stat_key = CHROOT_REPO_MD_DL_STAT_FMT.format( + copr_user=copr.owner.name, + copr_project_name=copr.name, + copr_chroot=chroot.name, + ) + chroot_rpms_dl_stat = TimedStatEvents.get_count( + rconnect=rcp.get_connection(), + name=chroot_rpms_dl_stat_key, + ) + if chroot.name_release not in repos_info: repos_info[chroot.name_release] = { "name_release": chroot.name_release, "name_release_human": chroot.name_release_human, "arch_list": [chroot.arch], "repo_file": "{}-{}-{}.repo".format(copr.owner.name, copr.name, chroot.name_release), - "dl_stat": repo_dl_stat[chroot.name_release] + "dl_stat": repo_dl_stat[chroot.name_release], + "rpm_dl_stat": { + chroot.arch: chroot_rpms_dl_stat + } } else: repos_info[chroot.name_release]["arch_list"].append(chroot.arch) + repos_info[chroot.name_release]["rpm_dl_stat"][chroot.arch] = chroot_rpms_dl_stat + repos_info_list = sorted(repos_info.values(), key=lambda rec: rec["name_release"]) return flask.render_template("coprs/detail/overview.html", diff --git a/frontend/coprs_frontend/coprs/views/misc.py b/frontend/coprs_frontend/coprs/views/misc.py index 6714f78..fd90267 100644 --- a/frontend/coprs_frontend/coprs/views/misc.py +++ b/frontend/coprs_frontend/coprs/views/misc.py @@ -261,7 +261,7 @@ def intranet_required(f): def decorated_function(*args, **kwargs): ip_addr = IPAddress(flask.request.remote_addr) if not any(ip_addr in IPNetwork(addr_or_net) - for addr_or_net in app.config.get(["INTRANET_IPS"], "127.0.0.1")): + for addr_or_net in app.config.get("INTRANET_IPS", ["127.0.0.1",])): return ("Stats can be update only from intranet hosts, " "not {}, check config\n".format(flask.request.remote_addr)), 403 diff --git a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py index 9b40e29..d1f5f15 100644 --- a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py +++ b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py @@ -1,12 +1,14 @@ # coding: utf-8 import flask +from coprs import rcp from coprs import app from coprs import db from coprs.helpers import REPO_DL_STAT_FMT, CounterStatType from ..misc import intranet_required from . import stats_rcv_ns -from ...logic.stat_logic import CounterStatLogic +from ...logic.stat_logic import CounterStatLogic, handle_logstash + @stats_rcv_ns.route("/") def ping(): @@ -27,22 +29,7 @@ def increment(counter_type, name): @intranet_required def logstash_handler(): try: - json = flask.request.json - if "request" in json: - # 0 1 2 3 4 5 - # "request": "/coprs/bob/foox/repo/epel-5/bob-foox-epel-5.repo", - req_split = json["request"].split("/") - kwargs = dict( - user=req_split[2], - copr=req_split[3], - name_release=req_split[5] - ) - name = REPO_DL_STAT_FMT.format(**kwargs) - app.logger.debug("kwargs: {}; name: {}".format(kwargs, name)) - - CounterStatLogic.incr(name=name, - counter_type=CounterStatType.REPO_DL) - db.session.commit() + handle_logstash(rcp.get_connection(), flask.request.json) except Exception as err: app.logger.exception(err) diff --git a/frontend/coprs_frontend/tests/test_rmodels.py b/frontend/coprs_frontend/tests/test_rmodels.py new file mode 100644 index 0000000..939426c --- /dev/null +++ b/frontend/coprs_frontend/tests/test_rmodels.py @@ -0,0 +1,53 @@ +# coding: utf-8 + +import time +import pytest + +from redis import StrictRedis + +from coprs.rmodels import TimedStatEvents + + +class TestRModels(object): + + def setup_method(self, method): + self.rc = StrictRedis() + self.prefix = "copr:test:r_models" + + self.time_now = time.time() + + def teardown_method(self, method): + keys = self.rc.keys('{}*'.format(self.prefix)) + if keys: + self.rc.delete(*keys) + + def test_nop(self): + pass + + def test_timed_stats_events(self): + TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix, + timestamp=self.time_now, ) + + assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 1 + TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix, + timestamp=self.time_now, count=2) + + assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 3 + + TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix, + timestamp=self.time_now - 1000000, count=2) + TimedStatEvents.add_event(self.rc, name="foobar", prefix=self.prefix, + timestamp=self.time_now - 3000000, count=3) + + assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix,) == 3 + assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix, + day_min=self.time_now - 2000000) == 5 + assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix, + day_min=self.time_now - 5000000) == 8 + + TimedStatEvents.trim_before(self.rc, name="foobar", + prefix=self.prefix, threshold_timestamp=self.time_now - 200000) + + assert TimedStatEvents.get_count(self.rc, name="foobar", prefix=self.prefix, + day_min=self.time_now - 5000000) == 3 +

1 0

[copr] master: [keygen] fix SELinux context for /var/log/copr-keygen/ to be accessable by httpd process (2b6fd88)
by vgologuz＠fedoraproject.org 23 Feb '15

23 Feb '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 2b6fd882a89dc359aeda34915cdd73b454c42b9c Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Mon Feb 23 14:19:56 2015 +0100 [keygen] fix SELinux context for /var/log/copr-keygen/ to be accessable by httpd process >--------------------------------------------------------------- keygen/copr-keygen.spec | 3 +++ 1 files changed, 3 insertions(+), 0 deletions(-) diff --git a/keygen/copr-keygen.spec b/keygen/copr-keygen.spec index 471231a..a3fcef2 100644 --- a/keygen/copr-keygen.spec +++ b/keygen/copr-keygen.spec @@ -150,6 +150,9 @@ getent passwd copr-signer >/dev/null || \ %post +semanage fcontext -a -t httpd_log_t '/var/log/copr-keygen(/.*)?' +restorecon -rv /var/log/copr-keygen/ + service httpd condrestart %postun

1 0

[copr] master: [backend] fixing pip requirements (e258b1a)
by vgologuz＠fedoraproject.org 23 Feb '15

23 Feb '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit e258b1a9b04344ab03f2b6011fbc5b958516c55f Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Mon Feb 23 13:18:21 2015 +0100 [backend] fixing pip requirements >--------------------------------------------------------------- backend/requirements.txt | 1 - 1 files changed, 0 insertions(+), 1 deletions(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index 0d72690..2b6438f 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,7 +1,6 @@ setproctitle PyYAML # ansible -setproctitle redis retask python-daemon

1 0

[copr] master: [backend] add pip requirement for: netaddr (1ca23b7)
by vgologuz＠fedoraproject.org 23 Feb '15

23 Feb '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 1ca23b75abdde8666066c8b7f550a141f126b29d Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Mon Feb 23 13:04:35 2015 +0100 [backend] add pip requirement for: netaddr >--------------------------------------------------------------- backend/requirements.txt | 1 + 1 files changed, 1 insertions(+), 0 deletions(-) diff --git a/backend/requirements.txt b/backend/requirements.txt index 71e1cdf..0d72690 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -10,3 +10,4 @@ IPy # documentation sphinx sphinx-argparse +netaddr

1 0

[copr] master: fixing test suite for jenkins (1be3ebc)
by vgologuz＠fedoraproject.org 23 Feb '15

23 Feb '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 1be3ebcf6fb5b83fae6134af6b3727865337593e Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Mon Feb 23 13:00:16 2015 +0100 fixing test suite for jenkins >--------------------------------------------------------------- test_suite.sh | 2 +- 1 files changed, 1 insertions(+), 1 deletions(-) diff --git a/test_suite.sh b/test_suite.sh index 63dcf2d..61112e9 100644 --- a/test_suite.sh +++ b/test_suite.sh @@ -8,7 +8,7 @@ source _venv/bin/activate # sphinx flask flask-script SQLAlchemy==0.8.7 flask-whooshee Flask-OpenID Flask-SQLAlchemy==1.0 Flask-WTF blinker pytz markdown pyLibravatar pydns flexmock whoosh decorator -pip install pytest mock pytest-cov ipdb redis bunch PyYAML +pip install pytest mock pytest-cov ipdb redis bunch PyYAML setproctitle ansible cp -rv /usr/lib/python2.7/site-packages/rpmUtils _venv/lib/python2.7/site-packages/

1 0

[copr] master: [frontend] safer /start_rcv/from_logstash handler (fedd2da)
by vgologuz＠fedoraproject.org 23 Feb '15

23 Feb '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit fedd2da3e6792b11b89f3f3598047feeb48c6e55 Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Mon Feb 23 12:58:40 2015 +0100 [frontend] safer /start_rcv/from_logstash handler >--------------------------------------------------------------- .../coprs/views/stats_ns/stats_receiver.py | 39 +++++++++++--------- 1 files changed, 21 insertions(+), 18 deletions(-) diff --git a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py index 3f811ba..9b40e29 100644 --- a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py +++ b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py @@ -22,25 +22,28 @@ def increment(counter_type, name): db.session.commit() return "", 201 + @stats_rcv_ns.route("/from_logstash", methods=['POST']) -@intranet_required # ? +@intranet_required def logstash_handler(): - # import ipdb; ipdb.set_trace() - json = flask.request.json - if "request" in json: - # 0 1 2 3 4 5 - # "request": "/coprs/bob/foox/repo/epel-5/bob-foox-epel-5.repo", - req_split = json["request"].split("/") - kwargs = dict( - user=req_split[2], - copr=req_split[3], - name_release=req_split[5] - ) - name = REPO_DL_STAT_FMT.format(**kwargs) - app.logger.debug("kwargs: {}; name: {}".format(kwargs, name)) - - CounterStatLogic.incr(name=name, - counter_type=CounterStatType.REPO_DL) - db.session.commit() + try: + json = flask.request.json + if "request" in json: + # 0 1 2 3 4 5 + # "request": "/coprs/bob/foox/repo/epel-5/bob-foox-epel-5.repo", + req_split = json["request"].split("/") + kwargs = dict( + user=req_split[2], + copr=req_split[3], + name_release=req_split[5] + ) + name = REPO_DL_STAT_FMT.format(**kwargs) + app.logger.debug("kwargs: {}; name: {}".format(kwargs, name)) + + CounterStatLogic.incr(name=name, + counter_type=CounterStatType.REPO_DL) + db.session.commit() + except Exception as err: + app.logger.exception(err) return "", 201

1 0

[copr] master: [frontend] views.misc.intranet_required use netaddr lib to compare addresses (ef0f12a)
by vgologuz＠fedoraproject.org 20 Feb '15

20 Feb '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit ef0f12a1216d3749b42302efb6509e67b0052c7e Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Feb 20 23:22:11 2015 +0100 [frontend] views.misc.intranet_required use netaddr lib to compare addresses >--------------------------------------------------------------- frontend/copr-frontend.spec | 2 ++ frontend/coprs_frontend/config/copr_devel.conf | 4 +++- frontend/coprs_frontend/coprs/views/misc.py | 14 ++++++-------- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/frontend/copr-frontend.spec b/frontend/copr-frontend.spec index 3df1de9..bf83869 100644 --- a/frontend/copr-frontend.spec +++ b/frontend/copr-frontend.spec @@ -51,6 +51,7 @@ Requires: python-pylibravatar Requires: python-whoosh >= 2.5.3 Requires: pytz Requires: python-six +Requires: python-netaddr # for tests: Requires: pytest Requires: python-flexmock @@ -70,6 +71,7 @@ BuildRequires: python-flask-openid BuildRequires: python-flask-whooshee BuildRequires: python-pylibravatar BuildRequires: python-flask-wtf +BuildRequires: python-netaddr BuildRequires: pytest BuildRequires: yum BuildRequires: python-flexmock diff --git a/frontend/coprs_frontend/config/copr_devel.conf b/frontend/coprs_frontend/config/copr_devel.conf index d6c5615..c586406 100644 --- a/frontend/coprs_frontend/config/copr_devel.conf +++ b/frontend/coprs_frontend/config/copr_devel.conf @@ -44,4 +44,6 @@ ENFORCE_PROTOCOL_FOR_FRONTEND_URL = "http" PUBLIC_COPR_HOSTNAME = "copr-fe-dev.cloud.fedoraproject.org" LOG_FILENAME="/tmp/copr_frontend.log" -INTRANET_IPS = ["127.0.0.1", "192.168.1.102"] + +# IP or subnet +INTRANET_IPS = ["127.0.0.1", "192.168.1.0/24"] diff --git a/frontend/coprs_frontend/coprs/views/misc.py b/frontend/coprs_frontend/coprs/views/misc.py index db29c29..6714f78 100644 --- a/frontend/coprs_frontend/coprs/views/misc.py +++ b/frontend/coprs_frontend/coprs/views/misc.py @@ -2,6 +2,7 @@ import base64 import datetime import functools +from netaddr import IPAddress, IPNetwork import re import flask @@ -258,14 +259,11 @@ def backend_authenticated(f): def intranet_required(f): @functools.wraps(f) def decorated_function(*args, **kwargs): - # app.logger.debug("MY: {}".format(flask.request.remote_addr)) - # Use, - # >>> import ipaddress - # >>> ipaddress.ip_address('192.168.0.1') in ipaddress.ip_network('192.168.0.0/24') - if not app.config["DEBUG"]: - if flask.request.remote_addr not in app.config["INTRANET_IPS"]: - return ("Stats can be update only from intranet hosts, " - "not {}, check config\n".format(flask.request.remote_addr)), 403 + ip_addr = IPAddress(flask.request.remote_addr) + if not any(ip_addr in IPNetwork(addr_or_net) + for addr_or_net in app.config.get(["INTRANET_IPS"], "127.0.0.1")): + return ("Stats can be update only from intranet hosts, " + "not {}, check config\n".format(flask.request.remote_addr)), 403 return f(*args, **kwargs) return decorated_function

1 0

[copr] master: [frontend] [rhbz:#1185959] RFE: Present statistics about project popularity: less logic in jinja template (64139b0)
by vgologuz＠fedoraproject.org 20 Feb '15

20 Feb '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 64139b0697134c8ce5f063d937d3f33fbdc3cdfe Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Feb 20 14:40:12 2015 +0100 [frontend] [rhbz:#1185959] RFE: Present statistics about project popularity: less logic in jinja template >--------------------------------------------------------------- frontend/coprs_frontend/coprs/config.py | 3 +- frontend/coprs_frontend/coprs/log.py | 2 +- frontend/coprs_frontend/coprs/models.py | 8 ++ .../coprs/templates/coprs/detail/overview.html | 75 +++++++------------- .../coprs/views/coprs_ns/coprs_general.py | 32 ++++++-- .../coprs/views/stats_ns/stats_receiver.py | 2 +- 6 files changed, 62 insertions(+), 60 deletions(-) diff --git a/frontend/coprs_frontend/coprs/config.py b/frontend/coprs_frontend/coprs/config.py index b8bb37a..15f3c08 100644 --- a/frontend/coprs_frontend/coprs/config.py +++ b/frontend/coprs_frontend/coprs/config.py @@ -42,8 +42,9 @@ class Config(object): # primary log file LOG_FILENAME = "/var/log/copr/frontend.log" - INTRANET_IPS = ["127.0.0.1"] + INTRANET_IPS = ["127.0.0.1"] + DEBUG = True class ProductionConfig(Config): DEBUG = False diff --git a/frontend/coprs_frontend/coprs/log.py b/frontend/coprs_frontend/coprs/log.py index 90d2022..8fd6d55 100644 --- a/frontend/coprs_frontend/coprs/log.py +++ b/frontend/coprs_frontend/coprs/log.py @@ -45,4 +45,4 @@ def setup_log(): handler.setLevel(log_level) app.logger.addHandler(handler) - app.logger.info("logging configuration finished") + app.logger.info("logging configuration finished, config: {}".format(app.config)) diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index d6cfe5d..1e011c3 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -429,6 +429,14 @@ class MockChroot(db.Model, helpers.Serializer): """ return "{}-{}".format(self.os_release, self.os_version) + + @property + def name_release_human(self): + """ + Textual representation of name of this or release + """ + return "{} {}".format(self.os_release, self.os_version) + @property def os(self): """ diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html index 6063a33..5c9629a 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html @@ -18,66 +18,42 @@ </div> <div class="dnf-enable-field"> # <input onClick="this.select();"type="text" value="dnf copr enable {{copr.owner.name}}/{{ copr.name}}" readonly="readonly"> More info about <a target="_blank" href="https://fedorahosted.org/copr/wiki/HowToEnableRepo">how to enable a repo on the wiki page.</a></div> + <table class="releases"> <tr> <th class="leftmost">Release</th> <th>Architecture</th> - - <th class="rightmost">Yum Repo <small>[Downloads]</small></th> + <th>#Downloads</th> + <th class="rightmost">Yum Repo </th> </tr> -  - {% for mock_chroot in copr.active_chroots %} - {% if loop.index < copr.active_chroots|length %} - {% if mock_chroot.os_release != copr.active_chroots[loop.index].os_release or - mock_chroot.os_version != copr.active_chroots[loop.index].os_version %} - {# next release is different => release-end #} - <tr class="release-end"> - {% else %} - <tr> - {% endif %} - {% else %}{# last line => release-end for sure #} - <tr class="release-end"> - {% endif %} - {% if mock_chroot.os_release != copr.active_chroots[loop.index0 - 1].os_release or - mock_chroot.os_version != copr.active_chroots[loop.index0 - 1].os_version or - loop.index0 == 0 %} - {# previous os_release-os_version were different or this is the first one #} - <td>{{ mock_chroot.os_release|capitalize }} {{ mock_chroot.os_version }}</td> - {% else %} - <td></td> - {% endif %} + {% for repo in repos_info_list %} + <tr class="release-end"> + <td class="leftmost"> + {{ repo.name_release_human|capitalize }} + </td> <td> - {{ mock_chroot.arch }} - {% if copr.buildroot_pkgs(mock_chroot): %} - <a id="modified-chroot-{{mock_chroot.name}}">[modified]</a> - {% endif %} + {{ repo.arch_list|join(", ") }} </td> <td> - {% if mock_chroot.os_release != copr.active_chroots[loop.index0 - 1].os_release or - mock_chroot.os_version != copr.active_chroots[loop.index0 - 1].os_version or - loop.index0 == 0 %} - {# previous os_release-os_version were different or this is the first one #} - - <a href="{{ - url_for( - 'coprs_ns.generate_repo_file', - username=copr.owner.name, - coprname=copr.name, - chroot=mock_chroot.os_release+"-"+mock_chroot.os_version, - repofile=copr.owner.name+'-'+copr.name+'-'+mock_chroot.os_release+"-"+mock_chroot.os_version+'.repo', - _external=True - )|fix_url_https_frontend}}"> - {{ copr.owner.name }}-{{ copr.name }}-{{mock_chroot.os_release+"-"+mock_chroot.os_version}}.repo - </a> -    <small>[ {{ repo_dl_stat[mock_chroot.name_release] }} ]</small> - {% endif %} + <center> {{ repo.dl_stat }} </center> </td> - - </tr> - {% else %} - <tr colspan="2"><td>No active releases</td></tr> + <td class="rightmost"> + <a href="{{ + url_for( + 'coprs_ns.generate_repo_file', + username=copr.owner.name, + coprname=copr.name, + name_release=repo.name_release, + repofile=repo.repo_file, + _external=True + )|fix_url_https_frontend}}"> + {{ repo.repo_file }} + </a> + </td> + </tr> {% endfor %} </table> + {% if copr.repos_list %} <h2>Repository List</h2> <ul class=repos-list> @@ -87,6 +63,7 @@ </ul> {% endif %} + {% if g.user and g.user.can_edit(copr) and copr and copr.owner and not copr.auto_createrepo %} <dt>  diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py index 75ade2f..de05b68 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py @@ -184,12 +184,28 @@ def copr_detail(username, coprname): except sqlalchemy.orm.exc.NoResultFound: return page_not_found( "Copr with name {0} does not exist.".format(coprname)) - from collections import defaultdict + repo_dl_stat = CounterStatLogic.get_copr_repo_dl_stat(copr) + + repos_info = {} + for chroot in copr.active_chroots: + if chroot.name_release not in repos_info: + repos_info[chroot.name_release] = { + "name_release": chroot.name_release, + "name_release_human": chroot.name_release_human, + "arch_list": [chroot.arch], + "repo_file": "{}-{}-{}.repo".format(copr.owner.name, copr.name, chroot.name_release), + "dl_stat": repo_dl_stat[chroot.name_release] + } + else: + repos_info[chroot.name_release]["arch_list"].append(chroot.arch) + repos_info_list = sorted(repos_info.values(), key=lambda rec: rec["name_release"]) + return flask.render_template("coprs/detail/overview.html", copr=copr, form=form, - repo_dl_stat=repo_dl_stat + repo_dl_stat=repo_dl_stat, + repos_info_list=repos_info_list, ) @@ -509,9 +525,9 @@ def copr_legal_flag(username, coprname): coprname=coprname)) -(a)coprs_ns.route("/<username>/<coprname>/repo/<chroot>/", defaults={"repofile": None}) -(a)coprs_ns.route("/<username>/<coprname>/repo/<chroot>/<repofile>") -def generate_repo_file(username, coprname, chroot, repofile): +(a)coprs_ns.route("/<username>/<coprname>/repo/<name_release>/", defaults={"repofile": None}) +(a)coprs_ns.route("/<username>/<coprname>/repo/<name_release>/<repofile>") +def generate_repo_file(username, coprname, name_release, repofile): """ Generate repo file for a given repo name. Reponame = username-coprname """ # This solution is used because flask splits off the last part after a @@ -520,7 +536,7 @@ def generate_repo_file(username, coprname, chroot, repofile): reponame = "{0}-{1}".format(username, coprname) - if repofile is not None and repofile != username + '-' + coprname + '-' + chroot + '.repo': + if repofile is not None and repofile != username + '-' + coprname + '-' + name_release + '.repo': return page_not_found( "Repository filename does not match expected: {0}" .format(repofile)) @@ -534,9 +550,9 @@ def generate_repo_file(username, coprname, chroot, repofile): return page_not_found( "Project {0}/{1} does not exist".format(username, coprname)) - mock_chroot = coprs_logic.MockChrootsLogic.get_from_name(chroot, noarch=True).first() + mock_chroot = coprs_logic.MockChrootsLogic.get_from_name(name_release, noarch=True).first() if not mock_chroot: - return page_not_found("Chroot {0} does not exist".format(chroot)) + return page_not_found("Chroot {0} does not exist".format(name_release)) url = "" for build in copr.builds: diff --git a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py index caf3039..3f811ba 100644 --- a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py +++ b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py @@ -23,7 +23,7 @@ def increment(counter_type, name): return "", 201 @stats_rcv_ns.route("/from_logstash", methods=['POST']) -#@intranet_required +@intranet_required # ? def logstash_handler(): # import ipdb; ipdb.set_trace() json = flask.request.json

1 0

[copr] master: [frontend] [rhbz:#1185959] RFE: Present statistics about project popularity : logstash config) (21b5faf)
by vgologuz＠fedoraproject.org 20 Feb '15

20 Feb '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit 21b5fafcd6e31146d47fa3ef56cc43452040680d Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Fri Feb 20 14:39:23 2015 +0100 [frontend] [rhbz:#1185959] RFE: Present statistics about project popularity : logstash config) >--------------------------------------------------------------- frontend/conf/logstash.conf | 41 +++++++++++++++++++++++++++++++++++++++++ frontend/copr-frontend.spec | 5 +++++ 2 files changed, 46 insertions(+), 0 deletions(-) diff --git a/frontend/conf/logstash.conf b/frontend/conf/logstash.conf new file mode 100644 index 0000000..82aa01d --- /dev/null +++ b/frontend/conf/logstash.conf @@ -0,0 +1,41 @@ +input { + file { + path => "/var/log/httpd/access_log" + type => "httpd-access" + } + # file { + # path => "/var/log/httpd/error_log" + # type => "httpd-error" + # } +} + +filter { + grok { + type => "httpd-access" + pattern => "%{COMBINEDAPACHELOG}" + } +} + +output { + if [type] == "httpd-access" { + if ".repo" in [request] and "stats_rcv" not in [request] { + http { + url => "http://127.0.0.1/stats_rcv/from_logstash" + format => "json" + http_method => "post" + message => "foobar" + } + } + #file { + # path => "/var/log/logstash/web.log" + # codec => rubydebug {} + # + #} + } + + # file { + # path => "/var/log/logstash/all.log" + # codec => rubydebug {} + # } + +} diff --git a/frontend/copr-frontend.spec b/frontend/copr-frontend.spec index 49b2b8d..3df1de9 100644 --- a/frontend/copr-frontend.spec +++ b/frontend/copr-frontend.spec @@ -57,6 +57,7 @@ Requires: python-flexmock Requires: python-mock Requires: python-decorator Requires: yum +Requires: logstash %if 0%{?rhel} < 7 && 0%{?rhel} > 0 BuildRequires: python-argparse %endif @@ -128,7 +129,9 @@ touch %{buildroot}%{_sharedstatedir}/copr/data/copr.db install -d %{buildroot}%{_var}/log/copr install -d %{buildroot}%{_sysconfdir}/logrotate.d +install -d %{buildroot}%{_sysconfdir}/logstash.d cp -a conf/logrotate %{buildroot}%{_sysconfdir}/logrotate.d/%{name} +cp -a conf/logstash.conf %{buildroot}%{_sysconfdir}/logstash.d/copr_frontend.conf touch %{buildroot}%{_var}/log/copr/frontend.log %check @@ -147,6 +150,7 @@ useradd -r -g copr-fe -G copr-fe -d %{_datadir}/copr/coprs_frontend -s /bin/bash %post service httpd condrestart +service logstash condrestart %files %license LICENSE @@ -157,6 +161,7 @@ service httpd condrestart %{_datadir}/copr/coprs_frontend %config(noreplace) %{_sysconfdir}/logrotate.d/%{name} +%config(noreplace) %{_sysconfdir}/logstash.d/copr_frontend.conf %defattr(-, copr-fe, copr-fe, -) %dir %{_sharedstatedir}/copr/data

1 0

[copr] master: [frontend] [rhbz:#1185959] RFE: Present statistics about project popularity (WIP todo: package logstash config) (c1a5794)
by vgologuz＠fedoraproject.org 19 Feb '15

19 Feb '15

Repository : http://git.fedorahosted.org/cgit/copr.git On branch : master >--------------------------------------------------------------- commit c1a5794e39b3d1ea433cccc1c1cb55e52b72fd4a Author: Valentin Gologuzov <vgologuz(a)redhat.com> Date: Thu Feb 19 23:17:38 2015 +0100 [frontend] [rhbz:#1185959] RFE: Present statistics about project popularity (WIP todo: package logstash config) >--------------------------------------------------------------- .../450fe5f7942d_added_table_counterstat.py | 31 +++++++ frontend/coprs_frontend/config/copr_devel.conf | 3 +- frontend/coprs_frontend/coprs/__init__.py | 2 + frontend/coprs_frontend/coprs/config.py | 1 + frontend/coprs_frontend/coprs/helpers.py | 7 ++ frontend/coprs_frontend/coprs/logic/stat_logic.py | 88 ++++++++++++++++++++ frontend/coprs_frontend/coprs/models.py | 19 ++++ .../coprs/templates/coprs/detail/overview.html | 6 +- .../coprs/views/coprs_ns/coprs_general.py | 8 ++- frontend/coprs_frontend/coprs/views/misc.py | 18 ++++- .../coprs/views/stats_ns/__init__.py | 4 + .../coprs/views/stats_ns/stats_receiver.py | 46 ++++++++++ .../tests/test_logic/test_stat_logic.py | 36 ++++++++ 13 files changed, 264 insertions(+), 5 deletions(-) diff --git a/frontend/coprs_frontend/alembic/versions/450fe5f7942d_added_table_counterstat.py b/frontend/coprs_frontend/alembic/versions/450fe5f7942d_added_table_counterstat.py new file mode 100644 index 0000000..dae6c40 --- /dev/null +++ b/frontend/coprs_frontend/alembic/versions/450fe5f7942d_added_table_counterstat.py @@ -0,0 +1,31 @@ +"""added table CounterStat + +Revision ID: 450fe5f7942d +Revises: bd0dab2e478 +Create Date: 2015-02-19 23:40:08.934834 + +""" + +# revision identifiers, used by Alembic. +revision = '450fe5f7942d' +down_revision = 'bd0dab2e478' + +from alembic import op +import sqlalchemy as sa + + +def upgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.create_table('counter_stat', + sa.Column('name', sa.String(length=127), nullable=False), + sa.Column('counter_type', sa.String(length=30), nullable=True), + sa.Column('counter', sa.Integer(), server_default='0', nullable=True), + sa.PrimaryKeyConstraint('name') + ) + ### end Alembic commands ### + + +def downgrade(): + ### commands auto generated by Alembic - please adjust! ### + op.drop_table('counter_stat') + ### end Alembic commands ### diff --git a/frontend/coprs_frontend/config/copr_devel.conf b/frontend/coprs_frontend/config/copr_devel.conf index 9f26dc2..d6c5615 100644 --- a/frontend/coprs_frontend/config/copr_devel.conf +++ b/frontend/coprs_frontend/config/copr_devel.conf @@ -26,7 +26,7 @@ SQLALCHEMY_DATABASE_URI = 'sqlite:////var/lib/copr/data/copr.db' #LOGGING_LEVEL = logging.ERROR DEBUG = True -SQLALCHEMY_ECHO = True +SQLALCHEMY_ECHO = False #CSRF_ENABLED = True # as of Flask-WTF 0.9+ @@ -44,3 +44,4 @@ ENFORCE_PROTOCOL_FOR_FRONTEND_URL = "http" PUBLIC_COPR_HOSTNAME = "copr-fe-dev.cloud.fedoraproject.org" LOG_FILENAME="/tmp/copr_frontend.log" +INTRANET_IPS = ["127.0.0.1", "192.168.1.102"] diff --git a/frontend/coprs_frontend/coprs/__init__.py b/frontend/coprs_frontend/coprs/__init__.py index de5fc5d..492dec3 100644 --- a/frontend/coprs_frontend/coprs/__init__.py +++ b/frontend/coprs_frontend/coprs/__init__.py @@ -46,6 +46,7 @@ from coprs.views import status_ns from coprs.views.status_ns import status_general from coprs.views import recent_ns from coprs.views.recent_ns import recent_general +from coprs.views.stats_ns import stats_receiver from .context_processors import include_banner @@ -58,5 +59,6 @@ app.register_blueprint(misc.misc) app.register_blueprint(backend_ns.backend_ns) app.register_blueprint(status_ns.status_ns) app.register_blueprint(recent_ns.recent_ns) +app.register_blueprint(stats_receiver.stats_rcv_ns) app.add_url_rule("/", "coprs_ns.coprs_show", coprs_general.coprs_show) diff --git a/frontend/coprs_frontend/coprs/config.py b/frontend/coprs_frontend/coprs/config.py index 557dcc6..b8bb37a 100644 --- a/frontend/coprs_frontend/coprs/config.py +++ b/frontend/coprs_frontend/coprs/config.py @@ -42,6 +42,7 @@ class Config(object): # primary log file LOG_FILENAME = "/var/log/copr/frontend.log" + INTRANET_IPS = ["127.0.0.1"] class ProductionConfig(Config): diff --git a/frontend/coprs_frontend/coprs/helpers.py b/frontend/coprs_frontend/coprs/helpers.py index dfa5137..f4f8efa 100644 --- a/frontend/coprs_frontend/coprs/helpers.py +++ b/frontend/coprs_frontend/coprs/helpers.py @@ -21,6 +21,13 @@ def generate_api_token(size=30): return ''.join(random.choice(string.ascii_lowercase) for x in range(size)) +REPO_DL_STAT_FMT = "{user}@{copr}:{name_release}" + + +class CounterStatType(object): + REPO_DL = "repo_dl" + + class EnumType(type): def __call__(self, attr): diff --git a/frontend/coprs_frontend/coprs/logic/stat_logic.py b/frontend/coprs_frontend/coprs/logic/stat_logic.py new file mode 100644 index 0000000..c1b2e0f --- /dev/null +++ b/frontend/coprs_frontend/coprs/logic/stat_logic.py @@ -0,0 +1,88 @@ +from collections import defaultdict +import json +import os +import pprint +import time +from sqlalchemy import or_ +from sqlalchemy import and_ + +from sqlalchemy.orm.exc import NoResultFound + +from coprs import app +from coprs import db +from coprs import exceptions +from coprs.models import CounterStat +from coprs import helpers +from coprs.helpers import REPO_DL_STAT_FMT +from coprs import signals +from coprs.helpers import CounterStatType + + + + +class CounterStatLogic(object): + + @classmethod + def get(cls, name): + """ + :param name: counter name + :return: + """ + return CounterStat.query.filter(CounterStat.name == name) + + @classmethod + def get_multiply_same_type(cls, counter_type, names_list): + return ( + CounterStat.query + .filter(CounterStat.counter_type == counter_type) + .filter(CounterStat.name.in_(names_list)) + ) + + @classmethod + def add(cls, name, counter_type): + csl = CounterStat(name=name, counter_type=counter_type) + db.session.add(csl) + return csl + + @classmethod + def incr(cls, name, counter_type): + """ + Warning: dirty method: does commit if missing stat record. + """ + try: + csl = CounterStatLogic.get(name).one() + csl.counter = CounterStat.counter + 1 + except NoResultFound: + csl = CounterStatLogic.add(name, counter_type) + csl.counter = 1 + + db.session.add(csl) + return csl + + @classmethod + def get_copr_repo_dl_stat(cls, copr): + # chroot -> stat_name + chroot_by_stat_name = {} + for chroot in copr.active_chroots: + kwargs = { + "user": copr.owner.name, + "copr": copr.name, + "name_release": chroot.name_release + } + chroot_by_stat_name[REPO_DL_STAT_FMT.format(**kwargs)] = chroot.name_release + + # [{counter: <value>, name: <stat_name>}, ...] + stats = cls.get_multiply_same_type(counter_type=helpers.CounterStatType.REPO_DL, + names_list=chroot_by_stat_name.keys()) + + # need: {chroot -> value, ... } + repo_dl_stats = defaultdict(int) + for stat in stats: + repo_dl_stats[chroot_by_stat_name[stat.name]] = stat.counter + + return repo_dl_stats + + + + + diff --git a/frontend/coprs_frontend/coprs/models.py b/frontend/coprs_frontend/coprs/models.py index ef8ad0e..d6cfe5d 100644 --- a/frontend/coprs_frontend/coprs/models.py +++ b/frontend/coprs_frontend/coprs/models.py @@ -423,6 +423,13 @@ class MockChroot(db.Model, helpers.Serializer): return "{0}-{1}-{2}".format(self.os_release, self.os_version, self.arch) @property + def name_release(self): + """ + Textual representation of name of this or release + """ + return "{}-{}".format(self.os_release, self.os_version) + + @property def os(self): """ Textual representation of the operating system name @@ -578,3 +585,15 @@ class Krb5Login(db.Model, helpers.Serializer): primary = db.Column(db.String(80), nullable=False, primary_key=True) user = db.relationship("User", backref=db.backref("krb5_logins")) + + +class CounterStat(db.Model, helpers.Serializer): + """ + Generic store for simple statistics. + """ + + name = db.Column(db.String(127), primary_key=True) + counter_type = db.Column(db.String(30)) + + counter = db.Column(db.Integer, default=0, server_default="0") + diff --git a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html index 7336814..6063a33 100644 --- a/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html +++ b/frontend/coprs_frontend/coprs/templates/coprs/detail/overview.html @@ -22,8 +22,10 @@ <tr> <th class="leftmost">Release</th> <th>Architecture</th> - <th class="rightmost">Yum Repo</th> + + <th class="rightmost">Yum Repo <small>[Downloads]</small></th> </tr> +  {% for mock_chroot in copr.active_chroots %} {% if loop.index < copr.active_chroots|length %} {% if mock_chroot.os_release != copr.active_chroots[loop.index].os_release or @@ -67,8 +69,10 @@ )|fix_url_https_frontend}}"> {{ copr.owner.name }}-{{ copr.name }}-{{mock_chroot.os_release+"-"+mock_chroot.os_version}}.repo </a> +    <small>[ {{ repo_dl_stat[mock_chroot.name_release] }} ]</small> {% endif %} </td> + </tr> {% else %} <tr colspan="2"><td>No active releases</td></tr> diff --git a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py index be91b69..75ade2f 100644 --- a/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py +++ b/frontend/coprs_frontend/coprs/views/coprs_ns/coprs_general.py @@ -16,6 +16,7 @@ from coprs import exceptions from coprs import forms from coprs import helpers from coprs import models +from coprs.logic.stat_logic import CounterStatLogic from coprs.views.misc import login_required, page_not_found @@ -183,10 +184,13 @@ def copr_detail(username, coprname): except sqlalchemy.orm.exc.NoResultFound: return page_not_found( "Copr with name {0} does not exist.".format(coprname)) - + from collections import defaultdict + repo_dl_stat = CounterStatLogic.get_copr_repo_dl_stat(copr) return flask.render_template("coprs/detail/overview.html", copr=copr, - form=form) + form=form, + repo_dl_stat=repo_dl_stat + ) @coprs_ns.route("/<username>/<coprname>/permissions/") diff --git a/frontend/coprs_frontend/coprs/views/misc.py b/frontend/coprs_frontend/coprs/views/misc.py index 7d6138a..db29c29 100644 --- a/frontend/coprs_frontend/coprs/views/misc.py +++ b/frontend/coprs_frontend/coprs/views/misc.py @@ -249,7 +249,23 @@ def backend_authenticated(f): def decorated_function(*args, **kwargs): auth = flask.request.authorization if not auth or auth.password != app.config["BACKEND_PASSWORD"]: - return "You have to provide the correct password", 401 + return "You have to provide the correct password\n", 401 + + return f(*args, **kwargs) + return decorated_function + + +def intranet_required(f): + @functools.wraps(f) + def decorated_function(*args, **kwargs): + # app.logger.debug("MY: {}".format(flask.request.remote_addr)) + # Use, + # >>> import ipaddress + # >>> ipaddress.ip_address('192.168.0.1') in ipaddress.ip_network('192.168.0.0/24') + if not app.config["DEBUG"]: + if flask.request.remote_addr not in app.config["INTRANET_IPS"]: + return ("Stats can be update only from intranet hosts, " + "not {}, check config\n".format(flask.request.remote_addr)), 403 return f(*args, **kwargs) return decorated_function diff --git a/frontend/coprs_frontend/coprs/views/stats_ns/__init__.py b/frontend/coprs_frontend/coprs/views/stats_ns/__init__.py new file mode 100644 index 0000000..33c09b2 --- /dev/null +++ b/frontend/coprs_frontend/coprs/views/stats_ns/__init__.py @@ -0,0 +1,4 @@ +# coding: utf-8 +import flask + +stats_rcv_ns = flask.Blueprint("stats_rcv_ns", __name__, url_prefix="/stats_rcv") diff --git a/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py new file mode 100644 index 0000000..caf3039 --- /dev/null +++ b/frontend/coprs_frontend/coprs/views/stats_ns/stats_receiver.py @@ -0,0 +1,46 @@ +# coding: utf-8 + +import flask +from coprs import app +from coprs import db +from coprs.helpers import REPO_DL_STAT_FMT, CounterStatType +from ..misc import intranet_required +from . import stats_rcv_ns +from ...logic.stat_logic import CounterStatLogic + +(a)stats_rcv_ns.route("/") +def ping(): + return "OK", 200 + + +(a)stats_rcv_ns.route("/<counter_type>/<name>/", methods=['POST']) +@intranet_required +def increment(counter_type, name): + app.logger.debug(flask.request.remote_addr) + + CounterStatLogic.incr(name, counter_type) + db.session.commit() + return "", 201 + +(a)stats_rcv_ns.route("/from_logstash", methods=['POST']) +#@intranet_required +def logstash_handler(): + # import ipdb; ipdb.set_trace() + json = flask.request.json + if "request" in json: + # 0 1 2 3 4 5 + # "request": "/coprs/bob/foox/repo/epel-5/bob-foox-epel-5.repo", + req_split = json["request"].split("/") + kwargs = dict( + user=req_split[2], + copr=req_split[3], + name_release=req_split[5] + ) + name = REPO_DL_STAT_FMT.format(**kwargs) + app.logger.debug("kwargs: {}; name: {}".format(kwargs, name)) + + CounterStatLogic.incr(name=name, + counter_type=CounterStatType.REPO_DL) + db.session.commit() + + return "", 201 diff --git a/frontend/coprs_frontend/tests/test_logic/test_stat_logic.py b/frontend/coprs_frontend/tests/test_logic/test_stat_logic.py new file mode 100644 index 0000000..14adb49 --- /dev/null +++ b/frontend/coprs_frontend/tests/test_logic/test_stat_logic.py @@ -0,0 +1,36 @@ +# coding: utf-8 +import pytest + +from coprs.exceptions import ActionInProgressException +from coprs.helpers import ActionTypeEnum +from coprs.logic.coprs_logic import CoprsLogic +from coprs.logic.stat_logic import CounterStatLogic +from coprs.helpers import CounterStatType +from coprs import models +from tests.coprs_test_case import CoprsTestCase + + +class TestStatLogic(CoprsTestCase): + + def setup_method(self, method): + super(TestStatLogic, self).setup_method(method) + + self.counter_type = CounterStatType.REPO_DL + self.counter_name = "{}:user/copr".format(CounterStatType.REPO_DL) + + def test_counter_basic(self): + CounterStatLogic.add(self.counter_name, self.counter_type) + self.db.session.commit() + CounterStatLogic.incr(self.counter_name, self.counter_type) + self.db.session.commit() + csl = CounterStatLogic.get(self.counter_name).one() + assert csl.counter == 1 + + def test_new_by_incr(self): + with pytest.raises(Exception): + CounterStatLogic.get(self.counter_name).one() + + CounterStatLogic.incr(self.counter_name, self.counter_type) + self.db.session.commit() + csl = CounterStatLogic.get(self.counter_name).one() + assert csl.counter == 1

1 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

copr-commits February 2015