data/: Redesign the contributors webpage

.ci/build.sh

@@ -27,6 +27,7 @@ python manage.py fetch_deployed_data _site $ISSUES_JSON \
 
 python manage.py migrate
 python manage.py import_contributors_data
+python manage.py create_org_cluster_map_and_activity_graph org_map
 python manage.py import_issues_data
 python manage.py import_merge_requests_data
 python manage.py create_config_data

.coafile

@@ -1,6 +1,6 @@
 [all]
 files = **.py, **.js, **.sh
-ignore = .git/**, **/__pycache__/**, gci/client.py, */migrations/**, private/*
+ignore = .git/**, **/__pycache__/**, gci/client.py, */migrations/**, private/*, openhub/**, **/leaflet_dist/**
 max_line_length = 80
 use_spaces = True
 preferred_quotation = '
@@ -42,6 +42,7 @@ files = static/**/*.js
 bears = JSHintBear
 allow_unused_variables = True
 javascript_strictness = False
+environment_jquery = True
 
 [all.yml]
 bears = YAMLLintBear

.gitignore

@@ -63,6 +63,7 @@ coverage.xml
 *.log
 local_settings.py
 db.sqlite3
+db.sqlite3-journal
 
 # Flask stuff:
 instance/
@@ -272,6 +273,7 @@ flycheck_*.el
 
 # Session
 Session.vim
+Sessionx.vim
 
 # Temporary
 .netrwhist
@@ -429,11 +431,7 @@ DerivedData/
 *.perspectivev3
 !default.perspectivev3
 
-## Xcode Patch
-*.xcodeproj/*
-!*.xcodeproj/project.pbxproj
-!*.xcodeproj/xcshareddata/
-!*.xcworkspace/contents.xcworkspacedata
+## Gcc Patch
 /*.gcno
 
 # Eclipse rules

.moban.yaml

@@ -9,13 +9,13 @@ packages:
   - gci
   - gsoc
   - gamification
-  - log
+  - ci_build
   - meta_review
   - model
-  - twitter
   - unassigned_issues
 
 dependencies:
+  - getorg~=0.3.1
   - git+https://gitlab.com/coala/coala-utils.git
   - git-url-parse
   - django>2.1,<2.2

.nocover.yaml

@@ -8,11 +8,10 @@ nocover_file_globs:
   - community/git.py
   - gci/*.py
   - gsoc/*.py
-  - log/*.py
+  - ci_build/*.py
   - meta_review/handler.py
   - model/*.py
   - openhub/*.py
-  - twitter/*.py
   # Optional coverage. Once off scripts.
   - inactive_issues/inactive_issues_scraper.py
   - unassigned_issues/unassigned_issues_scraper.py

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-python: 3.6
+python: 3.6.3
 
 cache:
   pip: true

activity/scraper.py

@@ -136,7 +136,7 @@ def get_data(self):
         return self.data
 
 
-def activity_json(request):
+def activity_json(filename):
 
     org_name = get_org_name()
 
@@ -152,4 +152,5 @@ def activity_json(request):
     real_data = Scraper(parsed_json['issues'], datetime.datetime.today())
     real_data = real_data.get_data()
 
-    return HttpResponse(json.dumps(real_data))
+    with open(filename, 'w+') as f:
+        json.dump(real_data, f, indent=4)

ci_build/view_log.py

@@ -0,0 +1,131 @@
+import re
+import json
+import os
+import sys
+
+from django.views.generic import TemplateView
+
+from community.views import get_header_and_footer
+from community.git import (
+    get_org_name,
+    get_owner,
+    get_deploy_url,
+    get_upstream_deploy_url
+)
+
+
+class BuildLogsView(TemplateView):
+    template_name = 'build_logs.html'
+
+    def copy_build_logs_json(self, ci_build_jsons):
+        """
+        :param ci_build_jsons: A dict of directories path
+        :return: A boolean, whether the build file is copied
+        """
+        if os.path.isfile(ci_build_jsons['public_path']):
+            if sys.platform == 'linux':
+                os.popen('cp {} {}'.format(
+                    ci_build_jsons['site_path'],
+                    ci_build_jsons['public_path']))
+                os.popen('cp {} {}'.format(
+                    ci_build_jsons['site_path'],
+                    ci_build_jsons['static_path']))
+            else:
+                os.popen('copy {} {}'.format(
+                    ci_build_jsons['site_path'],
+                    ci_build_jsons['public_path']))
+                os.popen('copy {} {}'.format(
+                    ci_build_jsons['site_path'],
+                    ci_build_jsons['static_path']))
+            return True
+        return False
+
+    def create_and_copy_build_logs_json(self, logs, level_specific_logs):
+        """
+        Create a build logs detailed json file in ./_site directory and copy
+        that file in the ./static and ./public/static directories
+        :param logs: A list of all lines in build log file
+        :param level_specific_logs: A dict containing logs divided in their
+        respective categories
+        :return: A boolean, whether the files were copied or not
+        """
+        ci_build_jsons = {
+            'site_path': './_site/ci-build-detailed-logs.json',
+            'public_path': './public/static/ci-build-detailed-logs.json',
+            'static_path': './static/ci-build-detailed-logs.json'
+        }
+        with open(ci_build_jsons['site_path'], 'w+') as build_logs_file:
+            data = {
+                'logs': logs,
+                'logs_level_Specific': level_specific_logs
+            }
+            json.dump(data, build_logs_file, indent=4)
+        return self.copy_build_logs_json(ci_build_jsons)
+
+    def get_build_logs(self, log_file_path):
+        """
+        :param log_file_path: build logs file path
+        :return: a tuple of two where the first element in tuple refers to
+        a list of build logs in the file, and the second element is a dict
+        which categorizes the build logs into 5 categories - INFO, DEBUG,
+        WARNING, ERROR nad CRITICAL
+        """
+        log_lines = []
+        log_level_specific_lines = {
+            'INFO': [],
+            'DEBUG': [],
+            'WARNING': [],
+            'ERROR': [],
+            'CRITICAL': []
+        }
+        with open(log_file_path) as log_file:
+            previous_found_level = None
+            for line in log_file:
+                log_lines.append(line)
+                levels = re.findall(r'\[[A-Z]+]', line)
+                if levels:
+                    level = levels[0]
+                    level = previous_found_level = level[1:-1]
+                    log_level_specific_lines[level].append(line)
+                elif previous_found_level:
+                    log_level_specific_lines[previous_found_level].append(
+                        line)
+        return log_lines, log_level_specific_lines
+
+    def check_build_logs_stored(self):
+        """
+        Check whether the build logs json file is copied to _site and public
+        directories or not
+        :return: A Boolean
+        """
+        log_file_path = './_site/community.log'
+        log_file_exists = os.path.isfile(log_file_path)
+        if log_file_exists:
+            logs, level_specific_logs = self.get_build_logs(log_file_path)
+            return self.create_and_copy_build_logs_json(logs,
+                                                        level_specific_logs)
+        return False
+
+    def get_build_info(self):
+        """
+        Get the information about build, like who deployed the website i.e.
+        owner, name of the organization or user etc.
+        :return: A dict having information about build related details
+        """
+        data = {
+            'Org name': get_org_name(),
+            'Owner': get_owner(),
+            'Deploy URL': get_deploy_url(),
+        }
+        try:
+            data['Upstream deploy URL'] = get_upstream_deploy_url()
+        except RuntimeError:
+            data['Upstream deploy URL'] = 'Not found'
+        return data
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        context = get_header_and_footer(context)
+        context['build_info'] = self.get_build_info()
+        context['logs_stored'] = self.check_build_logs_stored()
+        return context

community/git.py

@@ -49,7 +49,7 @@ def get_config_remote(name='origin'):
     raise KeyError('No git remotes found')
 
 
-def get_remote_url():
+def get_remote_url(name='origin'):
     """Obtain a parsed remote URL.
 
     Uses CI environment variables or git remotes.
@@ -58,7 +58,7 @@ def get_remote_url():
     # It only sets the REPOSITORY_URL
     url = os.environ.get('REPOSITORY_URL')
     if not url:
-        remote = get_config_remote()
+        remote = get_config_remote(name)
         assert remote[0][0] == 'url'
         url = remote[0][1]
 
@@ -146,7 +146,7 @@ def get_upstream_repo():
     """Obtain the parent slug of the repository.
     """
     try:
-        remote = get_config_remote(name='upstream')
+        remote = get_remote_url(name='origin')
     except KeyError:
         remote = None
 

community/urls.py

@@ -5,15 +5,12 @@
 from django_distill import distill_url
 from django.conf.urls.static import static
 from django.conf import settings
-from django.views.generic import TemplateView
 
-from community.views import HomePageView, info
-from gci.views import index as gci_index
+from community.views import HomePageView
+from gci.views import GCIStudentsList
 from gci.feeds import LatestTasksFeed as gci_tasks_rss
-from activity.scraper import activity_json
-from twitter.view_twitter import index as twitter_index
-from log.view_log import index as log_index
-from data.views import index as contributors_index
+from ci_build.view_log import BuildLogsView
+from data.views import ContributorsListView
 from gamification.views import index as gamification_index
 from meta_review.views import index as meta_review_index
 from inactive_issues.inactive_issues_scraper import inactive_issues_json
@@ -81,50 +78,26 @@ def get_organization():
         distill_func=get_index,
         distill_file='index.html',
     ),
-    distill_url(
-        'info.txt', info,
-        name='index',
-        distill_func=get_index,
-        distill_file='info.txt',
-    ),
-    distill_url(
-        r'static/activity-data.json', activity_json,
-        name='activity_json',
-        distill_func=get_index,
-        distill_file='static/activity-data.json',
-    ),
-    distill_url(
-        r'activity/', TemplateView.as_view(template_name='activity.html'),
-        name='activity',
-        distill_func=get_index,
-        distill_file='activity/index.html',
-    ),
     distill_url(
         r'gci/tasks/rss.xml', gci_tasks_rss(),
         name='gci-tasks-rss',
         distill_func=get_index,
         distill_file='gci/tasks/rss.xml',
     ),
     distill_url(
-        r'gci/', gci_index,
+        r'gci/', GCIStudentsList.as_view(),
         name='community-gci',
         distill_func=get_index,
         distill_file='gci/index.html',
     ),
     distill_url(
-        r'twitter/', twitter_index,
-        name='twitter',
-        distill_func=get_index,
-        distill_file='twitter/index.html',
-    ),
-    distill_url(
-        r'log/', log_index,
-        name='log',
+        r'ci/build/', BuildLogsView.as_view(),
+        name='ci_build',
         distill_func=get_index,
-        distill_file='log/index.html',
+        distill_file='ci/build/index.html',
     ),
     distill_url(
-        r'contributors/$', contributors_index,
+        r'contributors/$', ContributorsListView.as_view(),
         name='community-data',
         distill_func=get_index,
         distill_file='contributors/index.html',