Updated readme

.ci/build.sh

@@ -27,6 +27,7 @@ python manage.py fetch_deployed_data _site $ISSUES_JSON \
 
 python manage.py migrate
 python manage.py import_contributors_data
+python manage.py create_org_cluster_map_and_activity_graph org_map
 python manage.py import_issues_data
 python manage.py import_merge_requests_data
 python manage.py create_config_data
@@ -43,6 +44,8 @@ fi
 
 # Run meta review system
 python manage.py run_meta_review_system
+# Run openhub system
+python manage.py import_openhub_data
 
 rm _site/$ISSUES_JSON
 

.coafile

@@ -1,6 +1,6 @@
 [all]
 files = **.py, **.js, **.sh
-ignore = .git/**, **/__pycache__/**, gci/client.py, */migrations/**, private/*
+ignore = .git/**, **/__pycache__/**, gci/client.py, */migrations/**, private/*, **/leaflet_dist/**
 max_line_length = 80
 use_spaces = True
 preferred_quotation = '
@@ -42,6 +42,7 @@ files = static/**/*.js
 bears = JSHintBear
 allow_unused_variables = True
 javascript_strictness = False
+environment_jquery = True
 
 [all.yml]
 bears = YAMLLintBear
@@ -58,7 +59,7 @@ shell = bash
 # for use by other organizations.
 files = **
 # .coverage crashes AnnotationBear
-ignore += .coafile, *requirements.txt, .travis.yml, LICENSE, .nocover.yaml, .moban.yaml, .moban.dt/community-*.jj2, public/**, _site/**, .ci/check_moban.sh, .coverage
+ignore += .coafile, *requirements.txt, .travis.yml, LICENSE, .nocover.yaml, .moban.yaml, .moban.dt/community-*.jj2, public/**, _site/**, .ci/check_moban.sh, .coverage, static/js/main.js
 bears = KeywordBear
 language = python 3
 keywords = coala

.gitignore

@@ -63,6 +63,7 @@ coverage.xml
 *.log
 local_settings.py
 db.sqlite3
+db.sqlite3-journal
 
 # Flask stuff:
 instance/
@@ -272,6 +273,7 @@ flycheck_*.el
 
 # Session
 Session.vim
+Sessionx.vim
 
 # Temporary
 .netrwhist
@@ -429,11 +431,7 @@ DerivedData/
 *.perspectivev3
 !default.perspectivev3
 
-## Xcode Patch
-*.xcodeproj/*
-!*.xcodeproj/project.pbxproj
-!*.xcodeproj/xcshareddata/
-!*.xcworkspace/contents.xcworkspacedata
+## Gcc Patch
 /*.gcno
 
 # Eclipse rules

.moban.yaml

@@ -4,18 +4,15 @@ package_module: community
 packages:
   - community
   - activity
-  - inactive_issues
   - data
   - gci
   - gsoc
   - gamification
-  - log
+  - ci_build
   - meta_review
-  - model
-  - twitter
-  - unassigned_issues
 
 dependencies:
+  - getorg~=0.3.1
   - git+https://gitlab.com/coala/coala-utils.git
   - git-url-parse
   - django>2.1,<2.2
@@ -24,6 +21,7 @@ dependencies:
   - git+https://gitlab.com/gitmate/open-source/IGitt.git@1fa5a0a21ea4fb8739d467c06972f748717adbdc
   - requests
   - git+https://github.com/andrewda/trav.git@ce805d12d3d1db0a51b1aa26bba9cd9ecc0d96b8
+  - openhub-django~=0.1.0
   - python-dateutil
   - pillow
   - ruamel.yaml

.nocover.yaml

@@ -8,14 +8,8 @@ nocover_file_globs:
   - community/git.py
   - gci/*.py
   - gsoc/*.py
-  - log/*.py
+  - ci_build/*.py
   - meta_review/handler.py
-  - model/*.py
-  - openhub/*.py
-  - twitter/*.py
-  # Optional coverage. Once off scripts.
-  - inactive_issues/inactive_issues_scraper.py
-  - unassigned_issues/unassigned_issues_scraper.py
   # The following rules can remain here
   # django db
   - '*/migrations/*.py'

.travis.yml

@@ -1,5 +1,5 @@
 language: python
-python: 3.6
+python: 3.6.3
 
 cache:
   pip: true

README.md

@@ -4,27 +4,98 @@
 
 ## About
 
-* This repository is a static django website;
-* It uses the [GCI private API](https://developers.google.com/open-source/gci/resources/downloads/TaskAPISpec.pdf)
-* __Attention:__ This repository only re-publishes publicly available information
-* Uses IGitt, supporting both GitHub & GitLab
-
-## Deploy Netlify on your fork :
-
-Following are the steps to deploy netlify on your forked ``community`` repository :
-
-- Signup on [netlify](https://app.netlify.com/).
-- Click on **New site from Git**.
-- Choose **Git provider (Github) >** Choose a repository **< username >/community**.
-- Branch to deploy **master**.
-- Basic build settings :
-  * Set Build command to **.ci/build.sh**.
-  * Set Publish directory to **public**.
-  * Click on **Show advanced >** **New variable** to add the `environment variables`.
-
-After that to set the ``environment variables`` :
-
-- Put **GH_TOKEN** in `Key` column and your generated `GitHub Access Token`
-  in `Value` column.
-- Similarly add other `variables` like **OH_TOKEN, GL_TOKEN** if needed.
-- Click on **Deploy Site** to deploy the site.
+The repository generates an *open-source community* static website, using the
+publicly available information about contributors and the organization. The
+information used is fetched from the **Webservices API**.
+
+**Features:**
+  1. Creates a *static django website* using *django_distill*.
+  2. Community data is being fetched from an API, named Webservices
+    *(on GitLab)*. To use it, just **fork** the repository, and Change the
+     organization name in *settings.py* file.
+  3. Uses the  [GCI private API](https://developers.google.com/open-source/gci/resources/downloads/TaskAPISpec.pdf)
+     to fetch information about students who participated in Google Code-In
+     with the organization.
+  4. Uses IGitt, supporting both GitHub & GitLab
+  5. __Attention:__ This repository only re-publishes publicly available
+    information.
+
+## Environment variables used
+
+1. **``GH_TOKEN``**: A *GitHub Personal Access Token*, which can be generated
+ from your *account settings* under the **Developer Settings**.
+2. **``GL_TOKEN``**: A *GitLab Personal Access Token*, which can be generated
+ from your *account settings* under the tab **Access Tokens**.
+3. **``OH_TOKEN``**: An *OpenHub Personal Access Token*, which can be
+ generated from your *account settings*.
+
+## Netlify based Community Forms
+1. **``OSFORMS_NETLIFY_FORM_NAME``**: A form which receives submission
+    from community *developers*, if they want to share any open-source form
+    like a Google Doc which is to be shared with all developers or within
+    the community.
+2. **``CALENDAR_NETLIFY_FORM_NAME``**: A form which receives submission
+    from community *developers*, if they want to add an upcoming event like
+    project related events, software releases, survey result announcements
+    etc.
+3. **``MENTOR_FORM_NAME``**: A form which receives submission from
+    community *developers*, if they want to participate as an
+    organization mentor, who will be available to mentor projects in the
+    summer/winter programs like Google Summer of Code, Google Code-In etc.
+4. **``GSOC_STUDENT_FORM_NAME``**: A form which receives submission from
+    community *newcomers & developers*, if they had participated with the
+    organization in Google Summer of Code and completed the proposed
+    project.
+5. **``ISSUES_ASSIGN_REQUEST_FORM_NAME``**: Any contributor, who wishes to work
+    on an issue can fill-up this form and request to get assigned on that
+    issue.
+6. **``NEWCOMER_PROMOTION_REQUEST_FORM_NAME``**: Any newcomer who wishes to get
+    the **Developer Role** in th oragnization, can fill-up the form and request
+    the access.
+7. **``FEEDBACK_FORM_NAME``**: The name is self-understandable :innocent:
+
+__ATTENTION DEVs:__ To use the forms, set the above mentioned environment
+ variables. All the forms, have respective **cron-jobs** defined in
+ **Webservices**, which processes the contributor request and check if the
+ request is valid or not. If not valid request, the user will be sent an
+ email informing about the validation errors.
+
+## Deploy your fork on Netlify:
+
+Following are the steps to deploy your forked *community* repository on Netlify:
+
+1. Signup on [netlify](https://app.netlify.com/).
+2. Click on **New site from Git**.
+3. Choose the **Git provider** as *Github* and Choose your forked *community*
+repository, named **< username>/community**.
+4. Further, choose **master** branch to deploy and set the following *Basic*
+ *build settings*:
+    * Set Build command to **.ci/build.sh**.
+    * Set Publish directory to **public**.
+    * Click on **Show advanced** and add a **New variable** by choosing the
+     option. After that, add the **required Environment variables** according
+     to your need.
+    * **Note:** ``GH_TOKEN`` is a required environment variable, and you must
+    set that before your deploy the site.
+5. Click on **Deploy Site** to deploy the site.
+
+## Want to add Netlify OAuth (GitHub and GitLab)?
+
+The repository has already integrated Netlify OAuth, all is to be done is
+to register new GitHub and GitLab OAuth apps on your account. So, that the
+OAuth is set to working on your fork.
+
+For registering a new GitHub OAuth App, you can follow-up the [Netlify OAuth
+ Provider Documentation](https://www.netlify.com/docs/authentication-providers/).
+
+For registering a new GitLab OAuth App,
+1. Go to your account Settings, and click *Applications* under
+ the Settings (or use this [shortcut](https://gitlab.com/profile/applications)).
+2. After page opens, you will see a form for registering a new OAuth
+ application.
+3. Enter the *name of the Application*.
+4. For the Redirect URI, enter ``https://api.netlify.com/auth/done``.
+5. And, the last one is the most important field, which is the **Scopes**. Set
+ it according to your needs. This repository uses ***api*** scope. If you
+ choose a different scope other than the ***api***, make sure to change it in
+ the `static/js/main.js` JavaScript file.

activity/scraper.py

@@ -136,7 +136,7 @@ def get_data(self):
         return self.data
 
 
-def activity_json(request):
+def activity_json(filename):
 
     org_name = get_org_name()
 
@@ -152,4 +152,5 @@ def activity_json(request):
     real_data = Scraper(parsed_json['issues'], datetime.datetime.today())
     real_data = real_data.get_data()
 
-    return HttpResponse(json.dumps(real_data))
+    with open(filename, 'w+') as f:
+        json.dump(real_data, f, indent=4)

ci_build/view_log.py

@@ -0,0 +1,131 @@
+import re
+import json
+import os
+import sys
+
+from django.views.generic import TemplateView
+
+from community.views import get_header_and_footer
+from community.git import (
+    get_org_name,
+    get_owner,
+    get_deploy_url,
+    get_upstream_deploy_url
+)
+
+
+class BuildLogsView(TemplateView):
+    template_name = 'build_logs.html'
+
+    def copy_build_logs_json(self, ci_build_jsons):
+        """
+        :param ci_build_jsons: A dict of directories path
+        :return: A boolean, whether the build file is copied
+        """
+        if os.path.isfile(ci_build_jsons['public_path']):
+            if sys.platform == 'linux':
+                os.popen('cp {} {}'.format(
+                    ci_build_jsons['site_path'],
+                    ci_build_jsons['public_path']))
+                os.popen('cp {} {}'.format(
+                    ci_build_jsons['site_path'],
+                    ci_build_jsons['static_path']))
+            else:
+                os.popen('copy {} {}'.format(
+                    ci_build_jsons['site_path'],
+                    ci_build_jsons['public_path']))
+                os.popen('copy {} {}'.format(
+                    ci_build_jsons['site_path'],
+                    ci_build_jsons['static_path']))
+            return True
+        return False
+
+    def create_and_copy_build_logs_json(self, logs, level_specific_logs):
+        """
+        Create a build logs detailed json file in ./_site directory and copy
+        that file in the ./static and ./public/static directories
+        :param logs: A list of all lines in build log file
+        :param level_specific_logs: A dict containing logs divided in their
+        respective categories
+        :return: A boolean, whether the files were copied or not
+        """
+        ci_build_jsons = {
+            'site_path': './_site/ci-build-detailed-logs.json',
+            'public_path': './public/static/ci-build-detailed-logs.json',
+            'static_path': './static/ci-build-detailed-logs.json'
+        }
+        with open(ci_build_jsons['site_path'], 'w+') as build_logs_file:
+            data = {
+                'logs': logs,
+                'logs_level_Specific': level_specific_logs
+            }
+            json.dump(data, build_logs_file, indent=4)
+        return self.copy_build_logs_json(ci_build_jsons)
+
+    def get_build_logs(self, log_file_path):
+        """
+        :param log_file_path: build logs file path
+        :return: a tuple of two where the first element in tuple refers to
+        a list of build logs in the file, and the second element is a dict
+        which categorizes the build logs into 5 categories - INFO, DEBUG,
+        WARNING, ERROR nad CRITICAL
+        """
+        log_lines = []
+        log_level_specific_lines = {
+            'INFO': [],
+            'DEBUG': [],
+            'WARNING': [],
+            'ERROR': [],
+            'CRITICAL': []
+        }
+        with open(log_file_path) as log_file:
+            previous_found_level = None
+            for line in log_file:
+                log_lines.append(line)
+                levels = re.findall(r'\[[A-Z]+]', line)
+                if levels:
+                    level = levels[0]
+                    level = previous_found_level = level[1:-1]
+                    log_level_specific_lines[level].append(line)
+                elif previous_found_level:
+                    log_level_specific_lines[previous_found_level].append(
+                        line)
+        return log_lines, log_level_specific_lines
+
+    def check_build_logs_stored(self):
+        """
+        Check whether the build logs json file is copied to _site and public
+        directories or not
+        :return: A Boolean
+        """
+        log_file_path = './_site/community.log'
+        log_file_exists = os.path.isfile(log_file_path)
+        if log_file_exists:
+            logs, level_specific_logs = self.get_build_logs(log_file_path)
+            return self.create_and_copy_build_logs_json(logs,
+                                                        level_specific_logs)
+        return False
+
+    def get_build_info(self):
+        """
+        Get the information about build, like who deployed the website i.e.
+        owner, name of the organization or user etc.
+        :return: A dict having information about build related details
+        """
+        data = {
+            'Org name': get_org_name(),
+            'Owner': get_owner(),
+            'Deploy URL': get_deploy_url(),
+        }
+        try:
+            data['Upstream deploy URL'] = get_upstream_deploy_url()
+        except RuntimeError:
+            data['Upstream deploy URL'] = 'Not found'
+        return data
+
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        context = get_header_and_footer(context)
+        context['build_info'] = self.get_build_info()
+        context['logs_stored'] = self.check_build_logs_stored()
+        return context