Background task

.gitignore

@@ -1,4 +1,5 @@
 __pycache__
 .env
 .vscode
-.idea
+.idea
+*.pyc

Pipfile

@@ -0,0 +1,14 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+flask-inputs = "*"
+celery = "*"
+redis = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.6"

Procfile

@@ -1 +1,2 @@
-web: gunicorn run
+web: gunicorn run
+worker: celery worker -A tasks -l INFO

app/__init__.py

@@ -0,0 +1,15 @@
+import os
+from flask import Flask
+
+from app.celery import make_celery
+
+
+flask_app = Flask(__name__)
+flask_app.config.update(
+    CELERY_BROKER_URL=os.environ.get('CELERY_BROKER_URL', 'redis://localhost:6379'),
+    CELERY_RESULT_BACKEND=os.environ.get('CELERY_RESULT_BACKEND', 'redis://localhost:6379')
+)
+
+from app import routes
+
+celery = make_celery(flask_app)

app/celery.py

@@ -0,0 +1,11 @@
+from celery import Celery
+
+
+def make_celery(app):
+    celery = Celery(
+        app.import_name,
+        broker=app.config['CELERY_BROKER_URL']
+    )
+    celery.conf.update(app.config)
+
+    return celery

extractor/extractor.py → app/extractor.py

@@ -7,8 +7,10 @@
 from newspaper.configuration import Configuration
 from newspaper.cleaners import DocumentCleaner
 
+from logger import Logger
 
-def get_data_from_html(html):
+
+def get_data_from_html(article_url, html):
     result = {}
     parsed_html = Parser.fromstring(html)
 
@@ -18,16 +20,15 @@ def get_data_from_html(html):
     cleaner = DocumentCleaner(config)
 
     result['title'] = extractor.get_title(parsed_html)
-
     publishing_date = extractor.get_publishing_date('', parsed_html)
     if publishing_date is None:
         publishing_date = datetime.datetime.now()
 
     result['published_at'] = publishing_date.isoformat()
-
     cleaned_html = cleaner.clean(parsed_html)
     top_node = extractor.calculate_best_node(cleaned_html)
     top_node = extractor.post_cleanup(top_node)
     result['content'], _ = formatter.get_formatted(top_node)
+    result['meta_img'] = extractor.get_meta_img_url(article_url, parsed_html)
 
     return result

extractor/logger.py → app/logger.py

@@ -2,7 +2,8 @@
 
 logging.basicConfig(filename='info.log', format='%(levelname)s:%(message)s', level=logging.INFO)
 
-class Logger():
+
+class Logger:
     @staticmethod
     def info(msg):
         print("INFO: %s" % msg)

app/routes.py

@@ -0,0 +1,29 @@
+from flask import request
+from app import flask_app, extractor, utils
+from app.logger import Logger
+from tasks import extract_content
+
+@flask_app.route("/extract", methods=["POST"])
+def extract():
+    if not request.is_json:
+        return 'Invalid payload format. Should be application/json', 400
+
+    request_payload = request.get_json()
+    client_token = request_payload['token']
+    if not client_token:
+        Logger.error("Invalid [token] parameter")
+        return 'Invalid [token] parameter', 400
+    article_url = request_payload['url']
+    if not article_url:
+        Logger.error("Invalid [article_url] parameter")
+        return 'Invalid [article_url] parameter', 400
+    article_html = request_payload['pageHtml']
+    if not article_html:
+        Logger.error("Invalid [article_html] parameter")
+        return 'Invalid [article_html] parameter', 400
+
+    Logger.info('URL received for extraction:[%s]' % article_url)
+
+    extract_content.delay(article_url, article_html, client_token)
+
+    return 'Article content was sent to processing', 200

extractor/api.py → app/utils.py

@@ -4,20 +4,20 @@
 
 API_URL = 'https://svandis-api-prod.herokuapp.com'
 
-def upload_post_data(api_token, post_data):
+def upload_article_data(client_token, article_data):
     url = API_URL + '/api/post'
-    headers = _get_request_headers(api_token)
+    headers = _get_request_headers(client_token)
 
-    r = requests.post(url, headers=headers, data=json.dumps({'post': post_data}))
+    r = requests.post(url, headers=headers, data=json.dumps({'post': article_data}))
     r.raise_for_status()
     return r.status_code
 
 
-def invalidate_url(api_token, url):
+def invalidate_url(client_token, article_url):
     url = API_URL + '/api/post/invalidate-url'
-    headers = _get_request_headers(api_token)
+    headers = _get_request_headers(client_token)
 
-    r = requests.post(url, headers=headers, data=json.dumps({'url': url}))
+    r = requests.post(url, headers=headers, data=json.dumps({'url': article_url}))
     r.raise_for_status()
     return r.status_code
 

info.log

@@ -1,8 +0,0 @@
-INFO: * Running on http://0.0.0.0:5000/ (Press CTRL+C to quit)
-INFO: * Running on http://0.0.0.0:5000/ (Press CTRL+C to quit)
-INFO:127.0.0.1 - - [27/Jun/2018 17:03:14] "GET / HTTP/1.1" 404 -
-INFO:127.0.0.1 - - [27/Jun/2018 17:03:15] "GET / HTTP/1.1" 404 -
-INFO:127.0.0.1 - - [27/Jun/2018 17:03:18] "GET /extract HTTP/1.1" 400 -
-INFO: * Running on http://0.0.0.0:5000/ (Press CTRL+C to quit)
-INFO:127.0.0.1 - - [27/Jun/2018 17:03:50] "GET /extract HTTP/1.1" 405 -
-INFO:127.0.0.1 - - [27/Jun/2018 17:03:51] "GET /extract HTTP/1.1" 405 -