TRA-591 Market Metadata Service: Initial commit

.gitignore

@@ -34,6 +34,11 @@ override.tf.json
 # Ignore secret.tfvars files that hold sensitive terraform variables
 secret.tfvars
 
+# Ignore source dependencies for market_metadata_service
+market_metadata_service/src/*
+!market_metadata_service/src/lambda_function.py
+!market_metadata_service/src/requirements.txt
+
 # IDE
 .idea/*
 

market_metadata_service/README.md

@@ -0,0 +1,7 @@
+# Market Metadata Service
+
+This directory defines Terraform configs for the Market Metadata Service, a collection of Lambdas that poll, store, and serve metadata about different assets. 
+
+## Source Code Handling
+
+Lambda source code is included in this directory. When applied, `requirements.txt` is installed, and the source files + dependencies are zipped and uploaded. For now this is preferable to a container-based deployment, as it's easier to prototype and may offer improved cold start times (as the Lambdas need to serve live requests). 

market_metadata_service/iam.tf

@@ -0,0 +1,42 @@
+data "aws_iam_policy_document" "lambda_assume_role" {
+  statement {
+    effect = "Allow"
+
+    principals {
+      type        = "Service"
+      identifiers = ["lambda.amazonaws.com"]
+    }
+
+    actions = [
+      "sts:AssumeRole"
+    ]
+  }
+}
+
+data "aws_iam_policy_document" "lambda_permissions" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "s3:PutObject",
+      "s3:GetObject",
+      "s3:ListBucket"
+    ]
+
+    resources = [
+      aws_s3_bucket.market_metadata_bucket.arn,
+      "${aws_s3_bucket.market_metadata_bucket.arn}/*"
+    ]
+  }
+}
+
+resource "aws_iam_policy" "lambda_permissions_policy" {
+  name        = "market-metadata-lambda-permissions-policy"
+  description = "Permissions policy for the Market Metadata Service Lambda functions"
+  policy      = data.aws_iam_policy_document.lambda_permissions.json
+}
+
+resource "aws_iam_role" "lambda_executor" {
+  name                = "market-metadata-service-LambdaExecutionRole"
+  assume_role_policy  = data.aws_iam_policy_document.lambda_assume_role.json
+  managed_policy_arns = [aws_iam_policy.lambda_permissions_policy.arn]
+}

market_metadata_service/main.tf

@@ -0,0 +1,38 @@
+resource "null_resource" "install_python_dependencies" {
+  provisioner "local-exec" {
+    command = "pip3 install -r ./src/requirements.txt -t ./src"
+  }
+
+  triggers = {
+    trigger = timestamp()
+  }
+}
+
+data "archive_file" "lambda_zip" {
+  type        = "zip"
+  source_dir  = "${path.module}/src"
+  output_path = "${path.module}/src/lambda_function.zip"
+  excludes = [
+    "venv",
+    "_pycache_"
+  ]
+  depends_on = [
+    null_resource.install_python_dependencies
+  ]
+}
+
+resource "aws_lambda_function" "market_metadata_service" {
+  function_name    = "market-metadata-service"
+  filename         = data.archive_file.lambda_zip.output_path
+  source_code_hash = data.archive_file.lambda_zip.output_base64sha256
+
+  runtime = "python3.12"
+  handler = "lambda_function.lambda_handler"
+  timeout = 60
+
+  role = aws_iam_role.lambda_executor.arn
+
+  depends_on = [
+    data.archive_file.lambda_zip
+  ]
+}

market_metadata_service/providers.tf

@@ -0,0 +1,4 @@
+# Default provider.
+provider "aws" {
+  region = "ap-northeast-1"
+}

market_metadata_service/s3.tf

@@ -0,0 +1,3 @@
+resource "aws_s3_bucket" "market_metadata_bucket" {
+  bucket = "market-metadata"
+}

market_metadata_service/src/lambda_function.py

@@ -0,0 +1,137 @@
+import json
+import logging
+
+from typing import Any
+
+import boto3
+import requests
+
+S3_BUCKET = 'market-metadata'
+S3_INFO_OBJECT_KEY = 'info.json'
+S3_QUOTE_OBJECT_KEY = 'quote.json'
+
+CMC_API_KEY = ''
+CMC_BASE_API_URL = 'https://pro-api.coinmarketcap.com'
+CMC_INFO_API_URL = CMC_BASE_API_URL + '/v2/cryptocurrency/info'
+CMC_QUOTES_API_URL = CMC_BASE_API_URL + '/v2/cryptocurrency/quotes/latest'
+CMC_OHLCV_API_URL = CMC_BASE_API_URL + '/v2/cryptocurrency/ohlcv/historical'
+CMC_BASE_TOKEN_PAGE_URL = 'https://coinmarketcap.com/currencies'
+
+ASSET_TO_ID = {
+    'BTC': 1,
+    'ETH': 1027,
+    'SOL': 5426,
+    'DOGE': 74,
+    'PEPE': 24478,
+    'W': 29587,
+    'MANA': 1966,
+    'LPT': 3640,
+    'GLM': 1455,
+    'HNT': 5665,
+}
+
+ID_TO_ASSET = {
+    1: 'BTC',
+    1027: 'ETH',
+    5426: 'SOL',
+    74: 'DOGE',
+    24478: 'PEPE',
+    29587: 'W',
+    1966: 'MANA',
+    3640: 'LPT',
+    1455: 'GLM',
+    5665: 'HNT',
+}
+
+
+logger = logging.getLogger()
+logger.setLevel(logging.DEBUG)
+
+
+def init_session() -> requests.Session:
+    headers = {
+        'Accepts': 'application/json',
+        'Accept-Encoding': 'deflate, gzip',
+        'X-CMC_PRO_API_KEY': CMC_API_KEY
+    }
+    session = requests.Session()
+    session.headers.update(headers)
+    return session
+
+
+def make_get_request(session: requests.Session, url: str, params: Any) -> Any:
+    # TODO Handle errors, check 'status' field of response
+    try:
+        response = session.get(url, params=params)
+        data = json.loads(response.text)
+    except (ConnectionError, Timeout, TooManyRedirects) as e:
+        print(e)
+
+    return data
+
+
+def fetch_info(session: requests.Session, params: Any) -> Any:
+    data = make_get_request(session, CMC_INFO_API_URL, params)
+
+    res = {}
+    for cmc_id, cmc_data in data['data'].items():
+        res[ID_TO_ASSET[int(cmc_id)]] = {
+            'name': cmc_data['name'],
+            'logo': cmc_data['logo'],
+            'urls': {
+                # CMC returns a list of URLs for each category. For now, just take the first one
+                'website': cmc_data['urls']['website'][0] if cmc_data['urls']['website'] else None,
+                'technical_doc': cmc_data['urls']['technical_doc'][0] if cmc_data['urls']['technical_doc'] else None,
+                'cmc': CMC_BASE_TOKEN_PAGE_URL + '/' + cmc_data['slug']
+            },
+            'sector_tags': cmc_data['tags'],
+            'exchanges': []
+        }
+
+    return res
+
+
+def fetch_quotes(session: requests.Session, params: Any) -> Any:
+    data = make_get_request(session, CMC_QUOTES_API_URL, params)
+
+    res = {}
+    for cmc_id, cmc_data in data['data'].items():
+        quote = cmc_data['quote']['USD']
+        res[ID_TO_ASSET[int(cmc_id)]] = {
+            'price': quote['price'],
+            'percent_change_24h': quote['percent_change_24h'],
+            'volume_24h': quote['volume_24h'],
+            'market_cap': quote['market_cap'],
+        }
+
+    return res
+
+
+def write_to_s3(object_key: str, data: Any):
+    client = boto3.client('s3')
+    client.put_object(
+        Body=json.dumps(data),
+        Bucket=S3_BUCKET,
+        Key=object_key
+    )
+
+
+def main(run_type: str, ids: list[str]):
+    session = init_session()
+    params = {
+        'id': ','.join([str(x) for x in ASSET_TO_ID.values()])
+    }
+
+    info_data = fetch_info(session, params)
+    write_to_s3(S3_INFO_OBJECT_KEY, info_data)
+
+    quotes_data = fetch_quotes(session, params)
+    write_to_s3(S3_QUOTE_OBJECT_KEY, quotes_data)
+
+
+def lambda_handler(event, context):
+    main(None, None)
+
+
+if __name__ == "__main__":
+    main(None, None)

market_metadata_service/src/requirements.txt

@@ -0,0 +1,11 @@
+boto3==1.35.9
+botocore==1.35.9
+certifi==2024.8.30
+charset-normalizer==3.3.2
+idna==3.8
+jmespath==1.0.1
+python-dateutil==2.9.0.post0
+requests==2.32.3
+s3transfer==0.10.2
+six==1.16.0
+urllib3==2.2.2