From c5a1a50454951205ab2089e0206b988e53af7c4b Mon Sep 17 00:00:00 2001 From: pmbrull Date: Sun, 24 Oct 2021 19:09:27 +0200 Subject: [PATCH 1/4] Prepare JSON reader --- levy/config.py | 26 +++++++++++++++++++++----- tests/resources/test.json | 21 +++++++++++++++++++++ tests/test_config.py | 14 ++++++++++++-- 3 files changed, 54 insertions(+), 7 deletions(-) create mode 100644 tests/resources/test.json diff --git a/levy/config.py b/levy/config.py index 116ddc9..4e40c57 100644 --- a/levy/config.py +++ b/levy/config.py @@ -1,8 +1,10 @@ """ Config parser definition """ +import json import logging from collections import namedtuple +from pathlib import Path from typing import Any, Dict, Generic, List, Optional, TypeVar import yaml @@ -54,11 +56,25 @@ def read_file( :return: """ - with open(file, "r") as yml_file: - rendered = render_str(yml_file.read()) - cfg = cls.read_dict( - yaml.safe_load(rendered), name=name, list_id=list_id, datatype=datatype - ) + # Check file extension + ext = Path(file).suffix + + if ext == ".yaml": + with open(file, "r") as yml_file: + rendered = render_str(yml_file.read()) + cfg = cls.read_dict( + yaml.safe_load(rendered), + name=name, + list_id=list_id, + datatype=datatype, + ) + + if ext == ".json": + with open(file, "r") as json_file: + rendered = render_str(json_file.read()) + cfg = cls.read_dict( + json.loads(rendered), name=name, list_id=list_id, datatype=datatype + ) cfg._file = file # pylint: disable=attribute-defined-outside-init return cfg diff --git a/tests/resources/test.json b/tests/resources/test.json new file mode 100644 index 0000000..b85aede --- /dev/null +++ b/tests/resources/test.json @@ -0,0 +1,21 @@ +{ + "title": "Lévy the cat", + "colors": ["black", "white"], + "hobby": { + "eating": { + "what": "anything" + } + }, + "friends": [ + {% set friends = [ "cartman", "lima" ] %} + {% for friend in friends %} + { + "name": "${ friend }", + "type": "cat" + } + {% if loop.index0 < friends|length - 1%} + , + {% endif %} + {% endfor %} + ] +} diff --git a/tests/test_config.py b/tests/test_config.py index 3c233d0..e5a413f 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,13 +1,13 @@ import os -import pytest from typing import Dict, List, Optional from unittest import mock +import pytest from pydantic import BaseModel, ValidationError from levy.config import Config -from levy.renderer import render_reg from levy.exceptions import ListParseException +from levy.renderer import render_reg class TestConfig: @@ -17,6 +17,7 @@ def setup(self): self.resources = os.path.join(self.dir, "resources") self.file = os.path.join(self.resources, "test.yaml") + self.json_file = os.path.join(self.resources, "test.json") self.cfg = Config.read_file(file=self.file) def test_name(self): @@ -136,3 +137,12 @@ class Kitten(BaseModel): with pytest.raises(ValidationError): file = os.path.join(self.resources, "test_ko.yaml") Config.read_file(file, datatype=Kitten) + + def test_json(self): + """ + Validate JSON file read + """ + cfg = Config.read_file(self.json_file) + cfg_yaml = Config.read_file(self.file) + + assert cfg._vars == cfg_yaml._vars From ab71058c9cac7de34d8847273469a5ffbd4d84f9 Mon Sep 17 00:00:00 2001 From: pmbrull Date: Sun, 24 Oct 2021 19:23:13 +0200 Subject: [PATCH 2/4] Add JSON docs --- docs/index.md | 2 +- docs/json.md | 45 +++++++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 3 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 docs/json.md diff --git a/docs/index.md b/docs/index.md index 6fe977f..8671ee6 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,5 +1,5 @@

- Supercharge YAML configs with Jinja templates, typing and custom functions. + Supercharge configs with Jinja templates, typing and custom functions.

diff --git a/docs/json.md b/docs/json.md new file mode 100644 index 0000000..662d8b7 --- /dev/null +++ b/docs/json.md @@ -0,0 +1,45 @@ +## Rendering JSON Files + +`levy` supports JSON file as well as YAML. All the features remain the same, however, +we need to make sure that the JSON file we write is a correctly formatted JSON +after the render phase, which might be a bit tricky at times. + +> OBS: Note that the added difficulty is only for Jinja templating, as we need to + take care about putting all the quotes and commas. + +Let's revisit the first example we saw, and how it would look like as a JSON file: + +```json +{ + "title": "Lévy the cat", + "colors": ["black", "white"], + "hobby": { + "eating": { + "what": "anything" + } + }, + "friends": [ + {% set friends = [ "cartman", "lima" ] %} + {% for friend in friends %} + { + "name": "${ friend }", + "type": "cat" + } + {% if loop.index0 < friends|length - 1%} + , + {% endif %} + {% endfor %} + ] +} +``` + +As you can see, most of it is the same. However, in the `friends` list, we need +to add specific logic to add commas `,` if we have not reached the end of the loop. + +Afterwards, the API remains: + +```python +from levy.config import Config + +cfg = Config.read_file("test.json") +``` diff --git a/mkdocs.yml b/mkdocs.yml index 8228fa5..cb10ce6 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -34,6 +34,7 @@ nav: - Nested Config Lists: lists.md - Render Custom Functions: functions.md - Schema Validation: schema.md + - JSON Files: json.md - Contributing: contributing.md - References: references.md From 9d5aabb3c793a33d96acfdaf62f495ee5434cdc9 Mon Sep 17 00:00:00 2001 From: pmbrull Date: Sun, 24 Oct 2021 19:23:39 +0200 Subject: [PATCH 3/4] Update version --- levy/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/levy/__init__.py b/levy/__init__.py index 5eb5848..ff276be 100644 --- a/levy/__init__.py +++ b/levy/__init__.py @@ -1,3 +1,3 @@ """Dynamic python configuration parser""" -__version__ = "0.5.1" +__version__ = "0.6.1" From 5aced3cd6b86a8a459207f0f50287c66b61e2e2c Mon Sep 17 00:00:00 2001 From: pmbrull Date: Sun, 24 Oct 2021 19:30:10 +0200 Subject: [PATCH 4/4] Update docs --- README.md | 15 ++++++++------- docs/functions.md | 4 ++-- docs/index.md | 5 +++-- docs/schema.md | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 38274c4..e700dc7 100644 --- a/README.md +++ b/README.md @@ -19,14 +19,15 @@ Get up and running with pip install levy ``` -So far, it only supports YAML files or reading configurations directly from a `dict`. +It supports reading both JSON and YAML files, as well as getting configurations +directly from a `dict`. The interesting approach here is regarding handling multiple environments. Usually we need to pass different parameters depending on where we are (DEV, PROD, and any arbitrary environment name we might use). It is also common to have these specific parameters available as env variables, be it our infra or in a CI/CD process. -`levy` adds a `jinja2` layer on top of our YAML files, so that not only we can load +`levy` adds a `jinja2` layer on top of our config files, so that not only we can load env variables on the fly, but helps us leverage templating syntax to keep our configurations centralized and DRY. @@ -162,7 +163,7 @@ cfg("not in there") # AttributeError ### Environment Variables -With this templating approach on top of YAML, we can not only use default behaviors, but also +With this templating approach on top of our files, we can not only use default behaviors, but also define our own custom functionalities. The one we have provided by default is reading environment variables at render time: @@ -178,7 +179,7 @@ we'll get a `MissingEnvException`. ### Registering new functions -If we need to apply different functions when rendering the YAML, we can register them +If we need to apply different functions when rendering the files, we can register them by name before instantiating the `Config` class. Let's imagine the following YAML file: @@ -210,7 +211,7 @@ cfg.foo # 'X' Note how we registered `my_func` with the same name it appeared in the YAML. However, the name is completely arbitrary, and we can pass the function `upper` with the name `bar`. -With this approach one can add even further dynamism to the YAML config files. +With this approach one can add even further dynamism to both YAML and JSON config files. To peek into the registry state, we can run: @@ -228,7 +229,7 @@ Which in the example will show us ## Schema Validation -At some point it might be interesting to make sure that the YAML we are reading follows +At some point it might be interesting to make sure that the config we are reading follows some standards. That is why we have introduced the ability to pass a schema our file needs to follow. @@ -266,7 +267,7 @@ assert cfg.age is None assert cfg.friends.lima.fur == "soft" ``` -Note how this adds even another layer of flexibility, as after reading the YAML we will +Note how this adds even another layer of flexibility, as after reading the file we will have all the data we might require available to use. ## Contributing diff --git a/docs/functions.md b/docs/functions.md index 05d4191..78388bc 100644 --- a/docs/functions.md +++ b/docs/functions.md @@ -17,7 +17,7 @@ We can then access these values as usual. ## Registering new functions -If we need to apply different functions when rendering the YAML, we can register them +If we need to apply different functions when rendering the files, we can register them by name before instantiating the `Config` class. Let's imagine the following YAML file: @@ -60,7 +60,7 @@ $ cfg.foo Note how we registered `my_func` with the same name it appeared in the YAML. However, the name is completely arbitrary, and we can pass the function `upper` with the name `bar`. -With this approach one can add even further dynamism to the YAML config files. +With this approach one can add even further dynamism to our config files. ## Registry diff --git a/docs/index.md b/docs/index.md index 8671ee6..09e627e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -46,9 +46,10 @@ This will also bring to your environment `PyYAML`, `Jinja2` and `pydantic`. ## Quickstart -This project is a lightweight take on configuration parsing with a twist. So far, it only supports YAML files or reading configurations directly from a `dict`. +This project is a lightweight take on configuration parsing with a twist. It supports reading both JSON and YAML files, as well as getting configurations +directly from a `dict`. -`levy` adds a `jinja2` layer on top our YAML files, which allows us to run any Jinja templating syntax on them. Later on, we will also see how to register our own custom functions. +`levy` adds a `jinja2` layer on top our config files, which allows us to run any Jinja templating syntax on them. Later on, we will also see how to register our own custom functions. Let's suppose we have the following configuration: diff --git a/docs/schema.md b/docs/schema.md index 426acdf..e38ac7a 100644 --- a/docs/schema.md +++ b/docs/schema.md @@ -50,7 +50,7 @@ class Kitten(BaseModel): cfg = Config.read_file("", datatype=Kitten) ``` -Note how this adds even another layer of flexibility, as after reading the YAML we will +Note how this adds even another layer of flexibility, as after reading the config we will have all the data we might require available to use.