Skip to content

Commit

Permalink
Separate ConfigSpec parser (kivy#1639)
Browse files Browse the repository at this point in the history 
* Separate ConfigSpec parser (Address kivy#1636)

Separate ConfigParser code from Buildozer.

- Use subclass, rather than monkey-patching.
- Single implementation of list-handling code.
- Convenience functions to minimise impact of existing code (but instantly deprecated).
- Treat defaults consistently
- Apply the env overrides automatically (client doesn't need to trigger).
- Apply the env overrides once (read-time) rather than on every get.
- Avoid re-using term "raw" which has pre-existing definition in ConfigSpec.
   - Update android.py client to match,
- Unit tests.

Add comments to start and end of default.spec to explain the syntax (as discussed in kivy#1636)

* Add profile handling to configparser

Move it from Buildozer.
Rename it to be more meaningful.
Add support for whitespace trimming.
Update documentation in default.spec.
Update tests.
  • Loading branch information
@Julian-O
Julian-O authored Jul 29, 2023
1 parent 88a3b77 commit c7dee51
Show file tree
Hide file tree
Showing 6 changed files with 474 additions and 176 deletions.
155 changes: 10 additions & 145 deletions buildozer/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,6 @@
import codecs
import textwrap
import warnings
from buildozer.jsonstore import JsonStore
from buildozer.logger import Logger
from sys import stdout, stderr, exit
from re import search
from os.path import join, exists, dirname, realpath, splitext, expanduser
Expand All @@ -30,12 +28,16 @@
import pexpect

from urllib.request import FancyURLopener
from configparser import ConfigParser
try:
import fcntl
except ImportError:
# on windows, no fcntl
fcntl = None

from buildozer.jsonstore import JsonStore
from buildozer.logger import Logger
from buildozer.specparser import SpecParser

SIMPLE_HTTP_SERVER_PORT = 8000


Expand Down Expand Up @@ -74,25 +76,14 @@ def __init__(self, filename='buildozer.spec', target=None):
self.specfilename = filename
self.state = None
self.build_id = None
self.config_profile = ''
self.config = ConfigParser(allow_no_value=True)
self.config.optionxform = lambda value: value
self.config.getlist = self._get_config_list
self.config.getlistvalues = self._get_config_list_values
self.config.getdefault = self._get_config_default
self.config.getbooldefault = self._get_config_bool
self.config.getrawdefault = self._get_config_raw_default
self.config = SpecParser()

self.logger = Logger()

if exists(filename):
self.config.read(filename, "utf-8")
self.check_configuration_tokens()

# Check all section/tokens for env vars, and replace the
# config value if a suitable env var exists.
set_config_from_envs(self.config)

try:
self.logger.set_level(
int(self.config.getdefault(
Expand Down Expand Up @@ -908,6 +899,8 @@ def run_default(self):
self.run_command(cmd)

def run_command(self, args):
profile = None

while args:
if not args[0].startswith('-'):
break
Expand All @@ -921,13 +914,13 @@ def run_command(self, args):
exit(0)

elif arg in ('-p', '--profile'):
self.config_profile = args.pop(0)
profile = args.pop(0)

elif arg == '--version':
print('Buildozer {0}'.format(__version__))
exit(0)

self._merge_config_profile()
self.config.apply_profile(profile)

self.check_root()

Expand Down Expand Up @@ -1042,131 +1035,3 @@ def cmd_serve(self, *args):
print("Serving via HTTP at port {}".format(SIMPLE_HTTP_SERVER_PORT))
print("Press Ctrl+c to quit serving.")
httpd.serve_forever()

#
# Private
#

def _merge_config_profile(self):
profile = self.config_profile
if not profile:
return
for section in self.config.sections():

# extract the profile part from the section name
# example: [app@default,hd]
parts = section.split('@', 1)
if len(parts) < 2:
continue

# create a list that contain all the profiles of the current section
# ['default', 'hd']
section_base, section_profiles = parts
section_profiles = section_profiles.split(',')
if profile not in section_profiles:
continue

# the current profile is one available in the section
# merge with the general section, or make it one.
if not self.config.has_section(section_base):
self.config.add_section(section_base)
for name, value in self.config.items(section):
print('merged ({}, {}) into {} (profile is {})'.format(name,
value, section_base, profile))
self.config.set(section_base, name, value)

def _get_config_list_values(self, *args, **kwargs):
kwargs['with_values'] = True
return self._get_config_list(*args, **kwargs)

def _get_config_list(self, section, token, default=None, with_values=False):
# monkey-patch method for ConfigParser
# get a key as a list of string, separated from the comma

# check if an env var exists that should replace the file config
set_config_token_from_env(section, token, self.config)

# if a section:token is defined, let's use the content as a list.
l_section = '{}:{}'.format(section, token)
if self.config.has_section(l_section):
values = self.config.options(l_section)
if with_values:
return ['{}={}'.format(key, self.config.get(l_section, key)) for
key in values]
else:
return [x.strip() for x in values]

values = self.config.getdefault(section, token, '')
if not values:
return default
values = values.split(',')
if not values:
return default
return [x.strip() for x in values]

def _get_config_default(self, section, token, default=None):
# monkey-patch method for ConfigParser
# get an appropriate env var if it exists, else
# get a key in a section, or the default

# check if an env var exists that should replace the file config
set_config_token_from_env(section, token, self.config)

if not self.config.has_section(section):
return default
if not self.config.has_option(section, token):
return default
return self.config.get(section, token)

def _get_config_bool(self, section, token, default=False):
# monkey-patch method for ConfigParser
# get a key in a section, or the default

# check if an env var exists that should replace the file config
set_config_token_from_env(section, token, self.config)

if not self.config.has_section(section):
return default
if not self.config.has_option(section, token):
return default
return self.config.getboolean(section, token)

def _get_config_raw_default(self, section, token, default=None, section_sep="=", split_char=" "):
l_section = '{}:{}'.format(section, token)
if self.config.has_section(l_section):
return [section_sep.join(item) for item in self.config.items(l_section)]
if not self.config.has_option(section, token):
return default.split(split_char)
return self.config.get(section, token).split(split_char)


def set_config_from_envs(config):
'''Takes a ConfigParser, and checks every section/token for an
environment variable of the form SECTION_TOKEN, with any dots
replaced by underscores. If the variable exists, sets the config
variable to the env value.
'''
for section in config.sections():
for token in config.options(section):
set_config_token_from_env(section, token, config)


def set_config_token_from_env(section, token, config):
'''Given a config section and token, checks for an appropriate
environment variable. If the variable exists, sets the config entry to
its value.
The environment variable checked is of the form SECTION_TOKEN, all
upper case, with any dots replaced by underscores.
Returns True if the environment variable exists and was used, or
False otherwise.
'''
env_var_name = ''.join([section.upper(), '_',
token.upper().replace('.', '_')])
env_var = os.environ.get(env_var_name)
if env_var is None:
return False
config.set(section, token, env_var)
return True
73 changes: 46 additions & 27 deletions buildozer/default.spec
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,8 @@
# This .spec config file tells Buildozer an app's requirements for being built.
#
# It largely follows the syntax of an .ini file.
# See the end of the file for more details and warnings about common mistakes.

[app]

# (str) Title of your application
Expand Down Expand Up @@ -410,40 +415,54 @@ warn_on_root = 1
# (str) Path to build output (i.e. .apk, .aab, .ipa) storage
# bin_dir = ./bin

# -----------------------------------------------------------------------------
# List as sections
#-----------------------------------------------------------------------------
# Notes about using this file:
#
# You can define all the "list" as [section:key].
# Each line will be considered as a option to the list.
# Let's take [app] / source.exclude_patterns.
# Instead of doing:
# Buildozer uses a variant of Python's ConfigSpec to read this file.
# For the basic syntax, including interpolations, see
# https://docs.python.org/3/library/configparser.html#supported-ini-file-structure
#
#[app]
#source.exclude_patterns = license,data/audio/*.wav,data/images/original/*
# Warning: Comments cannot be used "inline" - i.e.
# [app]
# title = My Application # This is not a comment, it is part of the title.
#
# This can be translated into:
# Warning: Indented text is treated as a multiline string - i.e.
# [app]
# title = My Application
# package.name = myapp # This is all part of the title.
#
#[app:source.exclude_patterns]
#license
#data/audio/*.wav
#data/images/original/*
# Buildozer's .spec files have some additional features:
#


# -----------------------------------------------------------------------------
# Profiles
# Buildozer supports lists - i.e.
# [app]
# source.include_exts = py,png,jpg
# # ^ This is a list.
#
# [app:source.include_exts]
# py
# png
# jpg
# # ^ This is an alternative syntax for a list.
#
# Buildozer's option names are case-sensitive, unlike most .ini files.
#
# Buildozer supports overriding options through environment variables.
# Name an environment variable as SECTION_OPTION to override a value in a .spec
# file.
#
# Buildozer support overriding options through profiles.
# For example, you want to deploy a demo version of your application without
# HD content. You could first change the title to add "(demo)" in the name
# and extend the excluded directories to remove the HD content.
#
# You can extend section / key with a profile
# For example, you want to deploy a demo version of your application without
# HD content. You could first change the title to add "(demo)" in the name
# and extend the excluded directories to remove the HD content.
# [app@demo]
# title = My Application (demo)
#
#[app@demo]
#title = My Application (demo)
# [app:source.exclude_patterns@demo]
# images/hd/*
#
#[app:source.exclude_patterns@demo]
#images/hd/*
# Then, invoke the command line with the "demo" profile:
#
# Then, invoke the command line with the "demo" profile:
# buildozer --profile demo android debug
#
#buildozer --profile demo android debug
# Environment variable overrides have priority over profile overrides.
6 changes: 4 additions & 2 deletions buildozer/scripts/remote.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@

class BuildozerRemote(Buildozer):
def run_command(self, args):
profile = None

while args:
if not args[0].startswith('-'):
break
Expand All @@ -45,7 +47,7 @@ def run_command(self, args):
self.logger.log_level = 2

elif arg in ('-p', '--profile'):
self.config_profile = args.pop(0)
profile = args.pop(0)

elif arg in ('-h', '--help'):
self.usage()
Expand All @@ -55,7 +57,7 @@ def run_command(self, args):
print('Buildozer (remote) {0}'.format(__version__))
exit(0)

self._merge_config_profile()
self.config.apply_profile(profile)

if len(args) < 2:
self.usage()
Expand Down
Loading

0 comments on commit c7dee51

Please sign in to comment.