Commit 3d42bd55 authored by Langhammer, Jens's avatar Langhammer, Jens

lib: implement loading of environment variables in config loader without explicit config change

parent baa831b8
Pipeline #4022 failed with stage
in 5 minutes and 30 seconds
......@@ -3,25 +3,15 @@ kind: ConfigMap
metadata:
name: {{ include "p2.fullname" . }}-config
data:
config.yml: |
debug: false
error_report_enabled: {{ .Values.config.error_reporting }}
s3:
base_domain: {{ (index .Values.ingress.hosts 0) | quote }}
# Set this to true if you only want to use external authentication
external_auth_only: {{ .Values.config.external_auth_only }}
# Callback URL: <base url>/_/oidc/callback/
# oidc:
# enabled: false
# client_id: ""
# client_secret: ""
# auth_url: ""
# token_url: ""
# user_url: ""
oidc: {{ toYaml .Values.config.oidc | nindent 6 }}
secret_key: env://P2_SECRET_KEY
P2_DEBUG: 'false'
P2_ERROR_REPORT_ENABLED: {{ .Values.config.error_reporting }}
P2_S3__BASE_DOMAIN: {{ (index .Values.ingress.hosts 0) | quote }}
# Set this to true if you only want to use external authentication
P2_EXTERNAL_AUTH_ONLY: {{ .Values.config.external_auth_only }}
# Callback URL: <base url>/_/oidc/callback/
# P2_OIDC__ENABLED: false
# P2_OIDC__CLIENT_ID: ""
# P2_OIDC__CLIENT_SECRET: ""
# P2_OIDC__AUTH_URL: ""
# P2_OIDC__TOKEN_URL: ""
# P2_OIDC__USER_URL: ""
......@@ -28,9 +28,6 @@ spec:
securityContext:
fsGroup: 100
volumes:
- name: config-volume
configMap:
name: {{ include "p2.fullname" . }}-config
- name: media-storage
persistentVolumeClaim:
claimName: {{ include "p2.fullname" . }}-pvc-app-storage
......@@ -38,8 +35,10 @@ spec:
- name: {{ .Chart.Name }}
image: "docker.beryju.org/p2/server:{{ .Values.version }}"
imagePullPolicy: IfNotPresent
command: ["./manage.py"]
args: ["grpc"]
command:
- ./manage.py
args:
- grpc
ports:
- name: grpc
containerPort: 50051
......@@ -47,21 +46,24 @@ spec:
- name: prometheus
containerPort: 9102
protocol: TCP
envFrom:
- configMapRef:
name: {{ include "p2.fullname" . }}-config
env:
- name: P2_REDIS_HOST
- name: P2_REDIS__HOST
value: "{{ .Release.Name }}-redis-master"
- name: P2_REDIS_PASSWORD
- name: P2_REDIS__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-redis"
key: redis-password
- name: P2_PSQL_DBNAME
- name: P2_POSTGRESQL__NAME
value: "{{ .Values.postgresql.postgresqlDatabase }}"
- name: P2_PSQL_USERNAME
- name: P2_POSTGRESQL__USER
value: "{{ .Values.postgresql.postgresqlUsername }}"
- name: P2_PSQL_HOST
- name: P2_POSTGRESQL__HOST
value: "{{ .Release.Name }}-postgresql"
- name: P2_PSQL_PASSWORD
- name: P2_POSTGRESQL__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-postgresql"
......@@ -72,8 +74,6 @@ spec:
name: "{{ template "p2.fullname" . }}"
key: secret_key
volumeMounts:
- mountPath: /etc/p2
name: config-volume
- mountPath: /storage
name: media-storage
resources:
......
......@@ -28,9 +28,6 @@ spec:
securityContext:
fsGroup: 100
volumes:
- name: config-volume
configMap:
name: {{ include "p2.fullname" . }}-config
- name: media-storage
persistentVolumeClaim:
claimName: {{ include "p2.fullname" . }}-pvc-app-storage
......@@ -41,14 +38,24 @@ spec:
- ./manage.py
args:
- migrate
envFrom:
- configMapRef:
name: {{ include "p2.fullname" . }}-config
env:
- name: P2_PSQL_DBNAME
- name: P2_REDIS__HOST
value: "{{ .Release.Name }}-redis-master"
- name: P2_REDIS__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-redis"
key: redis-password
- name: P2_POSTGRESQL__NAME
value: "{{ .Values.postgresql.postgresqlDatabase }}"
- name: P2_PSQL_USERNAME
- name: P2_POSTGRESQL__USER
value: "{{ .Values.postgresql.postgresqlUsername }}"
- name: P2_PSQL_HOST
- name: P2_POSTGRESQL__HOST
value: "{{ .Release.Name }}-postgresql"
- name: P2_PSQL_PASSWORD
- name: P2_POSTGRESQL__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-postgresql"
......@@ -59,8 +66,6 @@ spec:
name: "{{ template "p2.fullname" . }}"
key: secret_key
volumeMounts:
- mountPath: /etc/p2
name: config-volume
- mountPath: /storage
name: media-storage
containers:
......@@ -71,6 +76,9 @@ spec:
- ./manage.py
args:
- web
envFrom:
- configMapRef:
name: {{ include "p2.fullname" . }}-config
env:
- name: P2_COMPONENT
value: web
......@@ -81,13 +89,13 @@ spec:
secretKeyRef:
name: "{{ .Release.Name }}-redis"
key: redis-password
- name: P2_PSQL_DBNAME
- name: P2_POSTGRESQL__NAME
value: "{{ .Values.postgresql.postgresqlDatabase }}"
- name: P2_PSQL_USERNAME
- name: P2_POSTGRESQL__USER
value: "{{ .Values.postgresql.postgresqlUsername }}"
- name: P2_PSQL_HOST
- name: P2_POSTGRESQL__HOST
value: "{{ .Release.Name }}-postgresql"
- name: P2_PSQL_PASSWORD
- name: P2_POSTGRESQL__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-postgresql"
......@@ -100,8 +108,6 @@ spec:
containerPort: 9102
protocol: TCP
volumeMounts:
- mountPath: /etc/p2
name: config-volume
- mountPath: /storage
name: media-storage
livenessProbe:
......
......@@ -25,9 +25,6 @@ spec:
securityContext:
fsGroup: 100
volumes:
- name: config-volume
configMap:
name: {{ include "p2.fullname" . }}-config
- name: media-storage
persistentVolumeClaim:
claimName: {{ include "p2.fullname" . }}-pvc-app-storage
......@@ -35,27 +32,31 @@ spec:
- name: {{ .Chart.Name }}
image: "docker.beryju.org/p2/server:{{ .Values.version }}"
imagePullPolicy: IfNotPresent
command: ["./manage.py", "worker"]
command:
- ./manage.py
args:
- worker
volumeMounts:
- mountPath: /etc/p2
name: config-volume
- mountPath: /storage
name: media-storage
envFrom:
- configMapRef:
name: {{ include "p2.fullname" . }}-config
env:
- name: P2_REDIS_HOST
- name: P2_REDIS__HOST
value: "{{ .Release.Name }}-redis-master"
- name: P2_REDIS_PASSWORD
- name: P2_REDIS__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-redis"
key: redis-password
- name: P2_PSQL_DBNAME
- name: P2_POSTGRESQL__NAME
value: "{{ .Values.postgresql.postgresqlDatabase }}"
- name: P2_PSQL_USERNAME
- name: P2_POSTGRESQL__USER
value: "{{ .Values.postgresql.postgresqlUsername }}"
- name: P2_PSQL_HOST
- name: P2_POSTGRESQL__HOST
value: "{{ .Release.Name }}-postgresql"
- name: P2_PSQL_PASSWORD
- name: P2_POSTGRESQL__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-postgresql"
......
"""supervisr core config loader"""
"""p2 core config loader"""
import os
from collections import Mapping
from contextlib import contextmanager
......@@ -17,16 +17,19 @@ SEARCH_PATHS = [
'',
] + glob('/etc/p2/config.d/*.yml', recursive=True)
LOGGER = get_logger()
ENVIRONMENT = os.getenv('P2_ENV', 'local')
ENV_PREFIX = 'P2'
ENVIRONMENT = os.getenv(f'{ENV_PREFIX}_ENV', 'local')
class ConfigLoader:
"""Search through SEARCH_PATHS and load configuration"""
"""Search through SEARCH_PATHS and load configuration. Environment variables starting with
`ENV_PREFIX` are also applied.
A variable like P2_POSTGRESQL__HOST would translate to postgresql.host"""
loaded_file = []
__config = {}
__context_default = None
__sub_dicts = []
def __init__(self):
......@@ -48,15 +51,7 @@ class ConfigLoader:
if os.path.isfile(env_file) and os.path.exists(env_file):
# Update config with env file
self.update_from_file(env_file)
self.handle_secret_key()
def handle_secret_key(self):
"""Handle `secret_key_file`"""
if 'secret_key_file' in self.__config:
secret_key_file = self.__config.get('secret_key_file')
if os.path.isfile(secret_key_file) and os.path.exists(secret_key_file):
with open(secret_key_file) as file:
self.__config['secret_key'] = file.read().replace('\n', '')
self.update_from_env()
def update(self, root, updatee):
"""Recursively update dictionary"""
......@@ -93,12 +88,26 @@ class ConfigLoader:
"""Update config from dict"""
self.__config.update(update)
@contextmanager
def default(self, value: Any):
"""Contextmanage that sets default"""
self.__context_default = value
yield
self.__context_default = None
def update_from_env(self):
"""Check environment variables"""
outer = {}
idx = 0
for key, value in os.environ.items():
if not key.startswith(ENV_PREFIX):
continue
relative_key = key.replace(f"{ENV_PREFIX}_", '').replace('__', '.').lower()
# Recursively convert path from a.b.c into outer[a][b][c]
current_obj = outer
dot_parts = relative_key.split('.')
for dot_part in dot_parts[:-1]:
if dot_part not in current_obj:
current_obj[dot_part] = {}
current_obj = current_obj[dot_part]
current_obj[dot_parts[-1]] = value
idx += 1
if idx > 0:
LOGGER.debug("Loaded environment variables", count=idx)
self.update_from_dict(outer)
@contextmanager
# pylint: disable=invalid-name
......@@ -110,8 +119,6 @@ class ConfigLoader:
def get(self, key: str, default=None) -> Any:
"""Get value from loaded config file"""
if default is None:
default = self.__context_default
config_copy = self.__config
for sub in self.__sub_dicts:
config_copy = config_copy.get(sub, None)
......@@ -125,8 +132,6 @@ class ConfigLoader:
# pylint: disable=invalid-name
def y(self, path: str, default=None, sep='.') -> Any:
"""Access attribute by using yaml path"""
if default is None:
default = self.__context_default
# Walk sub_dicts before parsing path
root = self.raw
for sub in self.__sub_dicts:
......@@ -142,7 +147,7 @@ class ConfigLoader:
CONFIG = ConfigLoader()
# pylint: disable=unused-argument
def signal_handler(sender, **kwargs):
def signal_handler(sender, **_):
"""Add all loaded config files to autoreload watcher"""
for path in CONFIG.loaded_file:
sender.watch_file(path)
......
# This is the default configuration file
postgresql:
name: env://P2_PSQL_DBNAME
user: env://P2_PSQL_USERNAME
password: env://P2_PSQL_PASSWORD
host: env://P2_PSQL_HOST
host: localhost
name: p2
user: postgres
password: ''
redis:
host: env://P2_REDIS_HOST
password: env://P2_REDIS_PASSWORD
host: localhost
password: ''
cache_db: 0
message_queue_db: 1
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment