...
 
Commits (34)
......@@ -4,13 +4,14 @@ stages:
- build-dev-image
- test
- build
- package
image: docker.beryju.org/p2/dev:latest
variables:
POSTGRES_DB: p2
POSTGRES_USER: p2
POSTGRES_PASSWORD: "EK-5jnKfjrGRm<77"
P2_POSTGRESQL__USER: p2
P2_POSTGRESQL__PASSWORD: "EK-5jnKfjrGRm<77"
cache:
key: ${CI_JOB_STAGE}
......@@ -80,6 +81,7 @@ pylint:
- redis:latest
coverage:
script:
- env
- coverage run manage.py test
- coverage report
- coverage html
......@@ -144,31 +146,15 @@ build-p2-tier0:
only:
- tags
- /^version/.*$/
package-helm:
image: debian:stretch-slim
stage: package
build-p2-operator:
stage: build
image:
name: gcr.io/kaniko-project/executor:debug
entrypoint: [""]
before_script:
- apt update && apt install -y curl
- curl https://raw.githubusercontent.com/helm/helm/master/scripts/get | bash
- echo "{\"auths\":{\"docker.beryju.org\":{\"auth\":\"$DOCKER_AUTH\"}}}" > /kaniko/.docker/config.json
script:
- helm init --client-only
- helm dependency build install/helm/p2
- helm package install/helm/p2
artifacts:
paths:
- p2-*.tgz
expire_in: 1 week
- /kaniko/executor --context $CI_PROJECT_DIR/operator --dockerfile $CI_PROJECT_DIR/operator/build/Dockerfile --destination docker.beryju.org/p2/operator:latest --destination docker.beryju.org/p2/operator:0.7.6
only:
- tags
- /^version/.*$/
# Manual tasks
# build-protos:
# stage: manual
# script:
# - python -m grpc_tools.protoc -I=. --python_out=p2/grpc/ --grpc_python_out=p2/grpc/ protos/*
# - protoc -I=. protos/*.proto --go_out=plugins=grpc:tier0/internal/
# # - sed -i -E 's/^\(import.*_pb2\)/from . \1/' p2/grpc/protos/*.py
# only:
# - never
---
apiVersion: k8s.beryju.org/v1alpha1
kind: P2
metadata:
name: example-p2
spec:
version: 0.7.6
secret_key: "r-9k#x4tkk2e8%=(9hf#^v4&=5z2)^gzn^)l*_=z+&0a97kwd8"
redis:
password: "ThisIsNotASecurePassword!"
postgresql:
postgresqlPassword: "ThisIsNotASecurePassword!"
config:
# Enable error reporting (errors are sent to sentry.beryju.org)
error_reporting: true
# Only allow single sign-on, configured below
external_auth_only: false
# OIDC Configuration
# Callback URL: <base url>/_/oidc/callback/
oidc:
enabled: false
client_id: ""
client_secret: ""
auth_url: ""
token_url: ""
user_url: ""
deployment:
webInstances: 1
workerInstances: 1
# To disable tier0, set the values below to 0
tier0Instances: 2
grpcInstances: 1
ingress:
enabled: true
serve:
hosts:
- "i.p2.local"
hosts:
- "p2.local"
tls:
- secretName: example-p2-tls
hosts:
- i.p2.local
- p2.local
......@@ -2,15 +2,10 @@
# p2 Install script
# Installs and updates a p2 instance using k3s and docker
# Supported enviormnet variables:
# - HOST: Hostname under which the will be accessible
# - SERVE_HOST: Optional; Hostname under which p2 will serve files.
# - STORAGE_BASE: Base directory in which p2 data will be storeed
# - LE_MAIL: Optional; Let's Encrypt E-Mail. If this is not set, Let's Encrypt is not enabled.
K3S_VERSION="0.6.1"
P2_VERSION="0.7.6"
export INSTALL_K3S_EXEC="--docker --no-deploy traefik"
export INSTALL_K3S_EXEC="--docker"
if [ "$EUID" -ne 0 ]; then
echo "Please run as root"
......@@ -65,57 +60,14 @@ curl -fsSL "https://raw.githubusercontent.com/rancher/k3s/v${K3S_VERSION}/instal
bash install.k3s.sh > /dev/null 2>&1
STORAGE_BASE="${STORAGE_BASE:-/srv/p2}"
P2_PASSWORD_FILE="${STORAGE_BASE}/password"
CPU_CORES=$(grep -c ^processor /proc/cpuinfo)
# Make sure storage directories exist
mkdir -p "${STORAGE_BASE}"
# Check if password has been generated, generate if not
if [[ ! -f "$P2_PASSWORD_FILE" ]]; then
openssl rand -hex 48 > "$P2_PASSWORD_FILE"
fi
# Make sure Password file can only be read by root
chown root: "$P2_PASSWORD_FILE"
chmod 600 "$P2_PASSWORD_FILE"
PASSWORD=$(cat $P2_PASSWORD_FILE)
# Download Helm Chart CRD for k3s, replace values and install
curl -fsSL -o p2_k3s_helm.yaml "https://git.beryju.org/BeryJu.org/p2/raw/version/${P2_VERSION}/install/k3s-helm.yaml"
curl -fsSL -o p2_k3s_storage.yaml "https://git.beryju.org/BeryJu.org/p2/raw/version/${P2_VERSION}/install/k3s-storage.yaml"
curl -fsSL -o p2_k3s_nginx.yaml "https://git.beryju.org/BeryJu.org/p2/raw/version/${P2_VERSION}/install/k3s-nginx-ingress.yaml"
# Replace variable in Helm CRD
sed -i "s|%HOST%|${HOST}|g" p2_k3s_helm.yaml
if [ -n "$SERVE_HOST" ]; then
sed -i "s|%SERVE_INSTANCES%|${CPU_CORES}|g" p2_k3s_helm.yaml
sed -i "s|%SERVE_HOST%|${SERVE_HOST}|g" p2_k3s_helm.yaml
else
sed -i "s|%SERVE_INSTANCES%|0|g" p2_k3s_helm.yaml
fi
sed -i "s|%PASSWORD%|${PASSWORD}|g" p2_k3s_helm.yaml
# Adjust webserver instances (1 instance per CPU)
sed -i "s|%WEB_INSTANCES%|${CPU_CORES}|g" p2_k3s_helm.yaml
# Replace variable in Storage
sed -i "s|%STORAGE_BASE%|${STORAGE_BASE}|g" p2_k3s_storage.yaml
# Run docker image pull in foreground to better show progress
docker image pull docker.beryju.org/p2/server:$P2_VERSION
docker image pull docker.beryju.org/p2/tier0:$P2_VERSION
docker image pull bitnami/postgresql:10.6.0
docker image pull bitnami/redis:4.0.11
sleep 30
mv p2_k3s_nginx.yaml /var/lib/rancher/k3s/server/manifests/p2-10-nginx.yaml
sleep 30
mv p2_k3s_storage.yaml /var/lib/rancher/k3s/server/manifests/p2-20-storage.yaml
sleep 30
mv p2_k3s_helm.yaml /var/lib/rancher/k3s/server/manifests/p2-30-helm.yaml
echo " * Your p2 instanace will be available at $INGRESS_HOST in a few minutes."
echo " * You can use the username admin with password admin to login."
# TODO: Download crd and operator
# echo " * Your p2 instanace will be available at $INGRESS_HOST in a few minutes."
# echo " * You can use the username admin with password admin to login."
rm -r "${TEMP_DIR}"
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: p2s.k8s.beryju.org
spec:
group: k8s.beryju.org
names:
kind: P2
listKind: P2List
plural: p2s
singular: p2
scope: Namespaced
subresources:
status: {}
version: v1alpha1
versions:
- name: v1alpha1
served: true
storage: true
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: p2-operator
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
creationTimestamp: null
name: p2-operator
rules:
- apiGroups:
- ""
resources:
- namespaces
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
- secrets
verbs:
- '*'
- apiGroups:
- ""
resources:
- configmaps
- persistentvolumeclaims
- secrets
- serviceaccounts
- services
verbs:
- '*'
- apiGroups:
- extensions
resources:
- deployments
- ingresses
verbs:
- '*'
- apiGroups:
- apps
resources:
- deployments
- statefulsets
verbs:
- '*'
- apiGroups:
- rbac.authorization.k8s.io
resources:
- rolebindings
- roles
verbs:
- '*'
- apiGroups:
- k8s.beryju.org
resources:
- '*'
verbs:
- '*'
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: p2-operator
subjects:
- kind: ServiceAccount
name: p2-operator
roleRef:
kind: Role
name: p2-operator
apiGroup: rbac.authorization.k8s.io
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: p2-operator
spec:
replicas: 1
selector:
matchLabels:
name: p2-operator
template:
metadata:
labels:
name: p2-operator
spec:
serviceAccountName: p2-operator
containers:
- name: p2-operator
image: docker.beryju.org/p2/operator:latest
imagePullPolicy: Always
env:
- name: WATCH_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: OPERATOR_NAME
value: "p2-operator"
Access your p² Install here:
{{- range .Values.ingress.hosts }}
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ . }}
{{- end }}
{{- if .Values.ingress.serve.enabled }}
tier0 has been configured for fast access on:
{{- range .Values.ingress.serve.hosts }}
http{{ if $.Values.ingress.tls }}s{{ end }}://{{ . }}
{{- end }}
{{- end }}
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "p2.fullname" . }}-config
data:
config.yml: |
databases:
default:
name: "{{ .Values.postgresql.postgresqlDatabase }}"
user: "{{ .Values.postgresql.postgresqlUsername }}"
password: "{{ .Values.postgresql.postgresqlPassword }}"
host: "{{ .Release.Name }}-postgresql"
port: ''
cache: "redis://:{{ .Values.redis.password }}@{{ .Release.Name }}-redis-master/0"
message_queue:
broker: "redis://:{{ .Values.redis.password }}@{{ .Release.Name }}-redis-master/1"
results: "redis://:{{ .Values.redis.password }}@{{ .Release.Name }}-redis-master/1"
debug: false
error_report_enabled: {{ .Values.config.error_reporting }}
s3:
base_domain: {{ (index .Values.ingress.hosts 0) | quote }}
# Set this to true if you only want to use external authentication
external_auth_only: {{ .Values.config.external_auth_only }}
# Callback URL: <base url>/_/oidc/callback/
# oidc:
# enabled: false
# client_id: ""
# client_secret: ""
# auth_url: ""
# token_url: ""
# user_url: ""
oidc: {{ toYaml .Values.config.oidc | nindent 6 }}
{{- if .Values.config.secret_key }}
secret_key: {{ .Values.config.secret_key }}
{{- else }}
secret_key: {{ randAlphaNum 50 }}
{{- end }}
---
apiVersion: helm.cattle.io/v1
kind: HelmChart
metadata:
name: p2
namespace: kube-system
spec:
chart: https://docker.beryju.org/chartrepo/p2/charts/p2-0.7.6.tgz
targetNamespace: p2
valuesContent: |-
deployment:
webInstances: %WEB_INSTANCES%
tier0Instances: %SERVE_INSTANCES%
grpcInstances: 1
ingress:
enabled: true
hosts:
- "%HOST%"
serve:
hosts:
- "%SERVE_HOST%"
postgresql:
postgresqlDatabase: p2
postgresqlUsername: p2
postgresqlPassword: '%PASSWORD%'
redis:
password: '%PASSWORD%'
cluster:
enabled: false
apiVersion: v1
kind: Namespace
metadata:
name: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: nginx-configuration
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: tcp-services
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
kind: ConfigMap
apiVersion: v1
metadata:
name: udp-services
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: nginx-ingress-clusterrole
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- endpoints
- nodes
- pods
- secrets
verbs:
- list
- watch
- apiGroups:
- ""
resources:
- nodes
verbs:
- get
- apiGroups:
- ""
resources:
- services
verbs:
- get
- list
- watch
- apiGroups:
- "extensions"
resources:
- ingresses
verbs:
- get
- list
- watch
- apiGroups:
- ""
resources:
- events
verbs:
- create
- patch
- apiGroups:
- "extensions"
resources:
- ingresses/status
verbs:
- update
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: Role
metadata:
name: nginx-ingress-role
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
rules:
- apiGroups:
- ""
resources:
- configmaps
- pods
- secrets
- namespaces
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
resourceNames:
# Defaults to "<election-id>-<ingress-class>"
# Here: "<ingress-controller-leader>-<nginx>"
# This has to be adapted if you change either parameter
# when launching the nginx-ingress-controller.
- "ingress-controller-leader-nginx"
verbs:
- get
- update
- apiGroups:
- ""
resources:
- configmaps
verbs:
- create
- apiGroups:
- ""
resources:
- endpoints
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: RoleBinding
metadata:
name: nginx-ingress-role-nisa-binding
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: nginx-ingress-role
subjects:
- kind: ServiceAccount
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: nginx-ingress-clusterrole-nisa-binding
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: nginx-ingress-clusterrole
subjects:
- kind: ServiceAccount
name: nginx-ingress-serviceaccount
namespace: ingress-nginx
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-ingress-controller
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
template:
metadata:
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
annotations:
prometheus.io/port: "10254"
prometheus.io/scrape: "true"
spec:
serviceAccountName: nginx-ingress-serviceaccount
containers:
- name: nginx-ingress-controller
image: quay.io/kubernetes-ingress-controller/nginx-ingress-controller:0.24.1
args:
- /nginx-ingress-controller
- --configmap=$(POD_NAMESPACE)/nginx-configuration
- --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
- --udp-services-configmap=$(POD_NAMESPACE)/udp-services
- --publish-service=$(POD_NAMESPACE)/ingress-nginx
- --annotations-prefix=nginx.ingress.kubernetes.io
securityContext:
allowPrivilegeEscalation: true
capabilities:
drop:
- ALL
add:
- NET_BIND_SERVICE
# www-data -> 33
runAsUser: 33
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
ports:
- name: http
containerPort: 80
- name: https
containerPort: 443
livenessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 10
readinessProbe:
failureThreshold: 3
httpGet:
path: /healthz
port: 10254
scheme: HTTP
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 10
---
apiVersion: v1
kind: Service
metadata:
name: ingress-nginx
namespace: ingress-nginx
labels:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
spec:
type: LoadBalancer
ports:
- name: http
port: 80
targetPort: 80
protocol: TCP
- name: https
port: 443
targetPort: 443
protocol: TCP
selector:
app.kubernetes.io/name: ingress-nginx
app.kubernetes.io/part-of: ingress-nginx
---
FROM quay.io/operator-framework/helm-operator:v0.8.1
COPY watches.yaml ${HOME}/watches.yaml
COPY helm-charts/ ${HOME}/helm-charts/
apiVersion: apps/v1
kind: Deployment
metadata:
name: p2-operator
spec:
replicas: 1
selector:
matchLabels:
name: p2-operator
template:
metadata:
labels:
name: p2-operator
spec:
serviceAccountName: p2-operator
containers:
- name: p2-operator
image: docker.beryju.org/p2/operator:latest
imagePullPolicy: Always
env:
- name: WATCH_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: OPERATOR_NAME
value: "p2-operator"
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
creationTimestamp: null
name: p2-operator
rules:
- apiGroups:
- ""
resources:
- namespaces
- pods
verbs:
- get
- apiGroups:
- ""
resources:
- configmaps
- secrets
verbs:
- '*'
- apiGroups:
- ""
resources:
- configmaps
- persistentvolumeclaims
- secrets
- serviceaccounts
- services
verbs:
- '*'
- apiGroups:
- extensions
resources:
- deployments
- ingresses
verbs:
- '*'
- apiGroups:
- apps
resources:
- deployments
- statefulsets
verbs:
- '*'
- apiGroups:
- rbac.authorization.k8s.io
resources:
- rolebindings
- roles
verbs:
- '*'
- apiGroups:
- k8s.beryju.org
resources:
- '*'
verbs:
- '*'
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
name: p2-operator
subjects:
- kind: ServiceAccount
name: p2-operator
roleRef:
kind: Role
name: p2-operator
apiGroup: rbac.authorization.k8s.io
apiVersion: v1
kind: ServiceAccount
metadata:
name: p2-operator
apiVersion: v1
appVersion: "0.7.6"
appVersion: 0.7.6
description: A Helm chart for p2.
icon: https://p2.beryju.org/images/logo.png
name: p2
version: 0.7.6
icon: https://p2.beryju.org/images/logo.png
dependencies:
- name: postgresql
version: 3.10.1
version: 6.2.1
repository: https://kubernetes-charts.storage.googleapis.com/
- name: redis
version: 5.1.0
version: 9.0.2
repository: https://kubernetes-charts.storage.googleapis.com/
apiVersion: v1
kind: ConfigMap
metadata:
name: {{ include "p2.fullname" . }}-config
data:
P2_DEBUG: 'false'
P2_ERROR_REPORT_ENABLED: {{ .Values.config.error_reporting | quote }}
P2_S3__BASE_DOMAIN: {{ (index .Values.ingress.hosts 0) | quote }}
# Set this to true if you only want to use external authentication
P2_EXTERNAL_AUTH_ONLY: {{ .Values.config.external_auth_only | quote }}
# Callback URL: <base url>/_/oidc/callback/
{{- if .Values.config.oidc.enabled -}}
P2_OIDC__ENABLED: 'true'
P2_OIDC__CLIENT_ID: '{{ .Values.config.oidc.client_id }}'
P2_OIDC__CLIENT_SECRET: '{{ .Values.config.oidc.client_secret }}'
P2_OIDC__AUTH_URL: '{{ .Values.config.oidc.auth_url }}'
P2_OIDC__TOKEN_URL: '{{ .Values.config.oidc.token_url }}'
P2_OIDC__USER_URL: '{{ .Values.config.oidc.user_url }}'
{{ end }}
......@@ -27,18 +27,17 @@ spec:
securityContext:
fsGroup: 100
volumes:
- name: config-volume
configMap:
name: {{ include "p2.fullname" . }}-config
- name: media-storage
persistentVolumeClaim:
claimName: {{ include "p2.fullname" . }}-pvc-app-storage
containers:
- name: {{ .Chart.Name }}
image: "docker.beryju.org/p2/server:{{ .Values.image.tag }}"
image: "docker.beryju.org/p2/server:{{ .Values.version }}"
imagePullPolicy: IfNotPresent
command: ["./manage.py"]
args: ["grpc"]
command:
- ./manage.py
args:
- grpc
ports:
- name: grpc
containerPort: 50051
......@@ -46,15 +45,40 @@ spec:
- name: prometheus
containerPort: 9102
protocol: TCP
envFrom:
- configMapRef:
name: {{ include "p2.fullname" . }}-config
env:
- name: P2_REDIS__HOST
value: "{{ .Release.Name }}-redis-master"
- name: P2_REDIS__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-redis"
key: redis-password
- name: P2_POSTGRESQL__NAME
value: "{{ .Values.postgresql.postgresqlDatabase }}"
- name: P2_POSTGRESQL__USER
value: "{{ .Values.postgresql.postgresqlUsername }}"
- name: P2_POSTGRESQL__HOST
value: "{{ .Release.Name }}-postgresql"
- name: P2_POSTGRESQL__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-postgresql"
key: postgresql-password
- name: P2_SECRET_KEY
valueFrom:
secretKeyRef:
name: "{{ template "p2.fullname" . }}"
key: secret_key
volumeMounts:
- mountPath: /etc/p2
name: config-volume
- mountPath: /storage
name: media-storage
resources:
requests:
cpu: 500m
cpu: 150m
memory: 200M
limits:
cpu: 1000m
cpu: 300m
memory: 250M
......@@ -8,8 +8,8 @@ metadata:
helm.sh/chart: {{ include "p2.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
k8s.p2.io/main-ingress: 'true'
annotations:
kubernetes.io/ingress.class: nginx
ingress.kubernetes.io/proxy-body-size: 5000m
nginx.ingress.kubernetes.io/proxy-body-size: 5000m
{{- with .Values.ingress.annotations }}
......
apiVersion: v1
kind: Secret
metadata:
name: {{ template "p2.fullname" . }}
labels:
app.kubernetes.io/name: {{ template "p2.name" . }}
helm.sh/chart: {{ template "p2.chart" . }}
app.kubernetes.io/managed-by: "{{ .Release.Service }}"
app.kubernetes.io/instance: "{{ .Release.Name }}"
app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
type: Opaque
data:
secret_key: {{ .Values.secret_key | b64enc | quote }}
......@@ -26,7 +26,7 @@ spec:
spec:
containers:
- name: {{ .Chart.Name }}-static
image: "docker.beryju.org/p2/static:{{ .Values.image.tag }}"
image: "docker.beryju.org/p2/static:{{ .Values.version }}"
imagePullPolicy: IfNotPresent
ports:
- name: http
......
......@@ -28,7 +28,7 @@ spec:
serviceAccountName: {{ include "p2.fullname" . }}-tier0
containers:
- name: {{ .Chart.Name }}-tier0
image: "docker.beryju.org/p2/tier0:{{ .Values.image.tag }}"
image: "docker.beryju.org/p2/tier0:{{ .Values.version }}"
imagePullPolicy: IfNotPresent
command: ['/tier0']
args: ['server']
......
......@@ -7,6 +7,7 @@ metadata:
helm.sh/chart: {{ include "p2.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
k8s.p2.io/component: tier0
spec:
type: ClusterIP
ports:
......
......@@ -27,35 +27,78 @@ spec:
securityContext:
fsGroup: 100
volumes:
- name: config-volume
configMap:
name: {{ include "p2.fullname" . }}-config
- name: media-storage
persistentVolumeClaim:
claimName: {{ include "p2.fullname" . }}-pvc-app-storage
initContainers:
- name: p2-database-migrations
image: "docker.beryju.org/p2/server:{{ .Values.image.tag }}"
image: "docker.beryju.org/p2/server:{{ .Values.version }}"
command:
- ./manage.py
args:
- migrate
envFrom:
- configMapRef:
name: {{ include "p2.fullname" . }}-config
env:
- name: P2_REDIS__HOST
value: "{{ .Release.Name }}-redis-master"
- name: P2_REDIS__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-redis"
key: redis-password
- name: P2_POSTGRESQL__NAME
value: "{{ .Values.postgresql.postgresqlDatabase }}"
- name: P2_POSTGRESQL__USER
value: "{{ .Values.postgresql.postgresqlUsername }}"
- name: P2_POSTGRESQL__HOST
value: "{{ .Release.Name }}-postgresql"
- name: P2_POSTGRESQL__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-postgresql"
key: postgresql-password
- name: P2_SECRET_KEY
valueFrom:
secretKeyRef:
name: "{{ template "p2.fullname" . }}"
key: secret_key
volumeMounts:
- mountPath: /etc/p2
name: config-volume
- mountPath: /storage
name: media-storage
containers:
- name: {{ .Chart.Name }}
image: "docker.beryju.org/p2/server:{{ .Values.image.tag }}"
image: "docker.beryju.org/p2/server:{{ .Values.version }}"
imagePullPolicy: IfNotPresent
command:
- ./manage.py
args:
- web
envFrom:
- configMapRef:
name: {{ include "p2.fullname" . }}-config
env:
- name: P2_COMPONENT
value: web
- name: P2_REDIS__HOST
value: "{{ .Release.Name }}-redis-master"
- name: P2_REDIS__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-redis"
key: redis-password
- name: P2_POSTGRESQL__NAME
value: "{{ .Values.postgresql.postgresqlDatabase }}"
- name: P2_POSTGRESQL__USER
value: "{{ .Values.postgresql.postgresqlUsername }}"
- name: P2_POSTGRESQL__HOST
value: "{{ .Release.Name }}-postgresql"
- name: P2_POSTGRESQL__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-postgresql"
key: postgresql-password
ports:
- name: http
containerPort: 8000
......@@ -64,8 +107,6 @@ spec:
containerPort: 9102
protocol: TCP
volumeMounts:
- mountPath: /etc/p2
name: config-volume
- mountPath: /storage
name: media-storage
livenessProbe:
......@@ -88,7 +129,7 @@ spec:
value: kubernetes-healthcheck-host
resources:
requests:
cpu: 300m
cpu: 100m
memory: 175M
limits:
cpu: 500m
......
......@@ -24,22 +24,47 @@ spec:
securityContext:
fsGroup: 100
volumes:
- name: config-volume
configMap:
name: {{ include "p2.fullname" . }}-config
- name: media-storage
persistentVolumeClaim:
claimName: {{ include "p2.fullname" . }}-pvc-app-storage
containers:
- name: {{ .Chart.Name }}
image: "docker.beryju.org/p2/server:{{ .Values.image.tag }}"
image: "docker.beryju.org/p2/server:{{ .Values.version }}"
imagePullPolicy: IfNotPresent
command: ["./manage.py", "worker"]
command:
- ./manage.py
args:
- worker
volumeMounts:
- mountPath: /etc/p2
name: config-volume
- mountPath: /storage
name: media-storage
envFrom:
- configMapRef:
name: {{ include "p2.fullname" . }}-config
env:
- name: P2_REDIS__HOST
value: "{{ .Release.Name }}-redis-master"
- name: P2_REDIS__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-redis"
key: redis-password
- name: P2_POSTGRESQL__NAME
value: "{{ .Values.postgresql.postgresqlDatabase }}"
- name: P2_POSTGRESQL__USER
value: "{{ .Values.postgresql.postgresqlUsername }}"
- name: P2_POSTGRESQL__HOST
value: "{{ .Release.Name }}-postgresql"
- name: P2_POSTGRESQL__PASSWORD
valueFrom:
secretKeyRef:
name: "{{ .Release.Name }}-postgresql"
key: postgresql-password
- name: P2_SECRET_KEY
valueFrom:
secretKeyRef:
name: "{{ template "p2.fullname" . }}"
key: secret_key
resources:
requests:
cpu: 100m
......
# Default values for p2.
version: 0.7.6
image:
tag: 0.7.6
# Replace this with your own secret_key (used to sign cookies and others)
secret_key: "y(qs_&z!7u+!7rq6z3fx=p)pfx_ah3l(i&#p(dx5cu=d3=knq3"
config:
# Optionally specify fixed secret_key, otherwise generated automatically
# secret_key: _k*@6h2u2@q-dku57hhgzb7tnx*ba9wodcb^s9g0j59@=y(@_o
# Enable error reporting (errors are sent to sentry.beryju.org)
error_reporting: true
......
---
- version: v1alpha1
group: k8s.beryju.org
kind: P2
chart: /opt/helm/helm-charts/p2
......@@ -4,6 +4,8 @@ from django.urls import include, path
from drf_yasg import openapi
from drf_yasg.views import get_schema_view
from rest_framework.routers import DefaultRouter
from rest_framework_jwt.views import (obtain_jwt_token, refresh_jwt_token,
verify_jwt_token)
from p2.api.permissions import CustomObjectPermissions
from p2.api.viewsets import APIKeyViewSet, UserViewSet
......@@ -30,13 +32,16 @@ ROUTER.register('core/volume', VolumeViewSet)
ROUTER.register('core/storage', StorageViewSet)
ROUTER.register('system/user', UserViewSet)
ROUTER.register('system/key', APIKeyViewSet)
ROUTER.register('serve/rule', ServeRuleViewSet)
ROUTER.register('tier0/policy', ServeRuleViewSet)
app_name = 'p2_api'
urlpatterns = [
path('v1/', include(ROUTER.urls)),
url(r'^swagger(?P<format>\.json|\.yaml)$',
SchemaView.without_ui(cache_timeout=0), name='schema-json'),
path('jwt/token/', obtain_jwt_token),
path('jwt/refresh/', refresh_jwt_token),
path('jwt/verify/', verify_jwt_token),
path('swagger/', SchemaView.with_ui('swagger',
cache_timeout=0), name='schema-swagger-ui'),
path('redoc/', SchemaView.with_ui('redoc',
......
"""supervisr core config loader"""
"""p2 core config loader"""
import os
from collections import Mapping
from contextlib import contextmanager
from glob import glob
from typing import Any
from urllib.parse import urlparse
import yaml
from django.conf import ImproperlyConfigured
......@@ -16,16 +17,19 @@ SEARCH_PATHS = [
'',
] + glob('/etc/p2/config.d/*.yml', recursive=True)
LOGGER = get_logger()
ENVIRONMENT = os.getenv('P2_ENV', 'local')
ENV_PREFIX = 'P2'
ENVIRONMENT = os.getenv(f'{ENV_PREFIX}_ENV', 'local')
class ConfigLoader:
"""Search through SEARCH_PATHS and load configuration"""
"""Search through SEARCH_PATHS and load configuration. Environment variables starting with
`ENV_PREFIX` are also applied.
A variable like P2_POSTGRESQL__HOST would translate to postgresql.host"""
loaded_file = []
__config = {}
__context_default = None
__sub_dicts = []
def __init__(self):
......@@ -47,15 +51,7 @@ class ConfigLoader:
if os.path.isfile(env_file) and os.path.exists(env_file):
# Update config with env file
self.update_from_file(env_file)
self.handle_secret_key()
def handle_secret_key(self):
"""Handle `secret_key_file`"""
if 'secret_key_file' in self.__config:
secret_key_file = self.__config.get('secret_key_file')
if os.path.isfile(secret_key_file) and os.path.exists(secret_key_file):
with open(secret_key_file) as file:
self.__config['secret_key'] = file.read().replace('\n', '')
self.update_from_env()
def update(self, root, updatee):
"""Recursively update dictionary"""
......@@ -70,9 +66,9 @@ class ConfigLoader:
def parse_uri(self, value):
"""Parse string values which start with a URI"""
# If value starts with env://, get variable from env
if value.startswith('env://'):
value = os.getenv(value.replace('env://', ''))
url = urlparse(value)
if url.scheme == 'env':
value = os.getenv(url.netloc, url.query)
return value
def update_from_file(self, path: str):
......@@ -92,12 +88,26 @@ class ConfigLoader:
"""Update config from dict"""
self.__config.update(update)
@contextmanager
def default(self, value: Any):
"""Contextmanage that sets default"""
self.__context_default = value
yield
self.__context_default = None
def update_from_env(self):
"""Check environment variables"""
outer = {}
idx = 0
for key, value in os.environ.items():
if not key.startswith(ENV_PREFIX):
continue
relative_key = key.replace(f"{ENV_PREFIX}_", '').replace('__', '.').lower()
# Recursively convert path from a.b.c into outer[a][b][c]
current_obj = outer
dot_parts = relative_key.split('.')
for dot_part in dot_parts[:-1]:
if dot_part not in current_obj:
current_obj[dot_part] = {}
current_obj = current_obj[dot_part]
current_obj[dot_parts[-1]] = value
idx += 1
if idx > 0:
LOGGER.debug("Loaded environment variables", count=idx)
self.update(self.__config, outer)
@contextmanager
# pylint: disable=invalid-name
......@@ -107,15 +117,6 @@ class ConfigLoader:
yield
self.__sub_dicts.pop()
def get(self, key: str, default=None) -> Any:
"""Get value from loaded config file"""
if default is None:
default = self.__context_default
config_copy = self.__config
for sub in self.__sub_dicts:
config_copy = config_copy.get(sub, None)
return config_copy.get(key, default)
@property
def raw(self) -> dict:
"""Get raw config dictionary"""
......@@ -124,8 +125,6 @@ class ConfigLoader:
# pylint: disable=invalid-name
def y(self, path: str, default=None, sep='.') -> Any:
"""Access attribute by using yaml path"""
if default is None:
default = self.__context_default
# Walk sub_dicts before parsing path
root = self.raw
for sub in self.__sub_dicts:
......@@ -138,10 +137,14 @@ class ConfigLoader:
return default
return root
def y_bool(self, path: str, default=False) -> bool:
"""Wrapper for y that converts value into boolean"""
return str(self.y(path, default)).lower() == 'true'
CONFIG = ConfigLoader()
# pylint: disable=unused-argument
def signal_handler(sender, **kwargs):
def signal_handler(sender, **_):
"""Add all loaded config files to autoreload watcher"""
for path in CONFIG.loaded_file:
sender.watch_file(path)
......
# This is the default configuration file
databases:
default:
name: p2
user: p2
password: ''
host: localhost
postgresql:
host: localhost
name: p2
user: postgres
password: ''
cache: redis://localhost/0
message_queue:
broker: redis://localhost/1
results: redis://localhost/1
redis:
host: localhost
password: ''
cache_db: 0
message_queue_db: 1
debug: false
......
......@@ -27,10 +27,10 @@ from p2.lib.sentry import before_send
BASE_DIR = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = CONFIG.get('secret_key',
'48e9z8tw=_z0e#m*x70&)u%cgo8#=16uzdze&i8q=*#**)@cp&') # noqa Debug
SECRET_KEY = CONFIG.y('secret_key',
'48e9z8tw=_z0e#m*x70&)u%cgo8#=16uzdze&i8q=*#**)@cp&') # noqa Debug
DEBUG = CONFIG.get('debug')
DEBUG = CONFIG.y_bool('debug')
TEST = any('test' in arg for arg in sys.argv)
CORS_ORIGIN_ALLOW_ALL = DEBUG
......@@ -73,9 +73,11 @@ REST_FRAMEWORK = {
'DEFAULT_PERMISSION_CLASSES': (
'p2.api.permissions.CustomObjectPermissions',
),
# 'DEFAULT_AUTHENTICATION_CLASSES': (
# 'rest_framework_jwt.authentication.JSONWebTokenAuthentication',
# ),
'DEFAULT_AUTHENTICATION_CLASSES': (
'rest_framework.authentication.SessionAuthentication',
'rest_framework.authentication.BasicAuthentication',
'rest_framework_jwt.authentication.JSONWebTokenAuthentication',
),
}
AUTHENTICATION_BACKENDS = [
......@@ -87,15 +89,18 @@ AUTHENTICATION_BACKENDS = [
# Redis settings
CACHES = {
"default": {
"BACKEND": "django_prometheus.cache.backends.redis.RedisCache",
"LOCATION": CONFIG.get('cache'),
# "BACKEND": "django_prometheus.cache.backends.redis.RedisCache",
"BACKEND": "django_redis.cache.RedisCache",
"LOCATION": f"redis://{CONFIG.y('redis.host')}:6379/{CONFIG.y('redis.cache_db')}",
"OPTIONS": {
"CLIENT_CLASS": "django_redis.client.DefaultClient",
"PASSWORD": CONFIG.y('redis.password')
}
}
}
DJANGO_REDIS_IGNORE_EXCEPTIONS = True
DJANGO_REDIS_LOG_IGNORED_EXCEPTIONS = True
SESSION_ENGINE = "django.contrib.sessions.backends.cache"
SESSION_CACHE_ALIAS = "default"
if os.getenv('P2_COMPONENT', "") == "web":
......@@ -109,8 +114,10 @@ CELERY_TASK_SOFT_TIME_LIMIT = 600
CELERY_BEAT_SCHEDULE = {}
CELERY_CREATE_MISSING_QUEUES = True
CELERY_TASK_DEFAULT_QUEUE = 'p2'
CELERY_BROKER_URL = CONFIG.y('message_queue.broker')
CELERY_RESULT_BACKEND = CONFIG.y('message_queue.results')
CELERY_BROKER_URL = (f"redis://:{CONFIG.y('redis.password')}@{CONFIG.y('redis.host')}"
f":6379/{CONFIG.y('redis.message_queue_db')}")
CELERY_RESULT_BACKEND = (f"redis://:{CONFIG.y('redis.password')}@{CONFIG.y('redis.host')}"
f":6379/{CONFIG.y('redis.message_queue_db')}")
CELERY_IMPORTS = (
'p2.core.tasks',
'p2.log.tasks',
......@@ -159,7 +166,7 @@ LOGIN_REDIRECT_URL = '/'
CRISPY_TEMPLATE_PACK = 'bootstrap4'
# Authentication - OIDC
OIDC_ENABLED = CONFIG.y('oidc.enabled')
OIDC_ENABLED = CONFIG.y_bool('oidc.enabled')
OIDC_RP_CLIENT_ID = CONFIG.y('oidc.client_id')
OIDC_RP_CLIENT_SECRET = CONFIG.y('oidc.client_secret')
OIDC_OP_AUTHORIZATION_ENDPOINT = CONFIG.y('oidc.auth_url')
......@@ -215,16 +222,15 @@ DATA_UPLOAD_MAX_MEMORY_SIZE = 536870912
# Database
# https://docs.djangoproject.com/en/1.11/ref/settings/#databases
DATABASES = {}
for db_alias, db_config in CONFIG.get('databases').items():
DATABASES[db_alias] = {
DATABASES = {
'default': {
'ENGINE': 'django_prometheus.db.backends.postgresql',
'HOST': db_config.get('host'),
'NAME': db_config.get('name'),
'USER': db_config.get('user'),
'PASSWORD': db_config.get('password'),
'OPTIONS': db_config.get('options', {}),
'HOST': CONFIG.y('postgresql.host'),
'NAME': CONFIG.y('postgresql.name'),
'USER': CONFIG.y('postgresql.user'),
'PASSWORD': CONFIG.y('postgresql.password'),
}
}
# Password validation
# https://docs.djangoproject.com/en/1.11/ref/settings/#auth-password-validators
......
......@@ -67,5 +67,5 @@
</div>
</div>
</div>
{% include 'blocks/pagination.html' with page_obj=object_list %}
{% include 'blocks/pagination.html' %}
{% endblock %}