...
 
Commits (5)
......@@ -15,8 +15,8 @@ variables:
before_script:
# Ensure all dependencies are installed, even those not included in p2/base
- pip install -r requirements.txt
- pip install -r requirements-dev.txt
- pipenv lock -r --dev > requirements-all.txt
- pip install -r requirements-all.txt
create-base-image:
image:
......
{
"python.pythonPath": "env/bin/python",
"python.pythonPath": "/Users/langhammerj/.local/share/virtualenvs/p2-cWKK25xq/bin/python",
"editor.tabSize": 4,
"[html]": {
"editor.tabSize": 2
......
FROM python:3.7-alpine
COPY ./requirements.txt /app/
COPY ./Pipfile /app/
COPY ./Pipfile.lock /app/
WORKDIR /app/
RUN apk update && \
apk add --no-cache openssl-dev libmagic build-base jpeg libffi-dev gcc musl-dev libgcc openssl-dev jpeg-dev zlib-dev freetype-dev lcms2-dev openjpeg-dev tiff-dev tk-dev tcl-dev postgresql-dev && \
pip install -r /app/requirements.txt --no-cache-dir && \
pipenv lock -r > requirements.txt && \
pip install -r requirements.txt --no-cache-dir && \
adduser -S p2 && \
chown -R p2 /app
FROM docker.beryju.org/p2/base:latest
COPY ./requirements-dev.txt /app/
RUN pip install -r /app/requirements-dev.txt --no-cache-dir
RUN pipenv lock --dev -r > requirements-dev.txt && \
pip install -r /app/requirements-dev.txt --no-cache-dir
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
coverage = "*"
isort = "*"
pylint = "*"
pylint-django = "*"
prospector = "*"
django-debug-toolbar = "*"
django-extensions = "*"
bumpversion = "*"
unittest-xml-reporting = "*"
grpcio-tools = "*"
colorama = "*"
pep8 = "*"
autopep8 = "*"
uwsgitop = "*"
[packages]
django-guardian = "*"
psycopg2 = "*"
structlog = "*"
django-prometheus = "*"
py-grpc-prometheus = "*"
python-magic = "*"
celery = "*"
django-redis = "*"
mozilla-django-oidc = "*"
sentry-sdk = "*"
grpcio = "*"
grpcio-reflection = "*"
protobuf = "*"
django-filter = "*"
djangorestframework = "==3.9.4"
djangorestframework-guardian = "*"
djangorestframework-jwt = "*"
drf-yasg = {extras = ["validation"],version = "*"}
django-crispy-forms = "*"
boto3 = "*"
Django = ">=2.2"
PyYAML = "*"
"ruamel.yaml" = "<0.16.0,>=0.15.34"
Pillow = "==5.2.0"
uwsgi = "*"
[requires]
python_version = "3.7"
This diff is collapsed.
......@@ -73,9 +73,16 @@ spec:
image: "docker.beryju.org/p2/server:{{ .Values.version }}"
imagePullPolicy: IfNotPresent
command:
- ./manage.py
- uwsgi
args:
- web
- --http 0.0.0.0:8000
- --wsgi-file p2/root/wsgi.py
- --master
- --processes 24
- --threads 2
- --offload-threads 4
- --stats 0.0.0.0:8001
- --stats-http
envFrom:
- configMapRef:
name: {{ include "p2.fullname" . }}-config
......@@ -105,6 +112,9 @@ spec:
- name: http
containerPort: 8000
protocol: TCP
- name: uwsgiStats
containerPort: 8000
protocol: TCP
- name: prometheus
containerPort: 9102
protocol: TCP
......@@ -112,23 +122,13 @@ spec:
- mountPath: /storage
name: media-storage
livenessProbe:
initialDelaySeconds: 10
timeoutSeconds: 5
httpGet:
path: /
port: http
httpHeaders:
- name: Host
value: kubernetes-healthcheck-host
port: uwsgiStats
readinessProbe:
initialDelaySeconds: 10
timeoutSeconds: 5
httpGet:
path: /
port: http
httpHeaders:
- name: Host
value: kubernetes-healthcheck-host
port: uwsgiStats
resources:
requests:
cpu: 100m
......
"""p2 core http responses"""
from wsgiref.util import FileWrapper
from django.http import StreamingHttpResponse
......@@ -13,5 +12,5 @@ class BlobResponse(StreamingHttpResponse):
def __init__(self, blob: Blob, chunk_size=8192):
super().__init__(FileWrapper(blob, chunk_size))
self['Content-Length'] = blob.attributes.get(ATTR_BLOB_SIZE_BYTES)
self['Content-Length'] = blob.attributes.get(ATTR_BLOB_SIZE_BYTES, 0)
self['Content-Type'] = blob.attributes.get(ATTR_BLOB_MIME, 'text/plain')
# Generated by Django 2.2.4 on 2019-08-29 11:50
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('p2_core', '0023_auto_20190718_1541'),
]
operations = [
migrations.AlterField(
model_name='volume',
name='name',
field=models.SlugField(max_length=63, unique=True),
),
]
......@@ -30,7 +30,7 @@ COMPONENT_MANAGER = ControllerManager('component.controllers', lazy=True)
class Volume(ExportModelOperationsMixin('volume'), UUIDModel, TagModel):
"""Folder-like object, holding a collection of blobs"""
name = models.SlugField(unique=True)
name = models.SlugField(unique=True, max_length=63)
storage = models.ForeignKey('Storage', on_delete=models.CASCADE)
@cached_property
......
......@@ -141,6 +141,7 @@ class ConfigLoader:
"""Wrapper for y that converts value into boolean"""
return str(self.y(path, default)).lower() == 'true'
CONFIG = ConfigLoader()
# pylint: disable=unused-argument
......@@ -148,4 +149,6 @@ def signal_handler(sender, **_):
"""Add all loaded config files to autoreload watcher"""
for path in CONFIG.loaded_file:
sender.watch_file(path)
autoreload_started.connect(signal_handler)
......@@ -6,12 +6,10 @@ from django.utils.translation import gettext_lazy as _
class InvalidYAMLInput(str):
"""Invalid YAML String type"""
pass
class YAMLString(str):
"""YAML String type"""
pass
class YAMLField(forms.CharField):
......
......@@ -16,6 +16,7 @@ class LogAdaptor:
def start_request(self, request):
"""Add unique ID to request and create logging method"""
request.uid = uuid4().hex
def request_logger(**kwargs):
if request.uid not in self.__cache:
self.__cache[request.uid] = []
......@@ -38,4 +39,5 @@ class LogAdaptor:
flattened['app'] = request.resolver_match.app_name
# write_log_record.delay(dict(flattened))
LOG_ADAPTOR = LogAdaptor()
......@@ -175,12 +175,12 @@ OIDC_OP_USER_ENDPOINT = CONFIG.y('oidc.user_url')
OIDC_USERNAME_ALGO = 'p2.root.oidc.generate_username'
MIDDLEWARE = [
'p2.core.middleware.HealthCheckMiddleware',
'django_prometheus.middleware.PrometheusBeforeMiddleware',
'p2.log.middleware.StartRequestMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'p2.s3.middleware.S3RoutingMiddleware',
'p2.core.middleware.HealthCheckMiddleware',
'django_prometheus.middleware.PrometheusBeforeMiddleware',
'p2.log.middleware.StartRequestMiddleware',
'django.middleware.security.SecurityMiddleware',
'django.middleware.common.CommonMiddleware',
'django.middleware.http.ConditionalGetMiddleware',
......@@ -286,7 +286,7 @@ structlog.configure_once(
structlog.stdlib.PositionalArgumentsFormatter(),
structlog.processors.TimeStamper(),
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
# structlog.processors.format_exc_info,
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
],
context_class=structlog.threadlocal.wrap_dict(dict),
......
......@@ -12,6 +12,7 @@ admin.site.site_title = 'p2'
admin.site.login = RedirectView.as_view(
pattern_name='p2_ui:index', permanent=True, query_string=True)
# pylint: disable=invalid-name
handler500 = ServerErrorView.as_view()
# S3 URLs get routed via middleware
......
......@@ -67,4 +67,5 @@ class WSGILogger:
size=content_length / 1000 if content_length > 0 else '-',
runtime=kwargs.get('runtime'))
application = WSGILogger(get_wsgi_application())
......@@ -22,8 +22,8 @@ class S3RoutingMiddleware:
def process_exception(self, request: HttpRequest, exception):
"""Catch AWS-specific exceptions and show them as XML response"""
if CONFIG.y_bool('debug'):
LOGGER.exception(exception)
LOGGER.debug("Request Body ", body=request.body)
LOGGER.exception("S3 Error", error=exception)
# LOGGER.debug("Request Body ", body=request.body)
if isinstance(exception, AWSError):
return AWSErrorView(exception)
return None
......@@ -94,8 +94,8 @@ class S3RoutingMiddleware:
# Set SECURE_PROXY_SSL_HEADER so SecurityMiddleware doesn't return a 302
request.META['HTTP_X_FORWARDED_PROTO'] = 'https'
response = self.get_response(request)
if CONFIG.y_bool('debug') and response.status_code > 300:
if response['Content-Type'] == 'text/xml':
LOGGER.debug("Request Body", body=request.body)
LOGGER.debug("Response Body", body=response.content)
# if CONFIG.y_bool('debug') and response.status_code > 300:
# if response['Content-Type'] == 'text/xml':
# LOGGER.debug("Request Body", body=request.body)
# LOGGER.debug("Response Body", body=response.content)
return response
......@@ -4,6 +4,7 @@ from xml.etree import ElementTree
from django.core.paginator import Paginator
from django.http import HttpResponse
from guardian.shortcuts import assign_perm, get_objects_for_user
from structlog import get_logger
from p2.core.constants import (ATTR_BLOB_HASH_MD5, ATTR_BLOB_IS_FOLDER,
ATTR_BLOB_SIZE_BYTES, ATTR_BLOB_STAT_MTIME)
......@@ -15,6 +16,7 @@ from p2.s3.errors import AWSAccessDenied
from p2.s3.http import XMLResponse
from p2.s3.views.common import S3View
LOGGER = get_logger()
class BucketView(S3View):
"""https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketOps.html"""
......@@ -65,7 +67,7 @@ class BucketView(S3View):
base_lookup = get_objects_for_user(self.request.user, 'p2_core.view_blob').filter(
prefix=make_absolute_prefix(requested_prefix),
volume=volume,
).order_by('path')
).order_by('path').select_related('volume__storage')
blobs = base_lookup.exclude(attributes__has_key=ATTR_BLOB_IS_FOLDER)
folders = base_lookup.filter(attributes__has_key=ATTR_BLOB_IS_FOLDER)
......@@ -114,6 +116,8 @@ class BucketView(S3View):
'tags__%s' % TAG_S3_DEFAULT_STORAGE: True
})
if not storages.exists():
LOGGER.warning(("No Storage marked as default. Add the Tag '%s: true'"
" to a storage instance."), TAG_S3_DEFAULT_STORAGE)
raise AWSAccessDenied
if not request.user.has_perm('p2_core.add_volume'):
raise AWSAccessDenied
......
"""p2 S3 Object views"""
from django.core.cache import cache
from django.http.response import HttpResponse
from guardian.shortcuts import assign_perm, get_objects_for_user
from structlog import get_logger
......@@ -43,17 +44,27 @@ class ObjectView(S3View):
def head(self, request, bucket, path):
"""https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html"""
blob = self.get_blob('view_blob', path=path, volume__name=bucket)
# blob = self.get_blob('view_blob', path=path, volume__name=bucket)
# We can't use self.get_blob here, since this endpoint needs to return
# a blank 404 response when the key wasn't found.
blobs = get_objects_for_user(self.request.user, 'view_blob', Blob).filter(
path=path, volume__name=bucket)
if not blobs.exists():
return HttpResponse(status=404)
# We're not using BlobResponse here since we only want the attributes
response = HttpResponse(status=200)
response['Content-Length'] = blob.attributes.get(ATTR_BLOB_SIZE_BYTES)
response['Content-Type'] = blob.attributes.get(ATTR_BLOB_MIME, 'text/plain')
response['Content-Length'] = blobs.first().attributes.get(ATTR_BLOB_SIZE_BYTES, 0)
response['Content-Type'] = blobs.first().attributes.get(ATTR_BLOB_MIME, 'text/plain')
return response
def get(self, request, bucket, path):
"""https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html"""
blob = self.get_blob('view_blob', path=path, volume__name=bucket)
return BlobResponse(blob)
cache_key = f"{self.request.user.pk}:{bucket}/{path}"
cached_data = cache.get(cache_key)
if not cached_data:
cached_data = self.get_blob('view_blob', path=path, volume__name=bucket)
cache.set(cache_key, cached_data)
return BlobResponse(cached_data)
def post(self, request, bucket, path):
"""Post handler"""
......@@ -87,6 +98,8 @@ class ObjectView(S3View):
def delete(self, request, bucket, path):
"""https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html"""
blob = self.get_blob('delete_blob', path=path, volume__name=bucket)
blob.delete()
blobs = get_objects_for_user(self.request.user, 'delete_blob', Blob).filter(
path=path, volume__name=bucket)
if blobs.exists():
blobs.delete()
return HttpResponse(status=204)
......@@ -44,7 +44,6 @@ class MockRequest:
def log(self, **kwargs):
"""Stub for p2.log's request logger"""
pass
class Serve(ServeServicer):
"""GRPC Service for Serve Application"""
......
......@@ -37,7 +37,6 @@ class S3StorageController(StorageController):
def collect_attributes(self, blob: Blob):
"""Collect attributes such as size and mime type"""
pass
def _ensure_bucket_exists(self, name):
"""Ensure bucket exists before we attempt any object operations"""
......
......@@ -8,6 +8,7 @@ from django.shortcuts import reverse
from p2.lib.config import CONFIG
from p2.lib.reflection import path_to_class
# pylint: disable=invalid-name
register = template.Library()
......
coverage
isort
astroid==2.0.4
pylint==2.1.1
pylint-django==2.0.2
prospector==1.1.5
django-debug-toolbar
django-extensions
werkzeug
pycodestyle<2.4.0,>=2.0.0
bumpversion
unittest-xml-reporting
grpcio-tools
colorama
pep8
# Root dependencies
django>=2.2
django-guardian
pyyaml
psycopg2
structlog
cherrypy
# Monitoring
django-prometheus
py-grpc-prometheus
# MIME Type
python-magic
# Task Queue & Cache
celery
django-redis
# SSO
mozilla-django-oidc
# Error Handling
sentry-sdk
# Protobuf for tier0
grpcio
grpcio-reflection
protobuf
# API dependencies
django-filter
djangorestframework==3.9.4
djangorestframework-guardian
djangorestframework-jwt
drf-yasg[validation]
packaging
# https://github.com/axnsan12/drf-yasg/issues/423
ruamel.yaml>=0.15.34,<0.16.0
# UI dependencies
django-crispy-forms
# S3 Storage dependencies
boto3
# Image Component dependencies
pillow==4.1.1