Commit d02faf24 authored by Jens Langhammer's avatar Jens Langhammer

Merge branch 'tier0-serve-grpc' into 'master'

Tier0 serve grpc

See merge request !3
parents 292b8575 6a1b88f8
Pipeline #3877 passed with stage
in 1 minute and 39 seconds
......@@ -10,6 +10,7 @@ omit =
p2/management/commands/web.py
p2/management/commands/worker.py
docs/
*_pb2.py
[report]
sort = Cover
......
......@@ -115,8 +115,8 @@ vendor/**
debian/p2/**
debian/files
debian/*.substvars
/storage
p2.crt
p2.key
static/**
storage/**
......@@ -15,6 +15,10 @@ variables:
POSTGRES_PASSWORD: "EK-5jnKfjrGRm<77"
before_script:
# Ensure all dependencies are installed, even those not included in p2/build-base
- pip install -r requirements-dev.txt
create-build-image:
image:
name: gcr.io/kaniko-project/executor:debug
......@@ -102,3 +106,13 @@ package-installer:
only:
- tags
- /^version/.*$/
# Manual tasks
# build-protos:
# stage: manual
# script:
# - python -m grpc_tools.protoc -I=. --python_out=p2/grpc/ --grpc_python_out=p2/grpc/ protos/*
# - protoc -I=. protos/*.proto --go_out=plugins=grpc:tier0/internal/
# # - sed -i -E 's/^\(import.*_pb2\)/from . \1/' p2/grpc/protos/*.py
# only:
# - never
......@@ -7,6 +7,7 @@ ignore-paths:
- migrations
- docs
- node_modules
- protos
uses:
- django
......@@ -2,6 +2,8 @@
disable=E0611,R0401,W0621,I0011,RP0401,I0021,arguments-differ,no-self-use,too-many-ancestors
load-plugins=pylint_django
ignore=migrations,protos
[SIMILARITIES]
# Minimum lines number of a similarity.
......
{{- if .Values.ingress.serve.enabled -}}
apiVersion: apps/v1beta2
kind: Deployment
metadata:
name: {{ include "p2.fullname" . }}-grpc
labels:
app.kubernetes.io/name: {{ include "p2.name" . }}
helm.sh/chart: {{ include "p2.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
spec:
replicas: {{ .Values.deployment.grpcInstances }}
selector:
matchLabels:
app.kubernetes.io/name: {{ include "p2.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
template:
metadata:
labels:
app.kubernetes.io/name: {{ include "p2.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
k8s.p2.io/component: grpc
spec:
securityContext:
fsGroup: 100
volumes:
- name: config-volume
configMap:
name: {{ include "p2.fullname" . }}-config
- name: media-storage
persistentVolumeClaim:
claimName: {{ include "p2.fullname" . }}-pvc-app-storage
containers:
- name: {{ .Chart.Name }}
image: "docker.beryju.org/p2/server:{{ .Values.image.tag }}"
imagePullPolicy: IfNotPresent
command: ["/bin/sh","-c"]
args: ["./manage.py migrate && ./manage.py grpc"]
ports:
- name: grpc
containerPort: 50051
protocol: TCP
volumeMounts:
- mountPath: /etc/p2
name: config-volume
- mountPath: /storage
name: media-storage
resources:
requests:
cpu: 50m
memory: 100M
limits:
cpu: 100m
memory: 200M
{{- end }}
{{- if .Values.ingress.serve.enabled -}}
apiVersion: v1
kind: Service
metadata:
name: {{ include "p2.fullname" . }}-grpc
labels:
app.kubernetes.io/name: {{ include "p2.name" . }}
helm.sh/chart: {{ include "p2.chart" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
k8s.p2.io/component: grpc
spec:
type: ClusterIP
ports:
- port: 50051
targetPort: grpc
protocol: TCP
name: grpc
selector:
app.kubernetes.io/name: {{ include "p2.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
k8s.p2.io/component: grpc
{{- end }}
......@@ -27,26 +27,25 @@ spec:
{{- end }}
{{- end }}
rules:
{{- $beta_enable_tier0 := .Values.beta_enable_tier0 -}}
{{- $fullname := include "p2.fullname" . -}}
{{- range .Values.ingress.hosts }}
{{- $fullname := include "p2.fullname" . -}}
{{- if .Values.ingress.serve.enabled -}}
{{- range .Values.ingress.serve.hosts }}
- host: {{ . | quote }}
http:
paths:
{{ if $beta_enable_tier0 -}}
- path: /_/
backend:
serviceName: {{ $fullname }}-web
servicePort: http
- path: /
backend:
serviceName: {{ $fullname }}-tier0
servicePort: http
{{ else }}
{{- end -}}
{{- end }}
{{- range .Values.ingress.hosts }}
- host: {{ . | quote }}
http:
paths:
- path: /
backend:
serviceName: {{ $fullname }}-web
servicePort: http
{{- end }}
{{- end }}
{{- end }}
{{- end }}
{{- if .Values.beta_enable_tier0 -}}
{{- if .Values.ingress.serve.enabled -}}
apiVersion: apps/v1beta2
kind: Deployment
metadata:
......
{{- if .Values.beta_enable_tier0 -}}
{{- if .Values.ingress.serve.enabled -}}
apiVersion: v1
kind: ServiceAccount
metadata:
......
......@@ -15,12 +15,11 @@ config:
# configure passbook then upgrade with external_auth_only = true.
external_auth_only: false
beta_enable_tier0: false
deployment:
webInstances: 2
workerInstances: 1
tier0Instances: 2
grpcInstances: 1
postgresql:
postgresqlUsername: p2
......@@ -28,9 +27,14 @@ postgresql:
ingress:
enabled: true
serve:
enabled: true
hosts:
- "i.p2.local"
hosts:
- p2.k8s.local
- "p2.local"
tls: []
# - secretName: chart-example-tls
# hosts:
# - p2.k8s.local
# - i.p2.local
# - p2.local
......@@ -3,7 +3,8 @@
# p2 Install script
# Installs and updates a p2 instance using k3s and docker
# Supported enviormnet variables:
# - INGRESS_HOST: Hostname under which p2 will be available
# - HOST: Hostname under which the will be accessible
# - SERVE_HOST: Optional; Hostname under which p2 will serve files.
# - STORAGE_BASE: Base directory in which p2 data will be storeed
# - LE_MAIL: Optional; Let's Encrypt E-Mail. If this is not set, Let's Encrypt is not enabled.
......@@ -87,7 +88,13 @@ curl -fsSL -o p2_k3s_nginx.yaml "https://git.beryju.org/BeryJu.org/p2/raw/versio
# curl -fsSL -o p2_k3s_cert.yaml "https://git.beryju.org/BeryJu.org/p2/raw/version/${P2_VERSION}/install/k3s-cert-manager.yaml"
# Replace variable in Helm CRD
sed -i "s|%INGRESS_HOST%|${INGRESS_HOST}|g" p2_k3s_helm.yaml
sed -i "s|%HOST%|${HOST}|g" p2_k3s_helm.yaml
if [ -n "$SERVE_HOST" ]; then
sed -i "s|%SERVE_ENABLED%|true|g" p2_k3s_helm.yaml
sed -i "s|%SERVE_HOST%|${SERVE_HOST}|g" p2_k3s_helm.yaml
else
sed -i "s|%SERVE_ENABLED%|false|g" p2_k3s_helm.yaml
fi
sed -i "s|%PASSWORD%|${PASSWORD}|g" p2_k3s_helm.yaml
# Adjust webserver instances (1 instance per CPU)
sed -i "s|%WEB_INSTANCES%|${CPU_CORES}|g" p2_k3s_helm.yaml
......
......@@ -12,8 +12,11 @@ spec:
ingress:
enabled: true
hosts:
- '%INGRESS_HOST%' # Replaced by install script
- '*.%INGRESS_HOST%' # Wildcard for S3
- "%HOST%"
serve:
enabled: %SERVE_ENABLED%
hosts:
- "%SERVE_HOST%"
# Since this is made for a single-node deployment, we disable most of the HA pods
postgresql:
postgresqlDatabase: p2
......
"""p2 GRPC management command"""
import time
from concurrent import futures
from contextlib import contextmanager
from logging import getLogger
import grpc
from django.core.management.base import BaseCommand
from django.utils import autoreload
from grpc_reflection.v1alpha import reflection
from p2.grpc.protos import serve_pb2, serve_pb2_grpc
from p2.serve.grpc import Serve
LOGGER = getLogger(__name__)
_ONE_DAY_IN_SECONDS = 60 * 60 * 24
@contextmanager
def serve_forever():
"""Run GRPC server, blocking"""
server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
serve_pb2_grpc.add_ServeServicer_to_server(Serve(), server)
service_names = (
serve_pb2.DESCRIPTOR.services_by_name['Serve'].full_name,
reflection.SERVICE_NAME,
)
reflection.enable_server_reflection(service_names, server)
server.add_insecure_port('[::]:50051')
server.start()
LOGGER.debug('Successfully started grpc server on port 50051')
while True:
time.sleep(_ONE_DAY_IN_SECONDS)
class Command(BaseCommand):
"""Run GRPC Server"""
def handle(self, *args, **options):
"""Start GRPC server and register services"""
autoreload.run_with_reloader(serve_forever)
# -*- coding: utf-8 -*-
# Generated by the protocol buffer compiler. DO NOT EDIT!
# source: protos/serve.proto
import sys
from google.protobuf import descriptor as _descriptor
from google.protobuf import message as _message
from google.protobuf import reflection as _reflection
from google.protobuf import symbol_database as _symbol_database
_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1'))
# @@protoc_insertion_point(imports)
_sym_db = _symbol_database.Default()
DESCRIPTOR = _descriptor.FileDescriptor(
name='protos/serve.proto',
package='p2',
syntax='proto3',
serialized_options=None,
serialized_pb=_b('\n\x12protos/serve.proto\x12\x02p2\"\x8c\x01\n\x0cServeRequest\x12\x0f\n\x07session\x18\x01 \x01(\t\x12\x0b\n\x03url\x18\x02 \x01(\t\x12.\n\x07headers\x18\x03 \x03(\x0b\x32\x1d.p2.ServeRequest.HeadersEntry\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x9b\x01\n\nServeReply\x12\x0f\n\x07session\x18\x01 \x01(\t\x12\x10\n\x08matching\x18\x02 \x01(\x08\x12\x0c\n\x04\x64\x61ta\x18\x03 \x01(\x0c\x12,\n\x07headers\x18\x04 \x03(\x0b\x32\x1b.p2.ServeReply.HeadersEntry\x1a.\n\x0cHeadersEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x32;\n\x05Serve\x12\x32\n\x0cRetrieveFile\x12\x10.p2.ServeRequest\x1a\x0e.p2.ServeReply\"\x00\x62\x06proto3')
)
_SERVEREQUEST_HEADERSENTRY = _descriptor.Descriptor(
name='HeadersEntry',
full_name='p2.ServeRequest.HeadersEntry',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='key', full_name='p2.ServeRequest.HeadersEntry.key', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='value', full_name='p2.ServeRequest.HeadersEntry.value', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=_b('8\001'),
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=121,
serialized_end=167,
)
_SERVEREQUEST = _descriptor.Descriptor(
name='ServeRequest',
full_name='p2.ServeRequest',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='session', full_name='p2.ServeRequest.session', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='url', full_name='p2.ServeRequest.url', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='headers', full_name='p2.ServeRequest.headers', index=2,
number=3, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[_SERVEREQUEST_HEADERSENTRY, ],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=27,
serialized_end=167,
)
_SERVEREPLY_HEADERSENTRY = _descriptor.Descriptor(
name='HeadersEntry',
full_name='p2.ServeReply.HeadersEntry',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='key', full_name='p2.ServeReply.HeadersEntry.key', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='value', full_name='p2.ServeReply.HeadersEntry.value', index=1,
number=2, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[],
enum_types=[
],
serialized_options=_b('8\001'),
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=121,
serialized_end=167,
)
_SERVEREPLY = _descriptor.Descriptor(
name='ServeReply',
full_name='p2.ServeReply',
filename=None,
file=DESCRIPTOR,
containing_type=None,
fields=[
_descriptor.FieldDescriptor(
name='session', full_name='p2.ServeReply.session', index=0,
number=1, type=9, cpp_type=9, label=1,
has_default_value=False, default_value=_b("").decode('utf-8'),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='matching', full_name='p2.ServeReply.matching', index=1,
number=2, type=8, cpp_type=7, label=1,
has_default_value=False, default_value=False,
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='data', full_name='p2.ServeReply.data', index=2,
number=3, type=12, cpp_type=9, label=1,
has_default_value=False, default_value=_b(""),
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
_descriptor.FieldDescriptor(
name='headers', full_name='p2.ServeReply.headers', index=3,
number=4, type=11, cpp_type=10, label=3,
has_default_value=False, default_value=[],
message_type=None, enum_type=None, containing_type=None,
is_extension=False, extension_scope=None,
serialized_options=None, file=DESCRIPTOR),
],
extensions=[
],
nested_types=[_SERVEREPLY_HEADERSENTRY, ],
enum_types=[
],
serialized_options=None,
is_extendable=False,
syntax='proto3',
extension_ranges=[],
oneofs=[
],
serialized_start=170,
serialized_end=325,
)
_SERVEREQUEST_HEADERSENTRY.containing_type = _SERVEREQUEST
_SERVEREQUEST.fields_by_name['headers'].message_type = _SERVEREQUEST_HEADERSENTRY
_SERVEREPLY_HEADERSENTRY.containing_type = _SERVEREPLY
_SERVEREPLY.fields_by_name['headers'].message_type = _SERVEREPLY_HEADERSENTRY
DESCRIPTOR.message_types_by_name['ServeRequest'] = _SERVEREQUEST
DESCRIPTOR.message_types_by_name['ServeReply'] = _SERVEREPLY
_sym_db.RegisterFileDescriptor(DESCRIPTOR)
ServeRequest = _reflection.GeneratedProtocolMessageType('ServeRequest', (_message.Message,), dict(
HeadersEntry = _reflection.GeneratedProtocolMessageType('HeadersEntry', (_message.Message,), dict(
DESCRIPTOR = _SERVEREQUEST_HEADERSENTRY,
__module__ = 'protos.serve_pb2'
# @@protoc_insertion_point(class_scope:p2.ServeRequest.HeadersEntry)
))
,
DESCRIPTOR = _SERVEREQUEST,
__module__ = 'protos.serve_pb2'
# @@protoc_insertion_point(class_scope:p2.ServeRequest)
))
_sym_db.RegisterMessage(ServeRequest)
_sym_db.RegisterMessage(ServeRequest.HeadersEntry)
ServeReply = _reflection.GeneratedProtocolMessageType('ServeReply', (_message.Message,), dict(
HeadersEntry = _reflection.GeneratedProtocolMessageType('HeadersEntry', (_message.Message,), dict(
DESCRIPTOR = _SERVEREPLY_HEADERSENTRY,
__module__ = 'protos.serve_pb2'
# @@protoc_insertion_point(class_scope:p2.ServeReply.HeadersEntry)
))
,
DESCRIPTOR = _SERVEREPLY,
__module__ = 'protos.serve_pb2'
# @@protoc_insertion_point(class_scope:p2.ServeReply)
))
_sym_db.RegisterMessage(ServeReply)
_sym_db.RegisterMessage(ServeReply.HeadersEntry)
_SERVEREQUEST_HEADERSENTRY._options = None
_SERVEREPLY_HEADERSENTRY._options = None
_SERVE = _descriptor.ServiceDescriptor(
name='Serve',
full_name='p2.Serve',
file=DESCRIPTOR,
index=0,
serialized_options=None,
serialized_start=327,
serialized_end=386,
methods=[
_descriptor.MethodDescriptor(
name='RetrieveFile',
full_name='p2.Serve.RetrieveFile',
index=0,
containing_service=None,
input_type=_SERVEREQUEST,
output_type=_SERVEREPLY,
serialized_options=None,
),
])
_sym_db.RegisterServiceDescriptor(_SERVE)
DESCRIPTOR.services_by_name['Serve'] = _SERVE
# @@protoc_insertion_point(module_scope)
# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
import grpc
from . import serve_pb2 as protos_dot_serve__pb2
class ServeStub(object):
# missing associated documentation comment in .proto file
pass
def __init__(self, channel):
"""Constructor.
Args:
channel: A grpc.Channel.
"""
self.RetrieveFile = channel.unary_unary(
'/p2.Serve/RetrieveFile',
request_serializer=protos_dot_serve__pb2.ServeRequest.SerializeToString,
response_deserializer=protos_dot_serve__pb2.ServeReply.FromString,
)
class ServeServicer(object):
# missing associated documentation comment in .proto file
pass
def RetrieveFile(self, request, context):
# missing associated documentation comment in .proto file
pass
context.set_code(grpc.StatusCode.UNIMPLEMENTED)
context.set_details('Method not implemented!')
raise NotImplementedError('Method not implemented!')
def add_ServeServicer_to_server(servicer, server):
rpc_method_handlers = {
'RetrieveFile': grpc.unary_unary_rpc_method_handler(
servicer.RetrieveFile,
request_deserializer=protos_dot_serve__pb2.ServeRequest.FromString,
response_serializer=protos_dot_serve__pb2.ServeReply.SerializeToString,
),
}
generic_handler = grpc.method_handlers_generic_handler(
'p2.Serve', rpc_method_handlers)
server.add_generic_rpc_handlers((generic_handler,))
......@@ -12,3 +12,6 @@ pyyaml
user-agents
colorlog
sentry-sdk
grpcio
grpcio-reflection
protobuf
......@@ -166,7 +166,6 @@ MIDDLEWARE = [
'p2.log.middleware.StartRequestMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.contrib.auth.middleware.AuthenticationMiddleware',
'p2.serve.middleware.ServeRoutingMiddleware',
'p2.s3.middleware.S3RoutingMiddleware',
'django.middleware.security.SecurityMiddleware',
'django.middleware.common.CommonMiddleware',
......@@ -275,6 +274,7 @@ with CONFIG.cd('log'):
'celery': 'WARNING',
'botocore': 'WARNING',
'werkzeug': 'DEBUG',
'grpc': 'DEBUG',
}
LOGGING = {
'version': 1,
......
"""p2 serve routing middleware"""
import hashlib
"""Serve GRPC functionality"""
from logging import getLogger
from django.core.cache import cache
from guardian.shortcuts import get_objects_for_user
from django.contrib.auth.models import User
from django.contrib.sessions.models import Session
from guardian.shortcuts import get_anonymous_user, get_objects_for_user
from p2.core.constants import ATTR_BLOB_HEADERS
from p2.core.http import BlobResponse
from p2.lib.shortcuts import get_object_for_user_or_404
from p2.grpc.protos.serve_pb2 import ServeReply, ServeRequest
from p2.grpc.protos.serve_pb2_grpc import ServeServicer
from p2.serve.models import ServeRule
LOGGER = getLogger(__name__)
LOGGER = getLogger(__name__)
# pylint: disable=too-few-public-methods
class ServeRoutingMiddleware:
"""Check if request matches any ServeRules and routes it to serve URLs"""
request = None
class Serve(ServeServicer):
"""GRPC Service for Serve Application"""
def __init__(self, get_response):
self.get_response = get_response
def fingerprint(self):
"""Return request's fingerprint"""
fingerprint_data = [
self.request.path,
str(self.request.user.pk or 0),
str(hash(frozenset(self.request.META.items()))),
]
_hash = hashlib.sha256()
_hash.update("".join(fingerprint_data).encode('utf-8'))
return _hash.hexdigest()
def rule_lookup(self, rule):
def rule_lookup(self, request: ServeRequest, rule):
"""Build blob lookup from rule"""
lookups = {}
# FIXME: Capture LOGGER output instead of returning a message array
......@@ -44,57 +25,54 @@ class ServeRoutingMiddleware:
debug_messages.append("Found new token '%s'" % lookup_token)
lookup_key, lookup_value = lookup_token.split('=')
lookups[lookup_key] = lookup_value.format(
path=self.request.path,
path_relative=self.request.path[1:],
host=self.request.META.get('HTTP_HOST', ''),
meta=self.request.META,
path=request.url,
path_relative=request.url[1:],
host=request.headers.get('Host', ''),
meta=request.headers,
)
debug_messages.append("Formatted to '%s'='%s'" % (lookup_key, lookups[lookup_key]))
debug_messages.append("Final lookup %r" % lookups)
return lookups, debug_messages
def get_blob_from_rule(self):
def get_user(self, request: ServeRequest) -> User:
"""Get user from cookie"""