Skip to content

Commit 6c8a1d8

Browse files
netsettlernetsettler
committed
Merge branch 'containerize' into kmp_C4-627_workaround_AcceptValidHeader_best_match_with_containerize
2 parents 2cdf587 + 22849a0 commit 6c8a1d8

24 files changed

Lines changed: 2314 additions & 517 deletions

‎Dockerfile‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
# Fourfront (Production) Dockerfile
2+
# Based off of the cgap-portal Dockerfile
3+
# Note that images are pinned via sha256 as opposed to tag
4+
# so that we don't pick up new images unintentionally
5+
6+
# Debian Buster with Python 3.6.15
7+
# Note image is updated from cgap-portal
8+
FROM python:3.6.15-slim-buster
9+
10+
MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu"
11+
12+
# Build Arguments
13+
ARG INI_BASE
14+
ENV INI_BASE=${INI_BASE:-"fourfront_any_alpha.ini"}
15+
16+
# Configure (global) Env
17+
ENV NGINX_USER=nginx
18+
ENV DEBIAN_FRONTEND=noninteractive
19+
ENV CRYPTOGRAPHY_DONT_BUILD_RUST=1
20+
ENV PYTHONFAULTHANDLER=1 \
21+
PYTHONUNBUFFERED=1 \
22+
PYTHONHASHSEED=random \
23+
PIP_NO_CACHE_DIR=off \
24+
PIP_DISABLE_PIP_VERSION_CHECK=on \
25+
PIP_DEFAULT_TIMEOUT=100 \
26+
POETRY_VERSION=1.1.12 \
27+
NODE_VERSION=12.22.9
28+
29+
# Install nginx, base system
30+
COPY deploy/docker/production/install_nginx.sh /
31+
RUN bash /install_nginx.sh && \
32+
apt-get update && \
33+
apt-get install -y curl vim emacs postgresql-client net-tools ca-certificates \
34+
gcc zlib1g-dev libpq-dev git make
35+
36+
# Configure Fourfront User (nginx)
37+
WORKDIR /home/nginx/.nvm
38+
39+
# Install Node
40+
ENV NVM_DIR=/home/nginx/.nvm
41+
RUN apt install -y curl
42+
RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.1/install.sh | bash
43+
RUN . "$NVM_DIR/nvm.sh" && nvm install ${NODE_VERSION}
44+
RUN . "$NVM_DIR/nvm.sh" && nvm use v${NODE_VERSION}
45+
RUN . "$NVM_DIR/nvm.sh" && nvm alias default v${NODE_VERSION}
46+
ENV PATH="/home/nginx/.nvm/versions/node/v${NODE_VERSION}/bin/:${PATH}"
47+
RUN node --version
48+
RUN npm --version
49+
50+
WORKDIR /home/nginx
51+
52+
# Configure venv
53+
ENV VIRTUAL_ENV=/opt/venv
54+
RUN python -m venv /opt/venv
55+
ENV PATH="$VIRTUAL_ENV/bin:$PATH"
56+
57+
# Upgrade pip, install in layer
58+
RUN pip install --upgrade pip && \
59+
pip install poetry==$POETRY_VERSION
60+
61+
# Adjust permissions
62+
RUN chown -R nginx:nginx /opt/venv && \
63+
mkdir -p /home/nginx/fourfront
64+
65+
WORKDIR /home/nginx/fourfront
66+
67+
# Do the back-end dependency install
68+
COPY pyproject.toml .
69+
COPY poetry.lock .
70+
RUN poetry install --no-root
71+
72+
# Do the front-end dependency install
73+
COPY package.json .
74+
COPY package-lock.json .
75+
RUN npm ci --no-fund --no-progress --no-optional --no-audit --python=/opt/venv/bin/python
76+
77+
# Build front-end
78+
# XXX: this fails but might be able to be made to succeed? - Will Oct 7 2021
79+
# COPY *.js .
80+
# COPY src/encoded/static .
81+
# RUN npm run build && \
82+
# npm run build-scss
83+
84+
# Copy over the rest of the code
85+
COPY . .
86+
87+
# Build front-end, remove node_modules when done
88+
RUN npm run build && \
89+
npm run build-scss && \
90+
rm -rf node_modules/
91+
92+
# Build remaining back-end
93+
RUN poetry install && \
94+
python setup_eb.py develop && \
95+
make fix-dist-info
96+
97+
# Misc
98+
RUN make aws-ip-ranges && \
99+
cat /dev/urandom | head -c 256 | base64 > session-secret.b64
100+
101+
# Copy config files in (down here for quick debugging)
102+
# Remove default configuration from Nginx
103+
RUN rm /etc/nginx/nginx.conf && \
104+
rm /etc/nginx/conf.d/default.conf
105+
COPY deploy/docker/production/nginx.conf /etc/nginx/nginx.conf
106+
107+
# nginx filesystem setup
108+
RUN chown -R nginx:nginx /var/cache/nginx && \
109+
chown -R nginx:nginx /var/log/nginx && \
110+
chown -R nginx:nginx /etc/nginx/conf.d && \
111+
touch /var/run/nginx.pid && \
112+
chown -R nginx:nginx /var/run/nginx.pid && \
113+
rm -f /var/log/nginx/* && \
114+
touch /var/log/nginx/access.log && \
115+
chown -R nginx:nginx /var/log/nginx/access.log && \
116+
touch /var/log/nginx/error.log && \
117+
chown -R nginx:nginx /var/log/nginx/error.log && \
118+
mkdir -p /data/nginx/cache && \
119+
chown -R nginx:nginx /data/nginx/cache
120+
121+
# Pull all required files
122+
# Note that *.ini must match the env name in secrets manager!
123+
# Note that deploy/docker/production/entrypoint.sh resolves which entrypoint to run
124+
# based on env variable "application_type".
125+
COPY deploy/docker/local/docker_development.ini development.ini
126+
COPY deploy/docker/local/entrypoint.bash entrypoint_local.bash
127+
RUN chown nginx:nginx development.ini
128+
RUN chmod +x entrypoint_local.bash
129+
130+
# Production setup
131+
RUN touch production.ini
132+
RUN chown nginx:nginx production.ini
133+
COPY deploy/docker/production/$INI_BASE deploy/ini_files/.
134+
COPY deploy/docker/production/entrypoint.bash .
135+
COPY deploy/docker/production/entrypoint_portal.bash .
136+
COPY deploy/docker/production/entrypoint_deployment.bash .
137+
COPY deploy/docker/production/entrypoint_indexer.bash .
138+
# Note that fourfront does not have an ingester
139+
# COPY deploy/docker/production/entrypoint_ingester.sh .
140+
COPY deploy/docker/production/assume_identity.py .
141+
RUN chmod +x entrypoint.bash
142+
RUN chmod +x entrypoint_deployment.bash
143+
RUN chmod +x entrypoint_deployment.bash
144+
RUN chmod +x entrypoint_indexer.bash
145+
RUN chmod +x assume_identity.py
146+
EXPOSE 8000
147+
148+
# Container does not run as root
149+
USER nginx
150+
151+
ENTRYPOINT ["/home/nginx/fourfront/entrypoint.bash"]

‎Makefile‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,40 @@ remote-test-unit: # Note this does the 'indexing' tests
145145
update: # updates dependencies
146146
poetry update
147147

148+
build-docker-local:
149+
docker-compose build
150+
151+
build-docker-local-clean:
152+
docker-compose build --no-cache
153+
154+
deploy-docker-local:
155+
docker-compose up -V
156+
157+
deploy-docker-local-daemon:
158+
docker-compose up -d -V
159+
160+
ENV_NAME ?= fourfront-mastertest
161+
AWS_ACCOUNT ?= 643366669028
162+
163+
ecr-login:
164+
@echo "Making ecr-login AWS_ACCOUNT=${AWS_ACCOUNT} ..."
165+
aws ecr get-login-password --region us-east-1 | docker login --username AWS --password-stdin ${AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com
166+
167+
build-docker-mastertest:
168+
scripts/build-docker-test --login --ecosystem fourfront-mastertest
169+
170+
build-docker-production:
171+
@echo "Making build-docker-production AWS_ACCOUNT=${AWS_ACCOUNT} ENV_NAME=${ENV_NAME} ..."
172+
docker build -t ${ENV_NAME}:latest .
173+
make tag-and-push-docker-production ENV_NAME=${ENV_NAME} AWS_ACCOUNT=${AWS_ACCOUNT}
174+
175+
tag-and-push-docker-production:
176+
@echo "Making tag-and-push-docker-production AWS_ACCOUNT=${AWS_ACCOUNT} ENV_NAME=${ENV_NAME} ..."
177+
docker tag ${ENV_NAME}:latest ${AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com/${ENV_NAME}:latest
178+
date
179+
docker push ${AWS_ACCOUNT}.dkr.ecr.us-east-1.amazonaws.com/${ENV_NAME}:latest
180+
date
181+
148182
help:
149183
@make info
150184

‎buildspec.yml‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
version: 0.2
2+
3+
phases:
4+
pre_build:
5+
commands:
6+
- touch deploy/docker/local/docker_development.ini # cheap substitute for prepare-docker to make ignored file
7+
- echo Logging in to Amazon ECR...
8+
- aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
9+
build:
10+
commands:
11+
- echo Build started on `date`
12+
- echo Building the Docker image...
13+
- docker build -t $IMAGE_REPO_NAME:$IMAGE_TAG .
14+
- docker tag $IMAGE_REPO_NAME:$IMAGE_TAG $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG
15+
post_build:
16+
commands:
17+
- echo Build completed on `date`
18+
- echo Pushing the Docker image...
19+
- docker push $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com/$IMAGE_REPO_NAME:$IMAGE_TAG

‎deploy/docker/elasticsearch/Dockerfile‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
FROM docker.elastic.co/elasticsearch/elasticsearch:6.8.14
2+
3+
MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu"
4+
5+
ENV ELASTICSEARCH_VERSION="6.8.14"
6+
ENV ELASTICSEARCH_SERVICE_PORT=9200
7+
EXPOSE $ELASTICSEARCH_SERVICE_PORT

‎deploy/docker/local/docker_development.ini‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
###
2+
# Docker App Configuration for local deployment
3+
# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/environment.html
4+
# NOTE: Still needs to be customized for (automated use by) multiple users
5+
###
6+
7+
[app:app]
8+
use = config:base.ini#app
9+
session.secret = %(here)s/session-secret.b64
10+
env.name = ffdkr
11+
sqlalchemy.url = postgres://postgres:postgres@db:5432/postgres
12+
elasticsearch.server = search-fourfront-testing-6-8-kncqa2za2r43563rkcmsvgn2fq.us-east-1.es.amazonaws.com:443
13+
elasticsearch.aws_auth = true
14+
#elasticsearch.server = es:9200
15+
blob_bucket = encoded-4dn-blobs
16+
load_test_only = true
17+
create_tables = true
18+
testing = true
19+
postgresql.statement_timeout = 20
20+
mpindexer = true
21+
indexer = true
22+
pyramid.reload_templates = true
23+
pyramid.debug_authorization = false
24+
pyramid.debug_notfound = false
25+
pyramid.debug_routematch = false
26+
pyramid.default_locale_name = en
27+
# this line determines which load function is used in load_data
28+
# most deployments use: "load_test_data = encoded.loadxl:load_test_data"
29+
load_test_data = encoded.loadxl:load_test_data
30+
encoded_version = 100.200.300
31+
snovault_version = 200.300.400
32+
utils_version = 300.400.500
33+
eb_app_version = app-v-development-simulation
34+
35+
[pipeline:debug]
36+
pipeline =
37+
egg:PasteDeploy#prefix
38+
egg:repoze.debug#pdbpm
39+
app
40+
set pyramid.includes =
41+
pyramid_translogger
42+
43+
[composite:main]
44+
use = egg:rutter#urlmap
45+
/ = debug
46+
/_indexer = indexer
47+
48+
[composite:indexer]
49+
use = config:base.ini#indexer
50+
51+
###
52+
# wsgi server configuration
53+
###
54+
55+
[server:main]
56+
use = egg:waitress#main
57+
host = 0.0.0.0
58+
port = 6543
59+
threads = 1
60+
61+
###
62+
# logging configuration
63+
# http://docs.pylonsproject.org/projects/pyramid/en/latest/narr/logging.html
64+
###
65+
66+
[loggers]
67+
keys = root, wsgi, encoded
68+
69+
[handlers]
70+
keys = console
71+
72+
[formatters]
73+
keys = generic
74+
75+
[logger_root]
76+
level = INFO
77+
handlers = console
78+
79+
[logger_wsgi]
80+
level = DEBUG
81+
handlers =
82+
qualname = wsgi
83+
84+
[logger_encoded]
85+
level = DEBUG
86+
handlers =
87+
qualname = encoded
88+
89+
[handler_console]
90+
class = StreamHandler
91+
args = (sys.stderr,)
92+
level = NOTSET
93+
formatter = generic
94+
95+
[formatter_generic]
96+
format = %(asctime)s %(levelname)-5.5s [%(name)s][%(threadName)s] %(message)s

‎deploy/docker/local/entrypoint.bash‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
#!/bin/bash
2+
3+
if [ -z ${TEST+x} ]; then
4+
5+
if [ ! -z ${LOAD+x} ]; then
6+
7+
# Clear db/es since this is the local entry point
8+
poetry run clear-db-es-contents development.ini --app-name app --env "$FOURFRONT_ENV_NAME"
9+
10+
# Create mapping
11+
poetry run create-mapping-on-deploy development.ini --app-name app
12+
13+
# Load Data (based on development.ini, for now just master-inserts)
14+
poetry run load-data development.ini --app-name app --prod
15+
16+
fi
17+
18+
# Start nginx proxy
19+
service nginx start
20+
21+
# Start application
22+
make deploy2
23+
24+
else
25+
26+
echo "Not starting serving application"
27+
echo "Enter the container with docker exec"
28+
sleep 100000000
29+
30+
fi

‎deploy/docker/postgres/Dockerfile‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# TODO: upgrade to latest version we can tolerate
2+
FROM postgres:12.3
3+
4+
MAINTAINER William Ronchetti "william_ronchetti@hms.harvard.edu"
5+
6+
# Install some system level dependencies
7+
RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates htop vim emacs curl \
8+
&& apt-get clean \
9+
&& rm -rf /var/lib/apt/lists/*
10+
11+
# Copy over our custom conf, enable inbound connections
12+
COPY postgresql.conf /etc/postgresql/postgresql.conf
13+
RUN echo "host all all 0.0.0.0/0 trust" >> /etc/postgresql/pg_hba.conf
14+
15+
ENV PGDATA=/var/lib/postgresql/data/pgdata

0 commit comments

Comments
 (0)