diff --git a/.gitignore b/.gitignore index bcab9f037..a7cb67059 100644 --- a/.gitignore +++ b/.gitignore @@ -25,8 +25,8 @@ node_modules /frontend/resources/public/* /exporter/target /exporter/.shadow-cljs -/docker/frontend/dist -/docker/backend/dist +/docker/testenv/bundle +/bundle* /media /deploy /web diff --git a/backend/deps.edn b/backend/deps.edn index d0c322f80..d05c2f335 100644 --- a/backend/deps.edn +++ b/backend/deps.edn @@ -98,6 +98,10 @@ {:extra-deps {olical/depot {:mvn/version "1.8.4"}} :main-opts ["-m" "depot.outdated.main"]} + :jar + {:extra-deps {seancorfield/depstar {:mvn/version "RELEASE"}} + :main-opts ["-m" "hf.depstar.jar" "-S" "target/app.jar"]} + :jmx-remote {:jvm-opts ["-Dcom.sun.management.jmxremote" "-Dcom.sun.management.jmxremote.port=9090" diff --git a/backend/resources/log4j2-bundle.xml b/backend/resources/log4j2-bundle.xml new file mode 100644 index 000000000..7f6a243ab --- /dev/null +++ b/backend/resources/log4j2-bundle.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/backend/scripts/build.sh b/backend/scripts/build.sh new file mode 100755 index 000000000..23f262aca --- /dev/null +++ b/backend/scripts/build.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +CLASSPATH=`(clojure -Spath)` +NEWCP="./resources:./app.jar" + +rm -rf ./target/dist +mkdir -p ./target/dist/deps + +for item in $(echo $CLASSPATH | tr ":" "\n"); do + if [ "${item: -4}" == ".jar" ]; then + cp $item ./target/dist/deps/; + BN="$(basename -- $item)" + NEWCP+=":./deps/$BN" + fi +done + +cp ./resources/log4j2-bundle.xml ./target/dist/log4j2.xml + +clojure -Ajar + +cp ./target/app.jar ./target/dist/app.jar +echo $NEWCP > ./target/dist/classpath; + +tee -a ./target/dist/run.sh >> /dev/null <&2 echo "Couldn't find 'java'. Please set JAVA_HOME." + exit 1 + fi +fi + +if [ -f ./environ ]; then + source ./environ +fi + +set -x +\$JAVA_CMD \$JVM_OPTS -classpath \$CP -Dlog4j.configurationFile=./log4j2.xml "\$@" clojure.main -m app.main +EOF + +chmod +x ./target/dist/run.sh + + + diff --git a/docker/testenv/Dockerfile-backend b/docker/testenv/Dockerfile-backend new file mode 100644 index 000000000..b84fef87d --- /dev/null +++ b/docker/testenv/Dockerfile-backend @@ -0,0 +1,5 @@ +FROM azul/zulu-openjdk-debian:14 +LABEL maintainer="Andrey Antukh " +ADD ./bundle/backend/ /opt/bundle/ +WORKDIR /opt/bundle +CMD ["/bin/bash", "run.sh"] diff --git a/docker/testenv/Dockerfile-exporter b/docker/testenv/Dockerfile-exporter new file mode 100644 index 000000000..40bce1cdd --- /dev/null +++ b/docker/testenv/Dockerfile-exporter @@ -0,0 +1,87 @@ +FROM debian:buster +LABEL maintainer="Andrey Antukh " + +ARG DEBIAN_FRONTEND=noninteractive + +ENV LANG=en_US.UTF-8 LC_ALL=C.UTF-8 + +RUN set -ex; \ + mkdir -p /etc/resolvconf/resolv.conf.d; \ + echo "nameserver 8.8.8.8" > /etc/resolvconf/resolv.conf.d/tail; + +RUN set -ex; \ + apt-get update && \ + apt-get install -yq \ + locales \ + gnupg2 \ + ca-certificates \ + wget \ + curl \ + bash \ + rlwrap \ + imagemagick \ + netpbm \ + potrace \ + gconf-service \ + libasound2 \ + libatk1.0-0 \ + libatk-bridge2.0-0 \ + libcairo2 \ + libcups2 \ + libdbus-1-3 \ + libexpat1 \ + libfontconfig1 \ + libgcc1 \ + libgconf-2-4 \ + libgdk-pixbuf2.0-0 \ + libglib2.0-0 \ + libgtk-3-0 \ + libnspr4 \ + libpango-1.0-0 \ + libpangocairo-1.0-0 \ + libx11-6 \ + libx11-xcb1 \ + libxcb1 \ + libxcb-dri3-0 \ + libxcomposite1 \ + libxcursor1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxi6 \ + libxrandr2 \ + libxrender1 \ + libxss1 \ + libxtst6 \ + fonts-liberation \ + libappindicator1 \ + libnss3 \ + libgbm1 \ + ; \ + rm -rf /var/lib/apt/lists/*; + +RUN set -ex; \ + wget https://github.com/RazrFalcon/svgcleaner/releases/download/v0.9.5/svgcleaner_linux_x86_64_0.9.5.tar.gz; \ + tar xvf svgcleaner_linux_x86_64_0.9.5.tar.gz; \ + mv svgcleaner /usr/local/bin/; \ + rm -rf svgcleaner_linux_x86_64_0.9.5.tar.gz; + +RUN set -ex; \ + mkdir -p /tmp/node; \ + cd /tmp/node; \ + export PATH="$PATH:/usr/local/node-v12.18.3/bin"; \ + wget https://nodejs.org/dist/v12.18.3/node-v12.18.3-linux-x64.tar.xz; \ + tar xvf node-v12.18.3-linux-x64.tar.xz; \ + mv /tmp/node/node-v12.18.3-linux-x64 /usr/local/node-v12.18.3; \ + /usr/local/node-v12.18.3/bin/npm install -g yarn; \ + rm -rf /tmp/node; + +WORKDIR /opt/app + +ADD ./bundle/exporter/ /opt/app/ + +RUN set -ex; \ + export PATH="$PATH:/usr/local/node-v12.18.3/bin"; \ + yarn install; + +CMD ["/usr/local/node-v12.18.3/bin/node", "app.js"] diff --git a/docker/testenv/Dockerfile-nginx b/docker/testenv/Dockerfile-nginx new file mode 100644 index 000000000..b9c083348 --- /dev/null +++ b/docker/testenv/Dockerfile-nginx @@ -0,0 +1,9 @@ +FROM nginx:latest +LABEL maintainer="Andrey Antukh " + +ADD ./bundle/frontend /var/www/app/ +ADD ./files/nginx.conf /etc/nginx/nginx.conf +ADD ./files/nginx-entrypoint.sh /entrypoint.sh + +ENTRYPOINT ["/bin/bash", "/entrypoint.sh"] +CMD ["nginx", "-g", "daemon off;"] diff --git a/docker/testenv/docker-compose.yaml b/docker/testenv/docker-compose.yaml new file mode 100644 index 000000000..e266d7d2a --- /dev/null +++ b/docker/testenv/docker-compose.yaml @@ -0,0 +1,112 @@ +--- +version: "3" + +networks: + default: + driver: bridge + ipam: + driver: default + config: + - subnet: 172.177.99.0/24 + +volumes: + postgres_data: + user_data: + backend_data: + +services: + nginx: + image: "uxbox-testenv-nginx" + build: + context: "." + dockerfile: "Dockerfile-nginx" + + ports: + - 8080:80 + + networks: + default: + ipv4_address: 172.177.99.2 + + backend: + image: "uxbox-testenv-backend" + build: + context: "." + dockerfile: "Dockerfile-backend" + + volumes: + - backend_data:/opt/data + + depends_on: + - postgres + - smtp + - redis + + environment: + - APP_DATABASE_URI=postgresql://postgres/uxbox + - APP_DATABASE_USERNAME=uxbox + - APP_DATABASE_PASSWORD=uxbox + - APP_SENDMAIL_BACKEND=smtp + - APP_SMTP_HOST=smtp + - APP_SMTP_PORT=25 + - APP_MEDIA_DIRECTORY=/opt/data/media + + networks: + default: + ipv4_address: 172.177.99.3 + + + exporter: + image: "uxbox-testenv-exporter" + build: + context: "." + dockerfile: "Dockerfile-exporter" + + environment: + - APP_PUBLIC_URI=http://nginx + + depends_on: + - backend + - nginx + + networks: + default: + ipv4_address: 172.177.99.4 + + smtp: + image: mwader/postfix-relay:latest + restart: always + environment: + - POSTFIX_myhostname=smtp.testing.uxbox.io + - OPENDKIM_DOMAINS=smtp.testing.uxbox.io + + networks: + default: + ipv4_address: 172.177.99.5 + + postgres: + image: "postgres:12" + restart: always + stop_signal: SIGINT + + environment: + - POSTGRES_INITDB_ARGS=--data-checksums + - POSTGRES_DB=uxbox + - POSTGRES_USER=uxbox + - POSTGRES_PASSWORD=uxbox + + volumes: + - postgres_data:/var/lib/postgresql/data + + networks: + default: + ipv4_address: 172.177.99.6 + + redis: + image: redis:6 + restart: always + + networks: + default: + ipv4_address: 172.177.99.7 + diff --git a/docker/testenv/files/exporter-entrypoint.sh b/docker/testenv/files/exporter-entrypoint.sh new file mode 100644 index 000000000..7e11a3cc9 --- /dev/null +++ b/docker/testenv/files/exporter-entrypoint.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -ex + +export PATH="/usr/local/node-v12.18.3/bin/:$PATH" +# yarn install + +exec "$@" diff --git a/docker/testenv/files/nginx-entrypoint.sh b/docker/testenv/files/nginx-entrypoint.sh new file mode 100644 index 000000000..e2fce2c44 --- /dev/null +++ b/docker/testenv/files/nginx-entrypoint.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +exec "$@"; diff --git a/docker/testenv/files/nginx.conf b/docker/testenv/files/nginx.conf new file mode 100644 index 000000000..af58933d4 --- /dev/null +++ b/docker/testenv/files/nginx.conf @@ -0,0 +1,90 @@ +user www-data; +worker_processes auto; +pid /run/nginx.pid; +include /etc/nginx/modules-enabled/*.conf; + +events { + worker_connections 768; + # multi_accept on; +} + +http { + sendfile on; + tcp_nopush on; + tcp_nodelay on; + keepalive_timeout 65; + types_hash_max_size 2048; + # server_tokens off; + + # server_names_hash_bucket_size 64; + # server_name_in_redirect off; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + error_log /dev/stdout; + access_log /dev/stdout; + + gzip on; + + gzip_vary on; + gzip_proxied any; + gzip_comp_level 4; + gzip_buffers 16 8k; + gzip_http_version 1.1; + + gzip_types text/plain text/css text/javascript application/javascript application/json application/transit+json; + + map $http_upgrade $connection_upgrade { + default upgrade; + '' close; + } + + server { + listen 80 default_server; + server_name _; + + client_max_body_size 5M; + charset utf-8; + + proxy_http_version 1.1; + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Scheme $scheme; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + etag off; + + location / { + root /var/www/app/; + try_files $uri /index.html; + + location ~* \.(js|css).*$ { + add_header Cache-Control "max-age=86400" always; # 24 hours + } + + location = /index.html { + add_header Cache-Control "no-cache, max-age=0"; + } + } + + location /api { + proxy_pass http://172.177.99.3:6060/api; + } + + location /export { + proxy_pass http://172.177.99.4:6061; + } + + location /ws/notifications { + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection 'upgrade'; + proxy_pass http://172.177.99.3:6060/ws/notifications; + } + + location /media { + alias /mount/backend/media; + } + } +} diff --git a/docs/01-Development-Environment.md b/docs/01-Development-Environment.md index 1ca7895f2..389decb0f 100644 --- a/docs/01-Development-Environment.md +++ b/docs/01-Development-Environment.md @@ -64,8 +64,15 @@ This will do the following: - Attaches to the **devenv** container and executes the tmux session. - The tmux session automatically starts all the necessary services. +You can execute the individual steps manully if you want: -## First steps with tmux ## +```bash +./manage.sh build-devenv # builds the devenv docker image +./manage.sh start-devenv # starts background running containers +./manage.sh run-devenv # enters to new tmux session inside of one of the running containers +./manage.sh stop-devenv # stops background running containers +./manage.sh drop-devenv # removes all the volumes, containers and networks used by the devenv +``` Now having the the container running and tmux open inside the container, you are free to execute any commands and open many shells @@ -78,16 +85,16 @@ current window. For more info: https://tmuxcheatsheet.com/ -## Inside the tmux session ## +### Inside the tmux session -### Styles ### +#### gulp The styles and many related tasks are executed thanks to gulp and they are executed in the tmux **window 0**. This is a normal gulp watcher with some additional tasks. -### Frontend ### +#### shadow-cljs The frontend build process is located on the tmux **window 1**. **Shadow-cljs** is used for build and serve the frontend code. For @@ -101,7 +108,7 @@ connected browser, by opening a third window with `Ctrl+c` and running `npx shadow-cljs cljs-repl main`. -### Exporter ### +#### exporter The exporter app (clojurescript app running in nodejs) is located in **window 2**, and you can go directly to it using `ctrl+b 2` shortcut. @@ -115,7 +122,7 @@ If some reason scripts does not stars correctly, you can manually execute `node target/app.js ` to start the exporter app. -### Backend ### +#### backend The backend related environment is located in the tmux **window 3**, and you can go directly to it using `ctrl+b 2` shortcut. @@ -137,3 +144,46 @@ later use `(restart)` again. For more information, please refer to: `03-Backend-Guide.md`. + +## Start the testenv ## + +The purpose of the testenv (Test Environment) is provide an easy way +to get uxbox running in local pc without getting into the full +development environment. + +As first step we still need to build devenv image because that image +is used to produce the production-like bundle of the application: + +```bash +./manage.sh build-devenv +``` + +Once the image is build, you no longer need to rebuilt it until the +devenv image is changed and this happens we make some structural +changes or upgrading some dependencies. + +Them, let's proceed to build the bundle (a directory that contains all +the sources and dependencies of the platform ready to be deployed): + +```bash +./manage.sh build-bundle +``` + +This will generate on current directory one file and one +directory. The most important is the file like +`uxbox-2020.09.09-1343.tar.xz`. + +Then, let's proceed to build the docker images with the bundle +generated from the previous step. + +```bash +./manage.sh build-testenv ./uxbox-2020.09.09-1343.tar.xz +``` + +This will generate the necessary docker images ready to be executed. + +And finally, start the docker-compose: + +```bash +./manage.sh start-devenv +``` diff --git a/frontend/scripts/build-app.sh b/frontend/scripts/build.sh similarity index 100% rename from frontend/scripts/build-app.sh rename to frontend/scripts/build.sh diff --git a/manage.sh b/manage.sh index 0c9bee9ef..624635411 100755 --- a/manage.sh +++ b/manage.sh @@ -45,53 +45,85 @@ function run-devenv { docker exec -ti uxbox-devenv-main /home/start-tmux.sh } -function build-frontend { +function build { build-devenv-if-not-exists; - local IMAGE=$DEVENV_IMGNAME:latest; + docker volume create uxboxdev_user_data; + echo "Running development image $IMAGE to build frontend." docker run -t --rm \ + --mount source=uxboxdev_user_data,type=volume,target=/home/uxbox/ \ --mount source=`pwd`,type=bind,target=/home/uxbox/uxbox \ - --mount source=${HOME}/.m2,type=bind,target=/home/uxbox/.m2 \ - -w /home/uxbox/uxbox/frontend \ - $IMAGE ./scripts/build-app.sh -} - -function build-exporter { - build-devenv-if-not-exists; - - local IMAGE=$DEVENV_IMGNAME:latest; - - echo "Running development image $IMAGE to build frontend." - docker run -t --rm \ - --mount source=`pwd`,type=bind,target=/home/uxbox/uxbox \ - --mount source=${HOME}/.m2,type=bind,target=/home/uxbox/.m2 \ - -w /home/uxbox/uxbox/exporter \ + -w /home/uxbox/uxbox/$1 \ $IMAGE ./scripts/build.sh } +function build-frontend { + build "frontend"; +} + +function build-exporter { + build "exporter"; +} + function build-backend { - rm -rf ./backend/target/dist - mkdir -p ./backend/target/dist + build "backend"; +} - rsync -ar \ - --exclude="/tests*" \ - --exclude="/resources/public/media" \ - --exclude="/file-uploads" \ - --exclude="/target" \ - --exclude="/scripts" \ - --exclude="/.*" \ - ./backend/ ./backend/target/dist/ +function build-bundle { - rsync -ar \ - ./common/ ./backend/target/dist/common/ + build "frontend"; + build "exporter"; + build "backend"; + + rm -rf ./bundle + mkdir -p ./bundle + mv ./frontend/target/dist ./bundle/frontend + mv ./backend/target/dist ./bundle/backend + mv ./exporter/target ./bundle/exporter + + NAME="uxbox-$(date '+%Y.%m.%d-%H%M')" + + pushd bundle/ + tar -cvf ../$NAME.tar *; + popd + + xz -vez4f -T4 $NAME.tar } function log-devenv { docker-compose -p uxboxdev -f docker/devenv/docker-compose.yaml logs -f --tail=50 } +function build-testenv { + local BUNDLE_FILE=$1; + local BUNDLE_FILE_PATH=`readlink -f $BUNDLE_FILE`; + + echo "Building testenv with bundle: $BUNDLE_FILE_PATH." + + if [ ! -f $BUNDLE_FILE ]; then + echo "File $BUNDLE_FILE does not exists." + fi + + rm -rf ./docker/testenv/bundle; + mkdir -p ./docker/testenv/bundle; + + pushd ./docker/testenv/bundle; + tar xvf $BUNDLE_FILE_PATH; + popd + + pushd ./docker/testenv; + docker-compose -p uxbox-testenv -f ./docker-compose.yaml build + popd +} + +function start-testenv { + pushd ./docker/testenv; + docker-compose -p uxbox-testenv -f ./docker-compose.yaml up + popd +} + function usage { echo "UXBOX build & release manager v$REV" echo "USAGE: $0 OPTION" @@ -131,6 +163,18 @@ case $1 in log-devenv ${@:2} ;; + + # Test Env + start-testenv) + start-testenv + ;; + + build-testenv) + build-testenv ${@:2} + ;; + + + ## testin related commands # run-all-tests) @@ -156,6 +200,10 @@ case $1 in build-exporter ;; + build-bundle) + build-bundle + ;; + *) usage ;;