deployment improvements and fixes

add faircoin service

.gitignore

 *.pyc
 *.pid
 *.log
+services/explorer/cce.conf
+services/explorer/server.conf

.gitlab-ci.yml

@@ -15,20 +15,9 @@ build-explorer:
     variables:
       - $CMD =~ /build/
   before_script:
-    - export RUNNER_UID=`id -u`
-    - export RUNNER_GID=`id -g`
-    - cat ${CCE_CONF} | sed -e s/MYSQL_USER/${MYSQL_USER}/g -e s/MYSQL_PASSWORD/${MYSQL_PASSWORD}/g -e s/MYSQL_DATABASE_PORT/${MYSQL_DATABASE_PORT}/g > ./cce.conf
-    - export LH_PORT=${LH_PORT_explorer} && curl https://git.fairkom.net/faircoin.co/admin/-/raw/master/nginx_proxy | sed -e s/DOMAIN/${CI_PROJECT_NAME}/g -e s/LH_PORT/${LH_PORT}/g > ${CI_PROJECT_NAME}
+    - chmod +x setenv && . setenv
   script:
-    - docker-compose stop
-    - docker-compose down -v
-    - docker-compose build app
-    - docker-compose up -d db
-    - sleep 20
-    - chmod +x create_db.sh
-    - docker-compose exec -T db /opt/explorer.faircoin.co/create_db.sh
-    - docker-compose up -d app
-    - docker-compose exec -T app python3 dbload.py -n -l
+    - deploy/build
   timeout: 6h
 
 start-explorer:
@@ -41,15 +30,9 @@ start-explorer:
     variables:
       - $CMD =~ /build|start/
   before_script:
-    - export RUNNER_UID=`id -u`
-    - export RUNNER_GID=`id -g`
-    - cat ${CCE_CONF} | sed -e s/MYSQL_USER/${MYSQL_USER}/g -e s/MYSQL_PASSWORD/${MYSQL_PASSWORD}/g -e s/MYSQL_DATABASE_PORT/${MYSQL_DATABASE_PORT}/g > ./cce.conf
-    - export LH_PORT=${LH_PORT_explorer} && curl https://git.fairkom.net/faircoin.co/admin/-/raw/master/nginx_proxy | sed -e s/DOMAIN/${CI_PROJECT_NAME}/g -e s/LH_PORT/${LH_PORT}/g > ${CI_PROJECT_NAME}
-    - cat ${SERVER_CONF} | sed -e s/LH_PORT_explorer/${LH_PORT_explorer}/g > ./server.conf
+    - chmod +x setenv && . setenv
   script:
-    - docker-compose stop
-    - docker-compose up -d
-    - docker-compose exec -T app python3 websrv.py
+    - deploy/start
 
 stop-explorer:
   stage: stop
@@ -60,8 +43,10 @@ stop-explorer:
       - master
     variables:
       - $CMD =~ /stop/
+  before_script:
+    - chmod +x setenv && . setenv
   script:
-    - docker-compose stop
+    - deploy/stop
 
 uninstall-explorer:
   stage: uninstall
@@ -72,6 +57,21 @@ uninstall-explorer:
       - master
     variables:
       - $CMD =~ /uninstall/
+  before_script:
+    - chmod +x setenv && . setenv
+  script:
+    - deploy/uninstall
+
+remove-explorer:
+  stage: uninstall
+  tags:
+    - FairCoin.Co
+  only:
+    refs:
+      - master
+    variables:
+      - $CMD =~ /remove/
+  before_script:
+    - chmod +x setenv && . setenv
   script:
-    - docker-compose stop
-    - docker-compose down -v
+    - deploy/remove

README.md

@@ -4,44 +4,70 @@ The service is made for/controlled by gitlab runner.
 
 ## Usage
 
-Go into gitlab **CI/CD** -> **Pipeline** and **Run Pipeline**
+### by Gitlab-Runner
 
-Enter variable name **CMD**
+1. Fork this project to your group/repositories where you have set up a gitlab-runner
+1. change the gitlab-runner tags in .gitlab-ci.yml
+1. add env files ( select Type = file ! ) to override defaults ./env_file , go to Gitlab **Settings** -> **CI/CD** -> **Variables**
+
+You can read a full developer report of server preparation with gitlab-runner usage > [DEV_REPORT.md](DEV_REPORT.md)
+
+~~~
+#### env files ( examples see env_file/... / select Type = file !) ######################
+CCE_CONF              # config dbload.py
+SERVER_CONF           # config websrv.py
+FAIRCHAINS_CONF       # conf file to run the faircoind or fairchains daemon
 
-#### CMD - commands
+#### env vars ###########################################################################
+MYSQL_USER            # mysql user ( used to run the explorer )
+MYSQL_PASSWORD        # mysql user password
+MYSQL_DATABASE_PORT   # mysql database port ( default 3306 )
+MYSQL_ROOT_PASSWORD   # mysql root user password ( used to admin the database )
 
+LH_PORT_explorer      # port running explorer by localhost ( default 8222 / use nginx proxy to make it available by remote   )  
+~~~
+1. run command, go to Gitlab **CI/CD** -> **Pipeline** and **Run Pipeline**
+Enter variable name **CMD**
 ~~~
 build        # build container ( changes of Dockerfile )
+init         # remove blockchain data and initialize sql database and faircoin/fairchains.conf
 start        # start container ( changes of scripts )
 stop         # stop container
-uninstall    # remove container
+install      # runs build, init, start stages
+uninstall    # remove container without to delete the data
+remove       # remove all data
 ~~~
 
-#### CI/CD Settings
-
-Go Gitlab **Settings** -> **CI/CD** -> **Variables**
+### by SSH access <small>( manually without gitlab-runner )</small>
 
+1. install docker and docker-compose ( https://docs.docker.com/compose/install/ )
+1. clone this project
+1. change configuration in ./env_file
+1. Initialize env vars
 ~~~
-#### FairCoin.Co group variables ######################
-CCE_CONF           # config dbload.py
-SERVER_CONF        # config websrv.py
+chmod +x setenv
+. setenv
 ~~~
 
-## Development <small>( manual usage )</small>
+1. run command from project root folder
+~~~
+deploy/build
+deploy/init
+deploy/start
+deploy/stop
+deploy/install
+deploy/uninstall
+deploy/remove
+~~~
 
-If you want create an instance manual then follow the  instructions.
-
-1. install docker and docker-compose ( https://docs.docker.com/compose/install/ )
-1. clone this project
-1. change configuration in ./env
-1. run services by ./control.sh
+#### Troubleshooting
 
+some commands can be helpful getting shell access to container or get some status info
 ~~~
-chmod +x ./control.sh
-./control.sh build
-./control.sh start
-./control.sh stop
-./control.sh uninstall
-sudo ./control.sh nginx-install
-sudo ./control.sh nginx-uninstall
+console/explorer                    # get access to container of explorer server app
+console/db                          # get access to container of mysql database
+console/faircoin                    # get access to container of faircoind
+
+console/ps                          # list containers status info
+console/status                      # list all ports / processes and faircoind info
 ~~~

console/db

+#!/bin/bash
+docker-compose exec db bash

console/explorer

+#!/bin/bash
+docker-compose exec explorer bash

console/faircoin

+#!/bin/bash
+docker-compose exec faircoin bash

console/help

+#!/bin/bash
+echo ""
+echo -e "console/explorer\t\t( run container shell )"
+echo -e "console/db \t( run container shell )"
+echo -e "console/faircoin \t\t( run container shell )"
+echo -e "console/help"
+echo -e "console/ps \t\t\t( list containers )"
+echo -e "console/status \t\t\t( network / processes / ... status overview )"
+echo ""

console/ps

+#!/bin/bash
+docker-compose ps

console/status

+#!/bin/bash
+echo ""
+echo "NETWORK"
+echo "======================================================================="
+echo "container:faircoin"
+echo ""
+
+docker-compose exec -T faircoin netstat -tulpen
+echo ""
+
+echo "_______________________________________________________________________"
+echo "container:explorer"
+echo ""
+docker-compose exec -T explorer netstat -tulpen
+echo ""
+echo "_______________________________________________________________________"
+echo "container:db"
+echo ""
+docker-compose exec -T db netstat -tulpen
+echo ""
+echo ""
+echo ""
+echo "PROCESSES"
+echo "======================================================================="
+echo "container:faircoin"
+echo ""
+docker-compose exec -T faircoin ps aux
+echo ""
+echo "_______________________________________________________________________"
+echo "container:explorer"
+echo ""
+docker-compose exec -T explorer ps aux
+echo ""
+echo "_______________________________________________________________________"
+echo "container:db"
+echo ""
+docker-compose exec -T db ps aux
+echo ""
+echo ""
+echo ""
+echo "FAIRCOIND"
+echo "======================================================================="
+echo "container:faircoin"
+echo ""
+docker-compose exec -T faircoin ./faircoin-cli getinfo

control.sh

-#!/bin/bash
-export RUNNER_UID=`id -u`
-export RUNNER_GID=`id -g`
-if [ -d ../${CI_PROJECT_NAME}.tmp ]; then
-  export `cat ./${CI_PROJECT_NAME}.tmp/*`
-else
-  export `cat ./env/*`
-fi
-
-case $1 in
-  "exec")
-      docker-compose exec $2 $3
-    ;;
-  "ps")
-      docker-compose ps
-    ;;
-  "build")
-      chmod +x create_db.sh
-      cat ${CCE_CONF} > ./cce.conf
-      docker-compose down
-      docker-compose build
-    ;;
-  "start")
-      cat ${CCE_CONF} > ./cce.conf
-      cat ${SERVER_CONF} > ./server.conf
-      docker-compose stop
-      docker-compose up -d
-      docker-compose exec -T app python3 websrv.py
-    ;;
-  "stop")
-      docker-compose stop $2
-    ;;
-  "uninstall")
-      docker-compose stop
-      docker-compose down
-    ;;
-  "nginx-install")
-      cp ${CI_PROJECT_NAME} > /etc/nginx/sites-available/
-      certbot certonly --nginx --domain ${CI_PROJECT_NAME}
-      ln -s /etc/nginx/sites-available/${CI_PROJECT_NAME} /etc/nginx/sites-enabled/
-      nginx -s reload
-    ;;
-  "nginx-uninstall")
-      rm /etc/nginx/sites-enabled/${CI_PROJECT_NAME}
-      nginx -s reload
-    ;;
-esac

deploy/build

+#!/bin/bash
+cat ${CCE_CONF} | sed -e s/MYSQL_USER/${MYSQL_USER}/g -e s/MYSQL_PASSWORD/${MYSQL_PASSWORD}/g -e s/MYSQL_DATABASE_PORT/${MYSQL_DATABASE_PORT}/g > ./services/explorer/cce.conf
+docker-compose stop
+docker-compose down
+chmod +x ./services/faircoin/entrypoint.sh
+chmod +x ./services/explorer/entrypoint.sh
+chmod +x ./services/db/create_db.sh
+docker-compose build

deploy/help

+#!/bin/bash
+echo ""
+echo -e "deploy/build \t\t( build container images )"
+echo -e "deploy/start \t\t( start container )"
+echo -e "deploy/stop \t\t( stop container )"
+echo -e "deploy/install \t( run steps build + start + new_ssl )"
+echo -e "deploy/uninstall \t( stop and remove container, named volumes will NOT removed )"
+echo -e "deploy/remove \t\t( stop and remove container and images, named volumes WILL REMOVED )"
+echo ""

deploy/init

+#!/bin/bash
+docker-compose down -v
+chmod +x ./services/db/create_db.sh
+docker-compose up -d db
+sleep 20
+docker-compose exec -T db /opt/explorer/create_db.sh
+cat ${FAIRCHAINS_CONF} > ./services/faircoin/scripts/fairchains.conf
+cat ${FAIRCHAINS_CONF} | sed -e s/netname/\#\ netname/g > ./services/faircoin/scripts/faircoin.conf
+export AUTOSTART=0
+docker-compose up -d faircoin
+docker-compose exec -T faircoin mv -f ./scripts/fairchains.conf .faircoin2/
+docker-compose exec -T faircoin mv -f ./scripts/faircoin.conf .faircoin2/
+docker-compose stop faircoin
+export AUTOSTART=1
+docker-compose up -d faircoin
+export AUTOSTART=0
+docker-compose up -d explorer
+sleep 20
+docker-compose exec -T explorer python3 dbload.py -n -l

deploy/install

+#!/bin/bash
+deploy/build
+deploy/init
+deploy/start

deploy/remove

+#!/bin/bash
+docker-compose stop
+docker-compose down -v --rmi all --remove-orphans

deploy/start

+#!/bin/bash
+cat ${CCE_CONF} | sed -e s/MYSQL_USER/${MYSQL_USER}/g -e s/MYSQL_PASSWORD/${MYSQL_PASSWORD}/g -e s/MYSQL_DATABASE_PORT/${MYSQL_DATABASE_PORT}/g > ./services/explorer/cce.conf
+cat ${SERVER_CONF} | sed -e s/LH_PORT_explorer/${LH_PORT_explorer}/g > ./services/explorer/server.conf
+docker-compose stop
+docker-compose up -d --remove-orphans
+# docker-compose exec -T --workdir /home/faircoin/explorer/scripts explorer python3 websrv.py

deploy/stop

+#!/bin/bash
+docker-compose stop

deploy/uninstall

+#!/bin/bash
+docker-compose stop
+docker-compose down

docker-compose.yml

@@ -2,41 +2,68 @@ version: '3.3'
 
 services:
 
-  app:
+  explorer:
+    restart: always
     build:
-      context: ./
+      context: services/explorer
       args:
-        RUNNER_UID: ${RUNNER_UID}
-        RUNNER_GID: ${RUNNER_GID}
+        RUNNER_UID: ${RUNNER_UID:-1000}
+        RUNNER_GID: ${RUNNER_GID:-1000}
+    environment:
+      AUTOSTART: ${AUTOSTART:-1}
     env_file:
-      - ${FAIRCOIN_CONF}
-    container_name: ${CI_PROJECT_NAME}-app
-    user: '${RUNNER_UID}:${RUNNER_GID}'
-    command: sleep inf
-    network_mode: host
+      - ${FAIRCHAINS_CONF}
+    container_name: explorer-app
+    user: '${RUNNER_UID:-1000}:${RUNNER_GID:-1000}'
+    # command: sleep inf
+    ports:
+      - ${LH_PORT_explorer:-8222}:8222
     depends_on:
       - db
+      - faircoin
     volumes:
-      - .:/home/faircoin/explorer.faircoin.co
+      - ./services/explorer:/home/faircoin/explorer
 
   db:
-    image: mysql:5.7
-    container_name: ${CI_PROJECT_NAME}-mysql
     restart: always
+    build:
+      context: services/db
+    container_name: explorer-db
     environment:
       MYSQL_DATABASE: explorer
       # So you don't have to use root, but you can if you like
-      MYSQL_USER: ${MYSQL_USER}
+      MYSQL_USER: ${MYSQL_USER:-user}
       # You can use whatever password you like
-      MYSQL_PASSWORD: ${MYSQL_PASSWORD}
+      MYSQL_PASSWORD: ${MYSQL_PASSWORD:-password}
       # Password for root access
-      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}
-    ports:
-      - ${MYSQL_DATABASE_PORT}:3306
+      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD:-rootpassword}
+    #ports:
+    #  - ${MYSQL_DATABASE_PORT:-3306}:3306
+    volumes:
+      - db:/var/lib/mysql
+      - ./services/db:/opt/explorer
 
+  faircoin:
+    restart: always
+    build:
+      context: services/faircoin
+      args:
+        source_repository: "https://github.com/faircoin/faircoin.git"
+        RUNNER_UID: "${RUNNER_UID:-1000}"
+        RUNNER_GID: "${RUNNER_GID:-1000}"
+    environment:
+      AUTOSTART: ${AUTOSTART:-1}
+    env_file:
+      - ${FAIRCHAINS_CONF}
+    container_name: explorer-faircoind
+    user: "${RUNNER_UID:-1000}:${RUNNER_GID:-1000}"
+    stop_grace_period: 30s
     volumes:
-      - my-db:/var/lib/mysql
-      - .:/opt/explorer.faircoin.co
+      - fairchains:/home/faircoin/.faircoin2
+      - ./services/faircoin/scripts:/home/faircoin/scripts
+      - ./services/explorer:/home/faircoin/explorer
+
 # Names our volume
 volumes:
-  my-db:
+  fairchains:
+  db:

env/CI_PROJECT_NAME

-CI_PROJECT_NAME=explorer.faircoin.co