Adjustments to get tests to work

. test env ansible needs to have the action plugins available
. specify default variable for when vpnweb_auth is empty
. remove example openvpn_tcp_network6
. don't run leap-prometheus role
. ensure roles are added to hosts in test environment
. set a specific MENSHEN_API for CI builds
. allow for an insecure http API call for internal tests

Update .gitlab-ci.yml for fixing test builds:

. Replace the copy with built-in --additional-config flags
. Set the vpn gateway Location in the inventory automatically
. Bring in a pre-fabricated site.yml that will work for CI
. Remove the backend group from the vpn host3

.gitlab-ci.yml

@@ -23,12 +23,19 @@ variables:
       --roles-path=config/roles
       --num-hosts=3
       --additional-host-group=openvpn=host3
+      --additional-config ${TEST_DIR}/group_vars/all/openvpn_config.yml
+      --additional-config ${TEST_DIR}/group_vars/all/custom_config.yml
+      --additional-config ${TEST_DIR}/group_vars/all/gateway_locations.yml
+      --additional-config ${TEST_DIR}/group_vars/all/provider_config.yml
       -e ansible_cfg.defaults.strategy=mitogen_linear
+      -e ansible_cfg.defaults.action_plugins=../float/plugins/action:../plugins/action
+      -e inventory.hosts.host3.location=Seattle
       -e libvirt.remote_host=${VMINE_SSH#*@}
       -e libvirt.remote_user=${VMINE_SSH%@*}
       ${APT_PROXY:+-e config.apt_proxy=${APT_PROXY}}
       $CREATE_ENV_VARS $BUILD_DIR
-    - cp -v ${TEST_DIR}/group_vars/all/*.yml ${BUILD_DIR}/group_vars/all/
+    - cp -v ${TEST_DIR}/site.yml ${BUILD_DIR}
+    - echo "$(awk '!/- backend/ || ++ctr != 2' ${BUILD_DIR}/hosts.yml)" > ${BUILD_DIR}/hosts.yml
     - with-ssh-key ./float/scripts/floatup.py --url $VMINE_URL --ssh $VMINE_SSH --inventory $BUILD_DIR/hosts.yml --ram 3072 --image ${VM_IMAGE:-bullseye} up
     - (cd ${BUILD_DIR} && with-ssh-key ../float/float run ../playbooks/init-credentials.yml)
     - with-ssh-key ./float/test-driver init --no-vagrant $BUILD_DIR

config/roles/openvpn/templates/50openvpn.firewall.j2

@@ -2,7 +2,7 @@
 add_rule4 -A user-input -p tcp -m tcp -d {{ ips | ansible.netcommon.ipv4 | first }} --dport 80 -j ACCEPT
 add_rule4 -A user-input -p udp -m udp -d  {{ ips | ansible.netcommon.ipv4 | first }} --dport 80 -j ACCEPT
 
-{% if openvpn_tcp_network6 is defined and openvpn_tcp_network6|length -%}
+{% if openvpn_tcp_network6 is defined and openvpn_tcp_network6|d('')|length > 0 -%}
 # Allow incoming connections to ipv6 port 80 for both udp and tcp
 add_rule6 -A user-input -p tcp -m tcp -d  {{ ips | ansible.netcommon.ipv6 | first }} --dport 80 -j ACCEPT
 add_rule6 -A user-input -p udp -m udp -d  {{ ips | ansible.netcommon.ipv6 | first }} --dport 80 -j ACCEPT
@@ -17,7 +17,7 @@ add_rule4 -A FORWARD -s {{ openvpn_udp_network | ipaddr('network/prefix') }} -o
 # allow re/established udp *inbound* to vpn hosts
 add_rule4 -A FORWARD -d {{ openvpn_udp_network | ipaddr('network/prefix') }} -m state --state RELATED,ESTABLISHED -j ACCEPT
 
-{% if openvpn_tcp_network6 is defined and openvpn_tcp_network6|length -%}
+{% if openvpn_tcp_network6 is defined and openvpn_tcp_network6|d('')|length > 0 -%}
 # let ipv6 tcp vpn hosts reach the internet
 add_rule6 -A FORWARD -i tun0 -o {{ ansible_default_ipv4.interface }} -s {{ openvpn_tcp_network6 }} -m state --state NEW -j ACCEPT
 # let ipv6 udp vpn hosts reach the internet
@@ -34,7 +34,7 @@ add_rule6 -A FORWARD -i {{ ansible_default_ipv4.interface }} -o tun1 -d {{ openv
 add_rule4 -A FORWARD -i tun0 -p tcp -o tun0 -j DROP
 # deny v4 udp client-to-client communication
 add_rule4 -A FORWARD -i tun1 -p udp -o tun0 -j DROP
-{% if openvpn_tcp_network6 is defined and openvpn_tcp_network6|length -%}
+{% if openvpn_tcp_network6 is defined and openvpn_tcp_network6|d('')|length > 0 -%}
 # deny v6 tcp client-to-client communication
 add_rule6 -A FORWARD -i tun0 -p tcp -o tun0 -j DROP
 # deny v6 udp client-to-client communication

config/roles/openvpn/templates/50openvpn_nat.firewall.j2

 # Set egress IP
-add_rule4 -A POSTROUTING -s {{ openvpn_tcp_network | ipaddr('network/prefix') }} -o {{ ansible_default_ipv4.interface }} -j SNAT --to-source {{ egress_ip }}
-add_rule4 -A POSTROUTING -s {{ openvpn_udp_network | ipaddr('network/prefix') }} -o {{ ansible_default_ipv4.interface }} -j SNAT --to-source {{ egress_ip }}
+add_rule4 -A POSTROUTING -s {{ openvpn_tcp_network | ipaddr('network/prefix') }} -o {{ ansible_default_ipv4.interface }} -j SNAT --to-source {{ egress_ip | default(ips | ansible.netcommon.ipv4 | first) }}
+add_rule4 -A POSTROUTING -s {{ openvpn_udp_network | ipaddr('network/prefix') }} -o {{ ansible_default_ipv4.interface }} -j SNAT --to-source {{ egress_ip | default(ips | ansible.netcommon.ipv4 | first) }}
 # Accept connections on ipv4 port 1194, redirecting them to openvpn
 add_rule4 -A PREROUTING -p tcp -d {{ ips | ansible.netcommon.ipv4 | first }} --dport 1194 -j DNAT --to-destination {{ ips | ansible.netcommon.ipv4 | first }}:80
 add_rule4 -A PREROUTING -p udp -d {{ ips | ansible.netcommon.ipv4 | first }} --dport 1194 -j DNAT --to-destination {{ ips | ansible.netcommon.ipv4 | first }}:80

config/roles/vpnweb/tasks/main.yml

@@ -47,4 +47,4 @@
     - "restart docker-vpnweb-vpnweb"
 
 - import_tasks: "sip.yml"
-  when: vpnweb_auth == "sip2"
+  when: vpnweb_auth | default('anon') == "sip2"

config/services.openvpn.yml

@@ -14,7 +14,8 @@ menshen:
       volumes:
         - /var/lib/GeoIP: /var/lib/GeoIP/
       env:
-        MENSHEN_API: "api.{{ domain_public[0] }}"
+        MENSHEN_API: "{{ menshen_api | default('api.{{ domain_public[0] }}') }}"
+        INSECURE_API: "{{ testing | lower }}"
   public_endpoints:
     - name: menshen
       port: 9001

test/test-full/group_vars/all/custom_config.yml

@@ -2,4 +2,6 @@
 float_limit_bind_to_known_interfaces: true
 geoip_account_id: "{{ lookup('env', 'GEOIP_ACCOUNT_ID') }}"
 geoip_license_key: "{{ lookup('env', 'GEOIP_LICENSE_KEY') }}"
- 
+# for testing only, because it is using http, it requires INSECURE_API env
+# passed to menshen init
+menshen_api: 'http://host2.vpnweb.float.hexacab.org:8000'

test/test-full/group_vars/all/openvpn_config.yml

@@ -16,5 +16,4 @@ openvpn_config:
 
 openvpn_tcp_network: "10.41.0.0/21"
 openvpn_udp_network: "10.42.0.0/21"
-openvpn_tcp_network6: "2001:db8:123::/64"
 

test/test-full/site.yml

 ---
-- import_playbook: "../../float/playbooks/all.yml"
+- import_playbook: "../float/playbooks/all.yml"
+
+- hosts: openvpn
+  roles:
+    - kresd
+    - openvpn
+
+- hosts: vpnweb
+  roles:
+    - vpnweb
+    - menshen
+
+- hosts: frontend
+  roles: [vpnweb-frontend]