virtualization:nested_lxc [NetGeekery]

This page is read only. You can view the source, but not change it. Ask your administrator if you think this is wrong.
====== Nested LXC Containers ======

*Don't actually recommend -- I did it just for fun

===== LXD CONFIGURATION (host) =====
    ## create lxd container
        containername: container1
    $ lxc launch ubuntu:x ${containername}

    ## exec lxd container
        exec:
            containername: container1
    $ lxc exec ${containername} -- /bin/bash

configure lxd container according to [[virtualization:lxd|LXD]]
    configuration:
        containername: container1
        username: ubuntu
        network:
            lbr0ipv4: 10.0.3.1

configure iptables according to [[linux:networking|Networking]]
    iptables:
        ## configure rules
        ##  -- forward packets on port 80 to container1 container's port 80
        chain: PREROUTING
        protocol: TCP
        localaddress: xx.xx.xx.xxx/32
        port: 80
        target: DNAT
        destaddress: 10.0.3.201:80
        ##  -- forward packets on port 443 to container1 container's port 443
        chain: PREROUTING
        protocol: TCP
        localaddress: xx.xx.xx.xxx/32
        port: 443
        target: DNAT
        destaddress: 10.0.3.201:443
        ##  -- forward packets on port 5232 to container1 container's port 5232
        chain: PREROUTING
        protocol: TCP
        localaddress: xx.xx.xx.xxx/32
        port: 5232
        target: DNAT
        destaddress: 10.0.3.201:5232
        ##  -- masquerade packets
        ## save rules
        ## show iptables rules
            # ubuntu-512mb-nyc2-01 = xx.xx.xx.xxx
            # Chain PREROUTING (policy ACCEPT)
            # target     prot opt source               destination
            # DNAT       tcp  --  anywhere             ubuntu-512mb-nyc2-01  tcp dpt:5232 to:10.0.3.201:5232
            # DNAT       tcp  --  anywhere             ubuntu-512mb-nyc2-01  tcp dpt:http to:10.0.3.201:80
            # DNAT       tcp  --  anywhere             ubuntu-512mb-nyc2-01  tcp dpt:https to:10.0.3.201:443
            
            # Chain INPUT (policy ACCEPT)
            # target     prot opt source               destination
            
            # Chain OUTPUT (policy ACCEPT)
            # target     prot opt source               destination
            
            # Chain POSTROUTING (policy ACCEPT)
            # target     prot opt source               destination
            # MASQUERADE  all  --  10.0.3.0/24         !10.0.3.0/24          /* generated for LXD network lbr0 */

    ## list containers
        # +--------+-----------+--------------------+-----------------------------------------------+------------+-----------+
        # |  NAME  |  STATE    |      IPV4          |                     IPV6                      |    TYPE    | SNAPSHOTS |
        # +--------+-----------+--------------------+-----------------------------------------------+------------+-----------+
        # | container1     | RUNNING   | 10.0.4.1 (lxcbr0)  | fd42:27e1:6a22:790c:216:3eff:fe58:61f4 (eth0) | PERSISTENT |     0     |
        # |        |           | 10.0.3.201 (eth0)  |                                               |            |           |
        # +--------+-----------+--------------------+-----------------------------------------------+------------+-----------+
    ## get container info
        Name: container1
        Remote: unix:/var/lib/lxd/unix.socket
        Architecture: x86_64
        Created: 2017/05/22 18:13 UTC
        Status: Running
        Type: persistent
        Profiles: default
        Pid: 28692
        Ips:
            eth0: inet  10.0.3.201  veth4LJSE5
            eth0: inet6 fd42:27e1:6a22:790c:216:3eff:fe58:61f4  veth4LJSE5
            eth0: inet6 fe80::216:3eff:fe58:61f4  veth4LJSE5
            lo: inet  127.0.0.1
            lo: inet6 ::1
        Resources:
            Processes: 29
            CPU usage:
                CPU usage (in seconds): 2
            Memory usage:
                Memory (current): 42.06MB
                Memory (peak): 42.06MB
            Network usage:
                eth0:
                    Bytes received: 3.63kB
                    Bytes sent: 1.57kB
                    Packets received: 32
                    Packets sent: 14
                lo:
                    Bytes received: 0B
                    Bytes sent: 0B
                    Packets received: 0
                    Packets sent: 0
    ## show container configuration
        architecture: x86_64
        config:
            image.architecture: amd64
            image.description: ubuntu 16.04 LTS amd64 (release) (20170516)
            image.label: release
            image.os: ubuntu
            image.release: xenial
            image.serial: "20170516"
            image.version: "16.04"
            security.nesting: "true"
            volatile.base_image: 8fa08537ae51c880966626561987153e72d073cbe19dfe5abc062713d929254d
            volatile.eth0.hwaddr: 00:16:3e:58:61:f4
            volatile.eth0.name: eth0
            volatile.idmap.base: "0"
            volatile.idmap.next: '[{"Isuid":true,"Isgid":false,"Hostid":100000,"Nsid":0,"Maprange":65536},{"Isuid":false,"Isgid":true,"Hostid":100000,"Nsid":0,"Maprange":65536}]'
            volatile.last_state.idmap: '[{"Isuid":true,"Isgid":false,"Hostid":100000,"Nsid":0,"Maprange":65536},{"Isuid":false,"Isgid":true,"Hostid":100000,"Nsid":0,"Maprange":65536}]'
            volatile.last_state.power: RUNNING
        devices:
            eth0:
                ipv4.address: 10.0.3.1
                nictype: bridged
                parent: lbr0
                type: nic
        ephemeral: false
        profiles:
        - default
    networking.notes socket statistics
        # Netid  State      Recv-Q Send-Q      Local Address:Port             Peer Address:Port
        # udp    UNCONN     0      0                10.0.3.1:domain                      *:*
        # udp    UNCONN     0      0                  *%lbr0:bootps                      *:*
        # udp    UNCONN     0      0            xx.xx.xx.xxx:ntp                         *:*
        # udp    UNCONN     0      0               127.0.0.1:ntp                         *:*
        # tcp    LISTEN     0      5                10.0.3.1:domain                      *:*

===== LXC CONFIGURATION (guest) =====
    root@container1:~#
        ## show socket statistics
        $ sudo ss -tula
            # Netid  State      Recv-Q Send-Q      Local Address:Port             Peer Address:Port
            # udp    UNCONN     0      0                10.0.4.1:domain                      *:*
            # udp    UNCONN     0      0                *%lxcbr0:bootps                      *:*
            # tcp    LISTEN     0      5                10.0.4.1:domain                      *:*

        ## show containers
        $ lxc-ls -f
            # NAME              STATE   AUTOSTART GROUPS IPV4       IPV6
            # innerContainer1   RUNNING 1         -      10.0.4.50  -
            # innerContainer2   RUNNING 1         -      10.0.4.51  -
            # innerContainer3   RUNNING 1         -      10.0.4.52  -
            # innerContainer4   RUNNING 1         -      10.0.4.53  -
            # innerContainer5   RUNNING 1         -      10.0.4.54  -
            # nginx             RUNNING 1         -      10.0.4.99  -

configure iptables according to [[linux:networking|Networking]]
            ## configure iptables
            ##  -- forward packets from hostContainer on port 80 to nginx container's port 80
            iptables:
                chain: PREROUTING
                protocol: TCP
                localaddress: 10.0.3.201/32
                port: 80
                target: DNAT
                destaddress: 10.0.4.99:80
            ##  -- forward packets from hostContainer on port 443 to nginx container's port 443
            iptables:
                chain: PREROUTING
                protocol: TCP
                localaddress: 10.0.3.201/32
                port: 443
                target: DNAT
                destaddress: 10.0.4.99:443
            ##  -- forward packets from hostContainer on port 5232 to nginx container's port 5232
            iptables:
                chain: PREROUTING
                protocol: TCP
                localaddress: 10.0.3.201/32
                port: 5232
                target: DNAT
                destaddress: 10.0.4.99:5232
            ## save rules
            ## show iptables rules
                # hostContainer.lxd = 10.0.3.201
                # Chain PREROUTING (policy ACCEPT)
                # target     prot opt source               destination
                # DNAT       tcp  --  anywhere             hostContainer.lxd  tcp dpt:5232 to:10.0.4.99:5232
                # DNAT       tcp  --  anywhere             hostContainer.lxd  tcp dpt:http to:10.0.4.99:80
                # DNAT       tcp  --  anywhere             hostContainer.lxd  tcp dpt:https to:10.0.4.99:443
                
                # Chain INPUT (policy ACCEPT)
                # target     prot opt source               destination
                
                # Chain OUTPUT (policy ACCEPT)
                # target     prot opt source               destination
                
                # Chain POSTROUTING (policy ACCEPT)
                # target     prot opt source               destination
                # MASQUERADE  all  --  10.0.4.0/24         !10.0.4.0/24

===== Sample Containers =====

==== innerContainer1 (simple static apache page) ====
    ## create container
    create:
        template: download
        config: ~/.config/lxc/default.conf
        containername: innerContainer1
        distro: ubuntu
        release: xenial
        arch: amd64

    $ lxc-create -t ${template} -f ${config} -n ${containername} -- -d ${distro} -r ${release} -a ${arch}
    $ lxc-start -n ${containername}
    $ lxc-attach -n ${containername}

Configure nginx according to [[selfhosted:nginx|Nginx]]
    ## configure nginx (simple_with_upstream_ssl)
        configuration:
            site: site1
            upstreamname: site1
            server: 10.0.4.51:80
            servername: hostname1
            cert: /etc/letsencrypt/live/hostname1/fullchain.pem
            key: /etc/letsencrypt/live/hostname1/privkey.pem

    ## show socket statistics
    $ sudo ss -tula
        # Netid  State     Recv-Q Send-Q      Local Address:Port             Peer Address:Port
        # udp    UCONN     0      0                       *:bootpc                      *:*
        # tcp    LISTEN    0      128                    :::http                       :::*

    ## show netstat
    $ sudo netstat -pltun
        # Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
        # tcp6       0      0 :::80                   :::*                    LISTEN      181/apache2
        # udp        0      0 0.0.0.0:68              0.0.0.0:*                           135/dhclient

==== innerContainer2 (mattermost) ====

    ## create container
    create:
        template: download
        config: ~/.config/lxc/default.conf
        containername: innerContainer1
        distro: ubuntu
        release: xenial
        arch: amd64

    $ lxc-create -t ${template} -f ${config} -n ${containername} -- -d ${distro} -r ${release} -a ${arch}
    $ lxc-start -n ${containername}
    $ lxc-attach -n ${containername}

Set up [[selfhosted:mattermost|Mattermost]] with the following settings:
    ## set up
    setup:
        database:
            user: mmsuser
            database: mattermost
            password: xxxxxxxxxxxxxxxxx
            mysql:
                host: %
            psql:
                host: 127.0.0.1
        email:
            email: xxxxxx@gmail.com
            server: smtp.gmail.com
            password:
            port: 587

Configure [[selfhosted:nginx|Nginx]] with the following settings:
    ## configure nginx (simple_with_upstream_ssl_cache)
        configuration:
            site: mattermost
            upstreamname: MM
            server: 10.0.4.52:8065
            servername: hostname2
            cert: /etc/letsencrypt/live/hostname2/fullchain.pem
            key: /etc/letsencrypt/live/hostname2/privkey.pem

    ## show socket statistics
    $ sudo ss -tula
        # Netid  State     Recv-Q Send-Q      Local Address:Port             Peer Address:Port
        # udp    UCONN     0      0                       *:bootpc                      *:*
        # udp    ESTAB     0      0               127.0.0.1:32965               127.0.0.1:32965
        # tcp    LISTEN    0      128             127.0.0.1:postgresql                  *:*
        # tcp    LISTEN    0      128                    :::8065                       :::*

    ## show netstat
    $ sudo netstat -pltun
        # Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
        # tcp        0      0 127.0.0.1:5432          0.0.0.0:*               LISTEN      185/postgres
        # tcp6       0      0 :::8065                      :::*               LISTEN      204/platform
        # udp        0      0 0.0.0.0:68              0.0.0.0:*                           153/dhclient

    ## test
    $ curl localhost:8065

==== innerContainer3 (nextcloud) ====

    ## create container
    create:
        template: download
        config: ~/.config/lxc/default.conf
        containername: innerContainer3
        distro: ubuntu
        release: xenial
        arch: amd64

    $ lxc-create -t ${template} -f ${config} -n ${containername} -- -d ${distro} -r ${release} -a ${arch}
    $ lxc-start -n ${containername}
    $ lxc-attach -n ${containername}

Configure [[selfhosted:nginx|Nginx]] with the following settings:
    ## configure nginx (simple_with_upstream_ssl)
        configuration:
            site: nextcloud
            upstreamname: nextcloud
            server: 10.0.4.53:80
            servername: hostname3
            cert: /etc/letsencrypt/live/hostname3/fullchain.pem
            key: /etc/letsencrypt/live/hostname3/privkey.pem

    ## show socket statistics
    $ sudo ss -tula
        # Netid  State     Recv-Q Send-Q      Local Address:Port             Peer Address:Port
        # udp    UCONN     0      0                       *:bootpc                      *:*
        # tcp    LISTEN    0      128             127.0.0.1:mysql                       *:*
        # tcp    LISTEN    0      128                    :::http                       :::*

    ## show netstat
    $ sudo netstat -pltun
        # Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
        # tcp        0      0 127.0.0.1:3306          0.0.0.0:*               LISTEN      -
        # tcp6       0      0 :::80                   :::*                    LISTEN      252/apache2
        # udp        0      0 0.0.0.0:68              0.0.0.0:*                           136/dhclient

    ## test
    ## -- redirects to http://localhost/index.php/login
    $ curl localhost

    ## information
    http://hostname3/remote.php/dav/principals/users/user999/
    http://hostname3/remote.php/dav/addressbooks/users/user999/contacts/
    
==== innerContainer4 (Seafile) ====

    ## create container
    create:
        template: download
        config: ~/.config/lxc/default.conf
        containername: innerContainer4
        distro: ubuntu
        release: xenial
        arch: amd64

    $ lxc-create -t ${template} -f ${config} -n ${containername} -- -d ${distro} -r ${release} -a ${arch}
    $ lxc-start -n ${containername}
    $ lxc-attach -n ${containername}

Configure [[selfhosted:nginx|Nginx]]

    ## show socket statistics
    $ sudo ss -tula
        # Netid  State     Recv-Q Send-Q      Local Address:Port             Peer Address:Port
        # udp    UCONN     0      0                       *:bootpc                      *:*
        # tcp    LISTEN    0      80              127.0.0.1:mysql                       *:*
        # tcp    LISTEN    0      5                       *:http-alt                    *:*
        # tcp    LISTEN    0      128                     *:8082                        *:*
        # tcp    LISTEN    0      128                     *:8000                        *:*
        # tcp    ESTAB     0      0               127.0.0.1:50100               127.0.0.1:mysql
        # tcp    ESTAB     0      0               127.0.0.1:mysql               127.0.0.1:50100

    ## show netstat
    $ sudo netstat -pltun
        # Proto Recv-Q Send-Q Local Address           Foreign Address         State       PID/Program name
        # tcp        0      0 127.0.0.1:3306          0.0.0.0:*               LISTEN      178/mysqld
        # tcp        0      0 0.0.0.0:8080            0.0.0.0:*               LISTEN      261/python2.7
        # tcp        0      0 0.0.0.0:8082            0.0.0.0:*               LISTEN      262/seaf-server
        # tcp        0      0 0.0.0.0:8000            0.0.0.0:*               LISTEN      352/python2.7
        # udp        0      0 0.0.0.0:68              0.0.0.0:*                           145/dhclient
        
 ==== innerContainer5 (Nginx) ====

    ## create container
    create:
        template: download
        config: ~/.config/lxc/default.conf
        containername: innerContainer1
        distro: ubuntu
        release: xenial
        arch: amd64

    $ lxc-create -t ${template} -f ${config} -n ${containername} -- -d ${distro} -r ${release} -a ${arch}
    $ lxc-start -n ${containername}
    $ lxc-attach -n ${containername}

    ## show socket statistics
    $ sudo ss -tula
            # Netid  State      Recv-Q Send-Q      Local Address:Port             Peer Address:Port
            # tcp    LISTEN     0      128                     *:http                        *:*
            # tcp    LISTEN     0      128                     *:https                       *:*

    ## list open files
    $ lsof -i
            # COMMAND  PID     USER   FD   TYPE  DEVICE SIZE/OFF NODE NAME
            # nginx   9442     root    8u  IPv4 5195802      0t0  TCP *:http (LISTEN)
            # nginx   9442     root    9u  IPv4 5195803      0t0  TCP *:https (LISTEN)
            # nginx   9444 www-data    8u  IPv4 5195802      0t0  TCP *:http (LISTEN)
            # nginx   9444 www-data    9u  IPv4 5195803      0t0  TCP *:https (LISTEN)