IP Transparency and Direct Server Return with NGINX and NGINX Plus as Transparent Proxy
nginx
ip rule add fwmark 1 lookup 100
ip route add local 0.0.0.0/0 dev lo table 100
iptables -t mangle -A PREROUTING -p tcp -s 10.11.1.200 –sport 443 -j MARK –set-xmark 0x1/0xffffffff
iptables -t mangle -A PREROUTING -p tcp -s 10.11.168.0/24 –sport 443 -j MARK –set-xmark 0x1/0xfffffffe
server {
listen 443 ssl;
server_name secure.homlish.net;
ssl_certificate /opt/etc/nginx/certs/secure.crt;
ssl_certificate_key /opt/etc/nginx/certs/secure.key;
location / {
proxy_bind $remote_addr transparent;
proxy_pass https://secure.homlish.net;
#proxy_set_header Host $host;
#proxy_set_header X-Real-IP $remote_addr;
#proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
#proxy_set_header X-Forwarded-Proto $scheme;
proxy_ssl_server_name on;
}
}
I added a bunch of these to the nginx and the
–set-mark
400=1024
note: I didn’t need to change the upstream servers
I did add this to the load balancers in k8s
spec:
externalTrafficPolicy: Local
root@fw:/opt/etc/nginx/conf.d# /opt/etc/init.d/S80nginx test
root@fw:/opt/etc/nginx/conf.d# curl -k https://secure.homlish.net:443/health
to each nginx server declaration add:
proxy_bind $remote_addr transparent;
my k8s prod pool for k0 is: 10.11.168.1-10.11.168.254
on the router add this to startup scripts
ip rule add fwmark 1 lookup 100
ip route add local 0.0.0.0/0 dev lo table 100
iptables -t mangle -A PREROUTING -p tcp -s 10.11.168.0/24 –sport 443 -j MARK –set-xmark 0x1/0xffffffff
from an external server:
phomlish@moe:~$ curl -k https://secure.homlish.net:443/health
Client Values:
Source ip:port 52.170.237.95:54024
Server Values:
Version: 1.2.17
Database Tripwire: ok
Database Web: ok
Images:
Total: 27123
Details: 0
Thumbnails: 0
Resize: 0
working configs 2020-12-11
root@fw:/opt/etc/nginx/conf.d# /opt/etc/init.d/S80nginx test
nginx: the configuration file /opt/etc/nginx/nginx.conf syntax is ok
nginx: configuration file /opt/etc/nginx/nginx.conf test is successful
root@fw:/opt/etc/nginx/conf.d# curl -k https://secure.homlish.net:443/health
Client Values:
Source ip:port 10.11.1.40:38750
Server Values:
Version: 1.2.17
Database Tripwire: ok
Database Web: ok
Images:
Total: 27123
Details: 0
Thumbnails: 0
Resize: 0
root@fw:/opt/etc/nginx/conf.d# iptables -t mangle -L -n
Chain PREROUTING (policy ACCEPT)
target prot opt source destination
MARK 0 — 0.0.0.0/0 71.162.237.5 MARK or 0x80000000
CONNMARK 0 — 0.0.0.0/0 0.0.0.0/0 CONNMARK save
Chain INPUT (policy ACCEPT)
target prot opt source destination
Chain FORWARD (policy ACCEPT)
target prot opt source destination
TCPMSS tcp — 0.0.0.0/0 0.0.0.0/0 tcp flags:0x06/0x02 TCPMSS clamp to PMTU
Chain OUTPUT (policy ACCEPT)
target prot opt source destination
Chain POSTROUTING (policy ACCEPT)
target prot opt source destination
root@fw:/opt/etc/nginx/conf.d# iptables -t mangle -L
Chain PREROUTING (policy ACCEPT)
target prot opt source destination
MARK 0 — anywhere homlish.net MARK or 0x80000000
CONNMARK 0 — anywhere anywhere CONNMARK save
Chain INPUT (policy ACCEPT)
target prot opt source destination
Chain FORWARD (policy ACCEPT)
target prot opt source destination
TCPMSS tcp — anywhere anywhere tcp flags:SYN,RST/SYN TCPMSS clamp to PMTU
Chain OUTPUT (policy ACCEPT)
target prot opt source destination
Chain POSTROUTING (policy ACCEPT)
target prot opt source destination
root@fw:/opt/etc/nginx/conf.d# cat http
#upstream www {
#server www.homlish.net;
#}
server {
listen 80;
server_name www.homlish.net;
location / {
proxy_pass http://www.homlish.net;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
}
server {
listen 443 ssl;
server_name www.homlish.net;
ssl_certificate /opt/etc/nginx/certs/www.crt;
ssl_certificate_key /opt/etc/nginx/certs/www.key;
location / {
proxy_pass https://www.homlish.net;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_ssl_server_name on;
}
}
server {
listen 443 ssl;
server_name secure.homlish.net;
ssl_certificate /opt/etc/nginx/certs/secure.crt;
ssl_certificate_key /opt/etc/nginx/certs/secure.key;
location / {
proxy_pass https://secure.homlish.net;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_ssl_server_name on;
}
}
server {
listen 443 ssl;
server_name images.homlish.net;
ssl_certificate /opt/etc/nginx/certs/images.crt;
ssl_certificate_key /opt/etc/nginx/certs/images.key;
location / {
proxy_pass https://images.homlish.net;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_ssl_server_name on;
}
}