added example

Healthcheck

Dockerfile

@@ -33,8 +33,7 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \
 
 # change to scratch and get comment the apk command for prod, i guess
 FROM alpine:latest AS runtime
-# RUN apk add --no-cache ca-certificates curl netcat-openbsd bind-tools strace
+RUN apk add --no-cache ca-certificates curl netcat-openbsd bind-tools strace
 WORKDIR /enginewhy
 COPY --from=builder /enginewhy/target/x86_64-unknown-linux-musl/release/l4lb /usr/bin/l4lb
-COPY config.yaml .
 ENTRYPOINT ["l4lb"]

README.md

@@ -1,128 +0,0 @@
-# nginy
-Production't graden't load balancer.
-
-## Quick links
-
-## Todo
-- [ ] architecture astronauting
-  - balancer module
-    - just the algorithms i guess
-- 
-  - backend module
-    - manages the backend pool
-    - deals with health / load check
-    - BackendPool for all the backends stored together
-    - Backend for individual backends
-    - has some methods used by balancer module to pick a suitable backend
-  - proxy module 
-    - all the different supported protocols to handle
-    - will create a session / stream context structure (ConnectionContext)
-      - not globally tracked (this might change for UDP!)
-      - mainly some metadata
-  - config module
-    - set up all the stuff or something
-- [ ] stream / session handling (i think wrapper around tokio TcpStream)
-- [ ] basic backend pooling
-- [ ] layer 4 load balancing
-
-## notes
-tcp, for nginx (and haproxy, its similar):
-```c
-// nginx
-struct ngx_connection_s {
-    void               *data;
-    ngx_event_t        *read;
-    ngx_event_t        *write;
-
-    ngx_socket_t        fd;
-
-    ngx_recv_pt         recv; // fn pointer to whatever recv fn used (different for idfferent platforms / protocol
-    ngx_send_pt         send; // ditto
-    ngx_recv_chain_pt   recv_chain;
-    ngx_send_chain_pt   send_chain;
-
-    ngx_listening_t    *listening;
-
-    off_t               sent;
-
-    ngx_log_t          *log;
-
-    ngx_pool_t         *pool;
-
-    int                 type;
-
-    struct sockaddr    *sockaddr;
-    socklen_t           socklen;
-    ngx_str_t           addr_text;
-
-    ngx_proxy_protocol_t  *proxy_protocol;
-
-#if (NGX_QUIC || NGX_COMPAT)
-    ngx_quic_stream_t     *quic;
-#endif
-
-#if (NGX_SSL || NGX_COMPAT)
-    ngx_ssl_connection_t  *ssl;
-#endif
-
-    ngx_udp_connection_t  *udp; // additional stuff for UDP (which is technically connectionless, but they use timeouts and a rbtree to store "sessions")
-
-    struct sockaddr    *local_sockaddr;
-    socklen_t           local_socklen;
-
-    ngx_buf_t          *buffer;
-
-    ngx_queue_t         queue;
-
-    ngx_atomic_uint_t   number;
-
-    ngx_msec_t          start_time;
-    ngx_uint_t          requests;
-
-    unsigned            buffered:8;
-
-    unsigned            log_error:3;     /* ngx_connection_log_error_e */
-
-    unsigned            timedout:1;
-    unsigned            error:1;
-    unsigned            destroyed:1;
-    unsigned            pipeline:1;
-
-    unsigned            idle:1;
-    unsigned            reusable:1;
-    unsigned            close:1;
-    unsigned            shared:1;
-
-    unsigned            sendfile:1;
-    unsigned            sndlowat:1;
-    unsigned            tcp_nodelay:2;   /* ngx_connection_tcp_nodelay_e */
-    unsigned            tcp_nopush:2;    /* ngx_connection_tcp_nopush_e */
-
-    unsigned            need_last_buf:1;
-    unsigned            need_flush_buf:1;
-
-#if (NGX_HAVE_SENDFILE_NODISKIO || NGX_COMPAT)
-    unsigned            busy_count:2;
-#endif
-
-#if (NGX_THREADS || NGX_COMPAT)
-    ngx_thread_task_t  *sendfile_task;
-#endif
-};
-```
-process to load balance:
-- accept incoming connection
-- create some kind of stream / session object
-  - nginx use this to abstract around tcp and udp layers
-  - for us we probably don't need as detailed as them, since we have tokio::net, so itll be a wrapper around TcpStream
-- ask the load balancing algorithm which server in the pool to route to
-- connect to the server
-- proxy the data (copy_bidirectional? maybe we want some metrics or logging, so might do manually)
-- cleanup when smoeone leavesr or something goes wrong (with TCP, OS / tokio will tell us, with UDP probably just timeout based, and a periodic sweep of all sessions)
-
-
-### UDP
-UDP is connectionless, and i don't think UdpSocket or UdpFramed implement the traits required for tokio copy_bidirectional
-but async write and read don't work on just regular datagrams, so probably not possible.
-
-Would require us to implement our own bidirectional copying / proxying, as well as tracking "active" connections.

config.yaml

@@ -1,15 +1,12 @@
-healthcheck_addr: "127.0.0.1:9000"
+healthcheck_addr: "0.0.0.0:8080"
 
-iperf_addr: "0.0.0.0:5200"
+iperf_addr: "0.0.0.0:5001"
 
 backends:
   - id: "srv-1"
-    ip: "127.0.0.1"
+    ip: "192.67.67.2:8080"
-    port: 8081
-
   - id: "srv-2"
-    ip: "127.0.0.1"
+    ip: "192.67.67.3:8080"
-    port: 8082
 
 clusters:
   main-api:
@@ -20,20 +17,11 @@ clusters:
 
 rules:
   - clients:
-      - "0.0.0.0/0:8888"
+      - "172.67.67.2/24:80"
     targets:
       - "main-api"
-    strategy:
-      type: "RoundRobin"
-
-  - clients:
-      - "0.0.0.0/0:6767"
-      - "0.0.0.0/0:6969"
-    targets: # no issues with duplicate servers or clusters
-      - "priority-api"
-      - "priority-api"
       - "priority-api"
     strategy:
       type: "Adaptive"
       coefficients: [ 1.5, 1.0, 0.5, 0.1 ]
-      alpha: 0.75
+      alpha: 0.75

docker-compose.yml

@@ -1,5 +1,4 @@
 services:
-  # two-arm load balancer  
   load-balancer:
     image: neoslhp/enginewhy-lb
     container_name: load-balancer

examples/loadbalancertest/config.yaml

@@ -0,0 +1,38 @@
+healthcheck_addr: "10.0.1.10:9000"
+
+iperf_addr: "10.0.1.10:5201"
+
+backends:
+  - id: "srv-1"
+    ip: "10.0.1.11:8081"
+  - id: "srv-2"
+    ip: "10.0.1.12:8082"
+  - id: "srv-3"
+    ip: "10.0.1.13:8083"
+  - id: "srv-4"
+    ip: "10.0.1.14:8084"
+
+clusters:
+  main-api:
+    - "srv-1"
+    - "srv-2"
+  priority-api:
+    - "srv-3"
+    - "srv-4"
+
+rules:
+  - clients:
+      - "0.0.0.0/0:8080"
+    targets:
+      - "main-api"
+    strategy:
+      type: "RoundRobin"
+
+  - clients:
+      - "10.0.0.0/24:8080"
+      - "10.0.0.0/24:25565"
+    targets:
+      - "main-api"
+      - "priority-api"
+    strategy:
+      type: "RoundRobin"

examples/loadbalancertest/docker-compose.yml

@@ -0,0 +1,110 @@
+services:
+  load-balancer:
+    image: enginewhy
+    container_name: load-balancer
+    tty: true
+    cap_add:
+      - NET_ADMIN
+      - SYS_ADMIN
+    volumes:
+      - ./config.yaml:/enginewhy/config.yaml
+    networks:
+      net_1:
+        ipv4_address: 10.0.1.10
+      net_2:
+        ipv4_address: 10.0.0.10
+      net_3:
+        ipv4_address: 10.0.2.10
+
+  srv-1:
+    image: nicolaka/netshoot
+    container_name: srv-1
+    tty: true
+    command: ["python3", "-m", "http.server", "8081", "--directory", "/root/www"]
+    networks:
+      net_1:
+        ipv4_address: 10.0.1.11
+    ports:
+      - "8081:8081"
+    volumes:
+      - ./srv1:/root/www
+    cap_add: [ NET_ADMIN ]
+
+  srv-2:
+    image: nicolaka/netshoot
+    container_name: srv-2
+    tty: true
+    command: ["python3", "-m", "http.server", "8082", "--directory", "/root/www"]
+    networks:
+      net_1:
+        ipv4_address: 10.0.1.12
+    ports:
+      - "8082:8082"
+    volumes:
+      - ./srv2:/root/www
+    cap_add: [ NET_ADMIN ]
+
+  srv-3:
+    image: nicolaka/netshoot
+    container_name: srv-3
+    tty: true
+    command: ["python3", "-m", "http.server", "8083", "--directory", "/root/www"]
+    networks:
+      net_1:
+        ipv4_address: 10.0.1.13
+    ports:
+      - "8083:8083"
+    volumes:
+      - ./srv3:/root/www
+    cap_add: [ NET_ADMIN ]
+
+  srv-4:
+    image: nicolaka/netshoot
+    container_name: srv-4
+    tty: true
+    command: ["python3", "-m", "http.server", "8084", "--directory", "/root/www"]
+    networks:
+      net_1:
+        ipv4_address: 10.0.1.14
+    ports:
+      - "8084:8084"
+    volumes:
+      - ./srv4:/root/www
+    cap_add: [ NET_ADMIN ]
+
+  client-net2:
+    image: nicolaka/netshoot
+    container_name: client-net2
+    tty: true
+    networks:
+      net_2:
+        ipv4_address: 10.0.0.11
+    cap_add: [ NET_ADMIN ]
+
+  client-net3:
+    image: nicolaka/netshoot
+    container_name: client-net3
+    tty: true
+    networks:
+      net_3:
+        ipv4_address: 10.0.2.11
+    cap_add: [ NET_ADMIN ]
+
+networks:
+  net_1:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 10.0.1.0/24
+
+  net_2:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 10.0.0.0/24
+
+  net_3:
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 10.0.2.0/24

examples/loadbalancertest/srv1/index.html

@@ -0,0 +1 @@
+Hello from server 1!

examples/loadbalancertest/srv2/index.html

@@ -0,0 +1 @@
+Hello from server 2!

examples/loadbalancertest/srv3/index.html

@@ -0,0 +1 @@
+Hello from server 3!

examples/loadbalancertest/srv4/index.html

@@ -0,0 +1 @@
+Hello from server 4!

src/config/loader.rs

@@ -8,6 +8,7 @@ use crate::backend::*;
 use crate::balancer::Balancer;
 use crate::balancer::adaptive_weight::AdaptiveWeightBalancer;
 use crate::balancer::round_robin::RoundRobinBalancer;
+use crate::balancer::ip_hashing::SourceIPHash;
 use crate::config::*;
 
 pub struct RoutingTable {
@@ -36,11 +37,9 @@ pub fn build_lb(
     let mut backends: HashMap<String, Arc<Backend>> = HashMap::new();
 
     for backend_cfg in &config.backends {
-        let ip: IpAddr = backend_cfg
+        let addr: SocketAddr = backend_cfg.ip.parse()
-            .ip
-            .parse()
             .map_err(|_| format!("bad ip: {}", backend_cfg.ip))?;
-        let addr = SocketAddr::new(ip, backend_cfg.port);
+        let ip = addr.ip();
 
         let health = healths
             .entry(ip)
@@ -102,6 +101,7 @@ pub fn build_lb(
 
             let balancer: Box<dyn Balancer + Send> = match &rule.strategy {
                 LoadBalancerStrategy::RoundRobin => Box::new(RoundRobinBalancer::new(pool)),
+                LoadBalancerStrategy::SourceIPHash => Box::new(SourceIPHash::new(pool)),
                 LoadBalancerStrategy::Adaptive {
                     coefficients,
                     alpha,
@@ -121,7 +121,7 @@ pub fn build_lb(
     for table in listeners.values_mut() {
         table
             .entries
-            .sort_by(|(a, _), (b, _)| a.network_length().cmp(&b.network_length()));
+            .sort_by(|(a, _), (b, _)| b.network_length().cmp(&a.network_length()));
     }
 
     Ok((listeners, healths))

src/config/mod.rs

@@ -44,7 +44,6 @@ pub struct AppConfig {
 pub struct BackendConfig {
     pub id: String,
     pub ip: String,
-    pub port: u16,
 }
 
 #[derive(Debug, Deserialize)]
@@ -58,5 +57,6 @@ pub struct RuleConfig {
 #[serde(tag = "type")]
 pub enum LoadBalancerStrategy {
     RoundRobin,
+    SourceIPHash,
     Adaptive { coefficients: [f64; 4], alpha: f64 },
 }