diff --git a/.idea/.gitignore b/.idea/.gitignore deleted file mode 100644 index 13566b8..0000000 --- a/.idea/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -# Default ignored files -/shelf/ -/workspace.xml -# Editor-based HTTP Client requests -/httpRequests/ -# Datasource local storage ignored files -/dataSources/ -/dataSources.local.xml diff --git a/.idea/l4lb.iml b/.idea/enginewhy.iml similarity index 84% rename from .idea/l4lb.iml rename to .idea/enginewhy.iml index cf84ae4..7c12fe5 100644 --- a/.idea/l4lb.iml +++ b/.idea/enginewhy.iml @@ -2,6 +2,7 @@ + diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..30bab2a --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,7 @@ + + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml index 1695888..47677bc 100644 --- a/.idea/modules.xml +++ b/.idea/modules.xml @@ -2,7 +2,7 @@ - + \ No newline at end of file diff --git a/.idea/workspace.xml b/.idea/workspace.xml new file mode 100644 index 0000000..f6edfcf --- /dev/null +++ b/.idea/workspace.xml @@ -0,0 +1,151 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 1766101214189 + + + + + + \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..b585d86 --- /dev/null +++ b/README.md @@ -0,0 +1,122 @@ +# nginy + +EngineWhy, also known as nginy, is a simple but flexible TCP load balancer. + +## Contents +- [Features](#features) +- [Getting started](#getting-started) + - [Running as a standalone binary](#running-as-a-standalone-binary) + - [Running as a dockerized application](#running-as-a-dockerized-application) +- [Configuration](#configuration) +- [Examples](#examples) + +## Features + +- Multiple different load balancing algorithms + - Round Robin + - IP Hashing + - Adaptive algorithm (based on [this paper](https://www.wcse.org/WCSE_2018/W110.pdf)) +- Cluster based backend grouping, and CIDR based client matching for fine-grained control. +- YAML based config for readability and simplicity. +- Automatic hot-reload of configuration without interruptions to service. + +## Getting started + +### Running as a standalone binary +You must have the latest stable Rust release (1.91 at the time of writing), and a valid configuration file (see [Configuration](#configuration) for details). + +1. Clone the repository: +```sh +git clone https://github.com/psun256/enginewhy.git +cd enginewhy +``` +2. Build the application: +```sh +cargo build --release +``` +3. Run the application: +```sh +./target/release/l4lb -c path/to/config.yaml +``` + +### Running as a dockerized application +You may also consider running the load balancer as a dockerized application. + +1. Clone the repository: +```sh +git clone https://github.com/psun256/enginewhy.git +cd enginewhy +``` +2. Build the application: +```sh +docker build -t enginewhy . +``` +3. Run the application: +```sh +docker run -v "path/to/config.yaml:/enginewhy/config.yaml" enginewhy +``` + +## Configuration +Configuration file is written in YAML. +By default, the program will look for a file named `config.yaml` in the working directory. +You can change this by specifying the path with `-c` or `--config` when running the program. + +The file consists of: +- Defining the health response and iperf port (IP + port). +- A set of backends (IP + Port for each). +- A set of clusters, which act as group aliases for a set of backends. +- A list of rules, which consist of: + - Clients: One or more CIDR + port number, used to match incoming client connection + - Targets: One or more clusters / backend names. + - Strategy: A load balancing algorithm to use. + - Some algorithms have additional configuration, like the Adaptive algorithm. + +Sample configuration: +```yml +healthcheck_addr: "10.0.1.10:9000" + +iperf_addr: "10.0.1.10:5201" + +backends: + - id: "srv-1" + ip: "10.0.1.11:8081" + - id: "srv-2" + ip: "10.0.1.12:8082" + - id: "srv-3" + ip: "10.0.1.13:8083" + - id: "srv-4" + ip: "10.0.1.14:8084" + +clusters: + main-api: + - "srv-1" + - "srv-2" + priority-api: + - "srv-3" + - "srv-4" + +rules: + - clients: + - "0.0.0.0/0:8080" + targets: + - "main-api" + strategy: + type: "RoundRobin" + + - clients: + - "10.0.0.0/24:8080" + - "10.0.0.0/24:25565" + targets: + - "main-api" + - "priority-api" + strategy: + type: "RoundRobin" +``` + +An incoming client will be matched with whatever rule has the longest matching prefix on the correct port. + +## Examples + +You can find some examples in the `examples` directory of the project. + +To run these, just run `docker compose up`. diff --git a/config.yaml b/config.yaml index 6025c39..f6c5e4f 100644 --- a/config.yaml +++ b/config.yaml @@ -1,27 +1,38 @@ -healthcheck_addr: "0.0.0.0:8080" +healthcheck_addr: "10.0.1.10:9000" -iperf_addr: "0.0.0.0:5001" +iperf_addr: "10.0.1.10:5201" backends: - id: "srv-1" - ip: "192.67.67.2:8080" + ip: "10.0.1.11:8081" - id: "srv-2" - ip: "192.67.67.3:8080" + ip: "10.0.1.12:8082" + - id: "srv-3" + ip: "10.0.1.13:8083" + - id: "srv-4" + ip: "10.0.1.14:8084" clusters: main-api: - "srv-1" - "srv-2" priority-api: - - "srv-1" + - "srv-3" + - "srv-4" rules: - clients: - - "172.67.67.2/24:80" + - "0.0.0.0/0:8080" + targets: + - "main-api" + strategy: + type: "RoundRobin" + + - clients: + - "10.0.0.0/24:8080" + - "10.0.0.0/24:25565" targets: - "main-api" - "priority-api" strategy: - type: "Adaptive" - coefficients: [ 1.5, 1.0, 0.5, 0.1 ] - alpha: 0.75 + type: "RoundRobin" diff --git a/report.md b/report.md deleted file mode 100644 index e69de29..0000000 diff --git a/src/backend/health.rs b/src/backend/health.rs index 3801020..5f9eeaf 100644 --- a/src/backend/health.rs +++ b/src/backend/health.rs @@ -1,10 +1,10 @@ +use rperf3::{Config, Server}; +use serde_json::Value; use std::collections::HashMap; use std::net::{IpAddr, SocketAddr}; use std::sync::{Arc, RwLock}; -use serde_json::Value; -use rperf3::{Config, Server}; -use tokio::net::{TcpListener, TcpSocket, TcpStream}; use tokio::io::AsyncBufReadExt; +use tokio::net::{TcpSocket, TcpStream}; // Physical server health statistics, used for certain load balancing algorithms #[derive(Debug, Default)] @@ -47,14 +47,14 @@ pub async fn start_healthcheck_listener( let listener = listener.ok_or_else(|| { eprintln!("health listener could not bind to port"); - std::io::Error::new(std::io::ErrorKind::Other, "health listener failed") + std::io::Error::other("health listener failed") })?; println!("healthcheck server listening on {}", addr); loop { - let (stream, remote_addr) = match listener.accept().await { + let (stream, _remote_addr) = match listener.accept().await { Ok(v) => v, - Err(e) => { + Err(_e) => { continue; } }; @@ -131,4 +131,4 @@ fn process_metrics( } Ok(()) -} \ No newline at end of file +} diff --git a/src/balancer/adaptive_weight.rs b/src/balancer/adaptive_weight.rs index 875ceea..9f2aca3 100644 --- a/src/balancer/adaptive_weight.rs +++ b/src/balancer/adaptive_weight.rs @@ -1,11 +1,10 @@ -use crate::backend::{Backend, BackendPool}; use crate::backend::health::ServerMetrics; +use crate::backend::{Backend, BackendPool}; use crate::balancer::{Balancer, ConnectionInfo}; use rand::prelude::*; use rand::rngs::SmallRng; use std::fmt::Debug; -use std::fs::Metadata; -use std::sync::{Arc, RwLock}; +use std::sync::Arc; #[derive(Debug)] struct AdaptiveNode { @@ -49,7 +48,7 @@ impl AdaptiveWeightBalancer { } impl Balancer for AdaptiveWeightBalancer { - fn choose_backend(&mut self, ctx: ConnectionInfo) -> Option> { + fn choose_backend(&mut self, _ctx: ConnectionInfo) -> Option> { if self.pool.is_empty() { return None; } @@ -72,7 +71,7 @@ impl Balancer for AdaptiveWeightBalancer { let safe_w_sum = w_sum.max(1e-12); let threshold = self.alpha * (r_sum / safe_w_sum); - + for idx in 0..self.pool.len() { let node = &self.pool[idx]; @@ -148,6 +147,7 @@ mod tests { use super::*; use crate::backend::Backend; use std::net::SocketAddr; + use std::sync::RwLock; fn backend_factory(id: &str, ip: &str, port: u16) -> Arc { Arc::new(Backend::new( diff --git a/src/balancer/round_robin.rs b/src/balancer/round_robin.rs index 83680ea..6dd807e 100644 --- a/src/balancer/round_robin.rs +++ b/src/balancer/round_robin.rs @@ -19,7 +19,7 @@ impl RoundRobinBalancer { } impl Balancer for RoundRobinBalancer { - fn choose_backend(&mut self, ctx: ConnectionInfo) -> Option> { + fn choose_backend(&mut self, _ctx: ConnectionInfo) -> Option> { let backends = self.pool.backends.clone(); if backends.is_empty() { return None; diff --git a/src/config/loader.rs b/src/config/loader.rs index 387a9ae..0733a92 100644 --- a/src/config/loader.rs +++ b/src/config/loader.rs @@ -7,8 +7,8 @@ use crate::backend::health::*; use crate::backend::*; use crate::balancer::Balancer; use crate::balancer::adaptive_weight::AdaptiveWeightBalancer; -use crate::balancer::round_robin::RoundRobinBalancer; use crate::balancer::ip_hashing::SourceIPHash; +use crate::balancer::round_robin::RoundRobinBalancer; use crate::config::*; pub struct RoutingTable { @@ -37,7 +37,9 @@ pub fn build_lb( let mut backends: HashMap> = HashMap::new(); for backend_cfg in &config.backends { - let addr: SocketAddr = backend_cfg.ip.parse() + let addr: SocketAddr = backend_cfg + .ip + .parse() .map_err(|_| format!("bad ip: {}", backend_cfg.ip))?; let ip = addr.ip(); @@ -87,7 +89,7 @@ pub fn build_lb( let mut port_groups: HashMap> = HashMap::new(); for client_def in &rule.clients { - let (cidr, port) = parse_client(&client_def)?; + let (cidr, port) = parse_client(client_def)?; port_groups.entry(port).or_default().push(cidr); } diff --git a/src/main.rs b/src/main.rs index 9f6bc8b..edede53 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,25 +3,22 @@ mod balancer; mod config; mod proxy; -use crate::backend::health::{start_healthcheck_listener, start_iperf_server, ServerMetrics}; +use crate::backend::health::{ServerMetrics, start_healthcheck_listener, start_iperf_server}; use crate::balancer::ConnectionInfo; -use crate::config::loader::{build_lb, RoutingTable}; +use crate::config::loader::{RoutingTable, build_lb}; use crate::proxy::tcp::proxy_tcp_connection; use anywho::Error; +use clap::Parser; +use notify::{Event, RecursiveMode, Watcher}; use std::collections::HashMap; use std::fs::File; -use std::hash::Hash; -use std::net::{IpAddr, SocketAddr}; +use std::net::IpAddr; use std::path::PathBuf; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex, RwLock}; use std::time::Duration; -use tokio::io::AsyncBufReadExt; use tokio::net::TcpListener; use tokio::sync::mpsc; -use clap::Parser; -use notify::{Event, RecursiveMode, Watcher}; -use std::cmp; static NEXT_CONN_ID: AtomicU64 = AtomicU64::new(1); @@ -72,10 +69,10 @@ async fn main() -> Result<(), Box> { let (tx, mut rx) = mpsc::channel(1); let mut watcher = notify::recommended_watcher(move |res: Result| { - if let Ok(event) = res { - if event.kind.is_modify() { - let _ = tx.blocking_send(()); - } + if let Ok(event) = res + && event.kind.is_modify() + { + let _ = tx.blocking_send(()); } }) .unwrap();