Initial impl

This commit is contained in:
Torbjörn Molin committed 2026-10-19 09:31:57 +02:00
commit 4b729c5d32
133 files changed
+20475

No files matched your search

+22
View File
@@ -0,0 +1,22 @@
[package]
name = "reqlite-http-curl"
description = "libcurl implementation of reqlite's async HttpClient."
version.workspace = true
edition.workspace = true
license.workspace = true
rust-version.workspace = true
publish.workspace = true
[dependencies]
reqlite-core.workspace = true
curl.workspace = true
tempfile.workspace = true
tokio.workspace = true
tracing.workspace = true
[dev-dependencies]
rcgen.workspace = true
rustls.workspace = true
tokio = { workspace = true, features = ["net", "io-util"] }
tokio-rustls.workspace = true
wiremock.workspace = true
+336
View File
@@ -0,0 +1,336 @@
//! libcurl implementation of [`reqlite_core::http::HttpClient`].
//!
//! v1 design (implementation plan §2.3): each request runs on Tokio's blocking
//! pool with its own curl `Easy2` handle. The async side races the blocking
//! task against the cancel token; curl's progress callback checks the same
//! token and aborts the transfer.
use std::fs::File;
use std::io::{Read, Write};
use std::sync::Arc;
use std::time::Duration;
use curl::easy::{Easy2, Form, Handler, HttpVersion, List, ReadError, WriteError};
use reqlite_core::http::{
canonical_reason, BodyStore, HttpClient, HttpError, Proxy, ResolvedBody, ResolvedPartBody, ResolvedRequest,
Response, Timings,
};
use reqlite_core::CancelToken;
/// The libcurl-backed client. Cheap to clone; holds no connections (each
/// request uses a fresh handle in v1).
#[derive(Debug, Clone, Default)]
pub struct CurlClient {
_priv: (),
}
impl CurlClient {
pub fn new() -> Self {
curl::init();
Self { _priv: () }
}
/// libcurl's version string, e.g. for an "about" box.
pub fn version() -> String {
let v = curl::Version::get();
format!(
"libcurl {} ({}{})",
v.version(),
v.ssl_version().unwrap_or("no TLS"),
if v.feature_http2() { ", HTTP/2" } else { "" }
)
}
}
impl HttpClient for CurlClient {
async fn send(&self, req: ResolvedRequest, cancel: CancelToken) -> Result<Response, HttpError> {
let token = cancel.clone();
let task = tokio::task::spawn_blocking(move || perform(req, token));
tokio::select! {
res = task => match res {
Ok(r) => r,
Err(e) => Err(HttpError::Other(format!("transfer task failed: {e}"))),
},
_ = cancel.cancelled() => Err(HttpError::Cancelled),
}
}
}
struct Collector {
cancel: CancelToken,
status_line: String,
headers: Vec<(String, String)>,
mem: Vec<u8>,
file: Option<tempfile::NamedTempFile>,
threshold: u64,
size: u64,
upload: Option<Box<dyn Read + Send>>,
io_error: Option<std::io::Error>,
}
impl Handler for Collector {
fn write(&mut self, data: &[u8]) -> Result<usize, WriteError> {
if self.file.is_none() && (self.mem.len() + data.len()) as u64 > self.threshold {
let spill = tempfile::Builder::new()
.prefix("reqlite-response-")
.tempfile()
.and_then(|mut f| f.write_all(&self.mem).map(|_| f));
match spill {
Ok(f) => {
self.file = Some(f);
self.mem = Vec::new();
}
Err(e) => {
self.io_error = Some(e);
return Ok(0);
}
}
}
if let Some(f) = &mut self.file {
if let Err(e) = f.write_all(data) {
self.io_error = Some(e);
return Ok(0);
}
} else {
self.mem.extend_from_slice(data);
}
self.size += data.len() as u64;
Ok(data.len())
}
fn read(&mut self, buf: &mut [u8]) -> Result<usize, ReadError> {
match &mut self.upload {
Some(r) => r.read(buf).map_err(|_| ReadError::Abort),
None => Ok(0),
}
}
fn header(&mut self, data: &[u8]) -> bool {
let line = String::from_utf8_lossy(data);
let line = line.trim_end_matches(['\r', '\n']);
if line.starts_with("HTTP/") {
// A new response (redirect hop or after `100 Continue`).
self.status_line = line.to_string();
self.headers.clear();
} else if let Some((n, v)) = line.split_once(':') {
self.headers.push((n.trim().to_string(), v.trim().to_string()));
}
true
}
fn progress(&mut self, _dltotal: f64, _dlnow: f64, _ultotal: f64, _ulnow: f64) -> bool {
!self.cancel.is_cancelled()
}
}
fn perform(req: ResolvedRequest, cancel: CancelToken) -> Result<Response, HttpError> {
let o = &req.options;
let mut easy = Easy2::new(Collector {
cancel: cancel.clone(),
status_line: String::new(),
headers: Vec::new(),
mem: Vec::new(),
file: None,
threshold: o.spill_threshold,
size: 0,
upload: None,
io_error: None,
});
match setup(&mut easy, &req) {
Ok(()) => {}
Err(SetupError::Curl(e)) => return Err(map_error(&e, &cancel, None)),
Err(SetupError::Io(e)) => return Err(HttpError::Io(e)),
}
tracing::debug!(method = %req.method, url = %req.url, "sending");
if let Err(e) = easy.perform() {
let io = easy.get_mut().io_error.take();
return Err(map_error(&e, &cancel, io));
}
let timings = Timings {
dns: easy.namelookup_time().unwrap_or_default(),
connect: easy.connect_time().unwrap_or_default(),
tls: easy.appconnect_time().unwrap_or_default(),
ttfb: easy.starttransfer_time().unwrap_or_default(),
total: easy.total_time().unwrap_or_default(),
};
let status = easy.response_code().unwrap_or(0) as u16;
let effective_url = easy.effective_url().ok().flatten().unwrap_or(&req.url).to_string();
let c = easy.get_mut();
let mut parts = c.status_line.splitn(3, ' ');
let http_version = parts.next().unwrap_or("").to_string();
let _code = parts.next();
let reason = match parts.next().map(str::trim) {
Some(r) if !r.is_empty() => r.to_string(),
_ => canonical_reason(status).to_string(),
};
let body = match c.file.take() {
Some(mut f) => {
f.flush().map_err(|e| HttpError::Io(e.to_string()))?;
BodyStore::TempFile(Arc::new(f.into_temp_path()))
}
None => BodyStore::InMemory(Arc::new(std::mem::take(&mut c.mem))),
};
Ok(Response {
status,
reason,
http_version,
headers: std::mem::take(&mut c.headers),
body,
timings,
size: c.size,
effective_url,
})
}
enum SetupError {
Curl(curl::Error),
Io(String),
}
impl From<curl::Error> for SetupError {
fn from(e: curl::Error) -> Self {
SetupError::Curl(e)
}
}
fn setup(easy: &mut Easy2<Collector>, req: &ResolvedRequest) -> Result<(), SetupError> {
let o = &req.options;
easy.url(&req.url)?;
easy.progress(true)?;
easy.signal(false)?;
let method = req.method.as_str();
let has_body = !matches!(req.body, ResolvedBody::None);
match &req.body {
ResolvedBody::None => {}
ResolvedBody::Bytes(b) => {
easy.post(true)?;
easy.post_fields_copy(b)?;
}
ResolvedBody::File(path) => {
let f = File::open(path).map_err(|e| SetupError::Io(format!("{}: {e}", path.display())))?;
let len = f.metadata().map(|m| m.len()).unwrap_or(0);
easy.get_mut().upload = Some(Box::new(f));
easy.post(true)?;
easy.post_field_size(len)?;
}
ResolvedBody::Multipart(parts) => {
let mut form = Form::new();
for p in parts {
let mut part = form.part(&p.name);
match &p.body {
ResolvedPartBody::Bytes(b) => {
part.contents(b);
}
ResolvedPartBody::File(f) => {
part.file(f);
}
}
if let Some(fname) = &p.filename {
part.filename(fname);
}
if let Some(ct) = &p.content_type {
part.content_type(ct);
}
part.add()
.map_err(|e| SetupError::Io(format!("multipart part `{}`: {e}", p.name)))?;
}
easy.httppost(form)?;
}
}
match (method, has_body) {
("GET", false) => easy.get(true)?,
("HEAD", false) => easy.nobody(true)?,
("POST", true) => {}
(m, _) => easy.custom_request(m)?,
}
let mut list = List::new();
let mut has_expect = false;
let mut has_accept_encoding = false;
let multipart = matches!(req.body, ResolvedBody::Multipart(_));
for (n, v) in &req.headers {
// curl generates the multipart Content-Type (with its own boundary).
if multipart && n.eq_ignore_ascii_case("content-type") {
continue;
}
has_expect |= n.eq_ignore_ascii_case("expect");
has_accept_encoding |= n.eq_ignore_ascii_case("accept-encoding");
if v.is_empty() {
list.append(&format!("{n};"))?;
} else {
list.append(&format!("{n}: {v}"))?;
}
}
if !has_expect {
list.append("Expect:")?;
}
easy.http_headers(list)?;
if !has_accept_encoding {
// Advertise everything curl can decode and decode it transparently.
easy.accept_encoding("")?;
}
easy.follow_location(o.follow_redirects)?;
easy.max_redirections(o.max_redirects)?;
if let Some(t) = o.timeout {
easy.timeout(t)?;
}
easy.connect_timeout(o.connect_timeout.unwrap_or(Duration::from_secs(30)))?;
easy.ssl_verify_peer(o.verify_tls)?;
easy.ssl_verify_host(o.verify_tls)?;
if let Some(ca) = &o.ca_file {
easy.cainfo(ca)?;
}
match &o.proxy {
Proxy::System => {}
Proxy::None => {
easy.proxy("")?;
easy.noproxy("*")?;
}
Proxy::Manual(p) => easy.proxy(p)?,
}
match o.http_version.as_deref() {
Some("HTTP/1.0") => easy.http_version(HttpVersion::V10)?,
Some("HTTP/1.1") => easy.http_version(HttpVersion::V11)?,
Some("HTTP/2" | "HTTP/2.0") => easy.http_version(HttpVersion::V2)?,
_ => easy.http_version(HttpVersion::V2TLS)?,
}
Ok(())
}
fn map_error(e: &curl::Error, cancel: &CancelToken, io: Option<std::io::Error>) -> HttpError {
let detail = || match e.extra_description() {
Some(x) => format!("{} ({x})", e.description()),
None => e.description().to_string(),
};
if cancel.is_cancelled() || e.is_aborted_by_callback() {
HttpError::Cancelled
} else if let Some(io) = io {
HttpError::Io(io.to_string())
} else if e.is_operation_timedout() {
HttpError::Timeout
} else if e.is_couldnt_resolve_host() || e.is_couldnt_resolve_proxy() {
HttpError::Dns(detail())
} else if e.is_couldnt_connect() {
HttpError::Connect(detail())
} else if e.is_ssl_connect_error()
|| e.is_peer_failed_verification()
|| e.is_ssl_certproblem()
|| e.is_ssl_cacert_badfile()
|| e.is_ssl_cipher()
|| e.is_ssl_issuer_error()
{
HttpError::Tls(detail())
} else if e.is_url_malformed() || e.is_unsupported_protocol() {
HttpError::InvalidUrl(detail())
} else if e.is_read_error() || e.is_write_error() {
HttpError::Io(detail())
} else {
HttpError::Other(detail())
}
}
+317
View File
@@ -0,0 +1,317 @@
use std::sync::Arc;
use std::time::{Duration, Instant};
use reqlite_core::http::{
BodyStore, HttpClient, HttpError, RequestOptions, ResolvedBody, ResolvedPart, ResolvedPartBody, ResolvedRequest,
};
use reqlite_core::CancelToken;
use reqlite_http_curl::CurlClient;
use wiremock::matchers::{body_string, body_string_contains, header, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
fn req(m: &str, url: String) -> ResolvedRequest {
ResolvedRequest {
method: m.into(),
url,
headers: Vec::new(),
body: ResolvedBody::None,
options: RequestOptions::default(),
}
}
async fn send(r: ResolvedRequest) -> Result<reqlite_core::http::Response, HttpError> {
CurlClient::new().send(r, CancelToken::new()).await
}
fn body_text(b: &BodyStore) -> String {
String::from_utf8(b.read_all().unwrap()).unwrap()
}
#[tokio::test]
async fn every_method_with_and_without_body() {
let server = MockServer::start().await;
for m in ["GET", "DELETE", "OPTIONS", "HEAD"] {
Mock::given(method(m))
.and(path("/m"))
.respond_with(ResponseTemplate::new(200).insert_header("x-method", m))
.mount(&server)
.await;
}
for m in ["POST", "PUT", "PATCH", "DELETE"] {
Mock::given(method(m))
.and(path("/b"))
.and(body_string(format!("body for {m}")))
.respond_with(ResponseTemplate::new(201).set_body_string(format!("got {m}")))
.mount(&server)
.await;
}
for m in ["GET", "DELETE", "OPTIONS", "HEAD"] {
let r = send(req(m, format!("{}/m", server.uri()))).await.unwrap();
assert_eq!(r.status, 200, "{m}");
assert_eq!(r.header("x-method"), Some(m));
}
for m in ["POST", "PUT", "PATCH", "DELETE"] {
let mut r = req(m, format!("{}/b", server.uri()));
r.body = ResolvedBody::Bytes(format!("body for {m}").into_bytes());
let resp = send(r).await.unwrap();
assert_eq!(resp.status, 201, "{m}");
assert_eq!(resp.reason, "Created");
assert_eq!(body_text(&resp.body), format!("got {m}"));
assert_eq!(resp.http_version, "HTTP/1.1");
assert!(resp.timings.total > Duration::ZERO);
}
}
#[tokio::test]
async fn sends_headers_and_file_and_multipart_bodies() {
let server = MockServer::start().await;
Mock::given(method("PUT"))
.and(path("/file"))
.and(header("x-test", "yes"))
.and(body_string("from disk"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
Mock::given(method("POST"))
.and(path("/multi"))
.and(body_string_contains("name=\"field\""))
.and(body_string_contains("value-1"))
.and(body_string_contains("filename=\"up.txt\""))
.and(body_string_contains("file content"))
.respond_with(ResponseTemplate::new(200))
.mount(&server)
.await;
let dir = tempfile::tempdir().unwrap();
let f = dir.path().join("up.txt");
std::fs::write(&f, "from disk").unwrap();
let mut r = req("PUT", format!("{}/file", server.uri()));
r.headers.push(("X-Test".into(), "yes".into()));
r.body = ResolvedBody::File(f.clone());
assert_eq!(send(r).await.unwrap().status, 200);
std::fs::write(&f, "file content").unwrap();
let mut r = req("POST", format!("{}/multi", server.uri()));
r.headers
.push(("Content-Type".into(), "multipart/form-data; boundary=X".into()));
r.body = ResolvedBody::Multipart(vec![
ResolvedPart {
name: "field".into(),
filename: None,
content_type: None,
body: ResolvedPartBody::Bytes(b"value-1".to_vec()),
},
ResolvedPart {
name: "upload".into(),
filename: Some("up.txt".into()),
content_type: Some("text/plain".into()),
body: ResolvedPartBody::File(f),
},
]);
assert_eq!(send(r).await.unwrap().status, 200);
}
#[tokio::test]
async fn redirects_follow_or_not() {
let server = MockServer::start().await;
Mock::given(path("/old"))
.respond_with(ResponseTemplate::new(302).insert_header("location", "/new"))
.mount(&server)
.await;
Mock::given(path("/new"))
.respond_with(ResponseTemplate::new(200).set_body_string("new"))
.mount(&server)
.await;
let r = send(req("GET", format!("{}/old", server.uri()))).await.unwrap();
assert_eq!(r.status, 200);
assert_eq!(body_text(&r.body), "new");
assert!(r.effective_url.ends_with("/new"));
assert!(r.header("location").is_none(), "headers are from the final hop");
let mut nr = req("GET", format!("{}/old", server.uri()));
nr.options.follow_redirects = false;
let r = send(nr).await.unwrap();
assert_eq!(r.status, 302);
assert_eq!(r.header("location"), Some("/new"));
}
#[tokio::test]
async fn timeout() {
let server = MockServer::start().await;
Mock::given(path("/slow"))
.respond_with(ResponseTemplate::new(200).set_delay(Duration::from_secs(3)))
.mount(&server)
.await;
let mut r = req("GET", format!("{}/slow", server.uri()));
r.options.timeout = Some(Duration::from_millis(300));
let start = Instant::now();
assert_eq!(send(r).await.unwrap_err(), HttpError::Timeout);
assert!(start.elapsed() < Duration::from_secs(2));
}
#[tokio::test]
async fn cancel_mid_transfer_ends_the_task_within_200ms() {
let server = MockServer::start().await;
Mock::given(path("/slow"))
.respond_with(ResponseTemplate::new(200).set_delay(Duration::from_secs(10)))
.mount(&server)
.await;
let cancel = CancelToken::new();
let task = tokio::spawn({
let cancel = cancel.clone();
let r = req("GET", format!("{}/slow", server.uri()));
async move { CurlClient::new().send(r, cancel).await }
});
tokio::time::sleep(Duration::from_millis(200)).await;
let cancelled_at = Instant::now();
cancel.cancel();
let res = task.await.unwrap();
assert_eq!(res.unwrap_err(), HttpError::Cancelled);
assert!(
cancelled_at.elapsed() < Duration::from_millis(200),
"took {:?}",
cancelled_at.elapsed()
);
}
#[tokio::test]
async fn twenty_concurrent_requests() {
let server = MockServer::start().await;
Mock::given(path("/c"))
.respond_with(ResponseTemplate::new(200).set_delay(Duration::from_millis(300)))
.expect(20)
.mount(&server)
.await;
let client = Arc::new(CurlClient::new());
let start = Instant::now();
let tasks: Vec<_> = (0..20)
.map(|_| {
let client = client.clone();
let r = req("GET", format!("{}/c", server.uri()));
tokio::spawn(async move { client.send(r, CancelToken::new()).await })
})
.collect();
for t in tasks {
assert_eq!(t.await.unwrap().unwrap().status, 200);
}
// In parallel, not one after the other (20 × 300 ms = 6 s).
assert!(start.elapsed() < Duration::from_secs(3), "{:?}", start.elapsed());
}
#[tokio::test]
async fn large_bodies_spill_to_a_temp_file() {
let server = MockServer::start().await;
let big = vec![b'x'; 6 * 1024 * 1024];
Mock::given(path("/big"))
.respond_with(ResponseTemplate::new(200).set_body_bytes(big.clone()))
.mount(&server)
.await;
Mock::given(path("/small"))
.respond_with(ResponseTemplate::new(200).set_body_string("small"))
.mount(&server)
.await;
let r = send(req("GET", format!("{}/big", server.uri()))).await.unwrap();
assert_eq!(r.size, big.len() as u64);
let BodyStore::TempFile(path) = &r.body else {
panic!("expected a spilled body")
};
let path = path.to_path_buf();
assert_eq!(r.body.read_all().unwrap().len(), big.len());
drop(r);
assert!(!path.exists(), "temp file is removed with the response");
let r = send(req("GET", format!("{}/small", server.uri()))).await.unwrap();
assert_eq!(r.body.in_memory(), Some(&b"small"[..]));
}
#[tokio::test]
async fn connection_errors_are_classified() {
// Nothing listens on port 9 (discard) on CI-less dev machines.
let mut r = req("GET", "http://127.0.0.1:9/".into());
r.options.connect_timeout = Some(Duration::from_secs(2));
assert!(matches!(send(r).await.unwrap_err(), HttpError::Connect(_)));
let r = req("GET", "http://does-not-exist.invalid/".into());
assert!(matches!(send(r).await.unwrap_err(), HttpError::Dns(_)));
let r = req("GET", "nope://x".into());
assert!(matches!(send(r).await.unwrap_err(), HttpError::InvalidUrl(_)));
}
mod tls {
use super::*;
use rcgen::{BasicConstraints, CertificateParams, CertifiedIssuer, DnType, IsCa, KeyPair};
use rustls::pki_types::{CertificateDer, PrivateKeyDer, PrivatePkcs8KeyDer};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
/// A one-shot-per-connection HTTPS server with a certificate signed by a
/// fresh CA. Returns the port and the CA certificate in PEM.
async fn https_server() -> (u16, String) {
let mut ca_params = CertificateParams::new(Vec::<String>::new()).unwrap();
ca_params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained);
// rcgen's default subject is the same for every cert; a CA with the
// leaf's name would make the leaf look self-signed.
ca_params.distinguished_name.push(DnType::CommonName, "reqlite test CA");
let ca = CertifiedIssuer::self_signed(ca_params, KeyPair::generate().unwrap()).unwrap();
let leaf_key = KeyPair::generate().unwrap();
let leaf = CertificateParams::new(vec!["localhost".to_string()])
.unwrap()
.signed_by(&leaf_key, &ca)
.unwrap();
let config = rustls::ServerConfig::builder()
.with_no_client_auth()
.with_single_cert(
vec![CertificateDer::from(leaf.der().to_vec())],
PrivateKeyDer::Pkcs8(PrivatePkcs8KeyDer::from(leaf_key.serialize_der())),
)
.unwrap();
let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(config));
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = listener.local_addr().unwrap().port();
tokio::spawn(async move {
loop {
let Ok((tcp, _)) = listener.accept().await else { return };
let acceptor = acceptor.clone();
tokio::spawn(async move {
let Ok(mut tls) = acceptor.accept(tcp).await else {
return;
};
let mut buf = vec![0u8; 4096];
let _ = tls.read(&mut buf).await;
let _ = tls
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 6\r\nConnection: close\r\n\r\nsecure")
.await;
let _ = tls.shutdown().await;
});
}
});
(port, ca.pem())
}
#[tokio::test]
async fn self_signed_certificate() {
let (port, ca_pem) = https_server().await;
let url = format!("https://localhost:{port}/");
// Untrusted by default.
let err = send(req("GET", url.clone())).await.unwrap_err();
assert!(matches!(err, HttpError::Tls(_)), "{err:?}");
// Verification off.
let mut r = req("GET", url.clone());
r.options.verify_tls = false;
let resp = send(r).await.unwrap();
assert_eq!(body_text(&resp.body), "secure");
assert!(resp.timings.tls >= resp.timings.connect);
// Custom CA file.
let dir = tempfile::tempdir().unwrap();
let ca = dir.path().join("ca.pem");
std::fs::write(&ca, ca_pem).unwrap();
let mut r = req("GET", url);
r.options.ca_file = Some(ca);
assert_eq!(send(r).await.unwrap().status, 200);
}
}