thumbor图片服务
protobuf相关处理
abi.proto
syntax = "proto3"; package abi; // 一个 ImageSpec 是一个有序的数组,服务器按照 spec 的顺序处理 message ImageSpec { repeated Spec specs = 1; } // 处理图片改变大小 message Resize { uint32 width = 1; uint32 height = 2; enum ResizeType { NORMAL = 0; SEAM_CARVE = 1; } ResizeType rtype = 3; enum SampleFilter { UNDEFINED = 0; NEAREST = 1; TRIANGLE = 2; CATMULL_ROM = 3; GAUSSIAN = 4; LANCZOS3 = 5; } SampleFilter filter = 4; } // 处理图片截取 message Crop { uint32 x1 = 1; uint32 y1 = 2; uint32 x2 = 3; uint32 y2 = 4; } // 处理水平翻转 message Fliph {} // 处理垂直翻转 message Flipv {} // 处理对比度 message Contrast { float contrast = 1; } // 处理滤镜 message Filter { enum Filter { UNSPECIFIED = 0; OCEANIC = 1; ISLANDS = 2; MARINE = 3; // more: https://docs.rs/photon-rs/0.3.1/photon_rs/filters/fn.filter.html } Filter filter = 1; } // 处理水印 message Watermark { uint32 x = 1; uint32 y = 2; } // 一个 spec 可以包含上述的处理方式之一 message Spec { oneof data { Resize resize = 1; Crop crop = 2; Flipv flipv = 3; Fliph fliph = 4; Contrast contrast = 5; Filter filter = 6; Watermark watermark = 7; } }
build.rs
use std::process::Command; fn main() { // 在编译时可选择检查环境变量。 let build_enabled = option_env!("BUILD_PROTO") .map(|v| v == "1") .unwrap_or(false); // 如果没有找到环境变量的对应值,就直接return,不再进行后续编译 if !build_enabled { println!("=== Skipped compiling protos ==="); return; } // 使用 prost_build 把 abi.proto 编译到 src/pb 目录下 prost_build::Config::new() .out_dir("src/pb") .compile_protos(&["abi.proto"], &["."]) .unwrap(); Command::new("cargo") .args(&["fmt", "--", "src/*.rs"]) .status() .expect("cargo fmt failed"); }
在编译时可选择检查环境变量。
关于rust的模块
可以参考这篇:Rust 模块系统理解 - 知乎
mod全认识
- mod(mod.rs或mod关键字)将代码分为多个逻辑模块,并管理这些模块的可见性(public / private)。
- 模块是项(item)的集合,项可以是:函数,结构体,trait,impl块,甚至其它模块。
- 一个目录下的所有代码,可以通过 mod.rs 声明
- Rust模块有三种形式:
- mod.rs: 一个目录下的所有代码,可以通过 mod.rs 声明
- 文件/目录即模块:编译器的机制决定,除了mod.rs外,每一个文件和目录都是一个模块。不允许只分拆文件,但是不声明mod,我们通常使用pub use,在父空间直接调用子空间的函数。
- mod关键字: 在文件内部分拆模块
- Rust编译器只接受一个源文件,输出一个crate
- 每一个crate都有一个匿名的根命名空间,命名空间可以无限嵌套
- “mod mod-name { … }“ 将大括号中的代码置于命名空间mod-name之下
- “use mod-name1::mod-name2;“ 可以打开命名空间,减少无休止的::操作符
- “mod mod-name;“ 可以指导编译器将多个文件组装成一个文件
- “pub use mod-nam1::mod-name2::item-name;“ 语句可以将mod-name2下的item-name提升到这条语句所在的空间,item-name通常是函数或者结构体。Rust社区通常用这个方法来缩短库API的命名空间深度 编译器规定use语句一定要在mod语句之前
mod文件定义与实现分离
在rust中,一般会在模块的mod.rs文件中对供外部使用的项进行实现, 项可以是:函数,结构体,trait,impl块,甚至其它模块. 这样有个好处,高内聚,可以在代码增长时,将变动局限在服务提供者内部,对外提供的api不变,不会造成破坏性更新。
pb模块: 处理protobuf
pb/abi.rs里面还有子模块
/// Nested message and enum types in `Spec`. pub mod spec { #[derive(Clone, PartialEq, ::prost::Oneof)] pub enum Data { #[prost(message, tag = "1")] Resize(super::Resize), #[prost(message, tag = "2")] Crop(super::Crop), #[prost(message, tag = "3")] Flipv(super::Flipv), #[prost(message, tag = "4")] Fliph(super::Fliph), #[prost(message, tag = "5")] Contrast(super::Contrast), #[prost(message, tag = "6")] Filter(super::Filter), #[prost(message, tag = "7")] Watermark(super::Watermark), } }
pb/abi.rs另外定义了spec::Data里面的各个元素结构体/嵌套模块mod
/// 一个 ImageSpec 是一个有序的数组,服务器按照 spec 的顺序处理 #[derive(Clone, PartialEq, ::prost::Message)] pub struct ImageSpec { #[prost(message, repeated, tag = "1")] pub specs: ::prost::alloc::vec::Vec<Spec>, } /// 处理图片改变大小 #[derive(Clone, PartialEq, ::prost::Message)] pub struct Resize { #[prost(uint32, tag = "1")] pub width: u32, #[prost(uint32, tag = "2")] pub height: u32, #[prost(enumeration = "resize::ResizeType", tag = "3")] pub rtype: i32, #[prost(enumeration = "resize::SampleFilter", tag = "4")] pub filter: i32, } /// Nested message and enum types in `Resize`. pub mod resize { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum ResizeType { Normal = 0, SeamCarve = 1, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum SampleFilter { Undefined = 0, Nearest = 1, Triangle = 2, CatmullRom = 3, Gaussian = 4, Lanczos3 = 5, } } /// 处理图片截取 #[derive(Clone, PartialEq, ::prost::Message)] pub struct Crop { #[prost(uint32, tag = "1")] pub x1: u32, #[prost(uint32, tag = "2")] pub y1: u32, #[prost(uint32, tag = "3")] pub x2: u32, #[prost(uint32, tag = "4")] pub y2: u32, } /// 处理水平翻转 #[derive(Clone, PartialEq, ::prost::Message)] pub struct Fliph {} /// 处理垂直翻转 #[derive(Clone, PartialEq, ::prost::Message)] pub struct Flipv {} /// 处理对比度 #[derive(Clone, PartialEq, ::prost::Message)] pub struct Contrast { #[prost(float, tag = "1")] pub contrast: f32, } /// 处理滤镜 #[derive(Clone, PartialEq, ::prost::Message)] pub struct Filter { #[prost(enumeration = "filter::Filter", tag = "1")] pub filter: i32, } /// Nested message and enum types in `Filter`. pub mod filter { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum Filter { Unspecified = 0, Oceanic = 1, Islands = 2, /// more: <https://docs.rs/photon-rs/0.3.1/photon_rs/filters/fn.filter.html> Marine = 3, } } /// 处理水印 #[derive(Clone, PartialEq, ::prost::Message)] pub struct Watermark { #[prost(uint32, tag = "1")] pub x: u32, #[prost(uint32, tag = "2")] pub y: u32, }
pb/abi.rs有个特殊结构体
/// 一个 spec 可以包含上述的处理方式之一 #[derive(Clone, PartialEq, ::prost::Message)] pub struct Spec { #[prost(oneof = "spec::Data", tags = "1, 2, 3, 4, 5, 6, 7")] pub data: ::core::option::Option<spec::Data>, }
ImageSpec
定义:有序数组
- pb/abi.rs
/// 一个 ImageSpec 是一个有序的数组,服务器按照 spec 的顺序处理 #[derive(Clone, PartialEq, ::prost::Message)] pub struct ImageSpec { #[prost(message, repeated, tag = "1")] pub specs: ::prost::alloc::vec::Vec<Spec>, }
实现:new方法、From&TryFrom实现类型转化
- pb/mod.rs
impl ImageSpec { pub fn new(specs: Vec<Spec>) -> Self { Self { specs } } } // 让 ImageSpec 可以生成一个字符串 impl From<&ImageSpec> for String { fn from(image_spec: &ImageSpec) -> Self { let data = image_spec.encode_to_vec(); encode_config(data, URL_SAFE_NO_PAD) } } // 让 ImageSpec 可以通过一个字符串创建。比如 s.parse().unwrap() impl TryFrom<&str> for ImageSpec { type Error = anyhow::Error; fn try_from(value: &str) -> Result<Self, Self::Error> { let data = decode_config(value, URL_SAFE_NO_PAD)?; Ok(ImageSpec::decode(&data[..])?) } }
Filter
定义:枚举体mod
- pb/abi.rs
/// Nested message and enum types in `Filter`. pub mod filter { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum Filter { Unspecified = 0, Oceanic = 1, Islands = 2, /// more: <https://docs.rs/photon-rs/0.3.1/photon_rs/filters/fn.filter.html> Marine = 3, } }
实现:双引号的使用、模式匹配
- pb/mod.rs
// 辅助函数,photon_rs 相应的方法里需要字符串 impl filter::Filter { pub fn to_str(self) -> Option<&'static str> { match self { filter::Filter::Unspecified => None, filter::Filter::Oceanic => Some("oceanic"), filter::Filter::Islands => Some("islands"), filter::Filter::Marine => Some("marine"), } } }
SampleFilter
定义:枚举体mod
- pb/abi.rs
/// Nested message and enum types in `Resize`. pub mod resize { #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum ResizeType { Normal = 0, SeamCarve = 1, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, ::prost::Enumeration)] #[repr(i32)] pub enum SampleFilter { Undefined = 0, Nearest = 1, Triangle = 2, CatmullRom = 3, Gaussian = 4, Lanczos3 = 5, } }
实现:mod使用双引号、From转为不同结果
- pb/mod.rs
impl From<resize::SampleFilter> for SamplingFilter { fn from(v: resize::SampleFilter) -> Self { match v { resize::SampleFilter::Undefined => SamplingFilter::Nearest, resize::SampleFilter::Nearest => SamplingFilter::Nearest, resize::SampleFilter::Triangle => SamplingFilter::Triangle, resize::SampleFilter::CatmullRom => SamplingFilter::CatmullRom, resize::SampleFilter::Gaussian => SamplingFilter::Gaussian, resize::SampleFilter::Lanczos3 => SamplingFilter::Lanczos3, } } }
Spec
定义:结构体
- pb/abi.rs
/// 一个 spec 可以包含上述的处理方式之一 #[derive(Clone, PartialEq, ::prost::Message)] pub struct Spec { #[prost(oneof = "spec::Data", tags = "1, 2, 3, 4, 5, 6, 7")] pub data: ::core::option::Option<spec::Data>, }
注意区别Self和self的使用:
注意区别Self和self的使用:
实现:类似面向对象中添加类方法Self
- pb/mod.rs
// 提供一些辅助函数,让创建一个 spec 的过程简单一些 impl Spec { pub fn new_resize_seam_carve(width: u32, height: u32) -> Self { Self { data: Some(spec::Data::Resize(Resize { width, height, rtype: resize::ResizeType::SeamCarve as i32, filter: resize::SampleFilter::Undefined as i32, })), } } pub fn new_resize(width: u32, height: u32, filter: resize::SampleFilter) -> Self { Self { data: Some(spec::Data::Resize(Resize { width, height, rtype: resize::ResizeType::Normal as i32, filter: filter as i32, })), } } pub fn new_filter(filter: filter::Filter) -> Self { Self { data: Some(spec::Data::Filter(Filter { filter: filter as i32, })), } } pub fn new_watermark(x: u32, y: u32) -> Self { Self { data: Some(spec::Data::Watermark(Watermark { x, y })), } } }
单元测试
单元测试
#[cfg(test)] mod tests { use super::*; use std::borrow::Borrow; #[test] fn encoded_spec_could_be_decoded() { let spec1 = Spec::new_resize(600, 600, resize::SampleFilter::CatmullRom); let spec2 = Spec::new_filter(filter::Filter::Marine); let image_spec = ImageSpec::new(vec![spec1, spec2]); let s: String = image_spec.borrow().into(); assert_eq!(image_spec, s.as_str().try_into().unwrap()); } }
engine模块: 处理图片
mod.rs: 定义统一的引擎trait
// Engine trait:未来可以添加更多的 engine,主流程只需要替换 engine pub trait Engine { // 对 engine 按照 specs 进行一系列有序的处理 fn apply(&mut self, specs: &[Spec]); // 从 engine 中生成目标图片,注意这里用的是 self,而非 self 的引用 fn generate(self, format: ImageOutputFormat) -> Vec<u8>; } // SpecTransform:未来如果添加更多的 spec,只需要实现它即可 pub trait SpecTransform<T> { // 对图片使用 op 做 transform fn transform(&mut self, op: T); }
photon.rs > 静态变量加载
lazy_static! { // 预先把水印文件加载为静态变量 static ref WATERMARK: PhotonImage = { let data = include_bytes!("../../rust-logo.png"); let watermark = open_image_from_bytes(data).unwrap(); transform::resize(&watermark, 64, 64, transform::SamplingFilter::Nearest) }; }
photon.rs > 具体引擎Photon的定义与转化TryFrom
pub struct Photon(PhotonImage); // 从 Bytes 转换成 Photon 结构 impl TryFrom<Bytes> for Photon { type Error = anyhow::Error; fn try_from(data: Bytes) -> Result<Self, Self::Error> { Ok(Self(open_image_from_bytes(&data)?)) } }
photon.rs > 具体引擎Photon的trait实现
Engine Trait
impl Engine for Photon { fn apply(&mut self, specs: &[Spec]) { for spec in specs.iter() { match spec.data { Some(spec::Data::Crop(ref v)) => self.transform(v), Some(spec::Data::Contrast(ref v)) => self.transform(v), Some(spec::Data::Filter(ref v)) => self.transform(v), Some(spec::Data::Fliph(ref v)) => self.transform(v), Some(spec::Data::Flipv(ref v)) => self.transform(v), Some(spec::Data::Resize(ref v)) => self.transform(v), Some(spec::Data::Watermark(ref v)) => self.transform(v), // 对于目前不认识的 spec,不做任何处理 _ => {} } } }
SpecTransform Trait
格式语义化
#![allow(unused)] fn main() { impl SpecTransform(&OpreationName) for SpecificEngine { fn transform(&mut self, _op: &OperationName) { transform::OperationMethod(&mut self.0) } } }
impl SpecTransform<&Crop> for Photon { fn transform(&mut self, op: &Crop) { let img = transform::crop(&mut self.0, op.x1, op.y1, op.x2, op.y2); self.0 = img; } } impl SpecTransform<&Contrast> for Photon { fn transform(&mut self, op: &Contrast) { effects::adjust_contrast(&mut self.0, op.contrast); } } impl SpecTransform<&Flipv> for Photon { fn transform(&mut self, _op: &Flipv) { transform::flipv(&mut self.0) } } impl SpecTransform<&Fliph> for Photon { fn transform(&mut self, _op: &Fliph) { transform::fliph(&mut self.0) } } impl SpecTransform<&Filter> for Photon { fn transform(&mut self, op: &Filter) { match filter::Filter::from_i32(op.filter) { Some(filter::Filter::Unspecified) => {} Some(f) => filters::filter(&mut self.0, f.to_str().unwrap()), _ => {} } } } impl SpecTransform<&Resize> for Photon { fn transform(&mut self, op: &Resize) { let img = match resize::ResizeType::from_i32(op.rtype).unwrap() { resize::ResizeType::Normal => transform::resize( &self.0, op.width, op.height, resize::SampleFilter::from_i32(op.filter).unwrap().into(), ), resize::ResizeType::SeamCarve => transform::seam_carve(&self.0, op.width, op.height), }; self.0 = img; } } impl SpecTransform<&Watermark> for Photon { fn transform(&mut self, op: &Watermark) { multiple::watermark(&mut self.0, &WATERMARK, op.x, op.y); } }
photon.rs > 在内存中对图片转换格式的方法
fn image_to_buf(img: PhotonImage, format: ImageOutputFormat) -> Vec<u8> { let raw_pixels = img.get_raw_pixels(); let width = img.get_width(); let height = img.get_height(); let img_buffer = ImageBuffer::from_vec(width, height, raw_pixels).unwrap(); let dynimage = DynamicImage::ImageRgba8(img_buffer); let mut buffer = Vec::with_capacity(32768); dynimage.write_to(&mut buffer, format).unwrap(); buffer }
main.rs
先引入mod,再use
// 参数使用 serde 做 Deserialize,axum 会自动识别并解析 #[derive(Deserialize)] struct Params { spec: String, url: String, }
主流程main函数
main()
#[tokio::main] async fn main() { // 初始化 tracing tracing_subscriber::fmt::init(); let cache: Cache = Arc::new(Mutex::new(LruCache::new(1024))); // 构建路由 let app = Router::new() // `GET /` 会执行 .route("/image/:spec/:url", get(generate)) .layer( ServiceBuilder::new() .load_shed() .concurrency_limit(1024) .timeout(Duration::from_secs(10)) .layer(TraceLayer::new_for_http()) .layer(AddExtensionLayer::new(cache)) .layer(CompressionLayer::new()) .into_inner(), ); // 运行 web 服务器 let addr = "127.0.0.1:3000".parse().unwrap(); print_test_url("https://images.pexels.com/photos/1562477/pexels-photo-1562477.jpeg?auto=compress&cs=tinysrgb&dpr=3&h=750&w=1260"); info!("Listening on {}", addr); axum::Server::bind(&addr) .serve(app.into_make_service()) .await .unwrap(); }
建造者模式
ServiceBuilder::new() .load_shed() .concurrency_limit(1024) .timeout(Duration::from_secs(10)) .layer(TraceLayer::new_for_http()) .layer(AddExtensionLayer::new(cache)) .layer(CompressionLayer::new()) .into_inner(),
笔记:类型转换总结
- 数字与字符串
i32
u32
f64
String*
i32
\
x as u32
x as f64
x.to_string()
u32
x as i32
\
x as f64
x.to_string()
f64
x as i32
x as u32
\
x.to_string()
String*
x.parse().unwrap()
x.parse().unwrap()
x.parse().unwrap()
\
- String 与 & str
\
String
&str
String
\
&*x
&str
x.to_string()
\
- 智能指针
\
Vec\
&[T]
Box<[T]>
Vec\
\
&x[...]
x.into_boxed_slice()
&[T]
x.to_vec()
\
Box::new(\*x)
Box<[T]>
x.to_vec()
&\*x
\
路由绑定的处理函数handler
// basic handler that responds with a static string async fn generate( Path(Params { spec, url }): Path<Params>, Extension(cache): Extension<Cache>, ) -> Result<(HeaderMap, Vec<u8>), StatusCode> { let spec: ImageSpec = spec .as_str() .try_into() .map_err(|_| StatusCode::BAD_REQUEST)?; let url: &str = &percent_decode_str(&url).decode_utf8_lossy(); let data = retrieve_image(url, cache) .await .map_err(|_| StatusCode::BAD_REQUEST)?; // 使用 image engine 处理 let mut engine: Photon = data .try_into() .map_err(|_| StatusCode::INTERNAL_SERVER_ERROR)?; engine.apply(&spec.specs); // TODO: 这里目前类型写死了,应该使用 content negotiation let image = engine.generate(ImageOutputFormat::Jpeg(85)); info!("Finished processing: image size {}", image.len()); let mut headers = HeaderMap::new(); headers.insert("content-type", HeaderValue::from_static("image/jpeg")); Ok((headers, image)) }
处理函数用到的图片获取方法
对于图片的网络请求,我们先把 URL 做个哈希,在 LRU 缓存中查找,找不到才用 reqwest 发送请求。
#[instrument(level = "info", skip(cache))] async fn retrieve_image(url: &str, cache: Cache) -> Result<Bytes> { let mut hasher = DefaultHasher::new(); url.hash(&mut hasher); let key = hasher.finish(); let g = &mut cache.lock().await; let data = match g.get(&key) { Some(v) => { info!("Match cache {}", key); v.to_owned() } None => { info!("Retrieve url"); let resp = reqwest::get(url).await?; let data = resp.bytes().await?; g.put(key, data.clone()); data } }; Ok(data) }
一个用于调试的辅助函数
// 调试辅助函数 fn print_test_url(url: &str) { use std::borrow::Borrow; let spec1 = Spec::new_resize(500, 800, resize::SampleFilter::CatmullRom); let spec2 = Spec::new_watermark(20, 20); let spec3 = Spec::new_filter(filter::Filter::Marine); let image_spec = ImageSpec::new(vec![spec1, spec2, spec3]); let s: String = image_spec.borrow().into(); let test_image = percent_encode(url.as_bytes(), NON_ALPHANUMERIC).to_string(); println!("test url: http://localhost:3000/image/{}/{}", s, test_image); }