From 3206be9ff8ff8cfc873b113af6829a5a70c97a76 Mon Sep 17 00:00:00 2001 From: Sean Lynch <42618346+swlynch99@users.noreply.github.com> Date: Fri, 30 Sep 2022 15:27:43 -0700 Subject: [PATCH 01/11] Introduce http protocol --- src/config/src/bloom.rs | 42 +++++++ src/config/src/bloomcache.rs | 190 ++++++++++++++++++++++++++++ src/entrystore/src/bloom/http.rs | 50 ++++++++ src/entrystore/src/bloom/mod.rs | 34 ++++++ src/protocol/http/Cargo.toml | 16 +++ src/protocol/http/src/error.rs | 62 ++++++++++ src/protocol/http/src/lib.rs | 27 ++++ src/protocol/http/src/request.rs | 149 ++++++++++++++++++++++ src/protocol/http/src/response.rs | 197 ++++++++++++++++++++++++++++++ src/protocol/http/src/util.rs | 41 +++++++ src/server/bloomcache/Cargo.toml | 16 +++ src/server/bloomcache/src/lib.rs | 51 ++++++++ src/server/bloomcache/src/main.rs | 20 +++ 13 files changed, 895 insertions(+) create mode 100644 src/config/src/bloom.rs create mode 100644 src/config/src/bloomcache.rs create mode 100644 src/entrystore/src/bloom/http.rs create mode 100644 src/entrystore/src/bloom/mod.rs create mode 100644 src/protocol/http/Cargo.toml create mode 100644 src/protocol/http/src/error.rs create mode 100644 src/protocol/http/src/lib.rs create mode 100644 src/protocol/http/src/request.rs create mode 100644 src/protocol/http/src/response.rs create mode 100644 src/protocol/http/src/util.rs create mode 100644 src/server/bloomcache/Cargo.toml create mode 100644 src/server/bloomcache/src/lib.rs create mode 100644 src/server/bloomcache/src/main.rs diff --git a/src/config/src/bloom.rs b/src/config/src/bloom.rs new file mode 100644 index 000000000..a9a938401 --- /dev/null +++ b/src/config/src/bloom.rs @@ -0,0 +1,42 @@ +// Copyright 2021 Twitter, Inc. +// Licensed under the Apache License, Version 2.0 +// http://www.apache.org/licenses/LICENSE-2.0 + +use crate::units::MB; + +use serde::{Deserialize, Serialize}; + +const BLOOM_DEFAULT_SIZE: usize = 16 * MB; +const BLOOM_DEFAULT_HASHES: usize = 64; + +fn size() -> usize { + BLOOM_DEFAULT_SIZE +} + +fn hashes() -> usize { + BLOOM_DEFAULT_HASHES +} + +#[derive(Serialize, Deserialize, Debug)] +pub struct Bloom { + /// The size of the bloom filter in bytes. + #[serde(default = "size")] + pub size: usize, + + /// The number of hash functions that are evaluated for each value inserted.F + #[serde(default = "hashes")] + pub hashes: usize, +} + +impl Default for Bloom { + fn default() -> Self { + Self { + size: BLOOM_DEFAULT_SIZE, + hashes: BLOOM_DEFAULT_HASHES, + } + } +} + +pub trait BloomConfig { + fn bloom(&self) -> &Bloom; +} diff --git a/src/config/src/bloomcache.rs b/src/config/src/bloomcache.rs new file mode 100644 index 000000000..8bcb61511 --- /dev/null +++ b/src/config/src/bloomcache.rs @@ -0,0 +1,190 @@ +// Copyright 2022 Twitter, Inc. +// Licensed under the Apache License, Version 2.0 +// http://www.apache.org/licenses/LICENSE-2.0 + +use crate::*; + +use serde::{Deserialize, Serialize}; + +use std::io::Read; + +// constants to define default values +const DAEMONIZE: bool = false; +const PID_FILENAME: Option = None; +const DLOG_INTERVAL: usize = 500; + +// helper functions +fn daemonize() -> bool { + DAEMONIZE +} + +fn pid_filename() -> Option { + PID_FILENAME +} + +fn dlog_interval() -> usize { + DLOG_INTERVAL +} + +// struct definitions +#[derive(Serialize, Deserialize, Debug)] +pub struct BloomcacheConfig { + // top-level + #[serde(default = "daemonize")] + daemonize: bool, + #[serde(default = "pid_filename")] + pid_filename: Option, + #[serde(default = "dlog_interval")] + dlog_interval: usize, + + // application modules + #[serde(default)] + admin: Admin, + #[serde(default)] + server: Server, + #[serde(default)] + worker: Worker, + #[serde(default)] + time: Time, + #[serde(default)] + tls: Tls, + #[serde(default)] + bloom: Bloom, + + // ccommon + #[serde(default)] + buf: Buf, + #[serde(default)] + debug: Debug, + #[serde(default)] + klog: Klog, + #[serde(default)] + sockio: Sockio, + #[serde(default)] + tcp: Tcp, +} + +impl AdminConfig for BloomcacheConfig { + fn admin(&self) -> &Admin { + &self.admin + } +} + +impl BufConfig for BloomcacheConfig { + fn buf(&self) -> &Buf { + &self.buf + } +} + +impl DebugConfig for BloomcacheConfig { + fn debug(&self) -> &Debug { + &self.debug + } +} + +impl KlogConfig for BloomcacheConfig { + fn klog(&self) -> &Klog { + &self.klog + } +} + +impl ServerConfig for BloomcacheConfig { + fn server(&self) -> &Server { + &self.server + } +} + +impl SockioConfig for BloomcacheConfig { + fn sockio(&self) -> &Sockio { + &self.sockio + } +} + +impl TcpConfig for BloomcacheConfig { + fn tcp(&self) -> &Tcp { + &self.tcp + } +} + +impl TimeConfig for BloomcacheConfig { + fn time(&self) -> &Time { + &self.time + } +} + +impl TlsConfig for BloomcacheConfig { + fn tls(&self) -> &Tls { + &self.tls + } +} + +impl WorkerConfig for BloomcacheConfig { + fn worker(&self) -> &Worker { + &self.worker + } + + fn worker_mut(&mut self) -> &mut Worker { + &mut self.worker + } +} + +impl BloomConfig for BloomcacheConfig { + fn bloom(&self) -> &Bloom { + &self.bloom + } +} + +// implementation +impl BloomcacheConfig { + pub fn load(file: &str) -> Result { + let mut file = std::fs::File::open(file)?; + let mut content = String::new(); + file.read_to_string(&mut content)?; + match toml::from_str(&content) { + Ok(t) => Ok(t), + Err(e) => { + error!("{}", e); + Err(std::io::Error::new( + std::io::ErrorKind::Other, + "Error parsing config", + )) + } + } + } + + pub fn daemonize(&self) -> bool { + self.daemonize + } + + pub fn pid_filename(&self) -> Option { + self.pid_filename.clone() + } + + pub fn dlog_interval(&self) -> usize { + self.dlog_interval + } +} + +// trait implementations +impl Default for BloomcacheConfig { + fn default() -> Self { + Self { + daemonize: daemonize(), + pid_filename: pid_filename(), + dlog_interval: dlog_interval(), + + admin: Default::default(), + server: Default::default(), + worker: Default::default(), + time: Default::default(), + bloom: Default::default(), + + buf: Default::default(), + debug: Default::default(), + klog: Default::default(), + sockio: Default::default(), + tcp: Default::default(), + tls: Default::default(), + } + } +} diff --git a/src/entrystore/src/bloom/http.rs b/src/entrystore/src/bloom/http.rs new file mode 100644 index 000000000..1cc430581 --- /dev/null +++ b/src/entrystore/src/bloom/http.rs @@ -0,0 +1,50 @@ +// Copyright 2021 Twitter, Inc. +// Licensed under the Apache License, Version 2.0 +// http://www.apache.org/licenses/LICENSE-2.0 + +use std::borrow::Cow; + +use protocol_common::Execute; +use protocol_http::{ + request::{Request, RequestData}, + Headers, ParseResult, Response, Storage, +}; + +use crate::Bloom; + +impl Execute for Bloom { + fn execute(&mut self, result: &ParseResult) -> Response { + let request = match result { + Ok(request) => request, + Err(e) => return e.to_response(), + }; + + let Request { headers, data } = request; + match data { + RequestData::Get(key) => self.get(key, headers), + RequestData::Put(key, value) => self.put(key, value, headers), + RequestData::Delete(key) => self.delete(key, headers), + } + } +} + +impl Storage for Bloom { + fn get(&mut self, key: &[u8], _headers: &Headers) -> Response { + if self.data.contains(key) { + Response::builder(204).empty() + } else { + Response::builder(404).empty() + } + } + + fn put(&mut self, key: &[u8], _value: &[u8], _headers: &Headers) -> Response { + self.data.insert(key); + Response::builder(204).empty() + } + + fn delete(&mut self, _key: &[u8], _headers: &Headers) -> Response { + let mut builder = Response::builder(405); + builder.header("Content-Type", b"text/plain"); + builder.body(Cow::Borrowed(b"DELETE method not supported")) + } +} diff --git a/src/entrystore/src/bloom/mod.rs b/src/entrystore/src/bloom/mod.rs new file mode 100644 index 000000000..65fa3e32a --- /dev/null +++ b/src/entrystore/src/bloom/mod.rs @@ -0,0 +1,34 @@ +// Copyright 2021 Twitter, Inc. +// Licensed under the Apache License, Version 2.0 +// http://www.apache.org/licenses/LICENSE-2.0 + +use bloom::BloomFilter; +use config::BloomConfig; + +use crate::EntryStore; + +mod http; + +/// A wrapper around [`bloom::BloomFilter`] which implements `EntryStore` +/// protocol traits. +pub struct Bloom { + data: BloomFilter<[u8]>, +} + +impl Bloom { + /// Create a bloom filter storage based on the config. + pub fn new(config: &T) -> Result { + // TODO: Validate the config here and return an error. + + let config = config.bloom(); + Ok(Self { + data: BloomFilter::new(config.size * 8, config.hashes), + }) + } +} + +impl EntryStore for Bloom { + fn clear(&mut self) { + self.data.clear(); + } +} diff --git a/src/protocol/http/Cargo.toml b/src/protocol/http/Cargo.toml new file mode 100644 index 000000000..4105efb78 --- /dev/null +++ b/src/protocol/http/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "protocol-http" +version = "0.1.0" +edition = "2021" + +[dependencies] +httparse = "1.8.0" +thiserror = "1.0.37" +urlencoding = "2.1.2" +arrayvec = "0.7.2" +bytes = "1.2.1" + +phf = { version = "0.11.1", features = ["macros"] } + +protocol-common = { path = "../common" } +logger = { path="../../logger" } diff --git a/src/protocol/http/src/error.rs b/src/protocol/http/src/error.rs new file mode 100644 index 000000000..383db6262 --- /dev/null +++ b/src/protocol/http/src/error.rs @@ -0,0 +1,62 @@ +// Copyright 2021 Twitter, Inc. +// Licensed under the Apache License, Version 2.0 +// http://www.apache.org/licenses/LICENSE-2.0 + +use crate::Response; + +#[derive(Debug, Error)] +pub enum Error { + #[error("unable to parse request")] + Unparseable(#[from] httparse::Error), + #[error("Content-Length header was invalid")] + BadContentLength, + #[error("Content-Length header was missing")] + MissingContentLength, + #[error("method was unsupported")] + BadRequestMethod, + + /// Contains the number of additional bytes needed to parse the rest of the + /// request, if known. + #[error("not enough data present to parse the whole request")] + PartialRequest(Option), + + #[error("an internal error occurred: {0}")] + InternalError(&'static str), +} + +impl Error { + pub fn to_response(&self) -> Response { + match self { + Self::Unparseable(e) => Response::builder(400) + .should_close(true) + .header("Content-Type", b"text/plain") + .body(format!("Unable to parse request: {}", e).as_bytes()), + Self::BadRequestMethod => Response::builder(405) + .should_close(true) + .header("Content-Type", b"text/plain") + .body( + format!("Unsupported method, only GET, PUT, and DELETE are supported") + .as_bytes(), + ), + Self::BadContentLength => Response::builder(400) + .should_close(true) + .header("Content-Type", b"text/plain") + .body(format!("Content-Length header was invalid").as_bytes()), + Self::MissingContentLength => Response::builder(411) + .should_close(true) + .header("Content-Type", b"text/plain") + .body( + format!("A Content-Length header is required for all PUT requests").as_bytes(), + ), + Self::InternalError(message) => Response::builder(500) + .should_close(true) + .header("Content-Type", b"text/plain") + .body(message.as_bytes()), + + Self::PartialRequest(_) => Response::builder(500) + .should_close(true) + .header("Content-Type", b"text/plain") + .body(b"internal server error"), + } + } +} diff --git a/src/protocol/http/src/lib.rs b/src/protocol/http/src/lib.rs new file mode 100644 index 000000000..a3c064e9d --- /dev/null +++ b/src/protocol/http/src/lib.rs @@ -0,0 +1,27 @@ +// Copyright 2021 Twitter, Inc. +// Licensed under the Apache License, Version 2.0 +// http://www.apache.org/licenses/LICENSE-2.0 + +//! + +#[macro_use] +extern crate thiserror; + +mod error; +pub mod request; +pub mod response; +mod util; + +pub use crate::error::Error; +pub use crate::request::Headers; +pub use crate::request::{Request, RequestData, RequestParser}; +pub use crate::response::Response; + +pub type Result = std::result::Result; +pub type ParseResult = Result; + +pub trait Storage { + fn get(&mut self, key: &[u8], headers: &Headers) -> Response; + fn put(&mut self, key: &[u8], value: &[u8], headers: &Headers) -> Response; + fn delete(&mut self, key: &[u8], headers: &Headers) -> Response; +} diff --git a/src/protocol/http/src/request.rs b/src/protocol/http/src/request.rs new file mode 100644 index 000000000..1db1278f7 --- /dev/null +++ b/src/protocol/http/src/request.rs @@ -0,0 +1,149 @@ +// Copyright 2021 Twitter, Inc. +// Licensed under the Apache License, Version 2.0 +// http://www.apache.org/licenses/LICENSE-2.0 + +use std::mem::MaybeUninit; + +use crate::{Error, ParseResult}; +use httparse::{Header, ParserConfig, Status}; +use protocol_common::{Parse, ParseOk}; + +pub struct Headers(Vec<(String, Vec)>); + +pub struct ParseData(pub Result); + +impl Headers { + fn from_httparse(headers: &[Header]) -> Self { + Self( + headers + .iter() + .map(|hdr| (hdr.name.to_owned(), hdr.value.to_owned())) + .collect(), + ) + } + + pub fn header(&self, hdr: &str) -> Option<&[u8]> { + self.0 + .iter() + .find(|(name, _)| name.eq_ignore_ascii_case(hdr)) + .map(|(_, value)| &**value) + } +} + +pub struct Request { + pub data: RequestData, + pub headers: Headers, +} + +impl Request { + pub fn data(&self) -> &RequestData { + &self.data + } + + pub fn header(&self, hdr: &str) -> Option<&[u8]> { + self.headers.header(hdr) + } +} + +pub enum RequestData { + Get(Vec), + Put(Vec, Vec), + Delete(Vec), +} + +#[derive(Clone)] +pub struct RequestParser { + config: ParserConfig, +} + +impl RequestParser { + pub fn new() -> Self { + Self { + config: ParserConfig::default(), + } + } + + fn do_parse(&self, buf: &mut &[u8]) -> ParseResult { + let mut headers = [MaybeUninit::uninit(); 32]; + let mut request = httparse::Request::new(&mut []); + let status = + self.config + .parse_request_with_uninit_headers(&mut request, *buf, &mut headers)?; + + let count = match status { + Status::Complete(count) => count, + Status::Partial => return Err(Error::PartialRequest(None)), + }; + + *buf = &buf[count..]; + + let method = request.method.ok_or(Error::InternalError( + "request was complete but had no method", + ))?; + let key = request + .path + .ok_or(Error::InternalError("request was complete but had no path"))?; + + let key = urlencoding::decode_binary(&key.as_bytes()).into_owned(); + let headers = Headers::from_httparse(request.headers); + + match method { + "GET" => Ok(Request { + data: RequestData::Get(key), + headers, + }), + "DELETE" => Ok(Request { + data: RequestData::Delete(key), + headers, + }), + "PUT" => { + let content_length = headers + .header("Content-Length") + .ok_or(Error::BadContentLength)?; + let len: usize = std::str::from_utf8(content_length) + .map_err(|_| Error::BadContentLength)? + .parse() + .map_err(|_| Error::BadContentLength)?; + + if buf.len() < len { + return Err(Error::PartialRequest(Some(len - buf.len()))); + } + + let (value, newbuf) = buf.split_at(len); + *buf = newbuf; + + Ok(Request { + data: RequestData::Put(key, value.to_owned()), + headers, + }) + } + _ => return Err(Error::BadRequestMethod), + } + } +} + +impl Parse for RequestParser { + fn parse(&self, buffer: &[u8]) -> Result, std::io::Error> { + let mut buf = buffer; + let result = self.do_parse(&mut buf); + + let consumed = match result.is_ok() { + true => unsafe { buf.as_ptr().offset_from(buffer.as_ptr()) as usize }, + false => 0, + }; + + if matches!(result, Err(Error::PartialRequest(_))) { + return Err(std::io::Error::from(std::io::ErrorKind::WouldBlock)); + } + + Ok(ParseOk::new(ParseData(result), consumed)) + } +} + +impl logger::Klog for ParseData { + type Response = crate::Response; + + fn klog(&self, _: &Self::Response) { + // todo: ignore for now + } +} diff --git a/src/protocol/http/src/response.rs b/src/protocol/http/src/response.rs new file mode 100644 index 000000000..53a5c3a17 --- /dev/null +++ b/src/protocol/http/src/response.rs @@ -0,0 +1,197 @@ +// Copyright 2021 Twitter, Inc. +// Licensed under the Apache License, Version 2.0 +// http://www.apache.org/licenses/LICENSE-2.0 + +use arrayvec::ArrayVec; +use phf::{phf_map, Map}; +use protocol_common::{BufMut, Compose}; +use std::io::Write; + +pub struct Response { + builder: ResponseBuilder, + body: Option>, +} + +impl Response { + pub fn builder(status: u16) -> ResponseBuilder { + ResponseBuilder::new(status) + } +} + +pub struct ResponseBuilder { + headers: Vec, + close: bool, +} + +impl ResponseBuilder { + pub fn new(status: u16) -> Self { + let mut data = Vec::with_capacity(2048); + write!( + &mut data, + "HTTP/1.1 {} {}\r\n", + status, + STATUSES.get(&status).copied().unwrap_or("") + ) + .unwrap(); + + Self { + headers: data, + close: false, + } + } + + pub fn header(&mut self, key: &str, value: &[u8]) -> &mut Self { + assert!(self.headers.is_empty()); + + self.headers.extend_from_slice(key.as_bytes()); + self.headers.extend_from_slice(b": "); + self.headers.extend_from_slice(value); + self.headers.extend_from_slice(b"\r\n"); + + self + } + + pub fn should_close(&mut self, close: bool) -> &mut Self { + self.close = close; + self + } + + /// Build a response with no body + pub fn empty(&mut self) -> Response { + assert!(!self.headers.is_empty()); + + Response { + builder: self.take(), + body: None, + } + } + + /// Build a response with the specified body, also appends a Content-Length + /// header. + pub fn body(&mut self, body: &[u8]) -> Response { + assert!(!self.headers.is_empty()); + + let body = body.to_owned(); + Response { + builder: self.take(), + body: Some(body), + } + } + + fn take(&mut self) -> Self { + Self { + headers: std::mem::take(&mut self.headers), + close: self.close, + } + } +} + +impl Compose for Response { + fn compose(&self, dst: &mut dyn BufMut) -> usize { + let mut dst = crate::util::CountingBuf::new(dst); + + dst.put_slice(&self.builder.headers); + + if self.builder.close { + dst.put_slice(b"Connection: close\r\n"); + } else { + dst.put_slice(b"Connection: keep-alive\r\n"); + dst.put_slice(b"Keep-Alive: timeout=60\r\n"); + } + + if let Some(body) = &self.body { + let mut lenbuf = ArrayVec::::new(); + write!(&mut lenbuf, "{}", body.len()).unwrap(); + + dst.put_slice(b"Content-Length: "); + dst.put_slice(&lenbuf); + dst.put_slice(b"\r\n"); + } + + dst.put_slice(b"\r\n"); + + if let Some(body) = &self.body { + dst.put_slice(body); + } + + dst.count() + } + + fn should_hangup(&self) -> bool { + self.builder.close + } +} + +const STATUSES: Map = phf_map! { + // Informational Responses + 100u16 => "Continue", + 101u16 => "Switching Protocols", + 102u16 => "Processing", + 103u16 => "Early Hints", + + // Successful Responses + 200u16 => "OK", + 201u16 => "Created", + 202u16 => "Accepted", + 203u16 => "Non-Authoritative Information", + 204u16 => "No Content", + 205u16 => "Reset Content", + 206u16 => "Partial Content", + 207u16 => "Multi-Status", + 208u16 => "Already Reported", + 226u16 => "IM Used", + + // Redirect Responses + 300u16 => "Multiple Choices", + 301u16 => "Moved Permanently", + 302u16 => "Found", + 303u16 => "See Other", + 304u16 => "Not Modified", + 305u16 => "Use Proxy", + 307u16 => "Temporary Redirect", + 308u16 => "Permanent Redirect", + + // Client Error Responses + 400u16 => "Bad Request", + 401u16 => "Unauthorized", + 402u16 => "Payment Required", + 403u16 => "Forbidden", + 404u16 => "Not Found", + 405u16 => "Method Not Allowed", + 406u16 => "Not Acceptable", + 407u16 => "Proxy Authentication Required", + 408u16 => "Request Timeout", + 409u16 => "Conflict", + 410u16 => "Gone", + 411u16 => "Length Required", + 412u16 => "Precondition Failed", + 413u16 => "Payload Too Large", + 414u16 => "URI Too Long", + 415u16 => "Unsupported Media Type", + 416u16 => "Range Not Satisfiable", + 417u16 => "Expectation Mailed", + 418u16 => "I'm a Teapot", + 421u16 => "Misdirected Request", + 422u16 => "Unprocessable Entity", + 423u16 => "Locked", + 424u16 => "Failed Dependency", + 425u16 => "Too Early", + 426u16 => "Upgrade Required", + 428u16 => "Precondition Required", + 429u16 => "Too Many Requests", + 431u16 => "Request Header Field Too Large", + 451u16 => "Unavailable For Legal Reasons", + + // Server Error Responses + 500u16 => "Internal Server Error", + 501u16 => "Not Implemented", + 502u16 => "Bad Gateway", + 503u16 => "Service Unavailable", + 504u16 => "Gateway Timeout", + 505u16 => "HTTP Version Not Supported", + 506u16 => "Variant Also Negotiates", + 507u16 => "Insufficient Storage", + 508u16 => "Loop Detected", + 510u16 => "Not Extended", + 511u16 => "Network Authentication Required", +}; diff --git a/src/protocol/http/src/util.rs b/src/protocol/http/src/util.rs new file mode 100644 index 000000000..b0899bf2a --- /dev/null +++ b/src/protocol/http/src/util.rs @@ -0,0 +1,41 @@ +use std::io::Write; + +use bytes::buf::UninitSlice; +use protocol_common::BufMut; + +pub(crate) struct CountingBuf { + buf: B, + count: usize, +} + +impl CountingBuf { + pub fn new(buf: B) -> Self { + Self { buf, count: 0 } + } + + pub fn count(&self) -> usize { + self.count + } +} + +unsafe impl BufMut for CountingBuf +where + B: BufMut, +{ + fn remaining_mut(&self) -> usize { + self.buf.remaining_mut() + } + + unsafe fn advance_mut(&mut self, cnt: usize) { + let data = self.chunk_mut(); + let slice = std::slice::from_raw_parts(data.as_mut_ptr(), cnt); + let _ = std::io::stdout().write_all(slice); + + self.count += cnt; + self.buf.advance_mut(cnt) + } + + fn chunk_mut(&mut self) -> &mut UninitSlice { + self.buf.chunk_mut() + } +} diff --git a/src/server/bloomcache/Cargo.toml b/src/server/bloomcache/Cargo.toml new file mode 100644 index 000000000..e87eed9c3 --- /dev/null +++ b/src/server/bloomcache/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "bloomcache" +version = "0.1.0" +edition = "2021" + +[dependencies] + +backtrace = "0.3.56" +clap = "4.0.4" +common = { path = "../../common" } +config = { path = "../../config" } +entrystore = { path = "../../entrystore" } +logger = { path = "../../logger" } +protocol-http = { path="../../protocol/http" } +rustcommon-metrics = { git = "https://github.com/twitter/rustcommon" } +server = { path = "../../core/server" } diff --git a/src/server/bloomcache/src/lib.rs b/src/server/bloomcache/src/lib.rs new file mode 100644 index 000000000..69f1ed49e --- /dev/null +++ b/src/server/bloomcache/src/lib.rs @@ -0,0 +1,51 @@ +// Copyright 2021 Twitter, Inc. +// Licensed under the Apache License, Version 2.0 +// http://www.apache.org/licenses/LICENSE-2.0 + +use config::*; +use entrystore::Bloom; +use logger::*; +use protocol_http::{ParseResult, RequestParser, Response}; +use server::{Process, ProcessBuilder}; + +type Parser = RequestParser; +type Storage = Bloom; + +pub struct BloomCache { + process: Process, +} + +impl BloomCache { + /// Create a new bloom cache from the provided config. + pub fn new(config: BloomcacheConfig) -> std::io::Result { + common::metrics::init(); + + let log = configure_logging(&config); + let storage = Storage::new(&config)?; + let parser = Parser::new(); + + let builder = ProcessBuilder::::new( + &config, log, parser, storage, + )? + .version(env!("CARGO_PKG_VERSION")); + + Ok(Self { + process: builder.spawn(), + }) + } + + /// Wait for all threads to complete. Blocks until the process has fully + /// terminated. Under normal conditions, this will block indefinitely. + pub fn wait(self) { + self.process.wait() + } + + /// Triggers a shutdown of the process and blocks until the process has + /// fully terminated. This is more likely to be used for running integration + /// tests or other automated testing. + pub fn shutdown(self) { + self.process.shutdown() + } +} + +common::metrics::test_no_duplicates!(); diff --git a/src/server/bloomcache/src/main.rs b/src/server/bloomcache/src/main.rs new file mode 100644 index 000000000..a010ba487 --- /dev/null +++ b/src/server/bloomcache/src/main.rs @@ -0,0 +1,20 @@ +// Copyright 2021 Twitter, Inc. +// Licensed under the Apache License, Version 2.0 +// http://www.apache.org/licenses/LICENSE-2.0 + +#[macro_use] +extern crate logger; + +use backtrace::Backtrace; +use config::PingserverConfig; +use rustcommon_metrics::*; +use server::PERCENTILES; + +fn main() { + // custom panic hook to terminate whole process after unwinding + std::panic::set_hook(Box::new(|s| { + error!("{}", s); + eprintln!("{:?}", Backtrace::new()); + std::process::exit(101); + })); +} From 086869465305bc68bd926a56f17d90352a0f3c7f Mon Sep 17 00:00:00 2001 From: Sean Lynch <42618346+swlynch99@users.noreply.github.com> Date: Fri, 30 Sep 2022 15:28:04 -0700 Subject: [PATCH 02/11] Add bloom config --- src/config/src/lib.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/config/src/lib.rs b/src/config/src/lib.rs index 3e143c700..de0f9c17b 100644 --- a/src/config/src/lib.rs +++ b/src/config/src/lib.rs @@ -7,6 +7,8 @@ extern crate log; mod admin; mod array; +mod bloom; +mod bloomcache; mod buf; mod dbuf; mod debug; @@ -28,6 +30,8 @@ mod worker; pub use admin::{Admin, AdminConfig}; pub use array::ArrayConfig; +pub use bloom::{Bloom, BloomConfig}; +pub use bloomcache::BloomcacheConfig; pub use buf::{Buf, BufConfig}; pub use dbuf::DbufConfig; pub use debug::{Debug, DebugConfig}; From 305e36695f413c5b8a0d9e2b972ce5b7011bec8c Mon Sep 17 00:00:00 2001 From: Sean Lynch <42618346+swlynch99@users.noreply.github.com> Date: Fri, 30 Sep 2022 15:28:29 -0700 Subject: [PATCH 03/11] Add entrystore entry for bloom server using http --- src/entrystore/Cargo.toml | 4 +++- src/entrystore/src/bloom/http.rs | 26 +++++++++++++++----------- src/entrystore/src/lib.rs | 2 ++ 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/src/entrystore/Cargo.toml b/src/entrystore/Cargo.toml index 2ba8c4c1e..875f2a69b 100644 --- a/src/entrystore/Cargo.toml +++ b/src/entrystore/Cargo.toml @@ -18,4 +18,6 @@ config = { path = "../config" } protocol-common = { path = "../protocol/common" } protocol-memcache = { path = "../protocol/memcache" } protocol-ping = { path = "../protocol/ping" } -seg = { path = "../storage/seg" } \ No newline at end of file +protocol-http = { path = "../protocol/http" } +seg = { path = "../storage/seg" } +bloom = { path = "../storage/bloom" } \ No newline at end of file diff --git a/src/entrystore/src/bloom/http.rs b/src/entrystore/src/bloom/http.rs index 1cc430581..70d3a2953 100644 --- a/src/entrystore/src/bloom/http.rs +++ b/src/entrystore/src/bloom/http.rs @@ -2,19 +2,17 @@ // Licensed under the Apache License, Version 2.0 // http://www.apache.org/licenses/LICENSE-2.0 -use std::borrow::Cow; - use protocol_common::Execute; use protocol_http::{ - request::{Request, RequestData}, - Headers, ParseResult, Response, Storage, + request::{ParseData, Request, RequestData}, + Headers, Response, Storage, }; use crate::Bloom; -impl Execute for Bloom { - fn execute(&mut self, result: &ParseResult) -> Response { - let request = match result { +impl Execute for Bloom { + fn execute(&mut self, data: &ParseData) -> Response { + let request = match &data.0 { Ok(request) => request, Err(e) => return e.to_response(), }; @@ -31,20 +29,26 @@ impl Execute for Bloom { impl Storage for Bloom { fn get(&mut self, key: &[u8], _headers: &Headers) -> Response { if self.data.contains(key) { - Response::builder(204).empty() + Response::builder(200).body(b"") } else { - Response::builder(404).empty() + Response::builder(404).body(b"") } } fn put(&mut self, key: &[u8], _value: &[u8], _headers: &Headers) -> Response { + let exists = self.data.contains(key); self.data.insert(key); - Response::builder(204).empty() + + if exists { + Response::builder(200).body(b"") + } else { + Response::builder(201).body(b"") + } } fn delete(&mut self, _key: &[u8], _headers: &Headers) -> Response { let mut builder = Response::builder(405); builder.header("Content-Type", b"text/plain"); - builder.body(Cow::Borrowed(b"DELETE method not supported")) + builder.body(b"DELETE method not supported") } } diff --git a/src/entrystore/src/lib.rs b/src/entrystore/src/lib.rs index ae3756838..569830fb2 100644 --- a/src/entrystore/src/lib.rs +++ b/src/entrystore/src/lib.rs @@ -7,9 +7,11 @@ //! addition to the base `EntryStore` trait. For example [`Seg`] implements both //! [`EntryStore`] and [`protocol::memcache::MemcacheStorage`]. +mod bloom; mod noop; mod seg; +pub use self::bloom::Bloom; pub use self::noop::*; pub use self::seg::*; From d2e780a4f79839cbd234e807307d1c41881138d7 Mon Sep 17 00:00:00 2001 From: Sean Lynch <42618346+swlynch99@users.noreply.github.com> Date: Fri, 30 Sep 2022 15:29:00 -0700 Subject: [PATCH 04/11] Add bloom cache server --- src/server/bloomcache/Cargo.toml | 4 +- src/server/bloomcache/src/lib.rs | 4 +- src/server/bloomcache/src/main.rs | 78 ++++++++++++++++++++++++++++++- 3 files changed, 81 insertions(+), 5 deletions(-) diff --git a/src/server/bloomcache/Cargo.toml b/src/server/bloomcache/Cargo.toml index e87eed9c3..020f1f517 100644 --- a/src/server/bloomcache/Cargo.toml +++ b/src/server/bloomcache/Cargo.toml @@ -1,12 +1,12 @@ [package] -name = "bloomcache" +name = "pelikan_bloomcache" version = "0.1.0" edition = "2021" [dependencies] backtrace = "0.3.56" -clap = "4.0.4" +clap = "2.33.3" common = { path = "../../common" } config = { path = "../../config" } entrystore = { path = "../../entrystore" } diff --git a/src/server/bloomcache/src/lib.rs b/src/server/bloomcache/src/lib.rs index 69f1ed49e..e63ec15aa 100644 --- a/src/server/bloomcache/src/lib.rs +++ b/src/server/bloomcache/src/lib.rs @@ -5,7 +5,7 @@ use config::*; use entrystore::Bloom; use logger::*; -use protocol_http::{ParseResult, RequestParser, Response}; +use protocol_http::{request::ParseData, RequestParser, Response}; use server::{Process, ProcessBuilder}; type Parser = RequestParser; @@ -24,7 +24,7 @@ impl BloomCache { let storage = Storage::new(&config)?; let parser = Parser::new(); - let builder = ProcessBuilder::::new( + let builder = ProcessBuilder::::new( &config, log, parser, storage, )? .version(env!("CARGO_PKG_VERSION")); diff --git a/src/server/bloomcache/src/main.rs b/src/server/bloomcache/src/main.rs index a010ba487..c5b87dfcf 100644 --- a/src/server/bloomcache/src/main.rs +++ b/src/server/bloomcache/src/main.rs @@ -6,7 +6,9 @@ extern crate logger; use backtrace::Backtrace; -use config::PingserverConfig; +use clap::{App, Arg}; +use config::BloomcacheConfig; +use pelikan_bloomcache::BloomCache; use rustcommon_metrics::*; use server::PERCENTILES; @@ -17,4 +19,78 @@ fn main() { eprintln!("{:?}", Backtrace::new()); std::process::exit(101); })); + + // parse command line options + let matches = App::new(env!("CARGO_BIN_NAME")) + .version(env!("CARGO_PKG_VERSION")) + .version_short("v") + .arg( + Arg::with_name("stats") + .short("s") + .long("stats") + .help("List all metrics in stats") + .takes_value(false), + ) + .arg( + Arg::with_name("CONFIG") + .help("Server configuration file") + .index(1), + ) + .get_matches(); + + if matches.is_present("stats") { + println!("{:<31} {:<15} DESCRIPTION", "NAME", "TYPE"); + + let mut metrics = Vec::new(); + + for metric in &rustcommon_metrics::metrics() { + let any = match metric.as_any() { + Some(any) => any, + None => { + continue; + } + }; + + if any.downcast_ref::().is_some() { + metrics.push(format!("{:<31} counter", metric.name())); + } else if any.downcast_ref::().is_some() { + metrics.push(format!("{:<31} gauge", metric.name())); + } else if any.downcast_ref::().is_some() { + for (label, _) in PERCENTILES { + let name = format!("{}_{}", metric.name(), label); + metrics.push(format!("{:<31} percentile", name)); + } + } else { + continue; + } + } + + metrics.sort(); + for metric in metrics { + println!("{}", metric); + } + std::process::exit(0); + } + + // load config from file + let config = if let Some(file) = matches.value_of("CONFIG") { + debug!("loading config: {}", file); + match BloomcacheConfig::load(file) { + Ok(c) => c, + Err(e) => { + println!("error launching bloomcache: {}", e); + std::process::exit(1); + } + } + } else { + Default::default() + }; + + match BloomCache::new(config) { + Ok(s) => s.wait(), + Err(e) => { + println!("error launching bloomcache: {}", e); + std::process::exit(1); + } + } } From ec6709f7722b76b8004d1cf70daab07186a62f86 Mon Sep 17 00:00:00 2001 From: Sean Lynch <42618346+swlynch99@users.noreply.github.com> Date: Fri, 30 Sep 2022 15:29:08 -0700 Subject: [PATCH 05/11] Update cargo files --- Cargo.lock | 191 +++++++++++++++++++++++++++++++++++++++++++++-------- Cargo.toml | 2 + 2 files changed, 167 insertions(+), 26 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2f636ab98..e1d365d1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,7 +31,7 @@ dependencies = [ "protocol-admin", "protocol-common", "queues", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", "session", "slab", "tiny_http", @@ -468,8 +468,8 @@ dependencies = [ "macros", "net", "rustcommon-logger", - "rustcommon-metrics", - "rustcommon-time", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-time 0.0.13 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", "serde", ] @@ -696,9 +696,11 @@ checksum = "90e5c1c8368803113bf0c9584fc495a58b86dc8a29edbf8fe877d21d9507e797" name = "entrystore" version = "0.3.0" dependencies = [ + "bloom", "common", "config", "protocol-common", + "protocol-http", "protocol-memcache", "protocol-ping", "seg", @@ -891,8 +893,18 @@ name = "heatmap" version = "0.0.0" source = "git+https://github.com/twitter/rustcommon?rev=87e0c87#87e0c87cc8ad42466ec0eeba0fee8742325e52c0" dependencies = [ - "histogram", - "rustcommon-time", + "histogram 0.0.0 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-time 0.0.13 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "thiserror", +] + +[[package]] +name = "heatmap" +version = "0.0.0" +source = "git+https://github.com/twitter/rustcommon#87e0c87cc8ad42466ec0eeba0fee8742325e52c0" +dependencies = [ + "histogram 0.0.0 (git+https://github.com/twitter/rustcommon)", + "rustcommon-time 0.0.13 (git+https://github.com/twitter/rustcommon)", "thiserror", ] @@ -922,6 +934,14 @@ dependencies = [ "thiserror", ] +[[package]] +name = "histogram" +version = "0.0.0" +source = "git+https://github.com/twitter/rustcommon#87e0c87cc8ad42466ec0eeba0fee8742325e52c0" +dependencies = [ + "thiserror", +] + [[package]] name = "http" version = "0.2.8" @@ -1339,7 +1359,7 @@ dependencies = [ "protocol-admin", "protocol-memcache", "protocol-resp", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", "session", "storage-types", "tokio", @@ -1366,7 +1386,7 @@ dependencies = [ "foreign-types-shared", "libc", "mio 0.8.4", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", ] [[package]] @@ -1512,6 +1532,21 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" +[[package]] +name = "pelikan_bloomcache" +version = "0.1.0" +dependencies = [ + "backtrace", + "clap 2.34.0", + "common", + "config", + "entrystore", + "logger", + "protocol-http", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon)", + "server", +] + [[package]] name = "pem" version = "1.1.0" @@ -1537,6 +1572,48 @@ dependencies = [ "indexmap", ] +[[package]] +name = "phf" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "928c6535de93548188ef63bb7c4036bd415cd8f36ad25af44b9789b2ee72a48c" +dependencies = [ + "phf_macros", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1181c94580fa345f50f19d738aaa39c0ed30a600d95cb2d3e23f94266f14fbf" +dependencies = [ + "phf_shared", + "rand", +] + +[[package]] +name = "phf_macros" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92aacdc5f16768709a569e913f7451034034178b05bdc8acda226659a3dccc66" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1fb5f6f826b772a8d4c0394209441e7d37cbbb967ae9c7e0e8134365c9ee676" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.0.12" @@ -1580,7 +1657,7 @@ dependencies = [ "logger", "protocol-ping", "proxy", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", ] [[package]] @@ -1595,7 +1672,7 @@ dependencies = [ "entrystore", "logger", "protocol-ping", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", "server", ] @@ -1715,7 +1792,7 @@ dependencies = [ "criterion 0.3.6", "logger", "protocol-common", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", "storage-types", ] @@ -1731,6 +1808,20 @@ dependencies = [ "storage-types", ] +[[package]] +name = "protocol-http" +version = "0.1.0" +dependencies = [ + "arrayvec 0.7.2", + "bytes 1.2.1", + "httparse", + "logger", + "phf", + "protocol-common", + "thiserror", + "urlencoding", +] + [[package]] name = "protocol-memcache" version = "0.3.0" @@ -1740,7 +1831,7 @@ dependencies = [ "logger", "nom 5.1.2", "protocol-common", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", ] [[package]] @@ -1752,7 +1843,7 @@ dependencies = [ "criterion 0.3.6", "logger", "protocol-common", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", "storage-types", ] @@ -1763,7 +1854,7 @@ dependencies = [ "common", "nom 5.1.2", "protocol-common", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", ] [[package]] @@ -1773,7 +1864,7 @@ dependencies = [ "common", "logger", "protocol-common", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", ] [[package]] @@ -1790,7 +1881,7 @@ dependencies = [ "protocol-admin", "protocol-common", "queues", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", "session", "slab", "waker", @@ -1961,8 +2052,8 @@ dependencies = [ "ahash", "log", "mpmc", - "rustcommon-metrics", - "rustcommon-time", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-time 0.0.13 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", ] [[package]] @@ -1970,12 +2061,25 @@ name = "rustcommon-metrics" version = "0.1.2" source = "git+https://github.com/twitter/rustcommon?rev=87e0c87#87e0c87cc8ad42466ec0eeba0fee8742325e52c0" dependencies = [ - "heatmap", + "heatmap 0.0.0 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", "linkme", "once_cell", "parking_lot", - "rustcommon-metrics-derive", - "rustcommon-time", + "rustcommon-metrics-derive 0.1.1 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-time 0.0.13 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", +] + +[[package]] +name = "rustcommon-metrics" +version = "0.1.2" +source = "git+https://github.com/twitter/rustcommon#87e0c87cc8ad42466ec0eeba0fee8742325e52c0" +dependencies = [ + "heatmap 0.0.0 (git+https://github.com/twitter/rustcommon)", + "linkme", + "once_cell", + "parking_lot", + "rustcommon-metrics-derive 0.1.1 (git+https://github.com/twitter/rustcommon)", + "rustcommon-time 0.0.13 (git+https://github.com/twitter/rustcommon)", ] [[package]] @@ -1989,6 +2093,17 @@ dependencies = [ "syn", ] +[[package]] +name = "rustcommon-metrics-derive" +version = "0.1.1" +source = "git+https://github.com/twitter/rustcommon#87e0c87cc8ad42466ec0eeba0fee8742325e52c0" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "rustcommon-time" version = "0.0.13" @@ -2001,6 +2116,18 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "rustcommon-time" +version = "0.0.13" +source = "git+https://github.com/twitter/rustcommon#87e0c87cc8ad42466ec0eeba0fee8742325e52c0" +dependencies = [ + "lazy_static", + "libc", + "mach", + "time 0.3.14", + "winapi 0.3.9", +] + [[package]] name = "rustls" version = "0.19.1" @@ -2103,7 +2230,7 @@ dependencies = [ "rand", "rand_chacha", "rand_xoshiro", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", "storage-types", "thiserror", ] @@ -2120,7 +2247,7 @@ dependencies = [ "entrystore", "logger", "protocol-memcache", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", "server", ] @@ -2179,7 +2306,7 @@ dependencies = [ "protocol-admin", "protocol-common", "queues", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", "session", "slab", "waker", @@ -2193,8 +2320,8 @@ dependencies = [ "log", "net", "protocol-common", - "rustcommon-metrics", - "rustcommon-time", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-time 0.0.13 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", ] [[package]] @@ -2224,6 +2351,12 @@ dependencies = [ "time 0.3.14", ] +[[package]] +name = "siphasher" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" + [[package]] name = "slab" version = "0.4.7" @@ -2360,7 +2493,7 @@ dependencies = [ "logger", "protocol-thrift", "proxy", - "rustcommon-metrics", + "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", ] [[package]] @@ -2723,6 +2856,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e8db7427f936968176eaa7cdf81b7f98b980b18495ec28f1b5791ac3bfe3eea9" + [[package]] name = "vec_map" version = "0.8.2" diff --git a/Cargo.toml b/Cargo.toml index 8499cf791..d08811f64 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ members = [ "src/net", "src/protocol/admin", "src/protocol/common", + "src/protocol/http", "src/protocol/memcache", "src/protocol/ping", "src/protocol/resp", @@ -27,6 +28,7 @@ members = [ "src/proxy/ping", "src/proxy/thrift", "src/queues", + "src/server/bloomcache", "src/server/pingserver", "src/server/segcache", "src/session", From 0f867031c99dc9a44157c22b218703c23e10b99b Mon Sep 17 00:00:00 2001 From: Sean Lynch <42618346+swlynch99@users.noreply.github.com> Date: Fri, 30 Sep 2022 15:29:47 -0700 Subject: [PATCH 06/11] Fix phantomdata in bloomfilter to keep bloomfilter sync --- src/storage/bloom/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/bloom/src/lib.rs b/src/storage/bloom/src/lib.rs index be69b75f1..acbec0c49 100644 --- a/src/storage/bloom/src/lib.rs +++ b/src/storage/bloom/src/lib.rs @@ -130,7 +130,7 @@ impl RawBloomFilter { pub struct BloomFilter { raw: RawBloomFilter, seed: u64, - _dummy: PhantomData<*const T>, + _dummy: PhantomData, } impl BloomFilter { From dcfc7fbdde116bee36e1859db395fbac0e511d4a Mon Sep 17 00:00:00 2001 From: Sean Lynch <42618346+swlynch99@users.noreply.github.com> Date: Fri, 30 Sep 2022 16:27:55 -0700 Subject: [PATCH 07/11] Add missing license to util.rs --- src/protocol/http/src/util.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/protocol/http/src/util.rs b/src/protocol/http/src/util.rs index b0899bf2a..b5277321d 100644 --- a/src/protocol/http/src/util.rs +++ b/src/protocol/http/src/util.rs @@ -1,3 +1,7 @@ +// Copyright 2021 Twitter, Inc. +// Licensed under the Apache License, Version 2.0 +// http://www.apache.org/licenses/LICENSE-2.0 + use std::io::Write; use bytes::buf::UninitSlice; From 7634047a2ac66b50f1c8642f86202ee2d2e30ff8 Mon Sep 17 00:00:00 2001 From: Sean Lynch <42618346+swlynch99@users.noreply.github.com> Date: Fri, 30 Sep 2022 16:28:49 -0700 Subject: [PATCH 08/11] Remove debug logging from util.rs --- src/protocol/http/src/util.rs | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/protocol/http/src/util.rs b/src/protocol/http/src/util.rs index b5277321d..4ac5dedbe 100644 --- a/src/protocol/http/src/util.rs +++ b/src/protocol/http/src/util.rs @@ -2,8 +2,6 @@ // Licensed under the Apache License, Version 2.0 // http://www.apache.org/licenses/LICENSE-2.0 -use std::io::Write; - use bytes::buf::UninitSlice; use protocol_common::BufMut; @@ -31,10 +29,6 @@ where } unsafe fn advance_mut(&mut self, cnt: usize) { - let data = self.chunk_mut(); - let slice = std::slice::from_raw_parts(data.as_mut_ptr(), cnt); - let _ = std::io::stdout().write_all(slice); - self.count += cnt; self.buf.advance_mut(cnt) } From 68464728d50428026828038fb4f1e78b131ceafb Mon Sep 17 00:00:00 2001 From: Sean Lynch <42618346+swlynch99@users.noreply.github.com> Date: Fri, 30 Sep 2022 17:20:01 -0700 Subject: [PATCH 09/11] Fix bad assert --- src/protocol/http/src/response.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/protocol/http/src/response.rs b/src/protocol/http/src/response.rs index 53a5c3a17..c0ad1f045 100644 --- a/src/protocol/http/src/response.rs +++ b/src/protocol/http/src/response.rs @@ -41,7 +41,7 @@ impl ResponseBuilder { } pub fn header(&mut self, key: &str, value: &[u8]) -> &mut Self { - assert!(self.headers.is_empty()); + assert!(!self.headers.is_empty()); self.headers.extend_from_slice(key.as_bytes()); self.headers.extend_from_slice(b": "); From 2ab7c1ebdfe404bd4e6e07fea0c9ae383304d253 Mon Sep 17 00:00:00 2001 From: Sean Lynch <42618346+swlynch99@users.noreply.github.com> Date: Thu, 6 Oct 2022 13:54:07 -0700 Subject: [PATCH 10/11] Update bloomcache's depedencies to be workspace dependencies --- Cargo.lock | 110 ++++++++----------------------- src/server/bloomcache/Cargo.toml | 17 +++-- 2 files changed, 38 insertions(+), 89 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4638612f0..5f7646b7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,7 +31,7 @@ dependencies = [ "protocol-admin", "protocol-common", "queues", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", "session", "slab", "tiny_http", @@ -486,8 +486,8 @@ dependencies = [ "macros", "net", "rustcommon-logger", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", - "rustcommon-time 0.0.13 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", + "rustcommon-time", "serde", ] @@ -911,18 +911,8 @@ name = "heatmap" version = "0.0.0" source = "git+https://github.com/twitter/rustcommon?rev=87e0c87#87e0c87cc8ad42466ec0eeba0fee8742325e52c0" dependencies = [ - "histogram 0.0.0 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", - "rustcommon-time 0.0.13 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", - "thiserror", -] - -[[package]] -name = "heatmap" -version = "0.0.0" -source = "git+https://github.com/twitter/rustcommon#87e0c87cc8ad42466ec0eeba0fee8742325e52c0" -dependencies = [ - "histogram 0.0.0 (git+https://github.com/twitter/rustcommon)", - "rustcommon-time 0.0.13 (git+https://github.com/twitter/rustcommon)", + "histogram", + "rustcommon-time", "thiserror", ] @@ -952,14 +942,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "histogram" -version = "0.0.0" -source = "git+https://github.com/twitter/rustcommon#87e0c87cc8ad42466ec0eeba0fee8742325e52c0" -dependencies = [ - "thiserror", -] - [[package]] name = "http" version = "0.2.8" @@ -1377,7 +1359,7 @@ dependencies = [ "protocol-admin", "protocol-memcache", "protocol-resp", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", "session", "storage-types", "tokio", @@ -1404,7 +1386,7 @@ dependencies = [ "foreign-types-shared", "libc", "mio 0.8.4", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", ] [[package]] @@ -1552,7 +1534,7 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "pelikan_bloomcache" -version = "0.1.0" +version = "0.3.0" dependencies = [ "backtrace", "clap 2.34.0", @@ -1561,7 +1543,7 @@ dependencies = [ "entrystore", "logger", "protocol-http", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon)", + "rustcommon-metrics", "server", ] @@ -1675,7 +1657,7 @@ dependencies = [ "logger", "protocol-ping", "proxy", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", ] [[package]] @@ -1690,7 +1672,7 @@ dependencies = [ "entrystore", "logger", "protocol-ping", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", "server", ] @@ -1810,7 +1792,7 @@ dependencies = [ "criterion 0.3.6", "logger", "protocol-common", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", "storage-types", ] @@ -1851,7 +1833,7 @@ dependencies = [ "logger", "nom 5.1.2", "protocol-common", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", ] [[package]] @@ -1863,7 +1845,7 @@ dependencies = [ "criterion 0.3.6", "logger", "protocol-common", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", "storage-types", ] @@ -1874,7 +1856,7 @@ dependencies = [ "common", "nom 5.1.2", "protocol-common", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", ] [[package]] @@ -1884,7 +1866,7 @@ dependencies = [ "common", "logger", "protocol-common", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", ] [[package]] @@ -1901,7 +1883,7 @@ dependencies = [ "protocol-admin", "protocol-common", "queues", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", "session", "slab", "waker", @@ -2072,8 +2054,8 @@ dependencies = [ "ahash", "log", "mpmc", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", - "rustcommon-time 0.0.13 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", + "rustcommon-time", ] [[package]] @@ -2081,25 +2063,12 @@ name = "rustcommon-metrics" version = "0.1.2" source = "git+https://github.com/twitter/rustcommon?rev=87e0c87#87e0c87cc8ad42466ec0eeba0fee8742325e52c0" dependencies = [ - "heatmap 0.0.0 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", - "linkme", - "once_cell", - "parking_lot", - "rustcommon-metrics-derive 0.1.1 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", - "rustcommon-time 0.0.13 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", -] - -[[package]] -name = "rustcommon-metrics" -version = "0.1.2" -source = "git+https://github.com/twitter/rustcommon#87e0c87cc8ad42466ec0eeba0fee8742325e52c0" -dependencies = [ - "heatmap 0.0.0 (git+https://github.com/twitter/rustcommon)", + "heatmap", "linkme", "once_cell", "parking_lot", - "rustcommon-metrics-derive 0.1.1 (git+https://github.com/twitter/rustcommon)", - "rustcommon-time 0.0.13 (git+https://github.com/twitter/rustcommon)", + "rustcommon-metrics-derive", + "rustcommon-time", ] [[package]] @@ -2113,17 +2082,6 @@ dependencies = [ "syn", ] -[[package]] -name = "rustcommon-metrics-derive" -version = "0.1.1" -source = "git+https://github.com/twitter/rustcommon#87e0c87cc8ad42466ec0eeba0fee8742325e52c0" -dependencies = [ - "proc-macro-crate", - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "rustcommon-time" version = "0.0.13" @@ -2136,18 +2094,6 @@ dependencies = [ "winapi 0.3.9", ] -[[package]] -name = "rustcommon-time" -version = "0.0.13" -source = "git+https://github.com/twitter/rustcommon#87e0c87cc8ad42466ec0eeba0fee8742325e52c0" -dependencies = [ - "lazy_static", - "libc", - "mach", - "time 0.3.14", - "winapi 0.3.9", -] - [[package]] name = "rustls" version = "0.19.1" @@ -2250,7 +2196,7 @@ dependencies = [ "rand", "rand_chacha", "rand_xoshiro", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", "storage-types", "thiserror", ] @@ -2267,7 +2213,7 @@ dependencies = [ "entrystore", "logger", "protocol-memcache", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", "server", ] @@ -2326,7 +2272,7 @@ dependencies = [ "protocol-admin", "protocol-common", "queues", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", "session", "slab", "waker", @@ -2340,8 +2286,8 @@ dependencies = [ "log", "net", "protocol-common", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", - "rustcommon-time 0.0.13 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", + "rustcommon-time", ] [[package]] @@ -2513,7 +2459,7 @@ dependencies = [ "logger", "protocol-thrift", "proxy", - "rustcommon-metrics 0.1.2 (git+https://github.com/twitter/rustcommon?rev=87e0c87)", + "rustcommon-metrics", ] [[package]] diff --git a/src/server/bloomcache/Cargo.toml b/src/server/bloomcache/Cargo.toml index 020f1f517..3a9e1c33a 100644 --- a/src/server/bloomcache/Cargo.toml +++ b/src/server/bloomcache/Cargo.toml @@ -1,16 +1,19 @@ [package] name = "pelikan_bloomcache" -version = "0.1.0" -edition = "2021" -[dependencies] +version = { workspace = true } +edition = { workspace = true } +homepage = { workspace = true } +repository = { workspace = true } +license = { workspace = true } -backtrace = "0.3.56" -clap = "2.33.3" +[dependencies] +backtrace = { workspace = true } +clap = { workspace = true } common = { path = "../../common" } config = { path = "../../config" } entrystore = { path = "../../entrystore" } logger = { path = "../../logger" } -protocol-http = { path="../../protocol/http" } -rustcommon-metrics = { git = "https://github.com/twitter/rustcommon" } +protocol-http = { path = "../../protocol/http" } +rustcommon-metrics = { workspace = true } server = { path = "../../core/server" } From eceae18934b9d131923fee99fa7f03bf63ef9b5c Mon Sep 17 00:00:00 2001 From: Sean Lynch <42618346+swlynch99@users.noreply.github.com> Date: Tue, 11 Oct 2022 16:22:37 -0700 Subject: [PATCH 11/11] Update error messages in bloomcache main to use eprintln --- src/server/bloomcache/src/main.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/server/bloomcache/src/main.rs b/src/server/bloomcache/src/main.rs index c5b87dfcf..1320f83e8 100644 --- a/src/server/bloomcache/src/main.rs +++ b/src/server/bloomcache/src/main.rs @@ -78,7 +78,7 @@ fn main() { match BloomcacheConfig::load(file) { Ok(c) => c, Err(e) => { - println!("error launching bloomcache: {}", e); + eprintln!("error launching bloomcache: {}", e); std::process::exit(1); } } @@ -89,7 +89,7 @@ fn main() { match BloomCache::new(config) { Ok(s) => s.wait(), Err(e) => { - println!("error launching bloomcache: {}", e); + eprintln!("error launching bloomcache: {}", e); std::process::exit(1); } }