From 14eb99f515da2c3627148a2ac2920a200140edff Mon Sep 17 00:00:00 2001 From: Manos Pitsidianakis Date: Fri, 13 Dec 2019 00:01:59 +0200 Subject: [PATCH] JMAP WIP #7 --- melib/src/backends/jmap/connection.rs | 35 +++++-- melib/src/backends/jmap/folder.rs | 18 +++- melib/src/backends/jmap/objects/email.rs | 117 ++++++++++++++++------- melib/src/backends/jmap/operations.rs | 22 +++-- melib/src/backends/jmap/protocol.rs | 96 +++++++++---------- melib/src/backends/jmap/rfc8620.rs | 100 ++++++++++++++++--- melib/src/email.rs | 14 +-- 7 files changed, 279 insertions(+), 123 deletions(-) diff --git a/melib/src/backends/jmap/connection.rs b/melib/src/backends/jmap/connection.rs index 66e91ca5..321aeb26 100644 --- a/melib/src/backends/jmap/connection.rs +++ b/melib/src/backends/jmap/connection.rs @@ -23,6 +23,7 @@ use super::*; #[derive(Debug)] pub struct JmapConnection { + pub session: JmapSession, pub request_no: Arc>, pub client: Arc>, pub online_status: Arc>, @@ -35,10 +36,6 @@ impl JmapConnection { pub fn new(server_conf: &JmapServerConf, online_status: Arc>) -> Result { use reqwest::header; let mut headers = header::HeaderMap::new(); - headers.insert( - header::AUTHORIZATION, - header::HeaderValue::from_static("fc32dffe-14e7-11ea-a277-2477037a1804"), - ); headers.insert( header::ACCEPT, header::HeaderValue::from_static("application/json"), @@ -51,12 +48,34 @@ impl JmapConnection { .danger_accept_invalid_certs(server_conf.danger_accept_invalid_certs) .default_headers(headers) .build()?; - - let res_text = client.get(&server_conf.server_hostname).send()?.text()?; + let req = client + .get(&server_conf.server_hostname) + .basic_auth( + &server_conf.server_username, + Some(&server_conf.server_password), + ) + .send()?; + let res_text = req.text()?; debug!(&res_text); + let session: JmapSession = serde_json::from_str(&res_text)?; + + if !session + .capabilities + .contains_key("urn:ietf:params:jmap:core") + { + return Err(MeliError::new(format!("Server {} did not return JMAP Core capability (urn:ietf:params:jmap:core). Returned capabilities were: {}", &server_conf.server_hostname, session.capabilities.keys().map(String::as_str).collect::>().join(", ")))); + } + if !session + .capabilities + .contains_key("urn:ietf:params:jmap:mail") + { + return Err(MeliError::new(format!("Server {} does not support JMAP Mail capability (urn:ietf:params:jmap:mail). Returned capabilities were: {}", &server_conf.server_hostname, session.capabilities.keys().map(String::as_str).collect::>().join(", ")))); + } + let server_conf = server_conf.clone(); Ok(JmapConnection { + session, request_no: Arc::new(Mutex::new(0)), client: Arc::new(Mutex::new(client)), online_status, @@ -65,4 +84,8 @@ impl JmapConnection { method_call_states: Arc::new(Mutex::new(Default::default())), }) } + + pub fn mail_account_id(&self) -> &Id { + &self.session.primary_accounts["urn:ietf:params:jmap:mail"] + } } diff --git a/melib/src/backends/jmap/folder.rs b/melib/src/backends/jmap/folder.rs index 1ce7da99..87e34c91 100644 --- a/melib/src/backends/jmap/folder.rs +++ b/melib/src/backends/jmap/folder.rs @@ -73,6 +73,22 @@ impl BackendFolder for JmapFolder { } fn special_usage(&self) -> SpecialUsageMailbox { - self.usage + match self.role.as_ref().map(String::as_str) { + Some("inbox") => SpecialUsageMailbox::Inbox, + Some("archive") => SpecialUsageMailbox::Archive, + Some("junk") => SpecialUsageMailbox::Junk, + Some("trash") => SpecialUsageMailbox::Trash, + Some("drafts") => SpecialUsageMailbox::Drafts, + Some("sent") => SpecialUsageMailbox::Sent, + Some(other) => { + debug!( + "unknown JMAP mailbox role for folder {}: {}", + self.path(), + other + ); + SpecialUsageMailbox::Normal + } + None => SpecialUsageMailbox::Normal, + } } } diff --git a/melib/src/backends/jmap/objects/email.rs b/melib/src/backends/jmap/objects/email.rs index ef252884..49a1ef60 100644 --- a/melib/src/backends/jmap/objects/email.rs +++ b/melib/src/backends/jmap/objects/email.rs @@ -139,17 +139,23 @@ pub struct EmailObject { #[serde(default)] received_at: String, #[serde(default)] + message_id: Vec, + #[serde(default)] to: Vec, #[serde(default)] - bcc: Vec, + bcc: Option>, #[serde(default)] - reply_to: Option, + reply_to: Option>, #[serde(default)] - cc: Vec, + cc: Option>, + #[serde(default)] + sender: Option>, #[serde(default)] from: Vec, #[serde(default)] - in_reply_to_email_id: Id, + in_reply_to: Option>, + #[serde(default)] + references: Option>, #[serde(default)] keywords: HashMap, #[serde(default)] @@ -166,13 +172,15 @@ pub struct EmailObject { #[serde(default)] preview: Option, #[serde(default)] - sent_at: String, + sent_at: Option, #[serde(default)] - subject: String, + subject: Option, #[serde(default)] text_body: Vec, #[serde(default)] thread_id: Id, + #[serde(flatten)] + extra: HashMap, } impl EmailObject { @@ -223,17 +231,19 @@ impl std::fmt::Display for EmailAddress { impl std::convert::From for crate::Envelope { fn from(mut t: EmailObject) -> crate::Envelope { let mut env = crate::Envelope::new(0); - env.set_date(std::mem::replace(&mut t.sent_at, String::new()).as_bytes()); + if let Some(ref mut sent_at) = t.sent_at { + env.set_date(std::mem::replace(sent_at, String::new()).as_bytes()); + } if let Ok(d) = crate::email::parser::date(env.date_as_str().as_bytes()) { env.set_datetime(d); } - if let Some(v) = t.headers.get("Message-ID").or(t.headers.get("Message-Id")) { + if let Some(v) = t.message_id.get(0) { env.set_message_id(v.as_bytes()); } - if let Some(v) = t.headers.get("In-Reply-To") { - env.set_in_reply_to(v.as_bytes()); - env.push_references(v.as_bytes()); + if let Some(ref in_reply_to) = t.in_reply_to { + env.set_in_reply_to(in_reply_to[0].as_bytes()); + env.push_references(in_reply_to[0].as_bytes()); } if let Some(v) = t.headers.get("References") { let parse_result = crate::email::parser::references(v.as_bytes()); @@ -251,7 +261,9 @@ impl std::convert::From for crate::Envelope { } } env.set_has_attachments(t.has_attachment); - env.set_subject(std::mem::replace(&mut t.subject, String::new()).into_bytes()); + if let Some(ref mut subject) = t.subject { + env.set_subject(std::mem::replace(subject, String::new()).into_bytes()); + } env.set_from( std::mem::replace(&mut t.from, Vec::new()) @@ -266,19 +278,23 @@ impl std::convert::From for crate::Envelope { .collect::>(), ); - env.set_cc( - std::mem::replace(&mut t.cc, Vec::new()) - .into_iter() - .map(|addr| addr.into()) - .collect::>(), - ); + if let Some(ref mut cc) = t.cc { + env.set_cc( + std::mem::replace(cc, Vec::new()) + .into_iter() + .map(|addr| addr.into()) + .collect::>(), + ); + } - env.set_bcc( - std::mem::replace(&mut t.bcc, Vec::new()) - .into_iter() - .map(|addr| addr.into()) - .collect::>(), - ); + if let Some(ref mut bcc) = t.bcc { + env.set_bcc( + std::mem::replace(bcc, Vec::new()) + .into_iter() + .map(|addr| addr.into()) + .collect::>(), + ); + } if env.references.is_some() { if let Some(pos) = env @@ -304,12 +320,21 @@ impl std::convert::From for crate::Envelope { #[serde(rename_all = "camelCase")] struct HtmlBody { blob_id: Id, + #[serde(default)] + charset: String, + #[serde(default)] cid: Option, - disposition: String, + #[serde(default)] + disposition: Option, + #[serde(default)] headers: Value, + #[serde(default)] language: Option>, + #[serde(default)] location: Option, + #[serde(default)] name: Option, + #[serde(default)] part_id: Option, size: u64, #[serde(alias = "type")] @@ -322,12 +347,21 @@ struct HtmlBody { #[serde(rename_all = "camelCase")] struct TextBody { blob_id: Id, + #[serde(default)] + charset: String, + #[serde(default)] cid: Option, - disposition: String, + #[serde(default)] + disposition: Option, + #[serde(default)] headers: Value, + #[serde(default)] language: Option>, + #[serde(default)] location: Option, + #[serde(default)] name: Option, + #[serde(default)] part_id: Option, size: u64, #[serde(alias = "type")] @@ -355,27 +389,34 @@ pub struct EmailQueryResponse { pub total: usize, } -#[derive(Deserialize, Serialize, Debug)] +#[derive(Serialize, Debug)] #[serde(rename_all = "camelCase")] -pub struct EmailQueryCall { - pub filter: EmailFilterCondition, /* "inMailboxes": [ folder.id ] },*/ +pub struct EmailQuery { + #[serde(flatten)] + pub query_call: Query, + //pub filter: EmailFilterCondition, /* "inMailboxes": [ folder.id ] },*/ pub collapse_threads: bool, - pub position: u64, - pub fetch_threads: bool, - pub fetch_messages: bool, - pub fetch_message_properties: Vec, } -impl Method for EmailQueryCall { +impl Method for EmailQuery { const NAME: &'static str = "Email/query"; } -impl EmailQueryCall { - pub const RESULT_FIELD_IDS: ResultField = - ResultField:: { +impl EmailQuery { + pub const RESULT_FIELD_IDS: ResultField = + ResultField:: { field: "/ids", _ph: PhantomData, }; + + pub fn new(query_call: Query) -> Self { + EmailQuery { + query_call, + collapse_threads: false, + } + } + + _impl!(collapse_threads: bool); } #[derive(Deserialize, Serialize, Debug)] @@ -388,10 +429,12 @@ pub struct EmailGet { #[serde(default = "bool_false")] pub fetch_text_body_values: bool, #[serde(default = "bool_false")] + #[serde(rename = "fetchHTMLBodyValues")] pub fetch_html_body_values: bool, #[serde(default = "bool_false")] pub fetch_all_body_values: bool, #[serde(default)] + #[serde(skip_serializing_if = "u64_zero")] pub max_body_value_bytes: u64, } diff --git a/melib/src/backends/jmap/operations.rs b/melib/src/backends/jmap/operations.rs index 29e6f04d..8ec3712f 100644 --- a/melib/src/backends/jmap/operations.rs +++ b/melib/src/backends/jmap/operations.rs @@ -71,12 +71,22 @@ impl BackendOp for JmapOp { && store_lck.byte_cache[&self.hash].bytes.is_some()) { let blob_id = &store_lck.blob_id_store[&self.hash]; - let res = self.connection - .client - .lock() - .unwrap() - .get(&format!("https://jmap-proxy.local/raw/fc32dffe-14e7-11ea-a277-2477037a1804/{blob_id}/{name}", blob_id = blob_id, name = "")) - .send(); + let res = self + .connection + .client + .lock() + .unwrap() + .get(&downloadRequestFormat( + &self.connection.session, + self.connection.mail_account_id(), + blob_id, + None, + )) + .basic_auth( + &self.connection.server_conf.server_username, + Some(&self.connection.server_conf.server_password), + ) + .send(); let res_text = res?.text()?; diff --git a/melib/src/backends/jmap/protocol.rs b/melib/src/backends/jmap/protocol.rs index 6ffc5316..4029e48c 100644 --- a/melib/src/backends/jmap/protocol.rs +++ b/melib/src/backends/jmap/protocol.rs @@ -104,32 +104,22 @@ impl Request { } } -#[derive(Serialize, Debug)] -#[serde(untagged)] -pub enum MethodCall { - #[serde(rename_all = "camelCase")] - EmailQuery { - filter: Vec, /* "inMailboxes": [ folder.id ] },*/ - collapse_threads: bool, - position: u64, - fetch_threads: bool, - fetch_messages: bool, - fetch_message_properties: Vec, - }, - MailboxGet {}, - Empty {}, -} - pub fn get_mailboxes(conn: &JmapConnection) -> Result> { let seq = get_request_no!(conn.request_no); let res = conn .client .lock() .unwrap() - .post("https://jmap-proxy.local/jmap/fc32dffe-14e7-11ea-a277-2477037a1804/") + .post(&conn.session.api_url) + .basic_auth( + &conn.server_conf.server_username, + Some(&conn.server_conf.server_password), + ) .json(&json!({ "using": ["urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"], - "methodCalls": [["Mailbox/get", {}, + "methodCalls": [["Mailbox/get", { + "accountId": conn.mail_account_id() + }, format!("#m{}",seq).as_str()]], })) .send(); @@ -191,14 +181,15 @@ pub struct JsonResponse<'a> { } pub fn get_message_list(conn: &JmapConnection, folder: &JmapFolder) -> Result> { - let email_call: EmailQueryCall = EmailQueryCall { - filter: EmailFilterCondition::new().in_mailbox(Some(folder.id.clone())), - collapse_threads: false, - position: 0, - fetch_threads: true, - fetch_messages: true, - fetch_message_properties: vec![], - }; + let email_call: EmailQuery = EmailQuery::new( + Query::new() + .account_id(conn.mail_account_id().to_string()) + .filter(Some( + EmailFilterCondition::new().in_mailbox(Some(folder.id.clone())), + )) + .position(0), + ) + .collapse_threads(false); let mut req = Request::new(conn.request_no.clone()); req.add_call(&email_call); @@ -207,7 +198,11 @@ pub fn get_message_list(conn: &JmapConnection, folder: &JmapFolder) -> Result Result>(), ))) - .account_id(conn.account_id.lock().unwrap().clone()), + .account_id(conn.mail_account_id().to_string()), ); let mut req = Request::new(conn.request_no.clone()); @@ -234,7 +229,11 @@ pub fn get_message(conn: &JmapConnection, ids: &[String]) -> Result>>, folder: &JmapFolder, ) -> Result> { - let email_query_call: EmailQueryCall = EmailQueryCall { - filter: EmailFilterCondition::new().in_mailbox(Some(folder.id.clone())), - collapse_threads: false, - position: 0, - fetch_threads: true, - fetch_messages: true, - fetch_message_properties: vec![ - MessageProperty::ThreadId, - MessageProperty::MailboxIds, - MessageProperty::IsUnread, - MessageProperty::IsFlagged, - MessageProperty::IsAnswered, - MessageProperty::IsDraft, - MessageProperty::HasAttachment, - MessageProperty::From, - MessageProperty::To, - MessageProperty::Subject, - MessageProperty::Preview, - ], - }; + let email_query_call: EmailQuery = EmailQuery::new( + Query::new() + .account_id(conn.mail_account_id().to_string()) + .filter(Some( + EmailFilterCondition::new().in_mailbox(Some(folder.id.clone())), + )) + .position(0), + ) + .collapse_threads(false); let mut req = Request::new(conn.request_no.clone()); let prev_seq = req.add_call(&email_query_call); @@ -282,9 +270,9 @@ pub fn get( Get::new() .ids(Some(JmapArgument::reference( prev_seq, - EmailQueryCall::RESULT_FIELD_IDS, + EmailQuery::RESULT_FIELD_IDS, ))) - .account_id(conn.account_id.lock().unwrap().clone()), + .account_id(conn.mail_account_id().to_string()), ); req.add_call(&email_call); @@ -293,7 +281,11 @@ pub fn get( .client .lock() .unwrap() - .post("https://jmap-proxy.local/jmap/fc32dffe-14e7-11ea-a277-2477037a1804/") + .post(&conn.session.api_url) + .basic_auth( + &conn.server_conf.server_username, + Some(&conn.server_conf.server_password), + ) .json(&req) .send(); diff --git a/melib/src/backends/jmap/rfc8620.rs b/melib/src/backends/jmap/rfc8620.rs index 69cf7b51..7f668418 100644 --- a/melib/src/backends/jmap/rfc8620.rs +++ b/melib/src/backends/jmap/rfc8620.rs @@ -20,6 +20,7 @@ */ use super::Id; +use crate::email::parser::BytesExt; use core::marker::PhantomData; use serde::de::DeserializeOwned; use serde::ser::{Serialize, SerializeStruct, Serializer}; @@ -41,30 +42,38 @@ pub trait Object { #[derive(Deserialize, Serialize, Debug)] #[serde(rename_all = "camelCase")] pub struct JmapSession { - capabilities: HashMap, - accounts: HashMap, - primary_accounts: Vec, - username: String, - api_url: String, - download_url: String, + pub capabilities: HashMap, + pub accounts: HashMap, + pub primary_accounts: HashMap, + pub username: String, + pub api_url: String, + pub download_url: String, - upload_url: String, - event_source_url: String, - state: String, + pub upload_url: String, + pub event_source_url: String, + pub state: String, #[serde(flatten)] - extra_properties: HashMap, + pub extra_properties: HashMap, } #[derive(Deserialize, Serialize, Debug)] #[serde(rename_all = "camelCase")] pub struct CapabilitiesObject { + #[serde(default)] max_size_upload: u64, + #[serde(default)] max_concurrent_upload: u64, + #[serde(default)] max_size_request: u64, + #[serde(default)] max_concurrent_requests: u64, + #[serde(default)] max_calls_in_request: u64, + #[serde(default)] max_objects_in_get: u64, + #[serde(default)] max_objects_in_set: u64, + #[serde(default)] collation_algorithms: Vec, } @@ -251,18 +260,19 @@ enum JmapError { #[derive(Serialize, Debug)] #[serde(rename_all = "camelCase")] -pub struct QueryCall, OBJ: Object> +pub struct Query, OBJ: Object> where OBJ: std::fmt::Debug + Serialize, { account_id: String, - filter: Option>, + filter: Option, sort: Option>, #[serde(default)] position: u64, #[serde(skip_serializing_if = "Option::is_none")] anchor: Option, #[serde(default)] + #[serde(skip_serializing_if = "u64_zero")] anchor_offset: u64, #[serde(skip_serializing_if = "Option::is_none")] limit: Option, @@ -272,7 +282,7 @@ where _ph: PhantomData<*const OBJ>, } -impl, OBJ: Object> QueryCall +impl, OBJ: Object> Query where OBJ: std::fmt::Debug + Serialize, { @@ -291,7 +301,7 @@ where } _impl!(account_id: String); - _impl!(filter: Option>); + _impl!(filter: Option); _impl!(sort: Option>); _impl!(position: u64); _impl!(anchor: Option); @@ -300,6 +310,10 @@ where _impl!(calculate_total: bool); } +pub fn u64_zero(num: &u64) -> bool { + *num == 0 +} + pub fn bool_false() -> bool { false } @@ -470,3 +484,61 @@ impl ChangesResponse { _impl!(get_mut updated_mut, updated: Vec); _impl!(get_mut destroyed_mut, destroyed: Vec); } + +pub fn downloadRequestFormat( + session: &JmapSession, + account_id: &Id, + blob_id: &Id, + name: Option, +) -> String { + // https://jmap.fastmail.com/download/{accountId}/{blobId}/{name} + let mut ret = String::with_capacity( + session.download_url.len() + + blob_id.len() + + name.as_ref().map(|n| n.len()).unwrap_or(0) + + account_id.len(), + ); + let mut prev_pos = 0; + + while let Some(pos) = session.download_url.as_bytes()[prev_pos..].find(b"{") { + ret.push_str(&session.download_url[prev_pos..prev_pos + pos]); + prev_pos += pos; + if session.download_url[prev_pos..].starts_with("{accountId}") { + ret.push_str(account_id); + prev_pos += "{accountId}".len(); + } else if session.download_url[prev_pos..].starts_with("{blobId}") { + ret.push_str(blob_id); + prev_pos += "{blobId}".len(); + } else if session.download_url[prev_pos..].starts_with("{name}") { + ret.push_str(name.as_ref().map(String::as_str).unwrap_or("")); + prev_pos += "{name}".len(); + } + } + if prev_pos != session.download_url.len() { + ret.push_str(&session.download_url[prev_pos..]); + } + ret +} + +pub fn uploadRequestFormat(session: &JmapSession, account_id: &Id) -> String { + //"uploadUrl": "https://jmap.fastmail.com/upload/{accountId}/", + let mut ret = String::with_capacity(session.upload_url.len() + account_id.len()); + let mut prev_pos = 0; + + while let Some(pos) = session.upload_url.as_bytes()[prev_pos..].find(b"{") { + ret.push_str(&session.upload_url[prev_pos..prev_pos + pos]); + prev_pos += pos; + if session.upload_url[prev_pos..].starts_with("{accountId}") { + ret.push_str(account_id); + prev_pos += "{accountId}".len(); + break; + } else { + ret.push('{'); + prev_pos += 1; + } + } + if prev_pos != session.upload_url.len() { + ret.push_str(&session.upload_url[prev_pos..]); + } + ret +} diff --git a/melib/src/email.rs b/melib/src/email.rs index 8c7e577b..48f7c76b 100644 --- a/melib/src/email.rs +++ b/melib/src/email.rs @@ -490,14 +490,14 @@ impl Envelope { self.subject = Some(new_val); } pub fn set_message_id(&mut self, new_val: &[u8]) { - let slice = match parser::message_id(new_val).to_full_result() { - Ok(v) => v, - Err(e) => { - debug!(e); - return; + match parser::message_id(new_val).to_full_result() { + Ok(slice) => { + self.message_id = MessageID::new(new_val, slice); } - }; - self.message_id = MessageID::new(new_val, slice); + Err(_) => { + self.message_id = MessageID::new(new_val, new_val); + } + } } pub fn push_references(&mut self, new_val: &[u8]) { let slice = match parser::message_id(new_val).to_full_result() {