melib/imap: implement OAUTH2 authentication

contrib/oauth2.py

@@ -0,0 +1,348 @@
+#!/usr/bin/env python3
+#
+# Copyright 2012 Google Inc.
+# Copyright 2020 Manos Pitsidianakis
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+     # http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Performs client tasks for testing IMAP OAuth2 authentication.
+
+To use this script, you'll need to have registered with Google as an OAuth
+application and obtained an OAuth client ID and client secret.
+See https://developers.google.com/identity/protocols/OAuth2 for instructions on
+registering and for documentation of the APIs invoked by this code.
+
+This script has 3 modes of operation.
+
+1. The first mode is used to generate and authorize an OAuth2 token, the
+first step in logging in via OAuth2.
+
+  oauth2 --user=xxx@gmail.com \
+    --client_id=1038[...].apps.googleusercontent.com \
+    --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ \
+    --generate_oauth2_token
+
+The script will converse with Google and generate an oauth request
+token, then present you with a URL you should visit in your browser to
+authorize the token. Once you get the verification code from the Google
+website, enter it into the script to get your OAuth access token. The output
+from this command will contain the access token, a refresh token, and some
+metadata about the tokens. The access token can be used until it expires, and
+the refresh token lasts indefinitely, so you should record these values for
+reuse.
+
+2. The script will generate new access tokens using a refresh token.
+
+  oauth2 --user=xxx@gmail.com \
+    --client_id=1038[...].apps.googleusercontent.com \
+    --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ \
+    --refresh_token=1/Yzm6MRy4q1xi7Dx2DuWXNgT6s37OrP_DW_IoyTum4YA
+
+3. The script will generate an OAuth2 string that can be fed
+directly to IMAP or SMTP. This is triggered with the --generate_oauth2_string
+option.
+
+  oauth2 --generate_oauth2_string --user=xxx@gmail.com \
+    --access_token=ya29.AGy[...]ezLg
+
+The output of this mode will be a base64-encoded string. To use it, connect to a
+IMAPFE and pass it as the second argument to the AUTHENTICATE command.
+
+  a AUTHENTICATE XOAUTH2 a9sha9sfs[...]9dfja929dk==
+"""
+
+import base64
+import imaplib
+import json
+from optparse import OptionParser
+import smtplib
+import sys
+import urllib.request, urllib.parse, urllib.error
+
+
+def SetupOptionParser():
+  # Usage message is the module's docstring.
+  parser = OptionParser(usage=__doc__)
+  parser.add_option('--generate_oauth2_token',
+                    action='store_true',
+                    dest='generate_oauth2_token',
+                    help='generates an OAuth2 token for testing')
+  parser.add_option('--generate_oauth2_string',
+                    action='store_true',
+                    dest='generate_oauth2_string',
+                    help='generates an initial client response string for '
+                         'OAuth2')
+  parser.add_option('--client_id',
+                    default=None,
+                    help='Client ID of the application that is authenticating. '
+                         'See OAuth2 documentation for details.')
+  parser.add_option('--client_secret',
+                    default=None,
+                    help='Client secret of the application that is '
+                         'authenticating. See OAuth2 documentation for '
+                         'details.')
+  parser.add_option('--access_token',
+                    default=None,
+                    help='OAuth2 access token')
+  parser.add_option('--refresh_token',
+                    default=None,
+                    help='OAuth2 refresh token')
+  parser.add_option('--scope',
+                    default='https://mail.google.com/',
+                    help='scope for the access token. Multiple scopes can be '
+                         'listed separated by spaces with the whole argument '
+                         'quoted.')
+  parser.add_option('--test_imap_authentication',
+                    action='store_true',
+                    dest='test_imap_authentication',
+                    help='attempts to authenticate to IMAP')
+  parser.add_option('--test_smtp_authentication',
+                    action='store_true',
+                    dest='test_smtp_authentication',
+                    help='attempts to authenticate to SMTP')
+  parser.add_option('--user',
+                    default=None,
+                    help='email address of user whose account is being '
+                         'accessed')
+  parser.add_option('--quiet',
+                    action='store_true',
+                    default=False,
+                    dest='quiet',
+                    help='Omit verbose descriptions and only print '
+                         'machine-readable outputs.')
+  return parser
+
+
+# The URL root for accessing Google Accounts.
+GOOGLE_ACCOUNTS_BASE_URL = 'https://accounts.google.com'
+
+
+# Hardcoded dummy redirect URI for non-web apps.
+REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'
+
+
+def AccountsUrl(command):
+  """Generates the Google Accounts URL.
+
+  Args:
+    command: The command to execute.
+
+  Returns:
+    A URL for the given command.
+  """
+  return '%s/%s' % (GOOGLE_ACCOUNTS_BASE_URL, command)
+
+
+def UrlEscape(text):
+  # See OAUTH 5.1 for a definition of which characters need to be escaped.
+  return urllib.parse.quote(text, safe='~-._')
+
+
+def UrlUnescape(text):
+  # See OAUTH 5.1 for a definition of which characters need to be escaped.
+  return urllib.parse.unquote(text)
+
+
+def FormatUrlParams(params):
+  """Formats parameters into a URL query string.
+
+  Args:
+    params: A key-value map.
+
+  Returns:
+    A URL query string version of the given parameters.
+  """
+  param_fragments = []
+  for param in sorted(iter(params.items()), key=lambda x: x[0]):
+    param_fragments.append('%s=%s' % (param[0], UrlEscape(param[1])))
+  return '&'.join(param_fragments)
+
+
+def GeneratePermissionUrl(client_id, scope='https://mail.google.com/'):
+  """Generates the URL for authorizing access.
+
+  This uses the "OAuth2 for Installed Applications" flow described at
+  https://developers.google.com/accounts/docs/OAuth2InstalledApp
+
+  Args:
+    client_id: Client ID obtained by registering your app.
+    scope: scope for access token, e.g. 'https://mail.google.com'
+  Returns:
+    A URL that the user should visit in their browser.
+  """
+  params = {}
+  params['client_id'] = client_id
+  params['redirect_uri'] = REDIRECT_URI
+  params['scope'] = scope
+  params['response_type'] = 'code'
+  return '%s?%s' % (AccountsUrl('o/oauth2/auth'),
+                    FormatUrlParams(params))
+
+
+def AuthorizeTokens(client_id, client_secret, authorization_code):
+  """Obtains OAuth access token and refresh token.
+
+  This uses the application portion of the "OAuth2 for Installed Applications"
+  flow at https://developers.google.com/accounts/docs/OAuth2InstalledApp#handlingtheresponse
+
+  Args:
+    client_id: Client ID obtained by registering your app.
+    client_secret: Client secret obtained by registering your app.
+    authorization_code: code generated by Google Accounts after user grants
+        permission.
+  Returns:
+    The decoded response from the Google Accounts server, as a dict. Expected
+    fields include 'access_token', 'expires_in', and 'refresh_token'.
+  """
+  params = {}
+  params['client_id'] = client_id
+  params['client_secret'] = client_secret
+  params['code'] = authorization_code
+  params['redirect_uri'] = REDIRECT_URI
+  params['grant_type'] = 'authorization_code'
+  request_url = AccountsUrl('o/oauth2/token')
+
+  response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode()).read()
+  return json.loads(response)
+
+
+def RefreshToken(client_id, client_secret, refresh_token):
+  """Obtains a new token given a refresh token.
+
+  See https://developers.google.com/accounts/docs/OAuth2InstalledApp#refresh
+
+  Args:
+    client_id: Client ID obtained by registering your app.
+    client_secret: Client secret obtained by registering your app.
+    refresh_token: A previously-obtained refresh token.
+  Returns:
+    The decoded response from the Google Accounts server, as a dict. Expected
+    fields include 'access_token', 'expires_in', and 'refresh_token'.
+  """
+  params = {}
+  params['client_id'] = client_id
+  params['client_secret'] = client_secret
+  params['refresh_token'] = refresh_token
+  params['grant_type'] = 'refresh_token'
+  request_url = AccountsUrl('o/oauth2/token')
+
+  response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode()).read()
+  return json.loads(response)
+
+
+def GenerateOAuth2String(username, access_token, base64_encode=True):
+  """Generates an IMAP OAuth2 authentication string.
+
+  See https://developers.google.com/google-apps/gmail/oauth2_overview
+
+  Args:
+    username: the username (email address) of the account to authenticate
+    access_token: An OAuth2 access token.
+    base64_encode: Whether to base64-encode the output.
+
+  Returns:
+    The SASL argument for the OAuth2 mechanism.
+  """
+  auth_string = 'user=%s\1auth=Bearer %s\1\1' % (username, access_token)
+  if base64_encode:
+    auth_string = base64.b64encode(bytes(auth_string, 'utf-8'))
+  return auth_string
+
+
+def TestImapAuthentication(user, auth_string):
+  """Authenticates to IMAP with the given auth_string.
+
+  Prints a debug trace of the attempted IMAP connection.
+
+  Args:
+    user: The Gmail username (full email address)
+    auth_string: A valid OAuth2 string, as returned by GenerateOAuth2String.
+        Must not be base64-encoded, since imaplib does its own base64-encoding.
+  """
+  print()
+  imap_conn = imaplib.IMAP4_SSL('imap.gmail.com')
+  imap_conn.debug = 4
+  imap_conn.authenticate('XOAUTH2', lambda x: auth_string)
+  imap_conn.select('INBOX')
+
+
+def TestSmtpAuthentication(user, auth_string):
+  """Authenticates to SMTP with the given auth_string.
+
+  Args:
+    user: The Gmail username (full email address)
+    auth_string: A valid OAuth2 string, not base64-encoded, as returned by
+        GenerateOAuth2String.
+  """
+  print()
+  smtp_conn = smtplib.SMTP('smtp.gmail.com', 587)
+  smtp_conn.set_debuglevel(True)
+  smtp_conn.ehlo('test')
+  smtp_conn.starttls()
+  smtp_conn.docmd('AUTH', 'XOAUTH2 ' + base64.b64encode(auth_string))
+
+
+def RequireOptions(options, *args):
+  missing = [arg for arg in args if getattr(options, arg) is None]
+  if missing:
+    print('Missing options: %s' % ' '.join(missing), file=sys.stderr)
+    sys.exit(-1)
+
+
+def main(argv):
+  options_parser = SetupOptionParser()
+  (options, args) = options_parser.parse_args()
+  if options.refresh_token:
+    RequireOptions(options, 'client_id', 'client_secret')
+    response = RefreshToken(options.client_id, options.client_secret,
+                            options.refresh_token)
+    if options.quiet:
+      print(response['access_token'])
+    else:
+      print('Access Token: %s' % response['access_token'])
+      print('Access Token Expiration Seconds: %s' % response['expires_in'])
+  elif options.generate_oauth2_string:
+    RequireOptions(options, 'user', 'access_token')
+    oauth2_string = GenerateOAuth2String(options.user, options.access_token)
+    if options.quiet:
+      print(oauth2_string.decode('utf-8'))
+    else:
+      print('OAuth2 argument:\n' + oauth2_string.decode('utf-8'))
+  elif options.generate_oauth2_token:
+    RequireOptions(options, 'client_id', 'client_secret')
+    print('To authorize token, visit this url and follow the directions:')
+    print('  %s' % GeneratePermissionUrl(options.client_id, options.scope))
+    authorization_code = input('Enter verification code: ')
+    response = AuthorizeTokens(options.client_id, options.client_secret,
+                                authorization_code)
+    print('Refresh Token: %s' % response['refresh_token'])
+    print('Access Token: %s' % response['access_token'])
+    print('Access Token Expiration Seconds: %s' % response['expires_in'])
+  elif options.test_imap_authentication:
+    RequireOptions(options, 'user', 'access_token')
+    TestImapAuthentication(options.user,
+        GenerateOAuth2String(options.user, options.access_token,
+                             base64_encode=False))
+  elif options.test_smtp_authentication:
+    RequireOptions(options, 'user', 'access_token')
+    TestSmtpAuthentication(options.user,
+        GenerateOAuth2String(options.user, options.access_token,
+                             base64_encode=False))
+  else:
+    options_parser.print_help()
+    print('Nothing to do, exiting.')
+    return
+
+
+if __name__ == '__main__':
+  main(sys.argv)

docs/meli.conf.5

@@ -235,11 +235,25 @@ Do not validate TLS certificates.
 Use IDLE extension.
 .\" default value
 .Pq Em true
+.It Ic use_condstore Ar boolean
+.Pq Em optional
+Use CONDSTORE extension.
+.\" default value
+.Pq Em true
 .It Ic use_deflate Ar boolean
 .Pq Em optional
 Use COMPRESS=DEFLATE extension (if built with DEFLATE support).
 .\" default value
 .Pq Em true
+.It Ic use_oauth2 Ar boolean
+.Pq Em optional
+Use OAUTH2 authentication.
+Can only be used with
+.Ic server_password_command
+which should return a base64-encoded OAUTH2 token ready to be passed to IMAP.
+For help on setup with Gmail, see Gmail section below.
+.\" default value
+.Pq Em false
 .It Ic timeout Ar integer
 .Pq Em optional
 Timeout to use for server connections in seconds.
@@ -247,6 +261,35 @@ A timeout of 0 seconds means there's no timeout.
 .\" default value
 .Pq Em 16
 .El
+.Ss Gmail
+Gmail has non-standard IMAP behaviors that need to be worked around.
+.Ss Gmail - sending mail
+Option
+.Ic store_sent_mail
+should be disabled since Gmail auto-saves sent mail by its own.
+.Ss Gmail OAUTH2
+To use OAUTH2, you must go through a process to register your own private "application" with Google that can use OAUTH2 tokens.
+For convenience in the meli repository under the
+.Pa contrib/
+directory you can find a python3 file named oauth2.py to generate and request the appropriate data to perform OAUTH2 authentication.
+Steps:
+.Bl -bullet -compact
+.It
+In Google APIs, create a custom OAuth client ID and note down the Client ID and Client Secret.
+You may need to create a consent screen; follow the steps described in the website.
+.It
+Run the oauth2.py script as follows (after adjusting binary paths and credentials):
+.Cm python3 oauth2.py --user=xxx@gmail.com --client_id=1038[...].apps.googleusercontent.com --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ --generate_oauth2_token
+and follow the instructions.
+Note down the refresh token.
+.It
+In
+.Ic server_password_command
+enter a command like this (after adjusting binary paths and credentials):
+.Cm TOKEN=$(python3 oauth2.py --user=xxx@gmail.com --quiet --client_id=1038[...].apps.googleusercontent.com --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ --refresh_token=1/Yzm6MRy4q1xi7Dx2DuWXNgT6s37OrP_DW_IoyTum4YA) && python3 oauth2.py --user=xxx@gmail.com --generate_oauth2_string --quiet --access_token=$TOKEN
+.It
+On startup, meli should evaluate this command which if successful must only return a base64-encoded token ready to be passed to IMAP.
+.El
 .Ss JMAP only
 JMAP specific options
 .Bl -tag -width 36n

melib/src/backends/imap.rs

@@ -64,6 +64,7 @@ pub type UIDVALIDITY = UID;
 pub type MessageSequenceNumber = ImapNum;
 
 pub static SUPPORTED_CAPABILITIES: &[&str] = &[
+    "AUTH=OAUTH2",
     #[cfg(feature = "deflate_compression")]
     "COMPRESS=DEFLATE",
     "CONDSTORE",
@@ -232,6 +233,7 @@ impl MailBackend for ImapType {
                     #[cfg(feature = "deflate_compression")]
                     deflate,
                     condstore,
+                    oauth2,
                 },
         } = self.server_conf.protocol
         {
@@ -273,6 +275,15 @@ impl MailBackend for ImapType {
                             };
                         }
                     }
+                    "AUTH=OAUTH2" => {
+                        if oauth2 {
+                            *status = MailBackendExtensionStatus::Enabled { comment: None };
+                        } else {
+                            *status = MailBackendExtensionStatus::Supported {
+                                comment: Some("Disabled by user configuration"),
+                            };
+                        }
+                    }
                     _ => {
                         if SUPPORTED_CAPABILITIES
                             .iter()
@@ -1218,7 +1229,14 @@ impl ImapType {
     ) -> Result<Box<dyn MailBackend>> {
         let server_hostname = get_conf_val!(s["server_hostname"])?;
         let server_username = get_conf_val!(s["server_username"])?;
+        let use_oauth2: bool = get_conf_val!(s["use_oauth2"], false)?;
         let server_password = if !s.extra.contains_key("server_password_command") {
+            if use_oauth2 {
+                return Err(MeliError::new(format!(
+                    "({}) `use_oauth2` use requires `server_password_command` set with a command that returns an OAUTH2 token. Consult documentation for guidance.",
+                    s.name,
+                )));
+            }
             get_conf_val!(s["server_password"])?.to_string()
         } else {
             let invocation = get_conf_val!(s["server_password_command"])?;
@@ -1275,6 +1293,7 @@ impl ImapType {
                     condstore: get_conf_val!(s["use_condstore"], true)?,
                     #[cfg(feature = "deflate_compression")]
                     deflate: get_conf_val!(s["use_deflate"], true)?,
+                    oauth2: use_oauth2,
                 },
             },
             timeout,
@@ -1463,7 +1482,14 @@ impl ImapType {
     pub fn validate_config(s: &AccountSettings) -> Result<()> {
         get_conf_val!(s["server_hostname"])?;
         get_conf_val!(s["server_username"])?;
+        let use_oauth2: bool = get_conf_val!(s["use_oauth2"], false)?;
         if !s.extra.contains_key("server_password_command") {
+            if use_oauth2 {
+                return Err(MeliError::new(format!(
+                    "({}) `use_oauth2` use requires `server_password_command` set with a command that returns an OAUTH2 token. Consult documentation for guidance.",
+                    s.name,
+                )));
+            }
             get_conf_val!(s["server_password"])?;
         } else if s.extra.contains_key("server_password") {
             return Err(MeliError::new(format!(

melib/src/backends/imap/connection.rs

@@ -63,6 +63,7 @@ pub struct ImapExtensionUse {
     pub idle: bool,
     #[cfg(feature = "deflate_compression")]
     pub deflate: bool,
+    pub oauth2: bool,
 }
 
 impl Default for ImapExtensionUse {
@@ -72,6 +73,7 @@ impl Default for ImapExtensionUse {
             idle: true,
             #[cfg(feature = "deflate_compression")]
             deflate: true,
+            oauth2: false,
         }
     }
 }
@@ -351,16 +353,39 @@ impl ImapStream {
             .set_err_kind(crate::error::ErrorKind::Authentication));
         }
 
-        let mut capabilities = None;
-        ret.send_command(
-            format!(
-                "LOGIN \"{}\" \"{}\"",
-                &server_conf.server_username, &server_conf.server_password
-            )
-            .as_bytes(),
-        )
-        .await?;
+        match server_conf.protocol {
+            ImapProtocol::IMAP {
+                extension_use: ImapExtensionUse { oauth2, .. },
+            } if oauth2 => {
+                if !capabilities
+                    .iter()
+                    .any(|cap| cap.eq_ignore_ascii_case(b"AUTH=XOAUTH2"))
+                {
+                    return Err(MeliError::new(format!(
+                                "Could not connect to {}: OAUTH2 is enabled but server did not return AUTH=XOAUTH2 capability. Returned capabilities were: {}",
+                                &server_conf.server_hostname,
+                                capabilities.iter().map(|capability|
+                                    String::from_utf8_lossy(capability).to_string()).collect::<Vec<String>>().join(" ")
+                    )));
+                }
+                ret.send_command(
+                    format!("AUTHENTICATE XOAUTH2 {}", &server_conf.server_password).as_bytes(),
+                )
+                .await?;
+            }
+            _ => {
+                ret.send_command(
+                    format!(
+                        "LOGIN \"{}\" \"{}\"",
+                        &server_conf.server_username, &server_conf.server_password
+                    )
+                    .as_bytes(),
+                )
+                .await?;
+            }
+        }
         let tag_start = format!("M{} ", (ret.cmd_id - 1));
+        let mut capabilities = None;
 
         loop {
             ret.read_lines(&mut res, &[], false).await?;
@@ -604,6 +629,7 @@ impl ImapConnection {
                             #[cfg(feature = "deflate_compression")]
                             deflate,
                             idle: _idle,
+                            oauth2: _,
                         },
                 } => {
                     if capabilities.contains(&b"CONDSTORE"[..]) && condstore {