melib/imap: implement OAUTH2 authentication

jmap-eventsource
Manos Pitsidianakis 2020-11-24 12:04:04 +02:00
parent 0a8a0c04c8
commit 92c12d3526
Signed by: Manos Pitsidianakis
GPG Key ID: 73627C2F690DF710
4 changed files with 452 additions and 9 deletions

348
contrib/oauth2.py 100755
View File

@ -0,0 +1,348 @@
#!/usr/bin/env python3
#
# Copyright 2012 Google Inc.
# Copyright 2020 Manos Pitsidianakis
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Performs client tasks for testing IMAP OAuth2 authentication.
To use this script, you'll need to have registered with Google as an OAuth
application and obtained an OAuth client ID and client secret.
See https://developers.google.com/identity/protocols/OAuth2 for instructions on
registering and for documentation of the APIs invoked by this code.
This script has 3 modes of operation.
1. The first mode is used to generate and authorize an OAuth2 token, the
first step in logging in via OAuth2.
oauth2 --user=xxx@gmail.com \
--client_id=1038[...].apps.googleusercontent.com \
--client_secret=VWFn8LIKAMC-MsjBMhJeOplZ \
--generate_oauth2_token
The script will converse with Google and generate an oauth request
token, then present you with a URL you should visit in your browser to
authorize the token. Once you get the verification code from the Google
website, enter it into the script to get your OAuth access token. The output
from this command will contain the access token, a refresh token, and some
metadata about the tokens. The access token can be used until it expires, and
the refresh token lasts indefinitely, so you should record these values for
reuse.
2. The script will generate new access tokens using a refresh token.
oauth2 --user=xxx@gmail.com \
--client_id=1038[...].apps.googleusercontent.com \
--client_secret=VWFn8LIKAMC-MsjBMhJeOplZ \
--refresh_token=1/Yzm6MRy4q1xi7Dx2DuWXNgT6s37OrP_DW_IoyTum4YA
3. The script will generate an OAuth2 string that can be fed
directly to IMAP or SMTP. This is triggered with the --generate_oauth2_string
option.
oauth2 --generate_oauth2_string --user=xxx@gmail.com \
--access_token=ya29.AGy[...]ezLg
The output of this mode will be a base64-encoded string. To use it, connect to a
IMAPFE and pass it as the second argument to the AUTHENTICATE command.
a AUTHENTICATE XOAUTH2 a9sha9sfs[...]9dfja929dk==
"""
import base64
import imaplib
import json
from optparse import OptionParser
import smtplib
import sys
import urllib.request, urllib.parse, urllib.error
def SetupOptionParser():
# Usage message is the module's docstring.
parser = OptionParser(usage=__doc__)
parser.add_option('--generate_oauth2_token',
action='store_true',
dest='generate_oauth2_token',
help='generates an OAuth2 token for testing')
parser.add_option('--generate_oauth2_string',
action='store_true',
dest='generate_oauth2_string',
help='generates an initial client response string for '
'OAuth2')
parser.add_option('--client_id',
default=None,
help='Client ID of the application that is authenticating. '
'See OAuth2 documentation for details.')
parser.add_option('--client_secret',
default=None,
help='Client secret of the application that is '
'authenticating. See OAuth2 documentation for '
'details.')
parser.add_option('--access_token',
default=None,
help='OAuth2 access token')
parser.add_option('--refresh_token',
default=None,
help='OAuth2 refresh token')
parser.add_option('--scope',
default='https://mail.google.com/',
help='scope for the access token. Multiple scopes can be '
'listed separated by spaces with the whole argument '
'quoted.')
parser.add_option('--test_imap_authentication',
action='store_true',
dest='test_imap_authentication',
help='attempts to authenticate to IMAP')
parser.add_option('--test_smtp_authentication',
action='store_true',
dest='test_smtp_authentication',
help='attempts to authenticate to SMTP')
parser.add_option('--user',
default=None,
help='email address of user whose account is being '
'accessed')
parser.add_option('--quiet',
action='store_true',
default=False,
dest='quiet',
help='Omit verbose descriptions and only print '
'machine-readable outputs.')
return parser
# The URL root for accessing Google Accounts.
GOOGLE_ACCOUNTS_BASE_URL = 'https://accounts.google.com'
# Hardcoded dummy redirect URI for non-web apps.
REDIRECT_URI = 'urn:ietf:wg:oauth:2.0:oob'
def AccountsUrl(command):
"""Generates the Google Accounts URL.
Args:
command: The command to execute.
Returns:
A URL for the given command.
"""
return '%s/%s' % (GOOGLE_ACCOUNTS_BASE_URL, command)
def UrlEscape(text):
# See OAUTH 5.1 for a definition of which characters need to be escaped.
return urllib.parse.quote(text, safe='~-._')
def UrlUnescape(text):
# See OAUTH 5.1 for a definition of which characters need to be escaped.
return urllib.parse.unquote(text)
def FormatUrlParams(params):
"""Formats parameters into a URL query string.
Args:
params: A key-value map.
Returns:
A URL query string version of the given parameters.
"""
param_fragments = []
for param in sorted(iter(params.items()), key=lambda x: x[0]):
param_fragments.append('%s=%s' % (param[0], UrlEscape(param[1])))
return '&'.join(param_fragments)
def GeneratePermissionUrl(client_id, scope='https://mail.google.com/'):
"""Generates the URL for authorizing access.
This uses the "OAuth2 for Installed Applications" flow described at
https://developers.google.com/accounts/docs/OAuth2InstalledApp
Args:
client_id: Client ID obtained by registering your app.
scope: scope for access token, e.g. 'https://mail.google.com'
Returns:
A URL that the user should visit in their browser.
"""
params = {}
params['client_id'] = client_id
params['redirect_uri'] = REDIRECT_URI
params['scope'] = scope
params['response_type'] = 'code'
return '%s?%s' % (AccountsUrl('o/oauth2/auth'),
FormatUrlParams(params))
def AuthorizeTokens(client_id, client_secret, authorization_code):
"""Obtains OAuth access token and refresh token.
This uses the application portion of the "OAuth2 for Installed Applications"
flow at https://developers.google.com/accounts/docs/OAuth2InstalledApp#handlingtheresponse
Args:
client_id: Client ID obtained by registering your app.
client_secret: Client secret obtained by registering your app.
authorization_code: code generated by Google Accounts after user grants
permission.
Returns:
The decoded response from the Google Accounts server, as a dict. Expected
fields include 'access_token', 'expires_in', and 'refresh_token'.
"""
params = {}
params['client_id'] = client_id
params['client_secret'] = client_secret
params['code'] = authorization_code
params['redirect_uri'] = REDIRECT_URI
params['grant_type'] = 'authorization_code'
request_url = AccountsUrl('o/oauth2/token')
response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode()).read()
return json.loads(response)
def RefreshToken(client_id, client_secret, refresh_token):
"""Obtains a new token given a refresh token.
See https://developers.google.com/accounts/docs/OAuth2InstalledApp#refresh
Args:
client_id: Client ID obtained by registering your app.
client_secret: Client secret obtained by registering your app.
refresh_token: A previously-obtained refresh token.
Returns:
The decoded response from the Google Accounts server, as a dict. Expected
fields include 'access_token', 'expires_in', and 'refresh_token'.
"""
params = {}
params['client_id'] = client_id
params['client_secret'] = client_secret
params['refresh_token'] = refresh_token
params['grant_type'] = 'refresh_token'
request_url = AccountsUrl('o/oauth2/token')
response = urllib.request.urlopen(request_url, urllib.parse.urlencode(params).encode()).read()
return json.loads(response)
def GenerateOAuth2String(username, access_token, base64_encode=True):
"""Generates an IMAP OAuth2 authentication string.
See https://developers.google.com/google-apps/gmail/oauth2_overview
Args:
username: the username (email address) of the account to authenticate
access_token: An OAuth2 access token.
base64_encode: Whether to base64-encode the output.
Returns:
The SASL argument for the OAuth2 mechanism.
"""
auth_string = 'user=%s\1auth=Bearer %s\1\1' % (username, access_token)
if base64_encode:
auth_string = base64.b64encode(bytes(auth_string, 'utf-8'))
return auth_string
def TestImapAuthentication(user, auth_string):
"""Authenticates to IMAP with the given auth_string.
Prints a debug trace of the attempted IMAP connection.
Args:
user: The Gmail username (full email address)
auth_string: A valid OAuth2 string, as returned by GenerateOAuth2String.
Must not be base64-encoded, since imaplib does its own base64-encoding.
"""
print()
imap_conn = imaplib.IMAP4_SSL('imap.gmail.com')
imap_conn.debug = 4
imap_conn.authenticate('XOAUTH2', lambda x: auth_string)
imap_conn.select('INBOX')
def TestSmtpAuthentication(user, auth_string):
"""Authenticates to SMTP with the given auth_string.
Args:
user: The Gmail username (full email address)
auth_string: A valid OAuth2 string, not base64-encoded, as returned by
GenerateOAuth2String.
"""
print()
smtp_conn = smtplib.SMTP('smtp.gmail.com', 587)
smtp_conn.set_debuglevel(True)
smtp_conn.ehlo('test')
smtp_conn.starttls()
smtp_conn.docmd('AUTH', 'XOAUTH2 ' + base64.b64encode(auth_string))
def RequireOptions(options, *args):
missing = [arg for arg in args if getattr(options, arg) is None]
if missing:
print('Missing options: %s' % ' '.join(missing), file=sys.stderr)
sys.exit(-1)
def main(argv):
options_parser = SetupOptionParser()
(options, args) = options_parser.parse_args()
if options.refresh_token:
RequireOptions(options, 'client_id', 'client_secret')
response = RefreshToken(options.client_id, options.client_secret,
options.refresh_token)
if options.quiet:
print(response['access_token'])
else:
print('Access Token: %s' % response['access_token'])
print('Access Token Expiration Seconds: %s' % response['expires_in'])
elif options.generate_oauth2_string:
RequireOptions(options, 'user', 'access_token')
oauth2_string = GenerateOAuth2String(options.user, options.access_token)
if options.quiet:
print(oauth2_string.decode('utf-8'))
else:
print('OAuth2 argument:\n' + oauth2_string.decode('utf-8'))
elif options.generate_oauth2_token:
RequireOptions(options, 'client_id', 'client_secret')
print('To authorize token, visit this url and follow the directions:')
print(' %s' % GeneratePermissionUrl(options.client_id, options.scope))
authorization_code = input('Enter verification code: ')
response = AuthorizeTokens(options.client_id, options.client_secret,
authorization_code)
print('Refresh Token: %s' % response['refresh_token'])
print('Access Token: %s' % response['access_token'])
print('Access Token Expiration Seconds: %s' % response['expires_in'])
elif options.test_imap_authentication:
RequireOptions(options, 'user', 'access_token')
TestImapAuthentication(options.user,
GenerateOAuth2String(options.user, options.access_token,
base64_encode=False))
elif options.test_smtp_authentication:
RequireOptions(options, 'user', 'access_token')
TestSmtpAuthentication(options.user,
GenerateOAuth2String(options.user, options.access_token,
base64_encode=False))
else:
options_parser.print_help()
print('Nothing to do, exiting.')
return
if __name__ == '__main__':
main(sys.argv)

View File

@ -235,11 +235,25 @@ Do not validate TLS certificates.
Use IDLE extension. Use IDLE extension.
.\" default value .\" default value
.Pq Em true .Pq Em true
.It Ic use_condstore Ar boolean
.Pq Em optional
Use CONDSTORE extension.
.\" default value
.Pq Em true
.It Ic use_deflate Ar boolean .It Ic use_deflate Ar boolean
.Pq Em optional .Pq Em optional
Use COMPRESS=DEFLATE extension (if built with DEFLATE support). Use COMPRESS=DEFLATE extension (if built with DEFLATE support).
.\" default value .\" default value
.Pq Em true .Pq Em true
.It Ic use_oauth2 Ar boolean
.Pq Em optional
Use OAUTH2 authentication.
Can only be used with
.Ic server_password_command
which should return a base64-encoded OAUTH2 token ready to be passed to IMAP.
For help on setup with Gmail, see Gmail section below.
.\" default value
.Pq Em false
.It Ic timeout Ar integer .It Ic timeout Ar integer
.Pq Em optional .Pq Em optional
Timeout to use for server connections in seconds. Timeout to use for server connections in seconds.
@ -247,6 +261,35 @@ A timeout of 0 seconds means there's no timeout.
.\" default value .\" default value
.Pq Em 16 .Pq Em 16
.El .El
.Ss Gmail
Gmail has non-standard IMAP behaviors that need to be worked around.
.Ss Gmail - sending mail
Option
.Ic store_sent_mail
should be disabled since Gmail auto-saves sent mail by its own.
.Ss Gmail OAUTH2
To use OAUTH2, you must go through a process to register your own private "application" with Google that can use OAUTH2 tokens.
For convenience in the meli repository under the
.Pa contrib/
directory you can find a python3 file named oauth2.py to generate and request the appropriate data to perform OAUTH2 authentication.
Steps:
.Bl -bullet -compact
.It
In Google APIs, create a custom OAuth client ID and note down the Client ID and Client Secret.
You may need to create a consent screen; follow the steps described in the website.
.It
Run the oauth2.py script as follows (after adjusting binary paths and credentials):
.Cm python3 oauth2.py --user=xxx@gmail.com --client_id=1038[...].apps.googleusercontent.com --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ --generate_oauth2_token
and follow the instructions.
Note down the refresh token.
.It
In
.Ic server_password_command
enter a command like this (after adjusting binary paths and credentials):
.Cm TOKEN=$(python3 oauth2.py --user=xxx@gmail.com --quiet --client_id=1038[...].apps.googleusercontent.com --client_secret=VWFn8LIKAMC-MsjBMhJeOplZ --refresh_token=1/Yzm6MRy4q1xi7Dx2DuWXNgT6s37OrP_DW_IoyTum4YA) && python3 oauth2.py --user=xxx@gmail.com --generate_oauth2_string --quiet --access_token=$TOKEN
.It
On startup, meli should evaluate this command which if successful must only return a base64-encoded token ready to be passed to IMAP.
.El
.Ss JMAP only .Ss JMAP only
JMAP specific options JMAP specific options
.Bl -tag -width 36n .Bl -tag -width 36n

View File

@ -64,6 +64,7 @@ pub type UIDVALIDITY = UID;
pub type MessageSequenceNumber = ImapNum; pub type MessageSequenceNumber = ImapNum;
pub static SUPPORTED_CAPABILITIES: &[&str] = &[ pub static SUPPORTED_CAPABILITIES: &[&str] = &[
"AUTH=OAUTH2",
#[cfg(feature = "deflate_compression")] #[cfg(feature = "deflate_compression")]
"COMPRESS=DEFLATE", "COMPRESS=DEFLATE",
"CONDSTORE", "CONDSTORE",
@ -232,6 +233,7 @@ impl MailBackend for ImapType {
#[cfg(feature = "deflate_compression")] #[cfg(feature = "deflate_compression")]
deflate, deflate,
condstore, condstore,
oauth2,
}, },
} = self.server_conf.protocol } = self.server_conf.protocol
{ {
@ -273,6 +275,15 @@ impl MailBackend for ImapType {
}; };
} }
} }
"AUTH=OAUTH2" => {
if oauth2 {
*status = MailBackendExtensionStatus::Enabled { comment: None };
} else {
*status = MailBackendExtensionStatus::Supported {
comment: Some("Disabled by user configuration"),
};
}
}
_ => { _ => {
if SUPPORTED_CAPABILITIES if SUPPORTED_CAPABILITIES
.iter() .iter()
@ -1218,7 +1229,14 @@ impl ImapType {
) -> Result<Box<dyn MailBackend>> { ) -> Result<Box<dyn MailBackend>> {
let server_hostname = get_conf_val!(s["server_hostname"])?; let server_hostname = get_conf_val!(s["server_hostname"])?;
let server_username = get_conf_val!(s["server_username"])?; let server_username = get_conf_val!(s["server_username"])?;
let use_oauth2: bool = get_conf_val!(s["use_oauth2"], false)?;
let server_password = if !s.extra.contains_key("server_password_command") { let server_password = if !s.extra.contains_key("server_password_command") {
if use_oauth2 {
return Err(MeliError::new(format!(
"({}) `use_oauth2` use requires `server_password_command` set with a command that returns an OAUTH2 token. Consult documentation for guidance.",
s.name,
)));
}
get_conf_val!(s["server_password"])?.to_string() get_conf_val!(s["server_password"])?.to_string()
} else { } else {
let invocation = get_conf_val!(s["server_password_command"])?; let invocation = get_conf_val!(s["server_password_command"])?;
@ -1275,6 +1293,7 @@ impl ImapType {
condstore: get_conf_val!(s["use_condstore"], true)?, condstore: get_conf_val!(s["use_condstore"], true)?,
#[cfg(feature = "deflate_compression")] #[cfg(feature = "deflate_compression")]
deflate: get_conf_val!(s["use_deflate"], true)?, deflate: get_conf_val!(s["use_deflate"], true)?,
oauth2: use_oauth2,
}, },
}, },
timeout, timeout,
@ -1463,7 +1482,14 @@ impl ImapType {
pub fn validate_config(s: &AccountSettings) -> Result<()> { pub fn validate_config(s: &AccountSettings) -> Result<()> {
get_conf_val!(s["server_hostname"])?; get_conf_val!(s["server_hostname"])?;
get_conf_val!(s["server_username"])?; get_conf_val!(s["server_username"])?;
let use_oauth2: bool = get_conf_val!(s["use_oauth2"], false)?;
if !s.extra.contains_key("server_password_command") { if !s.extra.contains_key("server_password_command") {
if use_oauth2 {
return Err(MeliError::new(format!(
"({}) `use_oauth2` use requires `server_password_command` set with a command that returns an OAUTH2 token. Consult documentation for guidance.",
s.name,
)));
}
get_conf_val!(s["server_password"])?; get_conf_val!(s["server_password"])?;
} else if s.extra.contains_key("server_password") { } else if s.extra.contains_key("server_password") {
return Err(MeliError::new(format!( return Err(MeliError::new(format!(

View File

@ -63,6 +63,7 @@ pub struct ImapExtensionUse {
pub idle: bool, pub idle: bool,
#[cfg(feature = "deflate_compression")] #[cfg(feature = "deflate_compression")]
pub deflate: bool, pub deflate: bool,
pub oauth2: bool,
} }
impl Default for ImapExtensionUse { impl Default for ImapExtensionUse {
@ -72,6 +73,7 @@ impl Default for ImapExtensionUse {
idle: true, idle: true,
#[cfg(feature = "deflate_compression")] #[cfg(feature = "deflate_compression")]
deflate: true, deflate: true,
oauth2: false,
} }
} }
} }
@ -351,16 +353,39 @@ impl ImapStream {
.set_err_kind(crate::error::ErrorKind::Authentication)); .set_err_kind(crate::error::ErrorKind::Authentication));
} }
let mut capabilities = None; match server_conf.protocol {
ret.send_command( ImapProtocol::IMAP {
format!( extension_use: ImapExtensionUse { oauth2, .. },
"LOGIN \"{}\" \"{}\"", } if oauth2 => {
&server_conf.server_username, &server_conf.server_password if !capabilities
) .iter()
.as_bytes(), .any(|cap| cap.eq_ignore_ascii_case(b"AUTH=XOAUTH2"))
) {
.await?; return Err(MeliError::new(format!(
"Could not connect to {}: OAUTH2 is enabled but server did not return AUTH=XOAUTH2 capability. Returned capabilities were: {}",
&server_conf.server_hostname,
capabilities.iter().map(|capability|
String::from_utf8_lossy(capability).to_string()).collect::<Vec<String>>().join(" ")
)));
}
ret.send_command(
format!("AUTHENTICATE XOAUTH2 {}", &server_conf.server_password).as_bytes(),
)
.await?;
}
_ => {
ret.send_command(
format!(
"LOGIN \"{}\" \"{}\"",
&server_conf.server_username, &server_conf.server_password
)
.as_bytes(),
)
.await?;
}
}
let tag_start = format!("M{} ", (ret.cmd_id - 1)); let tag_start = format!("M{} ", (ret.cmd_id - 1));
let mut capabilities = None;
loop { loop {
ret.read_lines(&mut res, &[], false).await?; ret.read_lines(&mut res, &[], false).await?;
@ -604,6 +629,7 @@ impl ImapConnection {
#[cfg(feature = "deflate_compression")] #[cfg(feature = "deflate_compression")]
deflate, deflate,
idle: _idle, idle: _idle,
oauth2: _,
}, },
} => { } => {
if capabilities.contains(&b"CONDSTORE"[..]) && condstore { if capabilities.contains(&b"CONDSTORE"[..]) && condstore {