meli/melib/src/sieve/compiler/lexer/string.rs

991 lines
37 KiB
Rust

/*
* Copyright (c) 2020-2023, Stalwart Labs Ltd.
*
* This file is part of the Stalwart Sieve Interpreter.
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
* in the LICENSE file at the top-level directory of this distribution.
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*
* You can be released from the requirements of the AGPLv3 license by
* purchasing a commercial license. Please contact licensing@stalw.art
* for more details.
*/
use std::convert::TryFrom;
use std::fmt::Display;
use mail_parser::HeaderName;
use crate::sieve::{
compiler::{
grammar::{
expr::{self, parser::ExpressionParser, tokenizer::Tokenizer},
instruction::CompilerState,
AddressPart,
},
ContentTypePart, ErrorType, HeaderPart, HeaderVariable, MessagePart, Number,
ReceivedHostname, ReceivedPart, Value, VariableType,
},
runtime::eval::IntoString,
Envelope, MAX_MATCH_VARIABLES,
};
enum State {
None,
Variable,
Encoded {
is_unicode: bool,
initial_buf_size: usize,
},
}
impl<'x> CompilerState<'x> {
pub(crate) fn tokenize_string(
&mut self,
bytes: &[u8],
parse_decoded: bool,
) -> Result<Value, ErrorType> {
let mut state = State::None;
let mut items = Vec::with_capacity(3);
let mut last_ch = 0;
let mut var_start_pos = usize::MAX;
let mut var_is_number = true;
let mut var_has_namespace = false;
let mut text_has_digits = true;
let mut text_has_dots = false;
let mut hex_start = usize::MAX;
let mut decode_buf = Vec::with_capacity(bytes.len());
let mut iter = bytes.iter().enumerate().peekable();
while let Some((mut pos, &ch)) = iter.next() {
let mut is_var_error = false;
match state {
State::None => match ch {
b'{' if last_ch == b'$' => {
decode_buf.pop();
var_start_pos = pos + 1;
var_is_number = true;
var_has_namespace = false;
state = State::Variable;
}
b'{' if last_ch == b'%' => {
decode_buf.pop();
var_start_pos = pos + 1;
// Add any text before the variable
if !decode_buf.is_empty() {
self.add_value(
&mut items,
&decode_buf,
parse_decoded,
text_has_digits,
text_has_dots,
)?;
decode_buf.clear();
text_has_digits = true;
text_has_dots = false;
}
match ExpressionParser::from_tokenizer(Tokenizer::from_iter(
iter,
|var_name, maybe_namespace| {
self.parse_expr_fnc_or_var(var_name, maybe_namespace)
},
))
.parse()
{
Ok(parser) => {
iter = parser.tokenizer.iter;
state = State::None;
if !parser.output.is_empty() {
items.push(Value::Expression(parser.output));
} else {
is_var_error = true;
pos = iter.peek().map(|(p, _)| *p).unwrap_or(bytes.len()) - 1;
}
}
Err(err) => {
return Err(ErrorType::InvalidExpression(format!(
"{}: {}",
std::str::from_utf8(bytes).unwrap_or_default(),
err
)))
}
}
}
b'.' => {
if text_has_dots {
text_has_digits = false;
} else {
text_has_dots = true;
}
decode_buf.push(ch);
}
b'0'..=b'9' => {
decode_buf.push(ch);
}
_ => {
text_has_digits = false;
decode_buf.push(ch);
}
},
State::Variable => match ch {
b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'[' | b']' | b'*' | b'-' => {
var_is_number = false;
}
b'.' => {
var_is_number = false;
var_has_namespace = true;
}
b'0'..=b'9' => {}
b'}' => {
if pos > var_start_pos {
// Add any text before the variable
if !decode_buf.is_empty() {
self.add_value(
&mut items,
&decode_buf,
parse_decoded,
text_has_digits,
text_has_dots,
)?;
decode_buf.clear();
text_has_digits = true;
text_has_dots = false;
}
// Parse variable type
let var_name = std::str::from_utf8(&bytes[var_start_pos..pos]).unwrap();
let var_type = if !var_is_number {
self.parse_variable(var_name, var_has_namespace)
} else {
self.parse_match_variable(var_name)
};
match var_type {
Ok(Some(var)) => items.push(Value::Variable(var)),
Ok(None) => {}
Err(
ErrorType::InvalidNamespace(_) | ErrorType::InvalidEnvelope(_),
) => {
is_var_error = true;
}
Err(e) => return Err(e),
}
state = State::None;
} else {
is_var_error = true;
}
}
b':' => {
if parse_decoded && !var_has_namespace {
match bytes.get(var_start_pos..pos) {
Some(enc) if enc.eq_ignore_ascii_case(b"hex") => {
state = State::Encoded {
is_unicode: false,
initial_buf_size: decode_buf.len(),
};
}
Some(enc) if enc.eq_ignore_ascii_case(b"unicode") => {
state = State::Encoded {
is_unicode: true,
initial_buf_size: decode_buf.len(),
};
}
_ => {
is_var_error = true;
}
}
} else if var_has_namespace {
var_is_number = false;
} else {
is_var_error = true;
}
}
_ => {
is_var_error = true;
}
},
State::Encoded {
is_unicode,
initial_buf_size,
} => match ch {
b'0'..=b'9' | b'a'..=b'f' | b'A'..=b'F' => {
if hex_start == usize::MAX {
hex_start = pos;
}
}
b' ' | b'\t' | b'\r' | b'\n' | b'}' => {
if hex_start != usize::MAX {
let code = std::str::from_utf8(&bytes[hex_start..pos]).unwrap();
hex_start = usize::MAX;
if !is_unicode {
if let Ok(ch) = u8::from_str_radix(code, 16) {
decode_buf.push(ch);
} else {
is_var_error = true;
}
} else if let Ok(ch) = u32::from_str_radix(code, 16) {
let mut buf = [0; 4];
decode_buf.extend_from_slice(
char::from_u32(ch)
.ok_or(ErrorType::InvalidUnicodeSequence(ch))?
.encode_utf8(&mut buf)
.as_bytes(),
);
} else {
is_var_error = true;
}
}
if ch == b'}' {
if decode_buf.len() != initial_buf_size {
state = State::None;
} else {
is_var_error = true;
}
}
}
_ => {
is_var_error = true;
}
},
}
if is_var_error {
if let State::Encoded {
initial_buf_size, ..
} = state
{
if initial_buf_size != decode_buf.len() {
decode_buf.truncate(initial_buf_size);
}
}
decode_buf.extend_from_slice(&bytes[var_start_pos - 2..pos + 1]);
hex_start = usize::MAX;
state = State::None;
}
last_ch = ch;
}
match state {
State::Variable => {
decode_buf.extend_from_slice(&bytes[var_start_pos - 2..bytes.len()]);
}
State::Encoded {
initial_buf_size, ..
} => {
if initial_buf_size != decode_buf.len() {
decode_buf.truncate(initial_buf_size);
}
decode_buf.extend_from_slice(&bytes[var_start_pos - 2..bytes.len()]);
}
State::None => (),
}
if !decode_buf.is_empty() {
self.add_value(
&mut items,
&decode_buf,
parse_decoded,
text_has_digits,
text_has_dots,
)?;
}
Ok(match items.len() {
1 => items.pop().unwrap(),
0 => Value::Text(String::new()),
_ => Value::List(items),
})
}
fn parse_match_variable(&mut self, var_name: &str) -> Result<Option<VariableType>, ErrorType> {
let num = var_name
.parse()
.map_err(|_| ErrorType::InvalidNumber(var_name.to_string()))?;
if num < MAX_MATCH_VARIABLES {
if self.register_match_var(num) {
let total_vars = num + 1;
if total_vars > self.vars_match_max {
self.vars_match_max = total_vars;
}
Ok(Some(VariableType::Match(num)))
} else {
Ok(None)
}
} else {
Err(ErrorType::InvalidMatchVariable(num))
}
}
pub fn parse_variable(
&self,
var_name: &str,
maybe_namespace: bool,
) -> Result<Option<VariableType>, ErrorType> {
if !maybe_namespace {
if self.is_var_global(var_name) {
Ok(Some(VariableType::Global(var_name.to_string())))
} else if let Some(var_id) = self.get_local_var(var_name) {
Ok(Some(VariableType::Local(var_id)))
} else {
Ok(None)
}
} else {
let var = match var_name.to_lowercase().split_once('.') {
Some(("global" | "t", var_name)) if !var_name.is_empty() => {
VariableType::Global(var_name.to_string())
}
Some(("env", var_name)) if !var_name.is_empty() => {
VariableType::Environment(var_name.to_string())
}
Some(("envelope", var_name)) if !var_name.is_empty() => {
let envelope = match var_name {
"from" => Envelope::From,
"to" => Envelope::To,
"by_time_absolute" => Envelope::ByTimeAbsolute,
"by_time_relative" => Envelope::ByTimeRelative,
"by_mode" => Envelope::ByMode,
"by_trace" => Envelope::ByTrace,
"notify" => Envelope::Notify,
"orcpt" => Envelope::Orcpt,
"ret" => Envelope::Ret,
"envid" => Envelope::Envid,
_ => {
return Err(ErrorType::InvalidEnvelope(var_name.to_string()));
}
};
VariableType::Envelope(envelope)
}
Some(("header", var_name)) if !var_name.is_empty() => {
self.parse_header_variable(var_name)?
}
Some(("body", var_name)) if !var_name.is_empty() => match var_name {
"text" => VariableType::Part(MessagePart::TextBody(false)),
"html" => VariableType::Part(MessagePart::HtmlBody(false)),
"to_text" => VariableType::Part(MessagePart::TextBody(true)),
"to_html" => VariableType::Part(MessagePart::HtmlBody(true)),
_ => return Err(ErrorType::InvalidNamespace(var_name.to_string())),
},
Some(("part", var_name)) if !var_name.is_empty() => match var_name {
"text" => VariableType::Part(MessagePart::Contents),
"raw" => VariableType::Part(MessagePart::Raw),
_ => return Err(ErrorType::InvalidNamespace(var_name.to_string())),
},
None => {
if self.is_var_global(var_name) {
VariableType::Global(var_name.to_string())
} else if let Some(var_id) = self.get_local_var(var_name) {
VariableType::Local(var_id)
} else {
return Ok(None);
}
}
_ => return Err(ErrorType::InvalidNamespace(var_name.to_string())),
};
Ok(Some(var))
}
}
fn parse_header_variable(&self, var_name: &str) -> Result<VariableType, ErrorType> {
#[derive(Debug)]
enum State {
Name,
Index,
Part,
PartIndex,
}
let mut name = vec![];
let mut has_name = false;
let mut has_wildcard = false;
let mut hdr_name = String::new();
let mut hdr_index = String::new();
let mut part = String::new();
let mut part_index = String::new();
let mut state = State::Name;
for ch in var_name.chars() {
match state {
State::Name => match ch {
'[' => {
state = if hdr_index.is_empty() {
State::Index
} else if part.is_empty() {
State::PartIndex
} else {
return Err(ErrorType::InvalidExpression(var_name.to_string()));
};
has_name = true;
}
'.' => {
state = State::Part;
has_name = true;
}
' ' | '\t' | '\r' | '\n' => {}
'*' if !has_wildcard && hdr_name.is_empty() && name.is_empty() => {
has_wildcard = true;
}
':' if !hdr_name.is_empty() && !has_wildcard => {
name.push(
HeaderName::parse(std::mem::take(&mut hdr_name)).ok_or_else(|| {
ErrorType::InvalidExpression(var_name.to_string())
})?,
);
}
_ if !has_name && !has_wildcard => {
hdr_name.push(ch);
}
_ => {
return Err(ErrorType::InvalidExpression(var_name.to_string()));
}
},
State::Index => match ch {
']' => {
state = State::Name;
}
' ' | '\t' | '\r' | '\n' => {}
_ => {
hdr_index.push(ch);
}
},
State::Part => match ch {
'[' => {
state = State::PartIndex;
}
' ' | '\t' | '\r' | '\n' => {}
_ => {
part.push(ch);
}
},
State::PartIndex => match ch {
']' => {
state = State::Name;
}
' ' | '\t' | '\r' | '\n' => {}
_ => {
part_index.push(ch);
}
},
}
}
if !hdr_name.is_empty() {
name.push(
HeaderName::parse(hdr_name)
.ok_or_else(|| ErrorType::InvalidExpression(var_name.to_string()))?,
);
}
if !name.is_empty() || has_wildcard {
Ok(VariableType::Header(HeaderVariable {
name,
part: HeaderPart::try_from(part.as_str())
.map_err(|_| ErrorType::InvalidExpression(var_name.to_string()))?,
index_hdr: match hdr_index.as_str() {
"" => {
if !has_wildcard {
-1
} else {
0
}
}
"*" => 0,
_ => hdr_index
.parse()
.map(|v| if v == 0 { 1 } else { v })
.map_err(|_| ErrorType::InvalidExpression(var_name.to_string()))?,
},
index_part: match part_index.as_str() {
"" => {
if !has_wildcard {
-1
} else {
0
}
}
"*" => 0,
_ => part_index
.parse()
.map(|v| if v == 0 { 1 } else { v })
.map_err(|_| ErrorType::InvalidExpression(var_name.to_string()))?,
},
}))
} else {
Err(ErrorType::InvalidExpression(var_name.to_string()))
}
}
pub fn parse_expr_fnc_or_var(
&self,
var_name: &str,
maybe_namespace: bool,
) -> Result<expr::Token, String> {
match self.parse_variable(var_name, maybe_namespace) {
Ok(Some(var)) => Ok(expr::Token::Variable(var)),
_ => {
if let Some((id, num_args)) = self.compiler.functions.get(var_name) {
Ok(expr::Token::Function {
name: var_name.to_string(),
id: *id,
num_args: *num_args,
})
} else {
Err(format!("Invalid variable or function name {var_name:?}"))
}
}
}
}
#[inline(always)]
fn add_value(
&mut self,
items: &mut Vec<Value>,
buf: &[u8],
parse_decoded: bool,
has_digits: bool,
has_dots: bool,
) -> Result<(), ErrorType> {
if !parse_decoded {
items.push(if has_digits {
if has_dots {
match std::str::from_utf8(buf)
.ok()
.and_then(|v| (v, v.parse::<f64>().ok()?).into())
{
Some((v, n)) if n.to_string() == v => Value::Number(Number::Float(n)),
_ => Value::Text(buf.to_vec().into_string()),
}
} else {
match std::str::from_utf8(buf)
.ok()
.and_then(|v| (v, v.parse::<i64>().ok()?).into())
{
Some((v, n)) if n.to_string() == v => Value::Number(Number::Integer(n)),
_ => Value::Text(buf.to_vec().into_string()),
}
}
} else {
Value::Text(buf.to_vec().into_string())
});
} else {
match self.tokenize_string(buf, false)? {
Value::List(new_items) => items.extend(new_items),
item => items.push(item),
}
}
Ok(())
}
}
impl TryFrom<&str> for HeaderPart {
type Error = ();
fn try_from(value: &str) -> Result<Self, Self::Error> {
let (value, subvalue) = value.split_once('.').unwrap_or((value, ""));
Ok(match value {
"" | "text" => HeaderPart::Text,
// Addresses
"name" => HeaderPart::Address(AddressPart::Name),
"addr" => {
if !subvalue.is_empty() {
HeaderPart::Address(AddressPart::try_from(subvalue)?)
} else {
HeaderPart::Address(AddressPart::All)
}
}
// Content-type
"type" => HeaderPart::ContentType(ContentTypePart::Type),
"subtype" => HeaderPart::ContentType(ContentTypePart::Subtype),
"attr" if !subvalue.is_empty() => {
HeaderPart::ContentType(ContentTypePart::Attribute(subvalue.to_string()))
}
// Received
"rcvd" => {
if !subvalue.is_empty() {
HeaderPart::Received(ReceivedPart::try_from(subvalue)?)
} else {
HeaderPart::Text
}
}
// Id
"id" => HeaderPart::Id,
// Raw
"raw" => HeaderPart::Raw,
// Date
"date" => HeaderPart::Date,
// Content-type attributes
_ => {
return Err(());
}
})
}
}
impl TryFrom<&str> for ReceivedPart {
type Error = ();
fn try_from(value: &str) -> Result<Self, Self::Error> {
Ok(match value {
// Received
"from" => ReceivedPart::From(ReceivedHostname::Any),
"from.name" => ReceivedPart::From(ReceivedHostname::Name),
"from.ip" => ReceivedPart::From(ReceivedHostname::Ip),
"ip" => ReceivedPart::FromIp,
"iprev" => ReceivedPart::FromIpRev,
"by" => ReceivedPart::By(ReceivedHostname::Any),
"by.name" => ReceivedPart::By(ReceivedHostname::Name),
"by.ip" => ReceivedPart::By(ReceivedHostname::Ip),
"for" => ReceivedPart::For,
"with" => ReceivedPart::With,
"tls" => ReceivedPart::TlsVersion,
"cipher" => ReceivedPart::TlsCipher,
"id" => ReceivedPart::Id,
"ident" => ReceivedPart::Ident,
"date" => ReceivedPart::Date,
"date.raw" => ReceivedPart::DateRaw,
_ => return Err(()),
})
}
}
impl TryFrom<&str> for AddressPart {
type Error = ();
fn try_from(value: &str) -> Result<Self, Self::Error> {
Ok(match value {
"name" => AddressPart::Name,
"addr" | "all" => AddressPart::All,
"addr.domain" => AddressPart::Domain,
"addr.local" => AddressPart::LocalPart,
"addr.user" => AddressPart::User,
"addr.detail" => AddressPart::Detail,
_ => return Err(()),
})
}
}
impl Display for Value {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
Value::Text(t) => f.write_str(t),
Value::List(l) => {
for i in l {
i.fmt(f)?;
}
Ok(())
}
Value::Number(n) => n.fmt(f),
Value::Variable(v) => v.fmt(f),
Value::Expression(_) => f.write_str("%{}"),
Value::Regex(r) => f.write_str(&r.expr),
}
}
}
impl Display for VariableType {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
VariableType::Local(v) => write!(f, "${{{v}}}"),
VariableType::Match(v) => write!(f, "${{{v}}}"),
VariableType::Global(v) => write!(f, "${{global.{v}}}"),
VariableType::Environment(v) => write!(f, "${{env.{v}}}"),
VariableType::Envelope(env) => f.write_str(match env {
Envelope::From => "${{envelope.from}}",
Envelope::To => "${{envelope.to}}",
Envelope::ByTimeAbsolute => "${{envelope.by_time_absolute}}",
Envelope::ByTimeRelative => "${{envelope.by_time_relative}}",
Envelope::ByMode => "${{envelope.by_mode}}",
Envelope::ByTrace => "${{envelope.by_trace}}",
Envelope::Notify => "${{envelope.notify}}",
Envelope::Orcpt => "${{envelope.orcpt}}",
Envelope::Ret => "${{envelope.ret}}",
Envelope::Envid => "${{envelope.envit}}",
}),
VariableType::Header(hdr) => {
write!(
f,
"${{header.{}",
hdr.name.first().map(|h| h.as_str()).unwrap_or_default()
)?;
if hdr.index_hdr != 0 {
write!(f, "[{}]", hdr.index_hdr)?;
} else {
f.write_str("[*]")?;
}
/*if hdr.part != HeaderPart::Text {
f.write_str(".")?;
f.write_str(match &hdr.part {
HeaderPart::Name => "name",
HeaderPart::Address => "address",
HeaderPart::Type => "type",
HeaderPart::Subtype => "subtype",
HeaderPart::Raw => "raw",
HeaderPart::Date => "date",
HeaderPart::Attribute(attr) => attr.as_str(),
HeaderPart::Text => unreachable!(),
})?;
}*/
if hdr.index_part != 0 {
write!(f, "[{}]", hdr.index_part)?;
} else {
f.write_str("[*]")?;
}
f.write_str("}")
}
VariableType::Part(part) => {
write!(
f,
"${{{}",
match part {
MessagePart::TextBody(true) => "body.to_text",
MessagePart::TextBody(false) => "body.text",
MessagePart::HtmlBody(true) => "body.to_html",
MessagePart::HtmlBody(false) => "body.html",
MessagePart::Contents => "part.text",
MessagePart::Raw => "part.raw",
}
)?;
f.write_str("}")
}
}
}
}
#[cfg(test)]
mod tests {
use mail_parser::HeaderName;
use super::Value;
use crate::sieve::compiler::grammar::instruction::{
Block, CompilerState, Instruction, MAX_PARAMS,
};
use crate::sieve::compiler::grammar::test::Test;
use crate::sieve::compiler::grammar::tests::test_string::TestString;
use crate::sieve::compiler::grammar::{Comparator, MatchType};
use crate::sieve::compiler::lexer::tokenizer::Tokenizer;
use crate::sieve::compiler::lexer::word::Word;
use crate::sieve::compiler::{AddressPart, HeaderPart, HeaderVariable, VariableType};
use crate::sieve::{AHashSet, Compiler};
use std::convert::{TryFrom, TryInto};
#[test]
fn tokenize_string() {
let c = Compiler::new();
let mut block = Block::new(Word::Not);
block.match_test_pos.push(0);
let mut compiler = CompilerState {
compiler: &c,
instructions: vec![Instruction::Test(Test::String(TestString {
match_type: MatchType::Regex(u64::MAX),
comparator: Comparator::AsciiCaseMap,
source: vec![Value::Variable(VariableType::Local(0))],
key_list: vec![Value::Variable(VariableType::Local(0))],
is_not: false,
}))],
block_stack: Vec::new(),
block,
last_block_type: Word::Not,
vars_global: AHashSet::new(),
vars_num: 0,
vars_num_max: 0,
vars_local: 0,
tokens: Tokenizer::new(&c, b""),
vars_match_max: usize::MAX,
param_check: [false; MAX_PARAMS],
includes_num: 0,
};
for (input, expected_result) in [
("$${hex:24 24}", Value::Text("$$$".to_string())),
("$${hex:40}", Value::Text("$@".to_string())),
("${hex: 40 }", Value::Text("@".to_string())),
("${HEX: 40}", Value::Text("@".to_string())),
("${hex:40", Value::Text("${hex:40".to_string())),
("${hex:400}", Value::Text("${hex:400}".to_string())),
("${hex:4${hex:30}}", Value::Text("${hex:40}".to_string())),
("${unicode:40}", Value::Text("@".to_string())),
("${ unicode:40}", Value::Text("${ unicode:40}".to_string())),
("${UNICODE:40}", Value::Text("@".to_string())),
("${UnICoDE:0000040}", Value::Text("@".to_string())),
("${Unicode:40}", Value::Text("@".to_string())),
(
"${Unicode:40 40 ",
Value::Text("${Unicode:40 40 ".to_string()),
),
(
"${Unicode:Cool}",
Value::Text("${Unicode:Cool}".to_string()),
),
("", Value::Text("".to_string())),
(
"${global.full}",
Value::Variable(VariableType::Global("full".to_string())),
),
(
"${BAD${global.Company}",
Value::List(vec![
Value::Text("${BAD".to_string()),
Value::Variable(VariableType::Global("company".to_string())),
]),
),
(
"${President, ${global.Company} Inc.}",
Value::List(vec![
Value::Text("${President, ".to_string()),
Value::Variable(VariableType::Global("company".to_string())),
Value::Text(" Inc.}".to_string()),
]),
),
(
"dear${hex:20 24 7b}global.Name}",
Value::List(vec![
Value::Text("dear ".to_string()),
Value::Variable(VariableType::Global("name".to_string())),
]),
),
(
"INBOX.lists.${2}",
Value::List(vec![
Value::Text("INBOX.lists.".to_string()),
Value::Variable(VariableType::Match(2)),
]),
),
(
"Ein unerh${unicode:00F6}rt gro${unicode:00DF}er Test",
Value::Text("Ein unerhört großer Test".to_string()),
),
("&%${}!", Value::Text("&%${}!".to_string())),
("${doh!}", Value::Text("${doh!}".to_string())),
(
"${hex: 20 }${global.hi}${hex: 20 }",
Value::List(vec![
Value::Text(" ".to_string()),
Value::Variable(VariableType::Global("hi".to_string())),
Value::Text(" ".to_string()),
]),
),
(
"${hex:20 24 7b z}${global.hi}${unicode:}${unicode: }${hex:20}",
Value::List(vec![
Value::Text("${hex:20 24 7b z}".to_string()),
Value::Variable(VariableType::Global("hi".to_string())),
Value::Text("${unicode:}${unicode: } ".to_string()),
]),
),
(
"${header.from}",
Value::Variable(VariableType::Header(HeaderVariable {
name: vec![HeaderName::From],
part: HeaderPart::Text,
index_hdr: -1,
index_part: -1,
})),
),
(
"${header.from.addr}",
Value::Variable(VariableType::Header(HeaderVariable {
name: vec![HeaderName::From],
part: HeaderPart::Address(AddressPart::All),
index_hdr: -1,
index_part: -1,
})),
),
(
"${header.from[1]}",
Value::Variable(VariableType::Header(HeaderVariable {
name: vec![HeaderName::From],
part: HeaderPart::Text,
index_hdr: 1,
index_part: -1,
})),
),
(
"${header.from[*]}",
Value::Variable(VariableType::Header(HeaderVariable {
name: vec![HeaderName::From],
part: HeaderPart::Text,
index_hdr: 0,
index_part: -1,
})),
),
(
"${header.from[20].name}",
Value::Variable(VariableType::Header(HeaderVariable {
name: vec![HeaderName::From],
part: HeaderPart::Address(AddressPart::Name),
index_hdr: 20,
index_part: -1,
})),
),
(
"${header.from[*].addr}",
Value::Variable(VariableType::Header(HeaderVariable {
name: vec![HeaderName::From],
part: HeaderPart::Address(AddressPart::All),
index_hdr: 0,
index_part: -1,
})),
),
(
"${header.from[-5].name[2]}",
Value::Variable(VariableType::Header(HeaderVariable {
name: vec![HeaderName::From],
part: HeaderPart::Address(AddressPart::Name),
index_hdr: -5,
index_part: 2,
})),
),
(
"${header.from[*].raw[*]}",
Value::Variable(VariableType::Header(HeaderVariable {
name: vec![HeaderName::From],
part: HeaderPart::Raw,
index_hdr: 0,
index_part: 0,
})),
),
] {
assert_eq!(
compiler.tokenize_string(input.as_bytes(), true).unwrap(),
expected_result,
"Failed for {input}"
);
}
for input in ["${unicode:200000}", "${Unicode:DF01}"] {
assert!(compiler.tokenize_string(input.as_bytes(), true).is_err());
}
}
}