--- /dev/null
+pub mod group;
+pub mod ident;
+pub mod literal;
+pub mod punct;
+pub mod span;
+pub mod token_stream;
+
+extern "C" {
+ fn bridge__is_available() -> bool;
+}
+
+pub fn is_available() -> bool {
+ unsafe { bridge__is_available() }
+}
--- /dev/null
+use bridge::span::Span;
+use bridge::token_stream::TokenStream;
+use std::fmt;
+use Delimiter;
+
+#[repr(C)]
+#[derive(Debug, Clone)]
+pub struct Group {
+ delimiter: Delimiter,
+ stream: TokenStream,
+}
+
+impl Group {
+ pub fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
+ Group { delimiter, stream }
+ }
+
+ pub fn delimiter(&self) -> Delimiter {
+ self.delimiter
+ }
+
+ pub fn span(&self) -> Span {
+ Span {}
+ }
+
+ pub fn set_span(&mut self, span: Span) {
+ let _ = span;
+ }
+
+ pub fn stream(&self) -> TokenStream {
+ self.stream.clone()
+ }
+}
+
+impl fmt::Display for Group {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self.delimiter {
+ Delimiter::Parenthesis => f.write_str("(")?,
+ Delimiter::Brace => f.write_str("{")?,
+ Delimiter::Bracket => f.write_str("[")?,
+ Delimiter::None => (),
+ }
+
+ self.stream.fmt(f)?;
+
+ match self.delimiter {
+ Delimiter::Parenthesis => f.write_str(")")?,
+ Delimiter::Brace => f.write_str("}")?,
+ Delimiter::Bracket => f.write_str("]")?,
+ Delimiter::None => (),
+ }
+
+ Ok(())
+ }
+}
--- /dev/null
+use bridge::span::Span;
+use std::convert::TryInto;
+use std::ffi::c_uchar;
+use std::fmt;
+
+extern "C" {
+ fn Ident__new(string: *const c_uchar, len: u64) -> Ident;
+ fn Ident__new_raw(string: *const c_uchar, len: u64) -> Ident;
+ fn Ident__drop(ident: *const Ident);
+ fn Ident__clone(ident: *const Ident) -> Ident;
+}
+
+#[repr(C)]
+#[derive(Debug)]
+pub struct Ident {
+ pub(crate) is_raw: bool,
+ pub(crate) val: *const c_uchar,
+ len: u64,
+}
+
+impl Ident {
+ pub fn new(string: &str, _span: Span) -> Self {
+ unsafe { Ident__new(string.as_ptr(), string.len().try_into().unwrap()) }
+ }
+
+ pub fn new_raw(string: &str, _span: Span) -> Self {
+ unsafe { Ident__new_raw(string.as_ptr(), string.len().try_into().unwrap()) }
+ }
+
+ pub fn span(&self) -> Span {
+ Span {}
+ }
+
+ pub fn set_span(&mut self, span: Span) {
+ let _ = span;
+ }
+}
+
+impl Drop for Ident {
+ fn drop(&mut self) {
+ unsafe { Ident__drop(self as *const Ident) }
+ }
+}
+
+impl fmt::Display for Ident {
+ /// Display as lossless converted string.
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ if self.is_raw {
+ f.write_str("r#")?;
+ }
+ fmt::Display::fmt(
+ unsafe {
+ std::str::from_utf8(std::slice::from_raw_parts(
+ self.val,
+ self.len.try_into().unwrap(),
+ ))
+ .unwrap()
+ },
+ f,
+ )
+ }
+}
+
+impl Clone for Ident {
+ fn clone(&self) -> Self {
+ unsafe { Ident__clone(self as *const Ident) }
+ }
+}
--- /dev/null
+use bridge::span::Span;
+use std::convert::{TryFrom, TryInto};
+use std::ffi::c_uchar;
+use std::fmt;
+use std::str::FromStr;
+use LexError;
+
+extern "C" {
+ fn Literal__drop(literal: *const Literal);
+ fn Literal__string(str: *const c_uchar, len: u64) -> Literal;
+ fn Literal__byte_string(bytes: *const u8, len: u64) -> Literal;
+ fn Literal__from_string(str: *const c_uchar, len: u64, lit: *mut Literal) -> bool;
+}
+
+#[repr(C)]
+#[derive(Debug, Clone, Copy)]
+pub enum Unsigned {
+ Unsigned8(u8),
+ Unsigned16(u16),
+ Unsigned32(u32),
+ Unsigned64(u64),
+ // u128 is not ffi safe, hence this representation
+ // https://github.com/rust-lang/rust/issues/54341
+ Unsigned128(u64, u64),
+}
+
+#[repr(C)]
+#[derive(Debug, Clone, Copy)]
+pub enum Signed {
+ Signed8(i8),
+ Signed16(i16),
+ Signed32(i32),
+ Signed64(i64),
+ // i128 is not ffi safe, hence this representation
+ // https://github.com/rust-lang/rust/issues/54341
+ Signed128(u64, u64),
+}
+
+#[repr(C)]
+#[derive(Debug)]
+pub enum Literal {
+ /// String literal internal representation
+ ///
+ /// # Note
+ /// This variant is constructed through FFI
+ #[allow(dead_code)]
+ String {
+ data: *const c_uchar,
+ len: u64,
+ },
+ /// Bytestring literal internal representation
+ ///
+ /// # Note
+ /// This variant is constructed through FFI
+ #[allow(dead_code)]
+ ByteString {
+ data: *const u8,
+ size: u64,
+ },
+ Char(u32),
+ Unsigned(Unsigned, bool),
+ Signed(Signed, bool),
+ Usize(u64, bool),
+ ISize(i64, bool),
+ Float32(f32, bool),
+ Float64(f64, bool),
+}
+
+impl Literal {
+ pub fn u8_suffixed(n: u8) -> Self {
+ Literal::Unsigned(Unsigned::Unsigned8(n), true)
+ }
+
+ pub fn u16_suffixed(n: u16) -> Self {
+ Literal::Unsigned(Unsigned::Unsigned16(n), true)
+ }
+
+ pub fn u32_suffixed(n: u32) -> Self {
+ Literal::Unsigned(Unsigned::Unsigned32(n), true)
+ }
+
+ pub fn u64_suffixed(n: u64) -> Self {
+ Literal::Unsigned(Unsigned::Unsigned64(n), true)
+ }
+
+ pub fn u128_suffixed(n: u128) -> Self {
+ Literal::Unsigned(
+ Unsigned::Unsigned128(
+ (n >> 64).try_into().unwrap(),
+ (n & 0xFFFFFFFFFFFFFFFF).try_into().unwrap(),
+ ),
+ true,
+ )
+ }
+
+ pub fn usize_suffixed(n: usize) -> Self {
+ Literal::Usize(n.try_into().expect("Cannot convert usize to u64"), true)
+ }
+
+ pub fn i8_suffixed(n: i8) -> Self {
+ Literal::Signed(Signed::Signed8(n), true)
+ }
+
+ pub fn i16_suffixed(n: i16) -> Self {
+ Literal::Signed(Signed::Signed16(n), true)
+ }
+
+ pub fn i32_suffixed(n: i32) -> Self {
+ Literal::Signed(Signed::Signed32(n), true)
+ }
+
+ pub fn i64_suffixed(n: i64) -> Self {
+ Literal::Signed(Signed::Signed64(n), true)
+ }
+
+ pub fn i128_suffixed(n: i128) -> Self {
+ Literal::Signed(
+ Signed::Signed128(
+ (n >> 64).try_into().unwrap(),
+ (n & 0xFFFFFFFFFFFFFFFF).try_into().unwrap(),
+ ),
+ true,
+ )
+ }
+
+ pub fn isize_suffixed(n: isize) -> Self {
+ Literal::ISize(n.try_into().expect("Cannot convert isize to i64"), true)
+ }
+
+ // Unsuffixed
+
+ pub fn u8_unsuffixed(n: u8) -> Self {
+ Literal::Unsigned(Unsigned::Unsigned8(n), false)
+ }
+
+ pub fn u16_unsuffixed(n: u16) -> Self {
+ Literal::Unsigned(Unsigned::Unsigned16(n), false)
+ }
+
+ pub fn u32_unsuffixed(n: u32) -> Self {
+ Literal::Unsigned(Unsigned::Unsigned32(n), false)
+ }
+
+ pub fn u64_unsuffixed(n: u64) -> Self {
+ Literal::Unsigned(Unsigned::Unsigned64(n), false)
+ }
+
+ pub fn u128_unsuffixed(n: u128) -> Self {
+ Literal::Unsigned(
+ Unsigned::Unsigned128(
+ (n >> 64).try_into().unwrap(),
+ (n & 0xFFFFFFFFFFFFFFFF).try_into().unwrap(),
+ ),
+ false,
+ )
+ }
+
+ pub fn usize_unsuffixed(n: usize) -> Self {
+ Literal::Usize(n.try_into().expect("Cannot convert usize to u64"), false)
+ }
+
+ pub fn i8_unsuffixed(n: i8) -> Self {
+ Literal::Signed(Signed::Signed8(n), false)
+ }
+
+ pub fn i16_unsuffixed(n: i16) -> Self {
+ Literal::Signed(Signed::Signed16(n), false)
+ }
+
+ pub fn i32_unsuffixed(n: i32) -> Self {
+ Literal::Signed(Signed::Signed32(n), false)
+ }
+
+ pub fn i64_unsuffixed(n: i64) -> Self {
+ Literal::Signed(Signed::Signed64(n), false)
+ }
+
+ pub fn i128_unsuffixed(n: i128) -> Self {
+ Literal::Signed(
+ Signed::Signed128(
+ (n >> 64).try_into().unwrap(),
+ (n & 0xFFFFFFFFFFFFFFFF).try_into().unwrap(),
+ ),
+ false,
+ )
+ }
+
+ pub fn isize_unsuffixed(n: isize) -> Self {
+ Literal::ISize(n.try_into().expect("Cannot convert isize to i64"), false)
+ }
+
+ pub fn f32_unsuffixed(n: f32) -> Self {
+ Literal::Float32(n, false)
+ }
+
+ pub fn f32_suffixed(n: f32) -> Self {
+ Literal::Float32(n, true)
+ }
+
+ pub fn f64_unsuffixed(n: f64) -> Self {
+ Literal::Float64(n, false)
+ }
+
+ pub fn f64_suffixed(n: f64) -> Self {
+ Literal::Float64(n, true)
+ }
+
+ pub fn string(string: &str) -> Self {
+ unsafe { Literal__string(string.as_ptr(), string.len().try_into().unwrap()) }
+ }
+
+ pub fn character(c: char) -> Self {
+ Literal::Char(c.into())
+ }
+
+ pub fn byte_string(bytes: &[u8]) -> Self {
+ unsafe { Literal__byte_string(bytes.as_ptr(), bytes.len().try_into().unwrap()) }
+ }
+
+ pub fn span(&self) -> Span {
+ Span {}
+ }
+
+ pub fn set_span(&mut self, span: Span) {
+ let _ = span;
+ }
+}
+
+impl Drop for Literal {
+ fn drop(&mut self) {
+ match self {
+ Literal::String { .. } | Literal::ByteString { .. } => unsafe {
+ Literal__drop(self as *const Literal)
+ },
+ _ => (),
+ }
+ }
+}
+
+impl fmt::Display for Literal {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ Literal::String { data, len } => {
+ let slice =
+ unsafe { std::slice::from_raw_parts(*data, (*len).try_into().unwrap()) };
+ f.write_str("\"")?;
+ f.write_str(std::str::from_utf8(slice).unwrap())?;
+ f.write_str("\"")?;
+ }
+ Literal::ByteString { data, size } => {
+ f.write_str("b\"")?;
+ let slice =
+ unsafe { std::slice::from_raw_parts(*data, (*size).try_into().unwrap()) };
+ for &byte in slice {
+ if byte != b'"' && (b' '..=b'z').contains(&byte) {
+ char::try_from(byte).unwrap().fmt(f)?;
+ } else {
+ write!(f, "\\x{:02x}", byte)?;
+ }
+ }
+ f.write_str("b\"")?;
+ }
+ Literal::Char(val) => {
+ let ch: char = (*val).try_into().unwrap();
+ match ch {
+ '\'' => f.write_str("'\\''")?,
+ '\0' => f.write_str("'\\0'")?,
+ '\n' => f.write_str("'\\n'")?,
+ ' '..='z' => write!(f, "'{}'", ch)?,
+ _ => write!(f, "'\\u{:x}'", val)?,
+ }
+ }
+ Literal::Unsigned(val, suffixed) => match val {
+ Unsigned::Unsigned8(val) => {
+ val.fmt(f)?;
+ if *suffixed {
+ f.write_str("u8")?;
+ }
+ }
+ Unsigned::Unsigned16(val) => {
+ val.fmt(f)?;
+ if *suffixed {
+ f.write_str("u16")?;
+ }
+ }
+ Unsigned::Unsigned32(val) => {
+ val.fmt(f)?;
+ if *suffixed {
+ f.write_str("u32")?;
+ }
+ }
+ Unsigned::Unsigned64(val) => {
+ val.fmt(f)?;
+ if *suffixed {
+ f.write_str("u64")?;
+ }
+ }
+ Unsigned::Unsigned128(h, l) => {
+ ((u128::from(*h) << 64) & u128::from(*l)).fmt(f)?;
+ if *suffixed {
+ f.write_str("u128")?;
+ }
+ }
+ },
+ Literal::Signed(val, suffixed) => match val {
+ Signed::Signed8(val) => {
+ val.fmt(f)?;
+ if *suffixed {
+ f.write_str("i8")?;
+ }
+ }
+ Signed::Signed16(val) => {
+ val.fmt(f)?;
+ if *suffixed {
+ f.write_str("i16")?;
+ }
+ }
+ Signed::Signed32(val) => {
+ val.fmt(f)?;
+ if *suffixed {
+ f.write_str("i32")?;
+ }
+ }
+ Signed::Signed64(val) => {
+ val.fmt(f)?;
+ if *suffixed {
+ f.write_str("i64")?;
+ }
+ }
+ Signed::Signed128(h, l) => {
+ ((i128::from(*h) << 64) & i128::from(*l)).fmt(f)?;
+ if *suffixed {
+ f.write_str("i128")?;
+ }
+ }
+ },
+ Literal::Usize(val, suffixed) => {
+ val.fmt(f)?;
+ if *suffixed {
+ f.write_str("usize")?;
+ }
+ }
+ Literal::ISize(val, suffixed) => {
+ val.fmt(f)?;
+ if *suffixed {
+ f.write_str("isize")?;
+ }
+ }
+ Literal::Float32(val, suffixed) => {
+ val.fmt(f)?;
+ if *suffixed {
+ f.write_str("f32")?;
+ }
+ }
+ Literal::Float64(val, suffixed) => {
+ val.fmt(f)?;
+ if *suffixed {
+ f.write_str("f64")?;
+ }
+ }
+ }
+ Ok(())
+ }
+}
+
+impl FromStr for Literal {
+ type Err = LexError;
+
+ fn from_str(string: &str) -> Result<Self, LexError> {
+ let mut lit = Literal::Char(0);
+ // TODO: We might want to pass a LexError by reference to retrieve
+ // error information
+ if unsafe {
+ Literal__from_string(
+ string.as_ptr(),
+ string.len().try_into().unwrap(),
+ &mut lit as *mut Literal,
+ )
+ } {
+ Err(LexError)
+ } else {
+ Ok(lit)
+ }
+ }
+}
+
+impl Clone for Literal {
+ fn clone(&self) -> Self {
+ match self {
+ Literal::String { data, len } => unsafe { Literal__string(*data, *len) },
+ Literal::ByteString { data, size } => unsafe { Literal__byte_string(*data, *size) },
+ Literal::Char(val) => Literal::Char(*val),
+ Literal::Unsigned(val, suffixed) => Literal::Unsigned(*val, *suffixed),
+ Literal::Signed(val, suffixed) => Literal::Signed(*val, *suffixed),
+ Literal::Usize(val, suffixed) => Literal::Usize(*val, *suffixed),
+ Literal::ISize(val, suffixed) => Literal::ISize(*val, *suffixed),
+ Literal::Float32(val, suffixed) => Literal::Float32(*val, *suffixed),
+ Literal::Float64(val, suffixed) => Literal::Float64(*val, *suffixed),
+ }
+ }
+}
--- /dev/null
+use bridge::span::Span;
+use std::convert::TryFrom;
+use std::fmt;
+use Spacing;
+
+#[repr(C)]
+#[derive(Clone, Debug)]
+pub struct Punct {
+ pub(crate) ch: u32,
+ pub(crate) spacing: Spacing,
+}
+
+impl Punct {
+ pub fn new(ch: char, spacing: Spacing) -> Self {
+ Punct {
+ ch: ch.into(),
+ spacing,
+ }
+ }
+
+ pub fn span(&self) -> Span {
+ Span {}
+ }
+
+ pub fn set_span(&mut self, span: Span) {
+ let _ = span;
+ }
+}
+
+impl fmt::Display for Punct {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ if let Spacing::Alone = self.spacing {
+ f.write_str(" ")?;
+ }
+ char::try_from(self.ch).unwrap().fmt(f)
+ }
+}
--- /dev/null
+//! Bridge internal span representation and functions
+//!
+//! # Note
+//!
+//! All methods accessing source location in rust are unstable, hence this
+//! implementation with an empty structure.
+
+#[derive(Copy, Clone, Debug)]
+#[repr(C)]
+pub struct Span {}
+
+impl Span {
+ pub fn call_site() -> Self {
+ Span {}
+ }
+
+ pub fn mixed_site() -> Self {
+ Span {}
+ }
+
+ pub fn resolved_at(&self, _other: Span) -> Self {
+ Span {}
+ }
+
+ pub fn located_at(&self, _other: Span) -> Self {
+ Span {}
+ }
+
+ pub fn source_text(&self) -> Option<String> {
+ None
+ }
+}
--- /dev/null
+use bridge::{group::Group, ident::Ident, literal::Literal, punct::Punct};
+use std::convert::TryInto;
+use std::ffi::c_uchar;
+use std::fmt;
+use std::slice;
+use std::str::FromStr;
+use LexError;
+
+type ExternalTokenTree = crate::TokenTree;
+type ExternalTokenStream = crate::TokenStream;
+
+extern "C" {
+ fn TokenStream__new() -> TokenStream;
+ fn TokenStream__with_capacity(capacity: u64) -> TokenStream;
+ fn TokenStream__push(stream: *mut TokenStream, tree: TokenTree);
+ fn TokenStream__from_string(str: *const c_uchar, len: u64, ts: *mut TokenStream) -> bool;
+ fn TokenStream__clone(ts: *const TokenStream) -> TokenStream;
+}
+
+#[repr(C)]
+#[derive(Clone)]
+pub enum TokenTree {
+ Group(Group),
+ Ident(Ident),
+ Punct(Punct),
+ Literal(Literal),
+}
+
+impl fmt::Display for TokenTree {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ TokenTree::Group(group) => group.fmt(f),
+ TokenTree::Ident(ident) => ident.fmt(f),
+ TokenTree::Punct(punct) => punct.fmt(f),
+ TokenTree::Literal(literal) => literal.fmt(f),
+ }
+ }
+}
+
+impl From<ExternalTokenTree> for TokenTree {
+ fn from(value: ExternalTokenTree) -> Self {
+ match value {
+ ExternalTokenTree::Group(g) => TokenTree::Group(g.0),
+ ExternalTokenTree::Ident(i) => TokenTree::Ident(i.0),
+ ExternalTokenTree::Punct(p) => TokenTree::Punct(p.0),
+ ExternalTokenTree::Literal(l) => TokenTree::Literal(l.0),
+ }
+ }
+}
+
+#[repr(C)]
+#[derive(Debug)]
+pub struct TokenStream {
+ pub(crate) data: *const TokenTree,
+ pub(crate) size: u64,
+ capacity: u64,
+}
+
+impl TokenStream {
+ pub fn new() -> Self {
+ unsafe { TokenStream__new() }
+ }
+
+ fn with_capacity(capacity: u64) -> Self {
+ unsafe { TokenStream__with_capacity(capacity) }
+ }
+
+ fn push(&mut self, tree: TokenTree) {
+ unsafe { TokenStream__push(self as *mut TokenStream, tree) }
+ }
+
+ pub fn is_empty(&self) -> bool {
+ 0 == self.size
+ }
+
+ pub fn from_iterator<I>(it: I) -> Self
+ where
+ I: IntoIterator<Item = ExternalTokenStream>,
+ {
+ let it = it.into_iter();
+ let mut result = TokenStream::with_capacity(it.size_hint().0.try_into().unwrap());
+ for stream in it {
+ for item in stream.into_iter() {
+ result.push(item.into());
+ }
+ }
+ result
+ }
+
+ pub fn from_tree_iterator<I>(it: I) -> Self
+ where
+ I: IntoIterator<Item = ExternalTokenTree>,
+ {
+ let it = it.into_iter();
+ let mut result = TokenStream::with_capacity(it.size_hint().0.try_into().unwrap());
+ for item in it {
+ result.push(item.into());
+ }
+ result
+ }
+}
+
+impl Extend<ExternalTokenTree> for TokenStream {
+ fn extend<I: IntoIterator<Item = ExternalTokenTree>>(&mut self, trees: I) {
+ for tt in trees {
+ self.push(tt.into())
+ }
+ }
+}
+
+impl Extend<ExternalTokenStream> for TokenStream {
+ fn extend<I: IntoIterator<Item = ExternalTokenStream>>(&mut self, streams: I) {
+ for stream in streams {
+ for tt in stream {
+ self.push(tt.into());
+ }
+ }
+ }
+}
+
+impl fmt::Display for TokenStream {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ for i in unsafe { slice::from_raw_parts(self.data, self.size.try_into().unwrap()) } {
+ i.fmt(f)?;
+ match i {
+ TokenTree::Punct(_) => (),
+ _ => f.write_str(" ")?,
+ }
+ }
+ Ok(())
+ }
+}
+
+impl FromStr for TokenStream {
+ type Err = LexError;
+ fn from_str(string: &str) -> Result<Self, LexError> {
+ let mut ts = TokenStream::new();
+ if unsafe {
+ TokenStream__from_string(
+ string.as_ptr(),
+ string.len().try_into().unwrap(),
+ &mut ts as *mut TokenStream,
+ )
+ } {
+ Err(LexError)
+ } else {
+ Ok(ts)
+ }
+ }
+}
+
+impl Clone for TokenStream {
+ fn clone(&self) -> Self {
+ unsafe { TokenStream__clone(self as *const TokenStream) }
+ }
+}
--- /dev/null
+use bridge;
+use std::fmt;
+use Span;
+use TokenStream;
+
+/// Describes how a sequence of token trees is delimited.
+#[repr(C)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum Delimiter {
+ /// The sequence is delimited by a parenthesis `(...)`.
+ Parenthesis,
+ /// The sequence is delimited by a brace `{...}`.
+ Brace,
+ /// The sequence is delimited by a bracket `[...]`.
+ Bracket,
+ /// Invisible delimiter to preserve operator priority.
+ None,
+}
+
+/// A delimited token stream.
+#[derive(Clone)]
+pub struct Group(pub(crate) bridge::group::Group);
+
+impl Group {
+ /// Creates a new `Group`.
+ ///
+ /// # Arguments
+ ///
+ /// * `delimiter` - The delimiter surrounding the inner [`TokenStream`].
+ /// * `stream` - The tokenstream for this `Group`.
+ pub fn new(delimiter: Delimiter, stream: TokenStream) -> Self {
+ Group(bridge::group::Group::new(delimiter, stream.0))
+ }
+
+ /// Get the delimiter of the `Group`.
+ pub fn delimiter(&self) -> Delimiter {
+ self.0.delimiter()
+ }
+
+ /// Get the stream of the `Group`.
+ ///
+ /// # Note
+ ///
+ /// The returned stream does not include the delimiters of this group.
+ pub fn stream(&self) -> TokenStream {
+ TokenStream(self.0.stream())
+ }
+
+ /// Get the span for the delimiters of this token stream, spanning the
+ /// entire group.
+ pub fn span(&self) -> Span {
+ Span(self.0.span())
+ }
+
+ /// Get the span pointing to the opening delimiter of this `Group`.
+ pub fn span_open(&self) -> Span {
+ Span(self.0.span())
+ }
+
+ /// Get the span pointing to the closing delimiter of this `Group`.
+ pub fn span_close(&self) -> Span {
+ Span(self.0.span())
+ }
+
+ /// Change the span for this `Group`'s delimiters, but not its internal
+ /// tokens.
+ ///
+ /// # Note
+ ///
+ /// This method will **not** set the span of all the internal tokens spanned
+ /// by this group, but rather it will only set the span of the delimiter
+ /// tokens at the level of the `Group`.
+ pub fn set_span(&mut self, span: Span) {
+ self.0.set_span(span.0)
+ }
+}
+
+impl fmt::Display for Group {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.0.fmt(f)
+ }
+}
+
+impl fmt::Debug for Group {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.0.fmt(f)
+ }
+}
--- /dev/null
+use bridge;
+use std::fmt;
+use Span;
+
+/// An identifier.
+#[derive(Clone)]
+pub struct Ident(pub(crate) bridge::ident::Ident);
+
+impl Ident {
+ /// Creates a new identifier.
+ ///
+ /// # Arguments
+ ///
+ /// * `string` - A valid identifier.
+ /// * `span` - The span of the identifier.
+ ///
+ /// # Panics
+ ///
+ /// The `string` argument must be a valid identifier permitted by the
+ /// language, otherwise the function will panic.
+ pub fn new(string: &str, span: Span) -> Self {
+ Ident(bridge::ident::Ident::new(string, span.0))
+ }
+
+ /// Creates a new raw identifier.
+ ///
+ /// # Arguments
+ ///
+ /// * `string` - A valid identifier.
+ /// * `span` - The span of the identifier.
+ ///
+ /// # Panics
+ ///
+ /// The `string` argument must be a valid identifier permitted by the
+ /// language. Furthermore, it should not be a keyword used in path
+ /// segments, otherwise this function will panic.
+ pub fn new_raw(string: &str, span: Span) -> Self {
+ Ident(bridge::ident::Ident::new_raw(string, span.0))
+ }
+
+ /// Return the span of the identifier
+ pub fn span(&self) -> Span {
+ Span(self.0.span())
+ }
+
+ /// Change the span of the identifier.
+ ///
+ /// # Arguments
+ ///
+ /// * `span` - The new span value.
+ pub fn set_span(&mut self, span: Span) {
+ self.0.set_span(span.0);
+ }
+}
+
+impl fmt::Display for Ident {
+ /// Display as lossless converted string.
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.0.fmt(f)
+ }
+}
+
+impl fmt::Debug for Ident {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.0.fmt(f)
+ }
+}
--- /dev/null
+pub use group::{Delimiter, Group};
+pub use ident::Ident;
+pub use literal::Literal;
+pub use punct::{Punct, Spacing};
+pub use span::Span;
+use std::error;
+use std::{fmt, iter, str::FromStr};
+
+mod bridge;
+mod group;
+mod ident;
+mod literal;
+mod punct;
+mod span;
+pub mod token_stream;
+
+/// Determines whether proc_macro has been made accessible to the currently
+/// running program.
+///
+/// # Note
+///
+/// This function provide a non panicking way to detect whether the API is
+/// invoked from inside of a procedural macro.
+pub fn is_available() -> bool {
+ bridge::is_available()
+}
+
+/// A single token or a delimited sequence of token trees.
+#[derive(Clone)]
+pub enum TokenTree {
+ Group(Group),
+ Ident(Ident),
+ Punct(Punct),
+ Literal(Literal),
+}
+
+type InternalTokenTree = bridge::token_stream::TokenTree;
+
+impl From<InternalTokenTree> for TokenTree {
+ fn from(value: InternalTokenTree) -> Self {
+ match value {
+ InternalTokenTree::Group(g) => TokenTree::Group(Group(g)),
+ InternalTokenTree::Ident(i) => TokenTree::Ident(Ident(i)),
+ InternalTokenTree::Punct(p) => TokenTree::Punct(Punct(p)),
+ InternalTokenTree::Literal(l) => TokenTree::Literal(Literal(l)),
+ }
+ }
+}
+
+impl TokenTree {
+ /// Get the [`Span`] for this TokenTree.
+ pub fn span(&self) -> Span {
+ match self {
+ TokenTree::Group(group) => group.span(),
+ TokenTree::Ident(ident) => ident.span(),
+ TokenTree::Punct(punct) => punct.span(),
+ TokenTree::Literal(literal) => literal.span(),
+ }
+ }
+
+ /// Set the span for this TokenTree.
+ ///
+ /// # Arguments
+ ///
+ /// * `span` - The new span value.
+ pub fn set_span(&mut self, span: Span) {
+ match self {
+ TokenTree::Group(group) => group.set_span(span),
+ TokenTree::Ident(ident) => ident.set_span(span),
+ TokenTree::Punct(punct) => punct.set_span(span),
+ TokenTree::Literal(literal) => literal.set_span(span),
+ }
+ }
+}
+
+impl fmt::Debug for TokenTree {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ TokenTree::Group(group) => group.fmt(f),
+ TokenTree::Ident(ident) => ident.fmt(f),
+ TokenTree::Punct(punct) => punct.fmt(f),
+ TokenTree::Literal(literal) => literal.fmt(f),
+ }
+ }
+}
+
+impl fmt::Display for TokenTree {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ match self {
+ TokenTree::Group(group) => group.fmt(f),
+ TokenTree::Ident(ident) => ident.fmt(f),
+ TokenTree::Punct(punct) => punct.fmt(f),
+ TokenTree::Literal(literal) => literal.fmt(f),
+ }
+ }
+}
+
+impl From<Group> for TokenTree {
+ fn from(g: Group) -> Self {
+ TokenTree::Group(g)
+ }
+}
+
+impl From<Ident> for TokenTree {
+ fn from(i: Ident) -> Self {
+ TokenTree::Ident(i)
+ }
+}
+
+impl From<Punct> for TokenTree {
+ fn from(p: Punct) -> Self {
+ TokenTree::Punct(p)
+ }
+}
+
+impl From<Literal> for TokenTree {
+ fn from(l: Literal) -> Self {
+ TokenTree::Literal(l)
+ }
+}
+
+/// Error returned from `from_str` functions.
+#[derive(Debug)]
+pub struct LexError;
+
+impl fmt::Display for LexError {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.write_str("cannot parse string into token stream")
+ }
+}
+
+impl error::Error for LexError {}
+
+/// An abstract sequence of token trees.
+///
+/// This type provides interfaces for iterating over those token trees. This
+/// is both the input and the output of `#[proc_macro]`,
+/// `#[proc_macro_attribute]` and `#[proc_macro_derive]` definitions.
+#[derive(Clone)]
+pub struct TokenStream(bridge::token_stream::TokenStream);
+
+impl TokenStream {
+ // TODO: Add experimental API functions for this type
+
+ /// Creates an empty `TokenStream` containing no token trees.
+ pub fn new() -> Self {
+ TokenStream(bridge::token_stream::TokenStream::new())
+ }
+
+ /// Checks if this `TokenStream` is empty.
+ pub fn is_empty(&self) -> bool {
+ self.0.is_empty()
+ }
+}
+
+impl fmt::Display for TokenStream {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.0.fmt(f)
+ }
+}
+
+impl fmt::Debug for TokenStream {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.0.fmt(f)
+ }
+}
+
+impl FromStr for TokenStream {
+ type Err = LexError;
+
+ fn from_str(src: &str) -> Result<Self, LexError> {
+ bridge::token_stream::TokenStream::from_str(src).map(TokenStream)
+ }
+}
+
+impl iter::FromIterator<TokenTree> for TokenStream {
+ fn from_iter<I: IntoIterator<Item = TokenTree>>(trees: I) -> Self {
+ TokenStream(bridge::token_stream::TokenStream::from_tree_iterator(trees))
+ }
+}
+
+impl iter::FromIterator<TokenStream> for TokenStream {
+ fn from_iter<I: IntoIterator<Item = TokenStream>>(streams: I) -> Self {
+ TokenStream(bridge::token_stream::TokenStream::from_iterator(streams))
+ }
+}
+
+impl Extend<TokenTree> for TokenStream {
+ fn extend<I: IntoIterator<Item = TokenTree>>(&mut self, trees: I) {
+ self.0.extend(trees);
+ }
+}
+
+impl Extend<TokenStream> for TokenStream {
+ fn extend<I: IntoIterator<Item = TokenStream>>(&mut self, streams: I) {
+ self.0.extend(streams)
+ }
+}
--- /dev/null
+use bridge;
+use std::fmt;
+use std::str::FromStr;
+use LexError;
+use Span;
+
+/// A type representing a literal value except `true` and `false`.
+///
+/// This could be one of the following:
+/// * literal string (`"hello"`)
+/// * byte string (`b"hello"`)
+/// * character (`'a'`)
+/// * byte character (`b'a'`)
+/// * unsuffixed integer (`42`)
+/// * suffixed integer (`42u8`)
+/// * unsuffixed floating point number (`1.618`)
+/// * suffixed floating point number (`1.618f32`)
+///
+/// # Note
+///
+/// Boolean literals like `true` and `false` are `Ident`s and do not belong
+/// here.
+#[derive(Clone)]
+pub struct Literal(pub(crate) bridge::literal::Literal);
+
+impl Literal {
+ // TODO: Add experimental API functions for this type
+ // TODO: Generate those constructor with 1/2 macros instead
+
+ pub fn u8_suffixed(n: u8) -> Self {
+ Literal(bridge::literal::Literal::u8_suffixed(n))
+ }
+
+ pub fn u16_suffixed(n: u16) -> Self {
+ Literal(bridge::literal::Literal::u16_suffixed(n))
+ }
+
+ pub fn u32_suffixed(n: u32) -> Self {
+ Literal(bridge::literal::Literal::u32_suffixed(n))
+ }
+
+ pub fn u64_suffixed(n: u64) -> Self {
+ Literal(bridge::literal::Literal::u64_suffixed(n))
+ }
+
+ pub fn u128_suffixed(n: u128) -> Self {
+ Literal(bridge::literal::Literal::u128_suffixed(n))
+ }
+
+ pub fn usize_suffixed(n: usize) -> Self {
+ Literal(bridge::literal::Literal::usize_suffixed(n))
+ }
+
+ pub fn i8_suffixed(n: i8) -> Self {
+ Literal(bridge::literal::Literal::i8_suffixed(n))
+ }
+
+ pub fn i16_suffixed(n: i16) -> Self {
+ Literal(bridge::literal::Literal::i16_suffixed(n))
+ }
+
+ pub fn i32_suffixed(n: i32) -> Self {
+ Literal(bridge::literal::Literal::i32_suffixed(n))
+ }
+
+ pub fn i64_suffixed(n: i64) -> Self {
+ Literal(bridge::literal::Literal::i64_suffixed(n))
+ }
+
+ pub fn i128_suffixed(n: i128) -> Self {
+ Literal(bridge::literal::Literal::i128_suffixed(n))
+ }
+
+ pub fn isize_suffixed(n: isize) -> Self {
+ Literal(bridge::literal::Literal::isize_suffixed(n))
+ }
+
+ // Unsuffixed
+
+ pub fn u8_unsuffixed(n: u8) -> Self {
+ Literal(bridge::literal::Literal::u8_unsuffixed(n))
+ }
+
+ pub fn u16_unsuffixed(n: u16) -> Self {
+ Literal(bridge::literal::Literal::u16_unsuffixed(n))
+ }
+
+ pub fn u32_unsuffixed(n: u32) -> Self {
+ Literal(bridge::literal::Literal::u32_unsuffixed(n))
+ }
+
+ pub fn u64_unsuffixed(n: u64) -> Self {
+ Literal(bridge::literal::Literal::u64_unsuffixed(n))
+ }
+
+ pub fn u128_unsuffixed(n: u128) -> Self {
+ Literal(bridge::literal::Literal::u128_unsuffixed(n))
+ }
+
+ pub fn usize_unsuffixed(n: usize) -> Self {
+ Literal(bridge::literal::Literal::usize_unsuffixed(n))
+ }
+
+ pub fn i8_unsuffixed(n: i8) -> Self {
+ Literal(bridge::literal::Literal::i8_unsuffixed(n))
+ }
+
+ pub fn i16_unsuffixed(n: i16) -> Self {
+ Literal(bridge::literal::Literal::i16_unsuffixed(n))
+ }
+
+ pub fn i32_unsuffixed(n: i32) -> Self {
+ Literal(bridge::literal::Literal::i32_unsuffixed(n))
+ }
+
+ pub fn i64_unsuffixed(n: i64) -> Self {
+ Literal(bridge::literal::Literal::i64_unsuffixed(n))
+ }
+
+ pub fn i128_unsuffixed(n: i128) -> Self {
+ Literal(bridge::literal::Literal::i128_unsuffixed(n))
+ }
+
+ pub fn isize_unsuffixed(n: isize) -> Self {
+ Literal(bridge::literal::Literal::isize_unsuffixed(n))
+ }
+
+ pub fn f32_unsuffixed(n: f32) -> Self {
+ Literal(bridge::literal::Literal::f32_unsuffixed(n))
+ }
+
+ pub fn f32_suffixed(n: f32) -> Self {
+ Literal(bridge::literal::Literal::f32_suffixed(n))
+ }
+
+ pub fn f64_unsuffixed(n: f64) -> Self {
+ Literal(bridge::literal::Literal::f64_unsuffixed(n))
+ }
+
+ pub fn f64_suffixed(n: f64) -> Self {
+ Literal(bridge::literal::Literal::f64_suffixed(n))
+ }
+
+ pub fn string(string: &str) -> Self {
+ Literal(bridge::literal::Literal::string(string))
+ }
+
+ pub fn character(c: char) -> Self {
+ Literal(bridge::literal::Literal::character(c))
+ }
+
+ pub fn byte_string(bytes: &[u8]) -> Self {
+ Literal(bridge::literal::Literal::byte_string(bytes))
+ }
+
+ /// Get the [`Span`] for this literal.
+ pub fn span(&self) -> Span {
+ Span(self.0.span())
+ }
+
+ /// Set the span for this literal.
+ ///
+ /// # Arguments
+ ///
+ /// * `span` - The new span value.
+ pub fn set_span(&mut self, span: Span) {
+ self.0.set_span(span.0);
+ }
+}
+
+impl fmt::Debug for Literal {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.0.fmt(f)
+ }
+}
+
+impl fmt::Display for Literal {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.0.fmt(f)
+ }
+}
+
+impl FromStr for Literal {
+ type Err = LexError;
+
+ fn from_str(src: &str) -> Result<Self, LexError> {
+ bridge::literal::Literal::from_str(src).map(Literal)
+ }
+}
--- /dev/null
+use bridge;
+use std::convert::TryInto;
+use std::fmt;
+use Span;
+
+/// Describes the context of a [`Punct`] relatively to the next token.
+#[repr(C)]
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum Spacing {
+ /// A [`Punct`] is not immediately followed by another `Punct`.
+ Alone,
+ /// A [`Punct`] is immediately followed by another `Punct` and can be
+ /// combined into a multi-character operator.
+ Joint,
+}
+
+/// Single punctuation character such as `+`, `-` or `#`.
+///
+/// Multi-character operators like `+=` are represented as two instances of
+/// `Punct` with different forms of `Spacing` returned.
+#[derive(Clone)]
+pub struct Punct(pub(crate) bridge::punct::Punct);
+
+impl Punct {
+ /// Creates a new `Punct` from a given character and spacing.
+ ///
+ /// # Arguments
+ ///
+ /// * `ch` - The punctuation character.
+ /// * `spacing` - The link between this character and the next one.
+ ///
+ /// # Panics
+ ///
+ /// This function will panic if the `ch` argument is not a valid
+ /// punctuation character allowed by the language.
+ pub fn new(ch: char, spacing: Spacing) -> Self {
+ Punct(bridge::punct::Punct::new(ch, spacing))
+ }
+
+ /// Get the value for this punctuation character as `char`.
+ pub fn as_char(&self) -> char {
+ self.0
+ .ch
+ .try_into()
+ .expect("Cannot convert from u32 to char")
+ }
+
+ /// Get the [`Spacing`] of this punctuation character, indicating whether
+ /// the following character can be combined into a multi-character operator
+ /// or not.
+ pub fn spacing(&self) -> Spacing {
+ self.0.spacing
+ }
+
+ /// Get the [`Span`] for this punctuation character.
+ pub fn span(&self) -> Span {
+ Span(self.0.span())
+ }
+
+ /// Set the span for this punctuation character.
+ ///
+ /// # Arguments
+ ///
+ /// * `span` - The new span value.
+ pub fn set_span(&mut self, span: Span) {
+ self.0.set_span(span.0);
+ }
+}
+
+impl fmt::Display for Punct {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.0.fmt(f)
+ }
+}
+
+impl fmt::Debug for Punct {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.0.fmt(f)
+ }
+}
+
+impl PartialEq<char> for Punct {
+ fn eq(&self, rhs: &char) -> bool {
+ self.as_char() == *rhs
+ }
+}
+
+impl PartialEq<Punct> for char {
+ fn eq(&self, rhs: &Punct) -> bool {
+ *self == rhs.as_char()
+ }
+}
--- /dev/null
+use bridge;
+use std::fmt;
+
+/// A region of source code along with macro expansion information.
+#[derive(Copy, Clone)]
+pub struct Span(pub(crate) bridge::span::Span);
+
+impl Span {
+ // TODO: Add experimental API functions for this type
+
+ /// Creates a new span that resolves at the macro call location.
+ pub fn call_site() -> Self {
+ Span(bridge::span::Span::call_site())
+ }
+
+ /// Creates a new span that resolved sometimes at macro call site, and
+ /// sometimes at macro definition site.
+ pub fn mixed_site() -> Self {
+ Span(bridge::span::Span::mixed_site())
+ }
+
+ /// Creates a new span with the same line/column informations but that
+ /// resolve symbols as though it were at `other`.
+ ///
+ /// # Arguments
+ ///
+ /// * `other` - Other span to resolve at.
+ pub fn resolved_at(&self, other: Span) -> Self {
+ Span(self.0.resolved_at(other.0))
+ }
+
+ /// Creates a new span with the same name resolution behavior as self, but
+ /// with the line/column information of `other`.
+ ///
+ /// # Arguments
+ ///
+ /// * `other` - Other span containing the line/column informations to use.
+ pub fn located_at(&self, other: Span) -> Self {
+ Span(self.0.located_at(other.0))
+ }
+
+ /// Return the source text behind a span.
+ pub fn source_text(&self) -> Option<String> {
+ self.0.source_text()
+ }
+}
+
+impl fmt::Debug for Span {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ self.0.fmt(f)
+ }
+}
--- /dev/null
+//! Public implementation details for the `TokenStream` type, such as iterators.
+use bridge;
+use std::convert::TryInto;
+
+use TokenStream;
+use TokenTree;
+
+/// An iterator over [`TokenStream`]'s [`TokenTree`]s.
+#[derive(Clone)]
+pub struct IntoIter {
+ current: *const bridge::token_stream::TokenTree,
+ end: *const bridge::token_stream::TokenTree,
+}
+
+impl Iterator for IntoIter {
+ type Item = TokenTree;
+
+ fn next(&mut self) -> Option<TokenTree> {
+ if self.current == self.end {
+ None
+ } else {
+ let result = self.current;
+ self.current = unsafe { self.current.add(1) };
+ Some(unsafe { std::ptr::read(result) }.into())
+ }
+ }
+
+ fn size_hint(&self) -> (usize, Option<usize>) {
+ // TODO: I'm not a fan of those casts, once #![feature(ptr_sub_ptr)]
+ // is implemented we may replace this line by the following:
+ // self.end.sub_ptr(self.current)
+ let remaining = self.end as usize - self.current as usize;
+ (remaining, Some(remaining))
+ }
+
+ fn count(self) -> usize {
+ self.end as usize - self.current as usize
+ }
+}
+
+impl IntoIterator for TokenStream {
+ type Item = TokenTree;
+ type IntoIter = IntoIter;
+
+ fn into_iter(self) -> IntoIter {
+ let capacity = self.0.size.try_into().unwrap();
+ IntoIter {
+ current: self.0.data,
+ end: unsafe { self.0.data.add(capacity) },
+ }
+ }
+}
projects / thirdparty / gcc.git / commitdiff
