Mike Gerwitz

Activist for User Freedom

aboutsummaryrefslogtreecommitdiffstats
path: root/tamer
diff options
context:
space:
mode:
authorMike Gerwitz <mike.gerwitz@ryansg.com>2022-03-22 13:56:43 -0400
committerMike Gerwitz <mike.gerwitz@ryansg.com>2022-03-22 14:06:52 -0400
commitb4a7591357077db4c9e88885d913ddaf229fd719 (patch)
treea373f71193ee9536ae14029332608057de707baa /tamer
parentf6957ff0286215bf091ee7d3d36e62ff61916c07 (diff)
downloadtame-b4a7591357077db4c9e88885d913ddaf229fd719.tar.gz
tame-b4a7591357077db4c9e88885d913ddaf229fd719.tar.bz2
tame-b4a7591357077db4c9e88885d913ddaf229fd719.zip
tamer: obj::xmlo::reader: Begin conversion to ParseState
This begins to transition XmloReader into a ParseState. Unlike previous changes where ParseStates were composed into a single ParseState, this is instead a lowering operation that will take the output of one Parser and provide it to another. The mess in ld::poc (...which still needs to be refactored and removed) shows the concept, which will be abstracted away. This won't actually get to the ASG in order to test that that this works with the wip-xmlo-xir-reader flag on (development hasn't gotten that far yet), but since it type-checks, it should conceptually work. Wiring lowering operations together is something that I've been dreading for months, but my approach of only abstracting after-the-fact has helped to guide a sane approach for this. For some definition of "sane". It's also worth noting that AsgBuilder will too become a ParseState implemented as another lowering operation, so: XIR -> XIRF -> XMLO -> ASG These steps will all be streaming, with iteration happening only at the topmost level. For this reason, it's important that ASG not be responsible for doing that pull, and further we should propagate Parsed::Incomplete rather than filtering it out and looping an indeterminate number of times outside of the toplevel. One final note: the choice of 64 for the maximum depth is entirely arbitrary and should be more than generous; it'll be finalized at some point in the future once I actually evaluate what maximum depth is reasonable based on how the system is used, with some added growing room. DEV-10863
Diffstat (limited to 'tamer')
-rw-r--r--tamer/src/ld/poc.rs39
-rw-r--r--tamer/src/obj/xmlo/error.rs2
-rw-r--r--tamer/src/obj/xmlo/reader.rs69
-rw-r--r--tamer/src/obj/xmlo/reader/test.rs67
-rw-r--r--tamer/src/parse.rs2
-rw-r--r--tamer/src/xir/flat.rs2
6 files changed, 100 insertions, 81 deletions
diff --git a/tamer/src/ld/poc.rs b/tamer/src/ld/poc.rs
index 2a8c994..1bd25ca 100644
--- a/tamer/src/ld/poc.rs
+++ b/tamer/src/ld/poc.rs
@@ -25,9 +25,6 @@ use super::xmle::{
xir::lower_iter,
XmleSections,
};
-use crate::sym::SymbolId;
-use crate::sym::{GlobalSymbolIntern, GlobalSymbolResolve};
-use crate::xir::writer::XmlWriter;
use crate::{
asg::{Asg, DefaultAsg, IdentObject},
xir::DefaultEscaper,
@@ -43,13 +40,19 @@ use crate::{
obj::xmlo::{AsgBuilder, AsgBuilderState, XmloReader},
xir::Escaper,
};
+use crate::{parse::ParseState, sym::SymbolId};
+use crate::{parse::Parsed, xir::writer::XmlWriter};
+use crate::{
+ sym::{GlobalSymbolIntern, GlobalSymbolResolve},
+ xir::flat,
+};
use fxhash::FxBuildHasher;
use petgraph_graphml::GraphMl;
-use std::error::Error;
use std::fs;
use std::io::Write;
use std::io::{BufReader, BufWriter};
use std::path::{Path, PathBuf};
+use std::{error::Error, iter};
type LinkerAsg = DefaultAsg<IdentObject>;
type LinkerAsgBuilderState = AsgBuilderState<FxBuildHasher>;
@@ -195,10 +198,32 @@ fn load_xmlo<'a, P: AsRef<Path>, S: Escaper>(
use crate::iter::into_iter_while_ok;
use crate::xir::reader::XmlXirReader;
+ // TODO: This entire block is a WIP and will be incrementally
+ // abstracted away.
into_iter_while_ok(XmlXirReader::new(file, escaper), |toks| {
- let xmlo: XmloReader<_> = toks.into();
- depgraph.import_xmlo(xmlo, state)
- })??
+ into_iter_while_ok(flat::State::<64>::parse(toks), |xirf| {
+ let mut xmlo = XmloReader::parse(iter::empty());
+
+ let foo = xirf.map(|parsed| match parsed {
+ Parsed::Incomplete => Ok(Parsed::Incomplete),
+ Parsed::Object(obj) => {
+ let item: flat::Object = obj;
+ xmlo.feed_tok(item)
+ }
+ });
+
+ into_iter_while_ok(foo, |xmlo_out| {
+ // TODO: Transitionary---we do not want to filter.
+ depgraph.import_xmlo(
+ xmlo_out.filter_map(|parsed| match parsed {
+ Parsed::Incomplete => None,
+ Parsed::Object(obj) => Some(Ok(obj)),
+ }),
+ state,
+ )
+ })
+ })
+ })????
}
};
diff --git a/tamer/src/obj/xmlo/error.rs b/tamer/src/obj/xmlo/error.rs
index 81720ce..2fb4e6e 100644
--- a/tamer/src/obj/xmlo/error.rs
+++ b/tamer/src/obj/xmlo/error.rs
@@ -34,7 +34,7 @@ use std::fmt::Display;
/// This drastically simplifies the reader and [`Result`] chaining.
///
/// TODO: These errors provide no context (byte offset).
-#[derive(Debug, PartialEq)]
+#[derive(Debug, PartialEq, Eq)]
pub enum XmloError {
/// XML parsing error (legacy, quick-xml).
XmlError(XmlError),
diff --git a/tamer/src/obj/xmlo/reader.rs b/tamer/src/obj/xmlo/reader.rs
index 7119870..74df3b0 100644
--- a/tamer/src/obj/xmlo/reader.rs
+++ b/tamer/src/obj/xmlo/reader.rs
@@ -44,68 +44,45 @@ mod new {
//! This module will be merged into [`super`] once complete;
//! it exists to make feature-flagging less confusing and error-prone.
- use super::{XmloError, XmloEvent, XmloResult};
+ use super::{XmloError, XmloEvent};
+ use crate::parse::{ParseState, Transition};
use crate::sym::st::*;
- use crate::xir::{Token, TokenStream};
+ use crate::xir::flat::Object as Xirf;
qname_const! {
QN_LV_PACKAGE: L_LV:L_PACKAGE,
QN_PACKAGE: :L_PACKAGE,
}
- #[derive(Debug)]
- pub struct XmloReader<I: TokenStream> {
- reader: I,
- seen_root: bool,
+ #[derive(Debug, Default, PartialEq, Eq)]
+ pub enum XmloReader {
+ #[default]
+ Ready,
}
- impl<I> XmloReader<I>
- where
- I: TokenStream,
- {
- pub fn from_reader(reader: I) -> Self {
- Self {
- reader,
- seen_root: false,
- }
- }
-
- pub fn read_event(&mut self) -> XmloResult<XmloEvent> {
- let token = self.reader.next().ok_or(XmloError::UnexpectedEof)?;
+ impl ParseState for XmloReader {
+ type Token = Xirf;
+ type Object = XmloEvent;
+ type Error = XmloError;
- if !self.seen_root {
- match token {
- Token::Open(QN_LV_PACKAGE | QN_PACKAGE, _) => {
- todo!("PackageAttrs removed");
- }
+ fn parse_token(
+ self,
+ tok: Self::Token,
+ ) -> crate::parse::TransitionResult<Self> {
+ use XmloReader::Ready;
- _ => return Err(XmloError::UnexpectedRoot),
+ match (self, tok) {
+ (Ready, Xirf::Open(QN_LV_PACKAGE | QN_PACKAGE, _, _)) => {
+ todo!("PackageAttrs removed");
}
- }
- match token {
- todo => todo!("read_event: {:?}", todo),
+ (Ready, _) => Transition(Ready).err(XmloError::UnexpectedRoot),
}
}
- }
-
- impl<I> Iterator for XmloReader<I>
- where
- I: TokenStream,
- {
- type Item = XmloResult<XmloEvent>;
-
- fn next(&mut self) -> Option<Self::Item> {
- Some(self.read_event())
- }
- }
- impl<I> From<I> for XmloReader<I>
- where
- I: TokenStream,
- {
- fn from(toks: I) -> Self {
- Self::from_reader(toks)
+ fn is_accepting(&self) -> bool {
+ // TODO
+ todo!("XmloReader::is_accepting")
}
}
}
diff --git a/tamer/src/obj/xmlo/reader/test.rs b/tamer/src/obj/xmlo/reader/test.rs
index 6c745c9..6f0d798 100644
--- a/tamer/src/obj/xmlo/reader/test.rs
+++ b/tamer/src/obj/xmlo/reader/test.rs
@@ -20,23 +20,33 @@
use std::assert_matches::assert_matches;
use super::*;
-use crate::{convert::ExpectInto, span::DUMMY_SPAN as DS, xir::Token};
+use crate::{
+ convert::ExpectInto,
+ parse::{ParseError, ParseState},
+ span::DUMMY_SPAN as DS,
+ xir::{
+ attr::Attr,
+ flat::{Depth, Object as Xirf},
+ },
+};
-type Sut<B> = XmloReader<B>;
-
-#[test]
-fn fail_unexpected_eof() {
- let mut sut = Sut::from_reader([].into_iter());
- assert_matches!(sut.next(), Some(Err(XmloError::UnexpectedEof)));
-}
+type Sut = XmloReader;
#[test]
fn fails_on_invalid_root() {
- let mut sut = Sut::from_reader(
- [Token::Open("not-a-valid-package-node".unwrap_into(), DS)].into_iter(),
+ let mut sut = Sut::parse(
+ [Xirf::Open(
+ "not-a-valid-package-node".unwrap_into(),
+ DS,
+ Depth(0),
+ )]
+ .into_iter(),
);
- assert_matches!(sut.next(), Some(Err(XmloError::UnexpectedRoot)));
+ assert_matches!(
+ sut.next(),
+ Some(Err(ParseError::StateError(XmloError::UnexpectedRoot)))
+ );
}
//#[test]
@@ -45,17 +55,25 @@ fn _parses_package_attrs() {
let relroot = "../../".into();
let elig = "elig-class-yields".into();
- let mut sut = Sut::from_reader(
+ let mut sut = Sut::parse(
[
- Token::Open("package".unwrap_into(), DS),
- Token::AttrName("name".unwrap_into(), DS),
- Token::AttrValue(name, DS),
- Token::AttrName("__rootpath".unwrap_into(), DS),
- Token::AttrValue(relroot, DS),
- Token::AttrName("program".unwrap_into(), DS),
- Token::AttrValue(crate::sym::st::raw::L_TRUE, DS),
- Token::AttrName(("preproc", "elig-class-yields").unwrap_into(), DS),
- Token::AttrValue(elig, DS),
+ Xirf::Open("package".unwrap_into(), DS, Depth(0)),
+ Xirf::Attr(Attr::new("name".unwrap_into(), name, (DS, DS))),
+ Xirf::Attr(Attr::new(
+ "__rootpath".unwrap_into(),
+ relroot,
+ (DS, DS),
+ )),
+ Xirf::Attr(Attr::new(
+ "program".unwrap_into(),
+ crate::sym::st::raw::L_TRUE,
+ (DS, DS),
+ )),
+ Xirf::Attr(Attr::new(
+ ("preproc", "elig-class-yields").unwrap_into(),
+ elig,
+ (DS, DS),
+ )),
]
.into_iter(),
);
@@ -70,11 +88,10 @@ fn _parses_package_attrs() {
fn _parses_package_attrs_with_ns_prefix() {
let name = "pkgrootns".into();
- let mut sut = Sut::from_reader(
+ let mut sut = Sut::parse(
[
- Token::Open(("lv", "package").unwrap_into(), DS),
- Token::AttrName("name".unwrap_into(), DS),
- Token::AttrValue(name, DS),
+ Xirf::Open(("lv", "package").unwrap_into(), DS, Depth(0)),
+ Xirf::Attr(Attr::new("name".unwrap_into(), name, (DS, DS))),
]
.into_iter(),
);
diff --git a/tamer/src/parse.rs b/tamer/src/parse.rs
index 0b95e21..10440ad 100644
--- a/tamer/src/parse.rs
+++ b/tamer/src/parse.rs
@@ -262,7 +262,7 @@ impl<S: ParseState, I: TokenStream<S::Token>> Parser<S, I> {
/// [`Iterator`] implementation.
/// The pull system also uses this method to provided data to the
/// parser.
- fn feed_tok(&mut self, tok: S::Token) -> ParsedResult<S> {
+ pub fn feed_tok(&mut self, tok: S::Token) -> ParsedResult<S> {
// Store the most recently encountered Span for error
// reporting in case we encounter an EOF.
self.last_span = Some(tok.span());
diff --git a/tamer/src/xir/flat.rs b/tamer/src/xir/flat.rs
index 13951c5..c5f5476 100644
--- a/tamer/src/xir/flat.rs
+++ b/tamer/src/xir/flat.rs
@@ -56,7 +56,7 @@ use std::{error::Error, fmt::Display};
/// Tag nesting depth
/// (`0` represents the root).
#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct Depth(usize);
+pub struct Depth(pub usize);
impl Display for Depth {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {