Moved scalar scanning-related parameters to a struct.

Renamed the valid/possible tokens to a single variable status with enums valid, invalid, and unverified.

regex.cpp

@@ -45,6 +45,21 @@ namespace YAML
 		delete m_pOp;
 	}
 
+	RegEx& RegEx::operator = (const RegEx& rhs)
+	{
+		delete m_pOp;
+		m_pOp = 0;
+
+		m_op = rhs.m_op;
+		m_a = rhs.m_a;
+		m_z = rhs.m_z;
+		m_params = rhs.m_params;
+
+		SetOp();
+
+		return *this;
+	}
+
 	void RegEx::SetOp()
 	{
 		delete m_pOp;

regex.h

@@ -60,6 +60,8 @@ namespace YAML
 		RegEx(const RegEx& rhs);
 		~RegEx();
 
+		RegEx& operator = (const RegEx& rhs);
+
 		bool Matches(char ch) const;
 		bool Matches(const std::string& str) const;
 		bool Matches(std::istream& in) const;

scanner.cpp

@@ -123,7 +123,7 @@ namespace YAML
 		ScanToNextToken();
 
 		// check the latest simple key
-		ValidateSimpleKey();
+		VerifySimpleKey();
 
 		// maybe need to end some blocks
 		PopIndentTo(INPUT.column);
@@ -213,7 +213,7 @@ namespace YAML
 			INPUT.EatLineBreak();
 
 			// oh yeah, and let's get rid of that simple key
-			ValidateSimpleKey();
+			VerifySimpleKey();
 
 			// new line - we may be able to accept a simple key now
 			if(m_flowLevel == 0)
@@ -272,16 +272,15 @@ namespace YAML
 			if(!m_tokens.empty())
 				pToken = m_tokens.front();
 
-			// ... that's possible
 			// (here's where we clean up the impossible tokens)
-			if(pToken && !pToken->isPossible) {
+			if(pToken && pToken->status == TS_INVALID) {
 				m_tokens.pop();
 				delete pToken;
 				continue;
 			}
 
-			// and valid
+			// on unverified tokens, we just have to wait
-			if(pToken && !pToken->isValid)
+			if(pToken && pToken->status == TS_UNVERIFIED)
 				pToken = 0;
 
 			// then that's what we want

scanner.h

@@ -26,8 +26,8 @@ namespace YAML
 		void PopIndentTo(int column);
 
 		void InsertSimpleKey();
-		bool ValidateSimpleKey();
+		bool VerifySimpleKey();
-		void ValidateAllSimpleKeys();
+		void VerifyAllSimpleKeys();
 
 		void Scan();
 

scanscalar.cpp

@@ -135,14 +135,20 @@ namespace YAML
 		//	scalar += info.Join();
 		//}
 
-		RegEx end = (m_flowLevel > 0 ? Exp::EndScalarInFlow : Exp::EndScalar) || (RegEx(' ') + Exp::Comment);
+		ScanScalarInfo info;
-		int indent = (m_flowLevel > 0 ? 0 : m_indents.top() + 1);
+		info.end = (m_flowLevel > 0 ? Exp::EndScalarInFlow : Exp::EndScalar) || (RegEx(' ') + Exp::Comment);
+		info.eatEnd = false;
+		info.indent = (m_flowLevel > 0 ? 0 : m_indents.top() + 1);
+		info.fold = true;
+		info.eatLeadingWhitespace = true;
+		info.trimTrailingSpaces = true;
+		info.chomp = CLIP;
 
 		// insert a potential simple key
 		if(m_simpleKeyAllowed)
 			InsertSimpleKey();
 
-		pToken->value = ScanScalar(INPUT, end, false, indent, 0, true, true, true, 0);
+		pToken->value = ScanScalar(INPUT, info);
 
 		m_simpleKeyAllowed = false;
 		if(true/*info.leadingBlanks*/)
@@ -224,14 +230,21 @@ namespace YAML
 		char quote = INPUT.GetChar();
 		pToken->single = (quote == '\'');
 
-		RegEx end = (pToken->single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
+		ScanScalarInfo info;
-		char escape = (pToken->single ? '\'' : '\\');
+		info.end = (pToken->single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
+		info.eatEnd = true;
+		info.escape = (pToken->single ? '\'' : '\\');
+		info.indent = 0;
+		info.fold = true;
+		info.eatLeadingWhitespace = true;
+		info.trimTrailingSpaces = false;
+		info.chomp = CLIP;
 
 		// insert a potential simple key
 		if(m_simpleKeyAllowed)
 			InsertSimpleKey();
 
-		pToken->value = ScanScalar(INPUT, end, true, 0, escape, true, true, false, 0);
+		pToken->value = ScanScalar(INPUT, info);
 		m_simpleKeyAllowed = false;
 
 		return pToken;
@@ -274,8 +287,14 @@ namespace YAML
 
 		GetBlockIndentation(INPUT, indent, info.trailingBreaks, m_indents.top());
 
-		bool eatLeadingWhitespace = false;
+		ScanScalarInfo sinfo;
-		pToken->value = ScanScalar(INPUT, RegEx(), false, indent, 0, info.fold, eatLeadingWhitespace, false, info.chomp);
+		sinfo.indent = indent;
+		sinfo.fold = info.fold;
+		sinfo.eatLeadingWhitespace = false;
+		sinfo.trimTrailingSpaces = false;
+		sinfo.chomp = (CHOMP) info.chomp;
+
+		pToken->value = ScanScalar(INPUT, sinfo);
 
 		// simple keys always ok after block scalars (since we're gonna start a new line anyways)
 		m_simpleKeyAllowed = true;
@@ -322,7 +341,7 @@ namespace YAML
 	}
 
 	// ScanScalar
-	std::string ScanScalar(Stream& INPUT, RegEx end, bool eatEnd, int indent, char escape, bool fold, bool eatLeadingWhitespace, bool trimTrailingSpaces, int chomp)
+	std::string ScanScalar(Stream& INPUT, ScanScalarInfo info)
 	{
 		bool emptyLine = false, moreIndented = false;
 		std::string scalar;
@@ -330,19 +349,19 @@ namespace YAML
 		while(INPUT) {
 			// ********************************
 			// Phase #1: scan until line ending
-			while(!end.Matches(INPUT) && !Exp::Break.Matches(INPUT)) {
+			while(!info.end.Matches(INPUT) && !Exp::Break.Matches(INPUT)) {
 				if(INPUT.peek() == EOF)
 					break;
 
 				// escaped newline? (only if we're escaping on slash)
-				if(escape == '\\' && Exp::EscBreak.Matches(INPUT)) {
+				if(info.escape == '\\' && Exp::EscBreak.Matches(INPUT)) {
 					int n = Exp::EscBreak.Match(INPUT);
 					INPUT.Eat(n);
 					continue;
 				}
 
 				// escape this?
-				if(INPUT.peek() == escape) {
+				if(INPUT.peek() == info.escape) {
 					scalar += Exp::Escape(INPUT);
 					continue;
 				}
@@ -353,15 +372,15 @@ namespace YAML
 
 			// eof? if we're looking to eat something, then we throw
 			if(INPUT.peek() == EOF) {
-				if(eatEnd)
+				if(info.eatEnd)
 					throw EOFInQuote();
 				break;
 			}
 
 			// are we done via character match?
-			int n = end.Match(INPUT);
+			int n = info.end.Match(INPUT);
 			if(n >= 0) {
-				if(eatEnd)
+				if(info.eatEnd)
 					INPUT.Eat(n);
 				break;
 			}
@@ -375,11 +394,11 @@ namespace YAML
 			// Phase #3: scan initial spaces
 
 			// first the required indentation
-			while(INPUT.peek() == ' ' && INPUT.column < indent)
+			while(INPUT.peek() == ' ' && INPUT.column < info.indent)
 				INPUT.Eat(1);
 
 			// and then the rest of the whitespace
-			if(eatLeadingWhitespace) {
+			if(info.eatLeadingWhitespace) {
 				while(Exp::Blank.Matches(INPUT))
 					INPUT.Eat(1);
 			}
@@ -388,7 +407,7 @@ namespace YAML
 			bool nextEmptyLine = Exp::Break.Matches(INPUT);
 			bool nextMoreIndented = (INPUT.peek() == ' ');
 
-			if(fold && !emptyLine && !nextEmptyLine && !moreIndented && !nextMoreIndented)
+			if(info.fold && !emptyLine && !nextEmptyLine && !moreIndented && !nextMoreIndented)
 				scalar += " ";
 			else
 				scalar += "\n"; 
@@ -397,22 +416,22 @@ namespace YAML
 			moreIndented = nextMoreIndented;
 
 			// are we done via indentation?
-			if(!emptyLine && INPUT.column < indent)
+			if(!emptyLine && INPUT.column < info.indent)
 				break;
 		}
 
 		// post-processing
-		if(trimTrailingSpaces) {
+		if(info.trimTrailingSpaces) {
 			unsigned pos = scalar.find_last_not_of(' ');
 			if(pos < scalar.size())
 				scalar.erase(pos + 1);
 		}
 
-		if(chomp <= 0) {
+		if(info.chomp <= 0) {
 			unsigned pos = scalar.find_last_not_of('\n');
-			if(chomp == 0 && pos + 1 < scalar.size())
+			if(info.chomp == 0 && pos + 1 < scalar.size())
 				scalar.erase(pos + 2);
-			else if(chomp == -1 && pos < scalar.size())
+			else if(info.chomp == -1 && pos < scalar.size())
 				scalar.erase(pos + 1);
 		}
 

scanscalar.h

@@ -6,8 +6,24 @@
 
 namespace YAML
 {
+	enum CHOMP { STRIP = -1, CLIP, KEEP };
+
+	struct ScanScalarInfo {
+		ScanScalarInfo(): eatEnd(false), indent(0), eatLeadingWhitespace(0), escape(0), fold(false), trimTrailingSpaces(0), chomp(CLIP) {}
+
+		RegEx end;                      // what condition ends this scalar?
+		bool eatEnd;                    // should we eat that condition when we see it?
+		int indent;                     // what level of indentation should be eaten and ignored?
+		bool eatLeadingWhitespace;      // should we continue eating this delicious indentation after 'indent' spaces?
+		char escape;                    // what character do we escape on (i.e., slash or single quote) (0 for none)
+		bool fold;                      // do we fold line ends?
+		bool trimTrailingSpaces;        // do we remove all trailing spaces (at the very end)
+		CHOMP chomp;                    // do we strip, clip, or keep trailing newlines (at the very end)
+		                                //   Note: strip means kill all, clip means keep at most one, keep means keep all
+	};
+
 	void GetBlockIndentation(Stream& INPUT, int& indent, std::string& breaks, int topIndent);
-	std::string ScanScalar(Stream& INPUT, RegEx end, bool eatEnd, int indent, char escape, bool fold, bool eatLeadingWhitespace, bool trimTrailingSpaces, int chomp);
+	std::string ScanScalar(Stream& INPUT, ScanScalarInfo info);
 
 	struct WhitespaceInfo {
 		WhitespaceInfo();

scantoken.cpp

@@ -26,7 +26,7 @@ namespace YAML
 			INPUT.column = 0;
 
 		PopIndentTo(-1);
-		ValidateAllSimpleKeys();
+		VerifyAllSimpleKeys();
 
 		m_simpleKeyAllowed = false;
 		m_endedStream = true;
@@ -38,7 +38,7 @@ namespace YAML
 	template <> DocumentStartToken *Scanner::ScanToken(DocumentStartToken *pToken)
 	{
 		PopIndentTo(INPUT.column);
-		ValidateAllSimpleKeys();
+		VerifyAllSimpleKeys();
 		m_simpleKeyAllowed = false;
 
 		// eat
@@ -50,7 +50,7 @@ namespace YAML
 	template <> DocumentEndToken *Scanner::ScanToken(DocumentEndToken *pToken)
 	{
 		PopIndentTo(-1);
-		ValidateAllSimpleKeys();
+		VerifyAllSimpleKeys();
 		m_simpleKeyAllowed = false;
 
 		// eat

simplekey.cpp

@@ -13,9 +13,9 @@ namespace YAML
 	void Scanner::SimpleKey::Validate()
 	{
 		if(pMapStart)
-			pMapStart->isValid = true;
+			pMapStart->status = TS_VALID;
 		if(pKey)
-			pKey->isValid = true;
+			pKey->status = TS_VALID;
 	}
 
 	void Scanner::SimpleKey::Invalidate()
@@ -24,9 +24,9 @@ namespace YAML
 			throw RequiredSimpleKeyNotFound();
 
 		if(pMapStart)
-			pMapStart->isPossible = false;
+			pMapStart->status = TS_INVALID;
 		if(pKey)
-			pKey->isPossible = false;
+			pKey->status = TS_INVALID;
 	}
 
 	// InsertSimpleKey
@@ -39,23 +39,22 @@ namespace YAML
 		// first add a map start, if necessary
 		key.pMapStart = PushIndentTo(INPUT.column, false);
 		if(key.pMapStart)
-			key.pMapStart->isValid = false;
+			key.pMapStart->status = TS_UNVERIFIED;
 //		else
 //			key.required = true;	// TODO: is this correct?
 
-		// then add the (now invalid) key
+		// then add the (now unverified) key
 		key.pKey = new KeyToken;
-		key.pKey->isValid = false;
+		key.pKey->status = TS_UNVERIFIED;
-
 		m_tokens.push(key.pKey);
 
 		m_simpleKeys.push(key);
 	}
 
-	// ValidateSimpleKey
+	// VerifySimpleKey
 	// . Determines whether the latest simple key to be added is valid,
 	//   and if so, makes it valid.
-	bool Scanner::ValidateSimpleKey()
+	bool Scanner::VerifySimpleKey()
 	{
 		m_isLastKeyValid = false;
 		if(m_simpleKeys.empty())
@@ -99,9 +98,9 @@ namespace YAML
 		return isValid;
 	}
 
-	void Scanner::ValidateAllSimpleKeys()
+	void Scanner::VerifyAllSimpleKeys()
 	{
 		while(!m_simpleKeys.empty())
-			ValidateSimpleKey();
+			VerifySimpleKey();
 	}
 }

test.yaml

@@ -9,4 +9,6 @@
   {
      std::cout << "Hello World!\n";
      return 0;
-  }
+  }
+- key1: value1
+  key2: value2

token.h

@@ -4,13 +4,15 @@
 
 namespace YAML
 {
+	enum TOKEN_STATUS { TS_VALID, TS_INVALID, TS_UNVERIFIED };
+
 	struct Token {
-		Token(): isValid(true), isPossible(true) {}
+		Token(): status(TS_VALID) {}
 		virtual ~Token() {}
 		virtual void Write(std::ostream& out) const {}
 
 		friend std::ostream& operator << (std::ostream& out, const Token& token) { token.Write(out); return out; }
-		bool isValid, isPossible;
+		TOKEN_STATUS status;
 	};
 
 	struct StreamStartToken: public Token {};