Small plain scalar scanning fixes.

document.cpp

@@ -34,10 +34,7 @@ namespace YAML
 		std::ifstream fin(fileName.c_str());
 		Scanner scanner(fin);
 
-		try {
 		scanner.Scan();
-		} catch(const Exception& e) {
-		}
 		getchar();
 //		if(!scanner)
 //			return;

scanner.cpp

@@ -37,6 +37,16 @@ namespace YAML
 		return ch;
 	}
 
+	// GetChar
+	// . Extracts 'n' characters from the stream and updates our position
+	std::string Scanner::GetChar(int n)
+	{
+		std::string ret;
+		for(int i=0;i<n;i++)
+			ret += GetChar();
+		return ret;
+	}
+
 	// Eat
 	// . Eats 'n' characters and updates our position.
 	void Scanner::Eat(int n)
@@ -199,11 +209,8 @@ namespace YAML
 		if(INPUT.peek() == Keys::FoldedScalar && m_flowLevel == 0)
 			return;
 
-		if(INPUT.peek() == '\'')
+		if(INPUT.peek() == '\'' || INPUT.peek() == '\"')
-			return;
+			return ScanAndEnqueue(new QuotedScalarToken);
-
-		if(INPUT.peek() == '\"')
-			return;
 
 		// plain scalars
 		if(IsPlainScalar())
@@ -290,7 +297,7 @@ namespace YAML
 			while(!m_tokens.empty()) {
 				Token *pToken = m_tokens.front();
 				m_tokens.pop();
-				std::cout << typeid(*pToken).name() << std::endl;
+				std::cout << typeid(*pToken).name() << ": " << *pToken << std::endl;
 				delete pToken;
 			}
 		}

scanner.h

@@ -8,7 +8,7 @@
 
 namespace YAML
 {
-	class Token;
+	struct Token;
 
 	class Scanner
 	{
@@ -25,6 +25,7 @@ namespace YAML
 
 	private:
 		char GetChar();
+		std::string GetChar(int n);
 		void Eat(int n = 1);
 		void EatLineBreak();
 

scantoken.cpp

@@ -199,6 +199,10 @@ namespace YAML
 	}
 
 	// PlainScalarToken
+	// . We scan these in passes of two steps each: First, grab all non-whitespace
+	//   characters we can, and then grab all whitespace characters we can.
+	// . This has the benefit of letting us handle leading whitespace (which is chomped)
+	//   and in-line whitespace (which is kept) separately.
 	template <> PlainScalarToken *Scanner::ScanToken(PlainScalarToken *pToken)
 	{
 		// TODO: "save simple key"
@@ -230,20 +234,21 @@ namespace YAML
 				if(m_flowLevel == 0 && Exp::EndScalar.Matches(INPUT))
 					break;
 
+				// join whitespace
 				if(leadingBlanks) {
-					if(!leadingBreaks.empty() && leadingBreaks[0] == '\n') {
+					if(Exp::Break.Matches(leadingBreaks)) {
 						// fold line break?
 						if(trailingBreaks.empty())
 							scalar += ' ';
-						else {
+						else
 							scalar += trailingBreaks;
-							trailingBreaks = "";
-						}
 					} else {
 						scalar += leadingBreaks + trailingBreaks;
+					}
+
+					leadingBlanks = false;
 					leadingBreaks = "";
 					trailingBreaks = "";
-					}
 				} else if(!whitespace.empty()) {
 					scalar += whitespace;
 					whitespace = "";
@@ -260,7 +265,8 @@ namespace YAML
 			// now eat blanks
 			while(INPUT && Exp::BlankOrBreak.Matches(INPUT)) {
 				if(Exp::Blank.Matches(INPUT)) {
-					if(leadingBlanks && m_column <= m_indents.top())
+					// can't use tabs as indentation! only spaces!
+					if(INPUT.peek() == '\t' && leadingBlanks && m_column <= m_indents.top())
 						throw IllegalTabInScalar();
 
 					// maybe store this character
@@ -269,13 +275,17 @@ namespace YAML
 					else
 						Eat(1);
 				} else {
+					// we know it's a line break; see how many characters to read
+					int n = Exp::Break.Match(INPUT);
+					std::string line = GetChar(n);
+
 					// where to store this character?
 					if(!leadingBlanks) {
 						leadingBlanks = true;
 						whitespace = "";
-						leadingBreaks += GetChar();
+						leadingBreaks += line;
 					} else
-						trailingBreaks += GetChar();
+						trailingBreaks += line;
 				}
 			}
 
@@ -285,10 +295,16 @@ namespace YAML
 		}
 
 		// now modify our token
-		pToken->SetValue(scalar);
+		pToken->value = scalar;
 		if(leadingBlanks)
 			m_simpleKeyAllowed = true;
 
 		return pToken;
 	}
+
+	// QuotedScalarToken
+	template <> QuotedScalarToken *Scanner::ScanToken(QuotedScalarToken *pToken)
+	{
+		return pToken;
+	}
 }

test.yaml

@@ -1,5 +1,10 @@
 ---
-- milk
+- green
+  eggs,
+  and
+  ham!
 - eggs             # this is really important!
-- cheese and bread
+- - cheddar cheese
+  - american cheese
+- bread
 ...

token.h

@@ -1,32 +1,40 @@
 #pragma once
 
+#include <ios>
+
 namespace YAML
 {
-	class Token { public: virtual ~Token() {} };
+	struct Token {
+		virtual ~Token() {}
+		virtual void Write(std::ostream& out) const {}
 
-	class StreamStartToken: public Token {};
+		friend std::ostream& operator << (std::ostream& out, const Token& token) { token.Write(out); return out; }
-	class StreamEndToken: public Token {};
-	class DocumentStartToken: public Token {};
-	class DocumentEndToken: public Token {};
-
-	class BlockSeqStartToken: public Token {};
-	class BlockMapStartToken: public Token {};
-	class BlockEndToken: public Token {};
-	class BlockEntryToken: public Token {};
-
-	class FlowSeqStartToken: public Token {};
-	class FlowMapStartToken: public Token {};
-	class FlowSeqEndToken: public Token {};
-	class FlowMapEndToken: public Token {};
-	class FlowEntryToken: public Token {};
-
-	class KeyToken: public Token {};
-	class ValueToken: public Token {};
-
-	class PlainScalarToken: public Token {
-	public:
-		void SetValue(const std::string& value) { m_value = value; }
-	protected:
-		std::string m_value;
 	};
+
+	struct StreamStartToken: public Token {};
+	struct StreamEndToken: public Token {};
+	struct DocumentStartToken: public Token {};
+	struct DocumentEndToken: public Token {};
+
+	struct BlockSeqStartToken: public Token {};
+	struct BlockMapStartToken: public Token {};
+	struct BlockEndToken: public Token {};
+	struct BlockEntryToken: public Token {};
+
+	struct FlowSeqStartToken: public Token {};
+	struct FlowMapStartToken: public Token {};
+	struct FlowSeqEndToken: public Token {};
+	struct FlowMapEndToken: public Token {};
+	struct FlowEntryToken: public Token {};
+
+	struct KeyToken: public Token {};
+	struct ValueToken: public Token {};
+
+	struct ScalarToken: public Token {
+		std::string value;
+		virtual void Write(std::ostream& out) const { out << value; }
+	};
+
+	struct PlainScalarToken: public ScalarToken {};
+	struct QuotedScalarToken: public ScalarToken {};
 }