Moved the three scalar token scanning functions back to scantoken.cpp, so scanscalar.cpp now only has the main scalar scanning function.

Renamed ScanScalarInfo to ScanScalarParams.

scanscalar.cpp

@@ -7,39 +7,48 @@
 namespace YAML
 {
 	// ScanScalar
-	std::string ScanScalar(Stream& INPUT, ScanScalarInfo& info)
+	// . This is where the scalar magic happens.
+	//
+	// . We do the scanning in three phases:
+	//   1. Scan until newline
+	//   2. Eat newline
+	//   3. Scan leading blanks.
+	//
+	// . Depending on the parameters given, we store or stop
+	//   and different places in the above flow.
+	std::string ScanScalar(Stream& INPUT, ScanScalarParams& params)
 	{
 		bool foundNonEmptyLine = false;
 		bool emptyLine = false, moreIndented = false;
 		std::string scalar;
-		info.leadingSpaces = false;
+		params.leadingSpaces = false;
 
 		while(INPUT) {
 			// ********************************
 			// Phase #1: scan until line ending
-			while(!info.end.Matches(INPUT) && !Exp::Break.Matches(INPUT)) {
+			while(!params.end.Matches(INPUT) && !Exp::Break.Matches(INPUT)) {
 				if(INPUT.peek() == EOF)
 					break;
 
 				// document indicator?
 				if(INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT)) {
-					if(info.onDocIndicator == BREAK)
+					if(params.onDocIndicator == BREAK)
 						break;
-					else if(info.onDocIndicator == THROW)
+					else if(params.onDocIndicator == THROW)
 						throw IllegalDocIndicator();
 				}
 
 				foundNonEmptyLine = true;
 
 				// escaped newline? (only if we're escaping on slash)
-				if(info.escape == '\\' && Exp::EscBreak.Matches(INPUT)) {
+				if(params.escape == '\\' && Exp::EscBreak.Matches(INPUT)) {
 					int n = Exp::EscBreak.Match(INPUT);
 					INPUT.Eat(n);
 					continue;
 				}
 
 				// escape this?
-				if(INPUT.peek() == info.escape) {
+				if(INPUT.peek() == params.escape) {
 					scalar += Exp::Escape(INPUT);
 					continue;
 				}
@@ -50,19 +59,19 @@ namespace YAML
 
 			// eof? if we're looking to eat something, then we throw
 			if(INPUT.peek() == EOF) {
-				if(info.eatEnd)
+				if(params.eatEnd)
 					throw IllegalEOF();
 				break;
 			}
 
 			// doc indicator?
-			if(info.onDocIndicator == BREAK && INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT))
+			if(params.onDocIndicator == BREAK && INPUT.column == 0 && Exp::DocIndicator.Matches(INPUT))
 				break;
 
 			// are we done via character match?
-			int n = info.end.Match(INPUT);
+			int n = params.end.Match(INPUT);
 			if(n >= 0) {
-				if(info.eatEnd)
+				if(params.eatEnd)
 					INPUT.Eat(n);
 				break;
 			}
@@ -76,20 +85,20 @@ namespace YAML
 			// Phase #3: scan initial spaces
 
 			// first the required indentation
-			while(INPUT.peek() == ' ' && (INPUT.column < info.indent || (info.detectIndent && !foundNonEmptyLine)))
+			while(INPUT.peek() == ' ' && (INPUT.column < params.indent || (params.detectIndent && !foundNonEmptyLine)))
 				INPUT.Eat(1);
 
 			// update indent if we're auto-detecting
-			if(info.detectIndent && !foundNonEmptyLine)
+			if(params.detectIndent && !foundNonEmptyLine)
-				info.indent = std::max(info.indent, INPUT.column);
+				params.indent = std::max(params.indent, INPUT.column);
 
 			// and then the rest of the whitespace
 			while(Exp::Blank.Matches(INPUT)) {
 				// we check for tabs that masquerade as indentation
-				if(INPUT.peek() == '\t'&& INPUT.column < info.indent && info.onTabInIndentation == THROW)
+				if(INPUT.peek() == '\t'&& INPUT.column < params.indent && params.onTabInIndentation == THROW)
 					throw IllegalTabInIndentation();
 
-				if(!info.eatLeadingWhitespace)
+				if(!params.eatLeadingWhitespace)
 					break;
 
 				INPUT.Eat(1);
@@ -101,7 +110,7 @@ namespace YAML
 
 			// TODO: for block scalars, we always start with a newline, so we should fold OR keep that
 
-			if(info.fold && !emptyLine && !nextEmptyLine && !moreIndented && !nextMoreIndented)
+			if(params.fold && !emptyLine && !nextEmptyLine && !moreIndented && !nextMoreIndented)
 				scalar += " ";
 			else
 				scalar += "\n";
@@ -110,146 +119,27 @@ namespace YAML
 			moreIndented = nextMoreIndented;
 
 			// are we done via indentation?
-			if(!emptyLine && INPUT.column < info.indent) {
+			if(!emptyLine && INPUT.column < params.indent) {
-				info.leadingSpaces = true;
+				params.leadingSpaces = true;
 				break;
 			}
 		}
 
 		// post-processing
-		if(info.trimTrailingSpaces) {
+		if(params.trimTrailingSpaces) {
 			unsigned pos = scalar.find_last_not_of(' ');
 			if(pos < scalar.size())
 				scalar.erase(pos + 1);
 		}
 
-		if(info.chomp <= 0) {
+		if(params.chomp <= 0) {
 			unsigned pos = scalar.find_last_not_of('\n');
-			if(info.chomp == 0 && pos + 1 < scalar.size())
+			if(params.chomp == 0 && pos + 1 < scalar.size())
 				scalar.erase(pos + 2);
-			else if(info.chomp == -1 && pos < scalar.size())
+			else if(params.chomp == -1 && pos < scalar.size())
 				scalar.erase(pos + 1);
 		}
 
 		return scalar;
 	}
-
-	// PlainScalarToken
-	template <> PlainScalarToken *Scanner::ScanToken(PlainScalarToken *pToken)
-	{
-		// set up the scanning parameters
-		ScanScalarInfo info;
-		info.end = (m_flowLevel > 0 ? Exp::EndScalarInFlow : Exp::EndScalar) || (RegEx(' ') + Exp::Comment);
-		info.eatEnd = false;
-		info.indent = (m_flowLevel > 0 ? 0 : m_indents.top() + 1);
-		info.fold = true;
-		info.eatLeadingWhitespace = true;
-		info.trimTrailingSpaces = true;
-		info.chomp = CLIP;
-		info.onDocIndicator = BREAK;
-		info.onTabInIndentation = THROW;
-
-		// insert a potential simple key
-		if(m_simpleKeyAllowed)
-			InsertSimpleKey();
-
-		pToken->value = ScanScalar(INPUT, info);
-
-		// can have a simple key only if we ended the scalar by starting a new line
-		m_simpleKeyAllowed = info.leadingSpaces;
-
-		// finally, we can't have any colons in a scalar, so if we ended on a colon, there
-		// had better be a break after it
-		if(Exp::IllegalColonInScalar.Matches(INPUT))
-			throw IllegalScalar();
-
-		return pToken;
-	}
-
-	// QuotedScalarToken
-	template <> QuotedScalarToken *Scanner::ScanToken(QuotedScalarToken *pToken)
-	{
-		// eat single or double quote
-		char quote = INPUT.GetChar();
-		pToken->single = (quote == '\'');
-
-		// setup the scanning parameters
-		ScanScalarInfo info;
-		info.end = (pToken->single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
-		info.eatEnd = true;
-		info.escape = (pToken->single ? '\'' : '\\');
-		info.indent = 0;
-		info.fold = true;
-		info.eatLeadingWhitespace = true;
-		info.trimTrailingSpaces = false;
-		info.chomp = CLIP;
-		info.onDocIndicator = THROW;
-
-		// insert a potential simple key
-		if(m_simpleKeyAllowed)
-			InsertSimpleKey();
-
-		pToken->value = ScanScalar(INPUT, info);
-		m_simpleKeyAllowed = false;
-
-		return pToken;
-	}
-
-	// BlockScalarToken
-	// . These need a little extra processing beforehand.
-	// . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
-	//   and then we need to figure out what level of indentation we'll be using.
-	template <> BlockScalarToken *Scanner::ScanToken(BlockScalarToken *pToken)
-	{
-		ScanScalarInfo info;
-		info.indent = 1;
-		info.detectIndent = true;
-
-		// eat block indicator ('|' or '>')
-		char indicator = INPUT.GetChar();
-		info.fold = (indicator == Keys::FoldedScalar);
-
-		// eat chomping/indentation indicators
-		int n = Exp::Chomp.Match(INPUT);
-		for(int i=0;i<n;i++) {
-			char ch = INPUT.GetChar();
-			if(ch == '+')
-				info.chomp = KEEP;
-			else if(ch == '-')
-				info.chomp = STRIP;
-			else if(Exp::Digit.Matches(ch)) {
-				info.indent = ch - '0';
-				info.detectIndent = false;
-				if(info.indent == 0)
-					throw ZeroIndentationInBlockScalar();
-			}
-		}
-
-		// now eat whitespace
-		while(Exp::Blank.Matches(INPUT))
-			INPUT.Eat(1);
-
-		// and comments to the end of the line
-		if(Exp::Comment.Matches(INPUT))
-			while(INPUT && !Exp::Break.Matches(INPUT))
-				INPUT.Eat(1);
-
-		// if it's not a line break, then we ran into a bad character inline
-		if(INPUT && !Exp::Break.Matches(INPUT))
-			throw UnexpectedCharacterInBlockScalar();
-
-		// set the initial indentation
-		if(m_indents.top() >= 0)
-			info.indent += m_indents.top();
-
-		info.eatLeadingWhitespace = false;
-		info.trimTrailingSpaces = false;
-		info.onTabInIndentation = THROW;
-
-		pToken->value = ScanScalar(INPUT, info);
-
-		// simple keys always ok after block scalars (since we're gonna start a new line anyways)
-		m_simpleKeyAllowed = true;
-		return pToken;
-	}
 }

scanscalar.h

@@ -9,8 +9,8 @@ namespace YAML
 	enum CHOMP { STRIP = -1, CLIP, KEEP };
 	enum ACTION { NONE, BREAK, THROW };
 
-	struct ScanScalarInfo {
+	struct ScanScalarParams {
-		ScanScalarInfo(): eatEnd(false), indent(0), detectIndent(false), eatLeadingWhitespace(0), escape(0), fold(false),
+		ScanScalarParams(): eatEnd(false), indent(0), detectIndent(false), eatLeadingWhitespace(0), escape(0), fold(false),
 			trimTrailingSpaces(0), chomp(CLIP), onDocIndicator(NONE), onTabInIndentation(NONE), leadingSpaces(false) {}
 
 		// input:
@@ -31,5 +31,5 @@ namespace YAML
 		bool leadingSpaces;
 	};
 
-	std::string ScanScalar(Stream& INPUT, ScanScalarInfo& info);
+	std::string ScanScalar(Stream& INPUT, ScanScalarParams& info);
 }

scantoken.cpp

@@ -2,6 +2,7 @@
 #include "token.h"
 #include "exceptions.h"
 #include "exp.h"
+#include "scanscalar.h"
 
 namespace YAML
 {
@@ -220,4 +221,124 @@ namespace YAML
 		pToken->value = tag;
 		return pToken;
 	}
+
+	// PlainScalarToken
+	template <> PlainScalarToken *Scanner::ScanToken(PlainScalarToken *pToken)
+	{
+		// set up the scanning parameters
+		ScanScalarParams params;
+		params.end = (m_flowLevel > 0 ? Exp::EndScalarInFlow : Exp::EndScalar) || (RegEx(' ') + Exp::Comment);
+		params.eatEnd = false;
+		params.indent = (m_flowLevel > 0 ? 0 : m_indents.top() + 1);
+		params.fold = true;
+		params.eatLeadingWhitespace = true;
+		params.trimTrailingSpaces = true;
+		params.chomp = CLIP;
+		params.onDocIndicator = BREAK;
+		params.onTabInIndentation = THROW;
+
+		// insert a potential simple key
+		if(m_simpleKeyAllowed)
+			InsertSimpleKey();
+
+		pToken->value = ScanScalar(INPUT, params);
+
+		// can have a simple key only if we ended the scalar by starting a new line
+		m_simpleKeyAllowed = params.leadingSpaces;
+
+		// finally, we can't have any colons in a scalar, so if we ended on a colon, there
+		// had better be a break after it
+		if(Exp::IllegalColonInScalar.Matches(INPUT))
+			throw IllegalScalar();
+
+		return pToken;
+	}
+
+	// QuotedScalarToken
+	template <> QuotedScalarToken *Scanner::ScanToken(QuotedScalarToken *pToken)
+	{
+		// eat single or double quote
+		char quote = INPUT.GetChar();
+		pToken->single = (quote == '\'');
+
+		// setup the scanning parameters
+		ScanScalarParams params;
+		params.end = (pToken->single ? RegEx(quote) && !Exp::EscSingleQuote : RegEx(quote));
+		params.eatEnd = true;
+		params.escape = (pToken->single ? '\'' : '\\');
+		params.indent = 0;
+		params.fold = true;
+		params.eatLeadingWhitespace = true;
+		params.trimTrailingSpaces = false;
+		params.chomp = CLIP;
+		params.onDocIndicator = THROW;
+
+		// insert a potential simple key
+		if(m_simpleKeyAllowed)
+			InsertSimpleKey();
+
+		pToken->value = ScanScalar(INPUT, params);
+		m_simpleKeyAllowed = false;
+
+		return pToken;
+	}
+
+	// BlockScalarToken
+	// . These need a little extra processing beforehand.
+	// . We need to scan the line where the indicator is (this doesn't count as part of the scalar),
+	//   and then we need to figure out what level of indentation we'll be using.
+	template <> BlockScalarToken *Scanner::ScanToken(BlockScalarToken *pToken)
+	{
+		ScanScalarParams params;
+		params.indent = 1;
+		params.detectIndent = true;
+
+		// eat block indicator ('|' or '>')
+		char indicator = INPUT.GetChar();
+		params.fold = (indicator == Keys::FoldedScalar);
+
+		// eat chomping/indentation indicators
+		int n = Exp::Chomp.Match(INPUT);
+		for(int i=0;i<n;i++) {
+			char ch = INPUT.GetChar();
+			if(ch == '+')
+				params.chomp = KEEP;
+			else if(ch == '-')
+				params.chomp = STRIP;
+			else if(Exp::Digit.Matches(ch)) {
+				if(ch == '0')
+					throw ZeroIndentationInBlockScalar();
+
+				params.indent = ch - '0';
+				params.detectIndent = false;
+			}
+		}
+
+		// now eat whitespace
+		while(Exp::Blank.Matches(INPUT))
+			INPUT.Eat(1);
+
+		// and comments to the end of the line
+		if(Exp::Comment.Matches(INPUT))
+			while(INPUT && !Exp::Break.Matches(INPUT))
+				INPUT.Eat(1);
+
+		// if it's not a line break, then we ran into a bad character inline
+		if(INPUT && !Exp::Break.Matches(INPUT))
+			throw UnexpectedCharacterInBlockScalar();
+
+		// set the initial indentation
+		if(m_indents.top() >= 0)
+			params.indent += m_indents.top();
+
+		params.eatLeadingWhitespace = false;
+		params.trimTrailingSpaces = false;
+		params.onTabInIndentation = THROW;
+
+		pToken->value = ScanScalar(INPUT, params);
+
+		// simple keys always ok after block scalars (since we're gonna start a new line anyways)
+		m_simpleKeyAllowed = true;
+		return pToken;
+	}
 }

test.yaml

@@ -1,17 +1,97 @@
 ---
-- here's a key: value
+model:
-  here's the first block: |
+    file: data/models/compound.model
-  after the block: value
+    textures: data/materials/compound
-  and here's a block: |-
+rooms:
-    What's going on?
+    - name: "Room #1"
-    How are you doing?
+      pos: [0, 0, 0]
-    Here's some code:
+      size: [1000, 1000, 500]
-    
+      height: 500
-      #include <iostream>
+      stairtype: none
-      int main()
+      display: []
-      {
+      pathfinding:
-         std::cout << "Hello World!\n";
+          tilesize: 50
-      }
+          size: [24, 24]
-    
+          map: |
-    I'm doing fine!
+              -----------------------
-  and last key: value
+              -+++++++++++++++++++++-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+---------------------
+              -+---------------------
+              -+---------------------
+              -+---------------------
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+++++++++++++++++++++-
+              -----------------------
+    - name: Doorway
+      pos: [1000, 400, 0]
+      size: [50, 200, 500]
+      height: 500
+      stairtype: none
+      display: []
+      pathfinding:
+          tilesize: 50
+          size: [5, 9]
+          map: |
+              -----
+              -+++-
+              -----
+              -----
+              -----
+              -----
+              -----
+              -+++-
+              -----
+    - name: "Room #2"
+      pos: [1050, 0, 0]
+      size: [1000, 1000, 500]
+      height: 500
+      stairtype: none
+      display: []
+      pathfinding:
+          tilesize: 50
+          size: [24, 24]
+          map: |
+              -----------------------
+              -+++++++++++++++++++++-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              ---------------------+-
+              ---------------------+-
+              ---------------------+-
+              ---------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+-------------------+-
+              -+++++++++++++++++++++-
+              -----------------------
+exits:
+    - room1: "Room #1"
+      room2: "Room #2"
+      dir: e
+      pos: [400, 600]