From c63ebbd878f729e7bdf38c224816b27c8f53cd5e Mon Sep 17 00:00:00 2001 From: Jesse Beder Date: Sat, 28 Jun 2008 16:46:37 +0000 Subject: [PATCH] Moved the simple key validation to before each token scan (plus at newlines of scalars). --- scanner.cpp | 68 ++++++++++++++++++++++++++++++++++----------------- scanner.h | 7 ++++-- scantoken.cpp | 10 ++++---- simplekey.cpp | 15 +++++++++--- 4 files changed, 66 insertions(+), 34 deletions(-) diff --git a/scanner.cpp b/scanner.cpp index c3477b7804..f6c844b836 100644 --- a/scanner.cpp +++ b/scanner.cpp @@ -146,17 +146,6 @@ namespace YAML m_limboTokens.insert(pToken); m_tokens.push(ScanToken(pToken)); m_limboTokens.erase(pToken); - - // then remove impossible tokens - std::queue temp; - while(!m_tokens.empty()) { - Token *pToken = m_tokens.front(); - m_tokens.pop(); - if(pToken->isPossible) - temp.push(pToken); - } - - m_tokens = temp; } /////////////////////////////////////////////////////////////////////// @@ -171,7 +160,7 @@ namespace YAML return ScanAndEnqueue(new StreamStartToken); ScanToNextToken(); - // TODO: remove "obsolete potential simple keys" + ValidateSimpleKey(); PopIndentTo(m_column); if(INPUT.peek() == EOF) @@ -318,23 +307,56 @@ namespace YAML } } + // GetNextToken + // . Returns the next token on the queue, and scans if only we need to. + Token *Scanner::GetNextToken() + { + while(1) { + Token *pToken = 0; + + // is there a token in the queue? + if(!m_tokens.empty()) + pToken = m_tokens.front(); + + // ... that's possible + // (here's where we clean up the impossible tokens) + if(pToken && !pToken->isPossible) { + m_tokens.pop(); + delete pToken; + continue; + } + + // and valid + if(pToken && !pToken->isValid) + pToken = 0; + + // then that's what we want + if(pToken) { + m_tokens.pop(); + return pToken; + } + + // no token? maybe we've actually finished + if(m_endedStream) + break; + + // no? then scan... + ScanNextToken(); + } + + return 0; + } + // temporary function for testing void Scanner::Scan() { while(1) { - ScanNextToken(); - if(m_tokens.empty()) + Token *pToken = GetNextToken(); + if(!pToken) break; - while(!m_tokens.empty()) { - Token *pToken = m_tokens.front(); - if(!pToken->isValid) // gotta wait on the invalid tokens - they might become valid! - break; - - m_tokens.pop(); - std::cout << typeid(*pToken).name() << ": " << *pToken << std::endl; - delete pToken; - } + std::cout << typeid(*pToken).name() << ": " << *pToken << std::endl; + delete pToken; } } } diff --git a/scanner.h b/scanner.h index 36362ac25b..8d0033baad 100644 --- a/scanner.h +++ b/scanner.h @@ -16,6 +16,8 @@ namespace YAML Scanner(std::istream& in); ~Scanner(); + Token *GetNextToken(); + void ScanNextToken(); void ScanToNextToken(); Token *PushIndentTo(int column, bool sequence); @@ -55,12 +57,12 @@ namespace YAML }; struct SimpleKey { - SimpleKey(int pos_, int line_, int column_); + SimpleKey(int pos_, int line_, int column_, int flowLevel_); void Validate(); void Invalidate(); - int pos, line, column; + int pos, line, column, flowLevel; bool required; Token *pMapStart, *pKey; }; @@ -81,6 +83,7 @@ namespace YAML bool m_startedStream, m_endedStream; bool m_simpleKeyAllowed; int m_flowLevel; // number of unclosed '[' and '{' indicators + bool m_isLastKeyValid; std::stack m_simpleKeys; std::stack m_indents; }; diff --git a/scantoken.cpp b/scantoken.cpp index ed69bd83bf..6b071c0ea6 100644 --- a/scantoken.cpp +++ b/scantoken.cpp @@ -99,7 +99,7 @@ namespace YAML // FlowMapEndToken template <> FlowMapEndToken *Scanner::ScanToken(FlowMapEndToken *pToken) { - ValidateSimpleKey(); + //ValidateSimpleKey(); DecreaseFlowLevel(); m_simpleKeyAllowed = false; @@ -111,7 +111,7 @@ namespace YAML // FlowEntryToken template <> FlowEntryToken *Scanner::ScanToken(FlowEntryToken *pToken) { - ValidateSimpleKey(); + //ValidateSimpleKey(); m_simpleKeyAllowed = true; // eat @@ -122,7 +122,7 @@ namespace YAML // BlockEntryToken template <> BlockEntryToken *Scanner::ScanToken(BlockEntryToken *pToken) { - ValidateSimpleKey(); + //ValidateSimpleKey(); // we better be in the block context! if(m_flowLevel == 0) { @@ -170,9 +170,9 @@ namespace YAML template <> ValueToken *Scanner::ScanToken(ValueToken *pToken) { // does this follow a simple key? - bool isValidKey = ValidateSimpleKey(); +// bool isValidKey = ValidateSimpleKey(); - if(isValidKey) { + if(m_isLastKeyValid) { // can't follow a simple key with another simple key (dunno why, though - it seems fine) m_simpleKeyAllowed = false; } else { diff --git a/simplekey.cpp b/simplekey.cpp index 990d24d128..cf90bd2163 100644 --- a/simplekey.cpp +++ b/simplekey.cpp @@ -5,8 +5,8 @@ namespace YAML { - Scanner::SimpleKey::SimpleKey(int pos_, int line_, int column_) - : pos(pos_), line(line_), column(column_), required(false), pMapStart(0), pKey(0) + Scanner::SimpleKey::SimpleKey(int pos_, int line_, int column_, int flowLevel_) + : pos(pos_), line(line_), column(column_), flowLevel(flowLevel_), required(false), pMapStart(0), pKey(0) { } @@ -34,7 +34,7 @@ namespace YAML // and saves it on a stack. void Scanner::InsertSimpleKey() { - SimpleKey key(INPUT.tellg(), m_line, m_column); + SimpleKey key(INPUT.tellg(), m_line, m_column, m_flowLevel); // first add a map start, if necessary key.pMapStart = PushIndentTo(m_column, false); @@ -57,11 +57,17 @@ namespace YAML // and if so, makes it valid. bool Scanner::ValidateSimpleKey() { + m_isLastKeyValid = false; if(m_simpleKeys.empty()) - return false; + return m_isLastKeyValid; // grab top key SimpleKey key = m_simpleKeys.top(); + + // only validate if we're in the correct flow level + if(key.flowLevel != m_flowLevel) + return false; + m_simpleKeys.pop(); bool isValid = true; @@ -89,6 +95,7 @@ namespace YAML if(!isValid && m_flowLevel == 0) m_indents.pop(); + m_isLastKeyValid = isValid; return isValid; }