[master] Merge branch 'trac2573'

fixed Conflicts:
	src/lib/datasrc/zone_loader.cc

src/lib/datasrc/tests/zone_loader_unittest.cc

@@ -246,6 +246,11 @@ TEST_F(ZoneLoaderTest, copyUnsigned) {
     // It gets the updater directly in the constructor
     ASSERT_EQ(1, destination_client_.provided_updaters_.size());
     EXPECT_EQ(Name::ROOT_NAME(), destination_client_.provided_updaters_[0]);
+
+    // Counter is initialized to 0, progress is "unknown" in case of copy.
+    EXPECT_EQ(0, loader.getRRCount());
+    EXPECT_EQ(ZoneLoader::PROGRESS_UNKNOWN, loader.getProgress());
+
     // Now load the whole zone
     loader.load();
     EXPECT_TRUE(destination_client_.commit_called_);
@@ -254,6 +259,12 @@ TEST_F(ZoneLoaderTest, copyUnsigned) {
 
     // The count is 34 because we expect the RRs to be separated.
     EXPECT_EQ(34, destination_client_.rrsets_.size());
+
+    // Check various counters.  getRRCount should be identical of the RRs
+    // we've seen. Progress is still "unknown" in the copy operation.
+    EXPECT_EQ(destination_client_.rrsets_.size(), loader.getRRCount());
+    EXPECT_EQ(ZoneLoader::PROGRESS_UNKNOWN, loader.getProgress());
+
     // Ensure known order.
     std::sort(destination_client_.rrset_texts_.begin(),
               destination_client_.rrset_texts_.end());
@@ -281,6 +292,11 @@ TEST_F(ZoneLoaderTest, copyUnsignedIncremental) {
     // Not committed yet, we didn't complete the loading
     EXPECT_FALSE(destination_client_.commit_called_);
 
+    // Check we can get intermediate counters. Progress is always "unknown"
+    // in case of copy.
+    EXPECT_EQ(destination_client_.rrsets_.size(), loader.getRRCount());
+    EXPECT_EQ(ZoneLoader::PROGRESS_UNKNOWN, loader.getProgress());
+
     // This is unusual, but allowed. Check it doesn't do anything
     loader.loadIncremental(0);
     EXPECT_EQ(10, destination_client_.rrsets_.size());
@@ -349,6 +365,11 @@ TEST_F(ZoneLoaderTest, classMismatch) {
 TEST_F(ZoneLoaderTest, loadUnsigned) {
     ZoneLoader loader(destination_client_, Name::ROOT_NAME(),
                       TEST_DATA_DIR "/root.zone");
+
+    // Counter and progress are initialized to 0.
+    EXPECT_EQ(0, loader.getRRCount());
+    EXPECT_EQ(0, loader.getProgress());
+
     // It gets the updater directly in the constructor
     ASSERT_EQ(1, destination_client_.provided_updaters_.size());
     EXPECT_EQ(Name::ROOT_NAME(), destination_client_.provided_updaters_[0]);
@@ -360,6 +381,12 @@ TEST_F(ZoneLoaderTest, loadUnsigned) {
 
     // The count is 34 because we expect the RRs to be separated.
     EXPECT_EQ(34, destination_client_.rrsets_.size());
+
+    // getRRCount should be identical of the RRs we've seen.  progress
+    // should reach 100% (= 1).
+    EXPECT_EQ(destination_client_.rrsets_.size(), loader.getRRCount());
+    EXPECT_EQ(1, loader.getProgress());
+
     // Ensure known order.
     std::sort(destination_client_.rrset_texts_.begin(),
               destination_client_.rrset_texts_.end());
@@ -380,6 +407,10 @@ TEST_F(ZoneLoaderTest, loadUnsignedIncremental) {
     ZoneLoader loader(destination_client_, Name::ROOT_NAME(),
                       TEST_DATA_DIR "/root.zone");
 
+    // Counters are initialized to 0.
+    EXPECT_EQ(0, loader.getRRCount());
+    EXPECT_EQ(0, loader.getProgress());
+
     // Try loading few RRs first.
     loader.loadIncremental(10);
     // We should get the 10 we asked for
@@ -390,11 +421,26 @@ TEST_F(ZoneLoaderTest, loadUnsignedIncremental) {
     EXPECT_EQ(10, destination_client_.rrsets_.size());
     EXPECT_FALSE(destination_client_.commit_called_);
 
+    // Check we can get intermediate counters. Expected progress is calculated
+    // based on the size of the zone file and the offset to the end of 10th RR
+    // (subject to future changes to the file, but we assume it's a rare
+    // event.).  The expected value should be the exact expression that
+    // getProgress() should do internally, so EXPECT_EQ() should work here,
+    // but floating-point comparison can be always tricky we use
+    // EXPECT_DOUBLE_EQ just in case.
+    EXPECT_EQ(destination_client_.rrsets_.size(), loader.getRRCount());
+    // file size = 1541, offset = 428 (27.77%).
+    EXPECT_DOUBLE_EQ(static_cast<double>(428) / 1541, loader.getProgress());
+
     // We can finish the rest
     loader.loadIncremental(30);
     EXPECT_EQ(34, destination_client_.rrsets_.size());
     EXPECT_TRUE(destination_client_.commit_called_);
 
+    // Counters are updated accordingly. Progress should reach 100%.
+    EXPECT_EQ(destination_client_.rrsets_.size(), loader.getRRCount());
+    EXPECT_EQ(1, loader.getProgress());
+
     // No more loading now
     EXPECT_THROW(loader.load(), isc::InvalidOperation);
     EXPECT_THROW(loader.loadIncremental(1), isc::InvalidOperation);

src/lib/datasrc/zone_loader.cc

@@ -34,17 +34,20 @@
 using isc::dns::Name;
 using isc::dns::ConstRRsetPtr;
 using isc::dns::MasterLoader;
+using isc::dns::MasterLexer;
 
 namespace isc {
 namespace datasrc {
 
+const double ZoneLoader::PROGRESS_UNKNOWN = -1;
+
 ZoneLoader::ZoneLoader(DataSourceClient& destination, const Name& zone_name,
                        DataSourceClient& source) :
     // Separate the RRsets as that is possibly faster (the data source doesn't
     // have to aggregate them) and also because our limit semantics.
     iterator_(source.getIterator(zone_name, true)),
     updater_(destination.getUpdater(zone_name, true, false)),
-    complete_(false)
+    complete_(false), rr_count_(0)
 {
     // The getIterator should never return NULL. So we check it.
     // Or should we throw instead?
@@ -65,11 +68,25 @@ ZoneLoader::ZoneLoader(DataSourceClient& destination, const Name& zone_name,
     }
 }
 
+namespace {
+// Unified callback to install RR and increment RR count at the same time.
+void
+addRR(ZoneUpdater* updater, size_t* rr_count,
+      const dns::Name& name, const dns::RRClass& rrclass,
+      const dns::RRType& type, const dns::RRTTL& ttl,
+      const dns::rdata::RdataPtr& data)
+{
+    isc::dns::BasicRRset rrset(name, rrclass, type, ttl);
+    rrset.addRdata(data);
+    updater->addRRset(rrset);
+    ++*rr_count;
+}
+}
+
 ZoneLoader::ZoneLoader(DataSourceClient& destination, const Name& zone_name,
                        const char* filename) :
     updater_(destination.getUpdater(zone_name, true, false)),
-    complete_(false),
-    loaded_ok_(true)
+    complete_(false), loaded_ok_(true), rr_count_(0)
 {
     if (updater_ == ZoneUpdaterPtr()) {
         isc_throw(DataSourceError, "Zone " << zone_name << " not found in "
@@ -83,7 +100,9 @@ ZoneLoader::ZoneLoader(DataSourceClient& destination, const Name& zone_name,
                                    createMasterLoaderCallbacks(zone_name,
                                        updater_->getFinder().getClass(),
                                        &loaded_ok_),
-                                   createMasterLoaderAddCallback(*updater_)));
+                                   boost::bind(addRR,
+                                               updater_.get(), &rr_count_,
+                                               _1, _2, _3, _4, _5)));
     }
 }
 
@@ -92,7 +111,7 @@ namespace {
 // Copy up to limit RRsets from source to destination
 bool
 copyRRsets(const ZoneUpdaterPtr& destination, const ZoneIteratorPtr& source,
-           size_t limit)
+           size_t limit, size_t& rr_count_)
 {
     size_t loaded = 0;
     while (loaded < limit) {
@@ -104,6 +123,7 @@ copyRRsets(const ZoneUpdaterPtr& destination, const ZoneIteratorPtr& source,
             destination->addRRset(*rrset);
         }
         ++loaded;
+        rr_count_ += rrset->getRdataCount();
     }
     return (false); // Not yet, there may be more
 }
@@ -133,8 +153,8 @@ ZoneLoader::loadIncremental(size_t limit) {
                   "Loading has been completed previously");
     }
 
-    if (iterator_ == ZoneIteratorPtr()) {
-        assert(loader_.get() != NULL);
+    if (!iterator_) {
+        assert(loader_);
         try {
             complete_ = loader_->loadIncremental(limit);
         } catch (const isc::dns::MasterLoaderError& e) {
@@ -144,7 +164,7 @@ ZoneLoader::loadIncremental(size_t limit) {
             isc_throw(MasterFileError, "Error while loading master file");
         }
     } else {
-        complete_ = copyRRsets(updater_, iterator_, limit);
+        complete_ = copyRRsets(updater_, iterator_, limit, rr_count_);
     }
 
     if (complete_) {
@@ -166,5 +186,36 @@ ZoneLoader::loadIncremental(size_t limit) {
     return (complete_);
 }
 
+size_t
+ZoneLoader::getRRCount() const {
+    return (rr_count_);
+}
+
+double
+ZoneLoader::getProgress() const {
+    if (!loader_) {
+        return (PROGRESS_UNKNOWN);
+    }
+
+    const size_t pos = loader_->getPosition();
+    const size_t total_size = loader_->getSize();
+
+    // If the current position is 0, progress should definitely be 0; we
+    // don't bother to check the total size even if it's "unknown".
+    if (pos == 0) {
+        return (0);
+    }
+
+    // These cases shouldn't happen with our usage of MasterLoader.  So, in
+    // theory, we could throw here; however, since this method is expected
+    // to be used for informational purposes only, that's probably too harsh.
+    // So we return "unknown" instead.
+    if (total_size == MasterLexer::SOURCE_SIZE_UNKNOWN || total_size == 0) {
+        return (PROGRESS_UNKNOWN);
+    }
+
+    return (static_cast<double>(pos) / total_size);
+}
+
 } // end namespace datasrc
 } // end namespace isc

src/lib/datasrc/zone_loader.h

@@ -154,10 +154,66 @@ public:
     /// \throw ZoneContentError when the zone doesn't pass sanity check.
     /// \note If the limit is exactly the number of RRs available to be loaded,
     ///     the method still returns false and true'll be returned on the next
-    ///     call (which will load 0 RRs). This is because the end of iterator or
-    ///     master file is detected when reading past the end, not when the last
-    ///     one is read.
+    ///     call (which will load 0 RRs). This is because the end of iterator
+    ///     or master file is detected when reading past the end, not when the
+    ///     last one is read.
     bool loadIncremental(size_t limit);
+
+    /// \brief Return the number of RRs loaded.
+    ///
+    /// This method returns the number of RRs loaded via this loader by the
+    /// time of the call.  Before starting the load it will return 0.
+    /// It will return the total number of RRs of the zone on and after
+    /// completing the load.
+    ///
+    /// \throw None
+    size_t getRRCount() const;
+
+    /// \brief Return the current progress of the loader.
+    ///
+    /// This method returns the current estimated progress of loader as a
+    /// value between 0 and 1 (inclusive); it's 0 before starting the load,
+    /// and 1 at the completion, and a value between these (exclusive) in the
+    /// middle of loading.  It's an implementation detail how to calculate
+    /// the progress, which may vary depending on how the loader is
+    /// constructed and may even be impossible to detect effectively.
+    ///
+    /// If the progress cannot be determined, this method returns a special
+    /// value of PROGRESS_UNKNOWN, which is not included in the range between
+    /// 0 and 1.
+    ///
+    /// As such, the application should use the return value only for
+    /// informational purposes such as logging.  For example, it shouldn't
+    /// be used to determine whether loading is completed by comparing it
+    /// to 1.  It should also expect the possibility of getting
+    /// \c PROGRESS_UNKNOWN at any call to this method; it shouldn't assume
+    /// the specific way of internal implementation as described below (which
+    /// is provided for informational purposes only).
+    ///
+    /// In this implementation, if the loader is constructed with a file
+    /// name, the progress value is measured by the number of characters
+    /// read from the zone file divided by the size of the zone file
+    /// (with taking into account any included files).  Note that due to
+    /// the possibility of intermediate included files, the total file size
+    /// cannot be fully fixed until the completion of the load.  And, due to
+    /// this possibility, return values from this method may not always
+    /// increase monotonically.
+    ///
+    /// If it's constructed with another data source client, this method
+    /// always returns \c PROGRESS_UNKNOWN; in future, however, it may become
+    /// possible to return something more useful, e.g, based on the result
+    /// of \c getRRCount() and the total number of RRs if the underlying data
+    /// source can provide the latter value efficiently.
+    ///
+    /// \throw None
+    double getProgress() const;
+
+    /// \brief A special value for \c getProgress, meaning the progress is
+    /// unknown.
+    ///
+    /// See the method description for details.
+    static const double PROGRESS_UNKNOWN;
+
 private:
     /// \brief The iterator used as source of data in case of the copy mode.
     const ZoneIteratorPtr iterator_;
@@ -169,9 +225,14 @@ private:
     bool complete_;
     /// \brief Was the loading successful?
     bool loaded_ok_;
+    size_t rr_count_;
 };
 
 }
 }
 
-#endif
+#endif  // DATASRC_ZONE_LOADER_H
+
+// Local Variables:
+// mode: c++
+// End:

src/lib/dns/master_lexer.cc

@@ -48,6 +48,7 @@ using namespace master_lexer_internal;
 
 struct MasterLexer::MasterLexerImpl {
     MasterLexerImpl() : source_(NULL), token_(MasterToken::NOT_STARTED),
+                        total_size_(0), popped_size_(0),
                         paren_count_(0), last_was_eol_(true),
                         has_previous_(false),
                         previous_paren_count_(0),
@@ -91,11 +92,28 @@ struct MasterLexer::MasterLexerImpl {
                 separators_.test(c & 0x7f));
     }
 
+    void setTotalSize() {
+        assert(source_ != NULL);
+        if (total_size_ != SOURCE_SIZE_UNKNOWN) {
+            const size_t current_size = source_->getSize();
+            if (current_size != SOURCE_SIZE_UNKNOWN) {
+                total_size_ += current_size;
+            } else {
+                total_size_ = SOURCE_SIZE_UNKNOWN;
+            }
+        }
+    }
+
     std::vector<InputSourcePtr> sources_;
     InputSource* source_;       // current source (NULL if sources_ is empty)
     MasterToken token_;         // currently recognized token (set by a state)
     std::vector<char> data_;    // placeholder for string data
 
+    // Keep track of the total size of all sources and characters that have
+    // been read from sources already popped.
+    size_t total_size_;         // accumulated size (# of chars) of sources
+    size_t popped_size_;        // total size of sources that have been popped
+
     // These are used in states, and defined here only as a placeholder.
     // The main lexer class does not need these members.
     size_t paren_count_;        // nest count of the parentheses
@@ -139,6 +157,7 @@ MasterLexer::pushSource(const char* filename, std::string* error) {
     impl_->source_ = impl_->sources_.back().get();
     impl_->has_previous_ = false;
     impl_->last_was_eol_ = true;
+    impl_->setTotalSize();
     return (true);
 }
 
@@ -154,6 +173,7 @@ MasterLexer::pushSource(std::istream& input) {
     impl_->source_ = impl_->sources_.back().get();
     impl_->has_previous_ = false;
     impl_->last_was_eol_ = true;
+    impl_->setTotalSize();
 }
 
 void
@@ -162,6 +182,7 @@ MasterLexer::popSource() {
         isc_throw(InvalidOperation,
                   "MasterLexer::popSource on an empty source");
     }
+    impl_->popped_size_ += impl_->source_->getPosition();
     impl_->sources_.pop_back();
     impl_->source_ = impl_->sources_.empty() ? NULL :
         impl_->sources_.back().get();
@@ -191,22 +212,12 @@ MasterLexer::getSourceLine() const {
 
 size_t
 MasterLexer::getTotalSourceSize() const {
-    size_t total_size = 0;
-    BOOST_FOREACH(InputSourcePtr& src, impl_->sources_) {
-        // If the size of any pushed source is unknown, the total is also
-        // considered unknown.
-        if (src->getSize() == SOURCE_SIZE_UNKNOWN) {
-            return (SOURCE_SIZE_UNKNOWN);
-        }
-
-        total_size += src->getSize();
-    }
-    return (total_size);
+    return (impl_->total_size_);
 }
 
 size_t
 MasterLexer::getPosition() const {
-    size_t position = 0;
+    size_t position = impl_->popped_size_;
     BOOST_FOREACH(InputSourcePtr& src, impl_->sources_) {
         position += src->getPosition();
     }

src/lib/dns/master_lexer.h

@@ -477,7 +477,7 @@ public:
     ///
     /// This method returns the sum of the size of sources that have been
     /// pushed to the lexer by the time of the call.  It would give the
-    /// caller of some hint about the amount of data the lexer is working on.
+    /// caller some hint about the amount of data the lexer is working on.
     ///
     /// The size of a normal file is equal to the file size at the time of
     /// the source is pushed.  The size of other type of input stream is
@@ -488,25 +488,42 @@ public:
     /// stream is unknown.  It happens, for example, if the standard input
     /// is associated with a pipe from the output of another process and it's
     /// specified as an input source.  If the size of some of the pushed
-    /// pushed source is unknown, this method returns SOURCE_SIZE_UNKNOWN.
+    /// source is unknown, this method returns SOURCE_SIZE_UNKNOWN.
     ///
-    /// If there is no source pushed in the lexer, it returns 0.
+    /// The total size won't change when a source is popped.  So the return
+    /// values of this method will monotonically increase or
+    /// \c SOURCE_SIZE_UNKNOWN; once it returns \c SOURCE_SIZE_UNKNOWN,
+    /// any subsequent call will also result in that value, by the above
+    /// definition.
+    ///
+    /// Before pushing any source, it returns 0.
     ///
     /// \throw None
     size_t getTotalSourceSize() const;
 
-    /// \brief Return the position of lexer in the currently pushed sources.
+    /// \brief Return the position of lexer in the pushed sources so far.
     ///
     /// This method returns the position in terms of the number of recognized
-    /// characters from all sources.  Roughly speaking, the position in a
-    /// single source is the offset from the beginning of the file or stream
-    /// to the current "read cursor" of the lexer, and the return value of
-    /// this method is the sum of the position in all the pushed sources.
+    /// characters from all sources that have been pushed by the time of the
+    /// call.  Conceptually, the position in a single source is the offset
+    /// from the beginning of the file or stream to the current "read cursor"
+    /// of the lexer.  The return value of this method is the sum of the
+    /// positions in all the pushed sources.  If any of the sources has
+    /// already been popped, the position of the source at the time of the
+    /// pop operation will be used for the calculation.
     ///
     /// If the lexer reaches the end for each of all the pushed sources,
     /// the return value should be equal to that of \c getTotalSourceSize().
+    /// It's generally expected that a source is popped when the lexer
+    /// reaches the end of the source.  So, when the application of this
+    /// class parses all contents of all sources, possibly with multiple
+    /// pushes and pops, the return value of this method and
+    /// \c getTotalSourceSize() should be identical (unless the latter
+    /// returns SOURCE_SIZE_UNKNOWN).  But this is not necessarily
+    /// guaranteed as the application can pop a source in the middle of
+    /// parsing it.
     ///
-    /// If there is no source pushed in the lexer, it returns 0.
+    /// Before pushing any source, it returns 0.
     ///
     /// The return values of this method and \c getTotalSourceSize() would
     /// give the caller an idea of the progress of the lexer at the time of
@@ -515,11 +532,10 @@ public:
     /// this way may not make much sense; it can only give an informational
     /// hint of the progress.
     ///
-    /// Note also that if a source is popped, this method will normally return
-    /// a smaller number by definition (and so will \c getTotalSourceSize()).
-    /// Likewise, the conceptual "read cursor" would move backward after a
+    /// Note that the conceptual "read cursor" would move backward after a
     /// call to \c ungetToken(), in which case this method will return a
-    /// smaller value, too.
+    /// smaller value.  That is, unlike \c getTotalSourceSize(), return
+    /// values of this method may not always monotonically increase.
     ///
     /// \throw None
     size_t getPosition() const;

src/lib/dns/master_loader.cc

@@ -76,7 +76,8 @@ public:
         previous_name_(false),
         complete_(false),
         seen_error_(false),
-        warn_rfc1035_ttl_(true)
+        warn_rfc1035_ttl_(true),
+        rr_count_(0)
     {}
 
     void pushSource(const std::string& filename, const Name& current_origin) {
@@ -103,6 +104,9 @@ public:
 
     bool loadIncremental(size_t count_limit);
 
+    size_t getSize() const { return (lexer_.getTotalSourceSize()); }
+    size_t getPosition() const { return (lexer_.getPosition()); }
+
 private:
     void reportError(const std::string& filename, size_t line,
                      const std::string& reason)
@@ -418,12 +422,14 @@ private:
     vector<IncludeInfo> include_info_;
     bool previous_name_; // True if there was a previous name in this file
                          // (false at the beginning or after an $INCLUDE line)
+
 public:
     bool complete_;             // All work done.
     bool seen_error_;           // Was there at least one error during the
                                 // load?
     bool warn_rfc1035_ttl_;     // should warn if implicit TTL determination
                                 // from the previous RR is used.
+    size_t rr_count_;    // number of RRs successfully loaded
 };
 
 // A helper method of loadIncremental, parsing the first token of a new line.
@@ -554,6 +560,7 @@ MasterLoader::MasterLoaderImpl::loadIncremental(size_t count_limit) {
                               rdata);
                 // Good, we loaded another one
                 ++count;
+                ++rr_count_;
             } else {
                 seen_error_ = true;
                 if (!many_errors_) {
@@ -630,5 +637,15 @@ MasterLoader::loadedSucessfully() const {
     return (impl_->complete_ && !impl_->seen_error_);
 }
 
+size_t
+MasterLoader::getSize() const {
+    return (impl_->getSize());
+}
+
+size_t
+MasterLoader::getPosition() const {
+    return (impl_->getPosition());
+}
+
 } // end namespace dns
 } // end namespace isc

src/lib/dns/master_loader.h

@@ -142,6 +142,44 @@ public:
     ///     finishing the load.
     bool loadedSucessfully() const;
 
+    /// \brief Return the total size of the zone files and streams.
+    ///
+    /// This method returns the size of the source of the zone to be loaded
+    /// (master zone files or streams) that is known at the time of the call.
+    /// For a zone file, it's the size of the file; for a stream, it's the
+    /// size of the data (in bytes) available at the start of the load.
+    /// If separate zone files are included via the $INCLUDE directive, the
+    /// sum of the sizes of these files are added.
+    ///
+    /// If the loader is constructed with a stream, the size can be
+    /// "unknown" as described for \c MasterLexer::getTotalSourceSize().
+    /// In this case this method always returns
+    /// \c MasterLexer::SOURCE_SIZE_UNKNOWN.
+    ///
+    /// If the loader is constructed with a zone file, this method
+    /// initially returns 0.  So until either \c load() or \c loadIncremental()
+    /// is called, the value is meaningless.
+    ///
+    /// Note that when the source includes separate files, this method
+    /// cannot take into account the included files that the loader has not
+    /// recognized at the time of call.  So it's possible that this method
+    /// returns different values at different times of call.
+    ///
+    /// \throw None
+    size_t getSize() const;
+
+    /// \brief Return the position of the loader in zone.
+    ///
+    /// This method returns a conceptual "position" of the loader in the
+    /// zone to be loaded.  Specifically, it returns the total number of
+    /// characters contained in the zone files and streams and recognized
+    /// by the loader.  Before starting the load it returns 0; on successful
+    /// completion it will be equal to the return value of \c getSize()
+    /// (unless the latter returns \c MasterLexer::SOURCE_SIZE_UNKNOWN).
+    ///
+    /// \throw None
+    size_t getPosition() const;
+
 private:
     class MasterLoaderImpl;
     MasterLoaderImpl* impl_;
@@ -151,3 +189,7 @@ private:
 } // end namespace isc
 
 #endif // MASTER_LOADER_H
+
+// Local Variables:
+// mode: c++
+// End:

src/lib/dns/tests/master_lexer_unittest.cc

@@ -52,7 +52,6 @@ void
 checkEmptySource(const MasterLexer& lexer) {
     EXPECT_TRUE(lexer.getSourceName().empty());
     EXPECT_EQ(0, lexer.getSourceLine());
-    EXPECT_EQ(0, lexer.getTotalSourceSize());
     EXPECT_EQ(0, lexer.getPosition());
 }
 
@@ -78,6 +77,7 @@ TEST_F(MasterLexerTest, pushStream) {
     lexer.popSource();
     EXPECT_EQ(0, lexer.getSourceCount());
     checkEmptySource(lexer);
+    EXPECT_EQ(4, lexer.getTotalSourceSize()); // this shouldn't change
 }
 
 TEST_F(MasterLexerTest, pushStreamFail) {
@@ -105,6 +105,7 @@ TEST_F(MasterLexerTest, pushFile) {
     lexer.popSource();
     checkEmptySource(lexer);
     EXPECT_EQ(0, lexer.getSourceCount());
+    EXPECT_EQ(143, lexer.getTotalSourceSize()); // this shouldn't change
 
     // If we give a non NULL string pointer, its content will be intact
     // if pushSource succeeds.
@@ -133,22 +134,44 @@ TEST_F(MasterLexerTest, pushFileFail) {
 }
 
 TEST_F(MasterLexerTest, nestedPush) {
-    ss << "test";
+    const string test_txt = "test";
+    ss << test_txt;
     lexer.pushSource(ss);
+
+    EXPECT_EQ(test_txt.size(), lexer.getTotalSourceSize());
+    EXPECT_EQ(0, lexer.getPosition());
+
     EXPECT_EQ(expected_stream_name, lexer.getSourceName());
 
+    // Read the string; getPosition() should reflect that.
+    EXPECT_EQ(MasterToken::STRING, lexer.getNextToken().getType());
+    EXPECT_EQ(test_txt.size(), lexer.getPosition());
+