stats collection: overlay stats

* ebuild count before/after overlay creation
* # of written ebuilds

2 files changed, 100 insertions, 36 deletions

diff --git a/roverlay/overlay/category.py b/roverlay/overlay/category.py
index 330058c..3126642 100644
--- a/roverlay/overlay/category.py
+++ b/roverlay/overlay/category.py
@@ -23,12 +23,63 @@ try:
 except ImportError:
    import Queue as queue
 
+
+import roverlay.stats.collector
+
 from roverlay.overlay import pkgdir
 
+class WriteQueueJob ( object ):
+
+   def __init__ ( self, write_queue, write_kw, catref, additions_dir ):
+      super ( WriteQueueJob, self ).__init__()
+      self.write_queue   = write_queue
+      self.write_kw      = write_kw
+      self.catref        = catref
+      self.additions_dir = additions_dir
+      self.stats         = catref.STATS.get_new()
+   # --- end of __init__ (...) ---
+
+   def run ( self ):
+      """Calls <package>.write for every <package> received from the queue.
+
+      arguments:
+      * q        -- queue
+      * write_kw -- keywords for write(...)
+      """
+      q             = self.write_queue
+      write_kw      = self.write_kw
+      stats         = self.stats
+      catref        = self.catref
+      additions_dir = self.additions_dir
+
+      while not q.empty() and not hasattr ( catref, 'RERAISE' ):
+         try:
+            pkg = q.get_nowait()
+            # remove manifest writing from threaded writing since it's
+            # single-threaded
+            pkg.write (
+               additions_dir = additions_dir.get_obj_subdir ( pkg ),
+               stats         = stats,
+               **write_kw
+            )
+            stats.ebuild_count.inc (
+               len ( list ( pkg.iter_packages_with_efile() ) )
+            )
+         except queue.Empty:
+            break
+         except ( Exception, KeyboardInterrupt ) as err:
+            catref.logger.exception ( err )
+            catref.RERAISE = sys.exc_info()
+   # --- end of run (...) ---
+
+# --- end of WriteQueueJob ---
+
 class Category ( object ):
 
    WRITE_JOBCOUNT = 3
 
+   STATS = roverlay.stats.collector.static.overlay
+
    def __init__ ( self,
       name, logger, directory, get_header, runtime_incremental
    ):
@@ -221,11 +272,12 @@ class Category ( object ):
 
    def scan ( self, **kw ):
       """Scans this category for existing ebuilds."""
+      stats = self.STATS
       for subdir in os.listdir ( self.physical_location ):
          if self.has_dir ( subdir ):
             pkgdir = self._get_package_dir ( subdir )
             try:
-               pkgdir.scan ( **kw )
+               pkgdir.scan ( stats=stats, **kw )
             finally:
                if pkgdir.empty():
                   del self._subdirs [subdir]
@@ -251,29 +303,6 @@ class Category ( object ):
 
       returns: None (implicit)
       """
-      def run_write_queue ( q, write_kw ):
-         """Calls <package>.write for every <package> received from the queue.
-
-         arguments:
-         * q        -- queue
-         * write_kw -- keywords for write(...)
-         """
-         while not q.empty() and not hasattr ( self, 'RERAISE' ):
-            try:
-               pkg = q.get_nowait()
-               # remove manifest writing from threaded writing since it's
-               # single-threaded
-               pkg.write (
-                  additions_dir = additions_dir.get_obj_subdir ( pkg ),
-                  **write_kw
-               )
-            except queue.Empty:
-               break
-            except ( Exception, KeyboardInterrupt ) as err:
-               self.logger.exception ( err )
-               self.RERAISE = sys.exc_info()
-      # --- end of run_write_queue (...) ---
-
       if len ( self._subdirs ) == 0: return
 
       # determine write keyword args
@@ -310,11 +339,13 @@ class Category ( object ):
             write_manifest and manifest_threadsafe
          )
 
+         jobs = frozenset (
+            WriteQueueJob ( write_queue, write_kwargs, self, additions_dir )
+            for n in range ( max_jobs )
+         )
+
          workers = frozenset (
-            threading.Thread (
-               target=run_write_queue,
-               args=( write_queue, write_kwargs )
-            ) for n in range ( max_jobs )
+            threading.Thread ( target=job.run ) for job in jobs
          )
 
          for w in workers: w.start()
@@ -344,15 +375,25 @@ class Category ( object ):
             for package in self._subdirs.values():
                package.write_manifest ( ignore_empty=True )
 
+
+         # merge stats from threads with self.(__class__.)STATS
+         for job in jobs:
+            self.STATS.merge_with ( job.stats )
       else:
+         stats = self.STATS
          for package in self._subdirs.values():
             package.write (
                additions_dir = additions_dir.get_obj_subdir ( package ),
+               stats         = stats,
                **write_kwargs
             )
+            stats.ebuild_count.inc (
+               len ( list ( package.iter_packages_with_efile() ) )
+            )
+
+         self.remove_empty()
       # -- end if;
 
-      self.remove_empty()
    # --- end of write (...) ---
 
    def write_manifest ( self, **manifest_kw ):
diff --git a/roverlay/overlay/pkgdir/packagedir_base.py b/roverlay/overlay/pkgdir/packagedir_base.py
index 8c0bd0b..7f1a0ba 100644
--- a/roverlay/overlay/pkgdir/packagedir_base.py
+++ b/roverlay/overlay/pkgdir/packagedir_base.py
@@ -48,7 +48,7 @@ class PackageDirBase ( object ):
    #DISTMAP  =
    #FETCHENV =
 
-   EBUILD_SUFFIX  = '.ebuild'
+   EBUILD_SUFFIX = '.ebuild'
 
    # MANIFEST_THREADSAFE (tri-state)
    # * None  -- unknown (e.g. write_manifest() not implemented)
@@ -124,10 +124,19 @@ class PackageDirBase ( object ):
       self._need_metadata    = False
    # --- end of __init__ (...) ---
 
-   def iter_package_info ( self ):
-      return self._packages.values()
+   def iter_package_info ( self, pkg_filter=None ):
+      if pkg_filter is None:
+         return self._packages.values()
+      else:
+         return ( p for p in self._packages.values() if pkg_filter ( p ) )
    # --- end of iter_package_info (...) --
 
+   def iter_packages_with_efile ( self ):
+      return (
+         p for p in self._packages.values() if p.has ( 'ebuild_file' )
+      )
+   # --- end of iter_packages_with_efile (...) ---
+
    def _remove_ebuild_file ( self, pkg_info ):
       """Removes the ebuild file of a pkg_info object.
       Returns True on success, else False.
@@ -420,7 +429,7 @@ class PackageDirBase ( object ):
       self.fs_cleanup()
    # --- end of fs_destroy (...) ---
 
-   def scan ( self, **kw ):
+   def scan ( self, stats, **kw ):
       """Scans the filesystem location of this package for existing
       ebuilds and adds them.
       """
@@ -457,6 +466,8 @@ class PackageDirBase ( object ):
                )
       # --- end of scan_ebuilds (...) ---
 
+      ebuild_found = stats.ebuilds_scanned.inc
+
       # ignore directories without a Manifest file
       if os.path.isfile ( self.physical_location + os.sep + 'Manifest' ):
          for pvr, efile in scan_ebuilds():
@@ -470,6 +481,8 @@ class PackageDirBase ( object ):
                      )
                   )
                   raise
+               else:
+                  ebuild_found()
    # --- end of scan (...) ---
 
    def show ( self, stream=sys.stderr ):
@@ -506,7 +519,7 @@ class PackageDirBase ( object ):
       additions_dir,
       overwrite_ebuilds=False,
       write_ebuilds=True, write_manifest=True, write_metadata=True,
-      cleanup=True, keep_n_ebuilds=None, cautious=True
+      cleanup=True, keep_n_ebuilds=None, cautious=True, stats=None,
    ):
       """Writes this directory to its (existent!) filesystem location.
 
@@ -526,6 +539,7 @@ class PackageDirBase ( object ):
       * cautious          -- be cautious when keeping the nth latest ebuilds,
                              this has some overhead
                              Defaults to True
+      * stats             --
 
       returns: success (True/False)
 
@@ -554,7 +568,8 @@ class PackageDirBase ( object ):
                # possible in this if-branch
                success = self.write_ebuilds (
                   additions_dir = additions_dir,
-                  overwrite     = bool ( overwrite_ebuilds )
+                  overwrite     = bool ( overwrite_ebuilds ),
+                  stats         = stats,
                )
 #                  overwrite = overwrite_ebuilds \
 #                     if overwrite_ebuilds is not None \
@@ -689,7 +704,9 @@ class PackageDirBase ( object ):
                   import_ebuild_efile ( pvr, efile, fname )
    # --- end of import_ebuilds (...) ---
 
-   def write_ebuilds ( self, overwrite, additions_dir, shared_fh=None ):
+   def write_ebuilds ( self,
+      overwrite, additions_dir, shared_fh=None, stats=None
+   ):
       """Writes all ebuilds.
 
       arguments:
@@ -697,6 +714,7 @@ class PackageDirBase ( object ):
       * additions_dir  --
       * shared_fh      -- if set and not None: don't use own file handles
                            (i.e. write files), write everything into shared_fh
+      * stats          --
       """
       ebuild_header = self.get_header()
 
@@ -798,6 +816,10 @@ class PackageDirBase ( object ):
 
       all_ebuilds_written = True
 
+      ebuild_written = (
+         stats.ebuilds_written.inc if stats is not None else ( lambda: None )
+      )
+
       # don't call dodir if shared_fh is set
       hasdir = bool ( shared_fh is not None )
 
@@ -831,6 +853,7 @@ class PackageDirBase ( object ):
                   ebuild_file=efile,
                   remove_auto='ebuild_written'
                )
+               ebuild_written()
 
                self.logger.info ( "Wrote ebuild {}.".format ( efile ) )
          else: