library: add support for smaps_rollup file, <pids> api

A couple of people have suggested that smaps_rollup be
added to the ps program and/or top program. This patch
is intended to set the stage for just such extensions.

There are currently 20 displayable items in the rollup
file. And newlib sometimes uses sscanf when populating
the target, sometimes hsearch and one customized gperf
approach. None of these fit well with the smaps items.

Thus, an approach using a simple table lookup was used
and, by disabling 1 code line, it could be made immune
from changes to the items order (unlike a sscanf call)
and doesn't carry the greater cost of a hsearch/gperf.

Note: The next patch will allow top to display some of
these new fields. Then, it'll be possible to determine
the colossal costs of accessing the smaps_rollup file.

Here is a small preview of just what you will discover
when using the command 'time top/top -d0 -n1000' while
configured with just two fields: PID + 1 memory field.

------------------------------------ as a regular user
    with only PID + RES (statm)
real       0m2.605s
user       0m1.060s
sys        0m1.377s
    with only PID + RSS (smaps)
real      0m26.397s                    10x more costly
user       0m1.253s
sys       0m24.915s

----------------- as a root (thus smaps for all tasks)
    with only PID + RES (statm)
real       0m2.651s
user       0m1.177s
sys        0m1.286s
    with only PID + RSS (smaps)
real      0m33.040s                    12x more costly
user       0m1.256s
sys       0m31.533s

Reference(s):
. ps: expose shared/private memory separately
https://gitlab.com/procps-ng/procps/-/issues/201
. top/ps: add support for PSS reporting
https://gitlab.com/procps-ng/procps/-/issues/112

Signed-off-by: Jim Warner <james.warner@comcast.net>

diff --git a/proc/pids.c b/proc/pids.c
index 172fd656781c2df5694a0b4c71b0bfd6055e1f05..472e6af94559c0f3ce6ea1ab01dd3e35bbda1dce 100644 (file)
--- a/proc/pids.c
+++ b/proc/pids.c
@@ -240,6 +240,26 @@ DUP_set(SIGCATCH,                  sigcatch)
 DUP_set(SIGIGNORE,                 sigignore)
 DUP_set(SIGNALS,                   signal)
 DUP_set(SIGPENDING,                _sigpnd)
+REG_set(SMAP_ANONYMOUS,   ul_int,  smap_Anonymous)
+REG_set(SMAP_HUGETBL_PRV, ul_int,  smap_Private_Hugetlb)
+REG_set(SMAP_HUGETBL_SHR, ul_int,  smap_Shared_Hugetlb)
+REG_set(SMAP_HUGE_ANON,   ul_int,  smap_AnonHugePages)
+REG_set(SMAP_HUGE_FILE,   ul_int,  smap_FilePmdMapped)
+REG_set(SMAP_HUGE_SHMEM,  ul_int,  smap_ShmemPmdMapped)
+REG_set(SMAP_LAZY_FREE,   ul_int,  smap_LazyFree)
+REG_set(SMAP_LOCKED,      ul_int,  smap_Locked)
+REG_set(SMAP_PRV_CLEAN,   ul_int,  smap_Private_Clean)
+REG_set(SMAP_PRV_DIRTY,   ul_int,  smap_Private_Dirty)
+REG_set(SMAP_PSS,         ul_int,  smap_Pss)
+REG_set(SMAP_PSS_ANON,    ul_int,  smap_Pss_Anon)
+REG_set(SMAP_PSS_FILE,    ul_int,  smap_Pss_File)
+REG_set(SMAP_PSS_SHMEM,   ul_int,  smap_Pss_Shmem)
+REG_set(SMAP_REFERENCED,  ul_int,  smap_Referenced)
+REG_set(SMAP_RSS,         ul_int,  smap_Rss)
+REG_set(SMAP_SHR_CLEAN,   ul_int,  smap_Shared_Clean)
+REG_set(SMAP_SHR_DIRTY,   ul_int,  smap_Shared_Dirty)
+REG_set(SMAP_SWAP,        ul_int,  smap_Swap)
+REG_set(SMAP_SWAP_PSS,    ul_int,  smap_SwapPss)
 REG_set(STATE,            s_ch,    state)
 STR_set(SUPGIDS,                   supgid)
 STR_set(SUPGROUPS,                 supgrp)
@@ -351,6 +371,7 @@ srtDECL(noop) {
 #define f_lxc      PROC_FILL_LXC
 #define f_ns       PROC_FILLNS
 #define f_oom      PROC_FILLOOM
+#define f_smaps    PROC_FILLSMAPS
 #define f_stat     PROC_FILLSTAT
 #define f_statm    PROC_FILLMEM
 #define f_status   PROC_FILLSTATUS
@@ -497,6 +518,26 @@ static struct {
     { RS(SIGIGNORE),         f_status,   FF(str),   QS(str),       0,        TS(str)     },
     { RS(SIGNALS),           f_status,   FF(str),   QS(str),       0,        TS(str)     },
     { RS(SIGPENDING),        f_status,   FF(str),   QS(str),       0,        TS(str)     },
+    { RS(SMAP_ANONYMOUS),    f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_HUGETBL_PRV),  f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_HUGETBL_SHR),  f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_HUGE_ANON),    f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_HUGE_FILE),    f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_HUGE_SHMEM),   f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_LAZY_FREE),    f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_LOCKED),       f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_PRV_CLEAN),    f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_PRV_DIRTY),    f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_PSS),          f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_PSS_ANON),     f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_PSS_FILE),     f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_PSS_SHMEM),    f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_REFERENCED),   f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_RSS),          f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_SHR_CLEAN),    f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_SHR_DIRTY),    f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_SWAP),         f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
+    { RS(SMAP_SWAP_PSS),     f_smaps,    NULL,      QS(ul_int),    0,        TS(ul_int)  },
     { RS(STATE),             f_either,   NULL,      QS(s_ch),      0,        TS(s_ch)    },
     { RS(SUPGIDS),           f_status,   FF(str),   QS(str),       0,        TS(str)     },
     { RS(SUPGROUPS),         x_supgrp,   FF(str),   QS(str),       0,        TS(str)     },
@@ -551,6 +592,7 @@ enum pids_item PIDS_logical_end = MAXTABLE(Item_table);
 #undef f_lxc
 #undef f_ns
 #undef f_oom
+#undef f_smaps
 //#undef f_stat                   // needed later
 #undef f_statm
 //#undef f_status                 // needed later

diff --git a/proc/pids.h b/proc/pids.h
index 750fb039e88ed407d5b30c42586f8897264d6970..40765025ebba9103285d0788435fb50a62bab6f2 100644 (file)
--- a/proc/pids.h
+++ b/proc/pids.h
@@ -125,6 +125,26 @@ enum pids_item {
     PIDS_SIGIGNORE,         //      str        status: SigIgn
     PIDS_SIGNALS,           //      str        status: ShdPnd
     PIDS_SIGPENDING,        //      str        status: SigPnd
+    PIDS_SMAP_ANONYMOUS,    //   ul_int        smaps_rollup: Anonymous
+    PIDS_SMAP_HUGETBL_PRV,  //   ul_int        smaps_rollup: Private_Hugetlb
+    PIDS_SMAP_HUGETBL_SHR,  //   ul_int        smaps_rollup: Shared_Hugetlb
+    PIDS_SMAP_HUGE_ANON,    //   ul_int        smaps_rollup: AnonHugePages
+    PIDS_SMAP_HUGE_FILE,    //   ul_int        smaps_rollup: FilePmdMapped
+    PIDS_SMAP_HUGE_SHMEM,   //   ul_int        smaps_rollup: ShmemPmdMapped
+    PIDS_SMAP_LAZY_FREE,    //   ul_int        smaps_rollup: LazyFree
+    PIDS_SMAP_LOCKED,       //   ul_int        smaps_rollup: Locked
+    PIDS_SMAP_PRV_CLEAN,    //   ul_int        smaps_rollup: Private_Clean
+    PIDS_SMAP_PRV_DIRTY,    //   ul_int        smaps_rollup: Private_Dirty
+    PIDS_SMAP_PSS,          //   ul_int        smaps_rollup: Pss
+    PIDS_SMAP_PSS_ANON,     //   ul_int        smaps_rollup: Pss_Anon
+    PIDS_SMAP_PSS_FILE,     //   ul_int        smaps_rollup: Pss_File
+    PIDS_SMAP_PSS_SHMEM,    //   ul_int        smaps_rollup: Pss_Shmem
+    PIDS_SMAP_REFERENCED,   //   ul_int        smaps_rollup: Referenced
+    PIDS_SMAP_RSS,          //   ul_int        smaps_rollup: Rss
+    PIDS_SMAP_SHR_CLEAN,    //   ul_int        smaps_rollup: Shared_Clean
+    PIDS_SMAP_SHR_DIRTY,    //   ul_int        smaps_rollup: Shared_Dirty
+    PIDS_SMAP_SWAP,         //   ul_int        smaps_rollup: Swap
+    PIDS_SMAP_SWAP_PSS,     //   ul_int        smaps_rollup: SwapPss
     PIDS_STATE,             //     s_ch        stat: state or status: State
     PIDS_SUPGIDS,           //      str        status: Groups
     PIDS_SUPGROUPS,         //      str        derived from SUPGIDS, see getgrgid(3)

diff --git a/proc/readproc.c b/proc/readproc.c
index 7d6e722fdb99a0eb05b31c0d909f6ca6297b2184..4975981d553306da46eadefe9cb8dd6b9c92d98d 100644 (file)
--- a/proc/readproc.c
+++ b/proc/readproc.c
@@ -648,6 +648,72 @@ static void io2proc(const char* s, proc_t *restrict P) {
             &P->syscw, &P->read_bytes, &P->write_bytes, &P->cancelled_write_bytes);
 }
 
+    // Assuming permissions have allowed the read of smaps_rollup, this
+    // guy will extract some %lu data. Considering the number of items,
+    // we are between small enough to use a sscanf and large enough for
+    // a search.h approach. Thus we roll (get it?) our own custom code.
+static void smaps2proc (const char* s, proc_t *restrict P) {
+  #define enMAX (int)((sizeof(smaptab) / sizeof(smaptab[0])))
+    // 1st proc_t data field
+  #define fZERO tid
+    // a smaptab entry generator
+  #define mkENT(F) { #F ":", -1, (int)((void*)&q->smap_ ## F - (void*)&q->fZERO) }
+    // make a target field
+  #define mkOBJ(e) ( (unsigned long *)((void *)&P->fZERO + smaptab[e].offs) )
+    static const proc_t *q;
+    static struct {
+        const char *item;
+        int slen;
+        int offs;
+    } smaptab[] =  {
+        /*    Size                 smaps only, not rollup */
+        /*    KernelPageSize                 "            */
+        /*    MMUPageSize                    "            */
+        mkENT(Rss),
+        mkENT(Pss),
+        mkENT(Pss_Anon),        /* rollup only, not smaps */
+        mkENT(Pss_File),        /*            "           */
+        mkENT(Pss_Shmem),       /*            "           */
+        mkENT(Shared_Clean),
+        mkENT(Shared_Dirty),
+        mkENT(Private_Clean),
+        mkENT(Private_Dirty),
+        mkENT(Referenced),
+        mkENT(Anonymous),
+        mkENT(LazyFree),
+        mkENT(AnonHugePages),
+        mkENT(ShmemPmdMapped),
+        mkENT(FilePmdMapped),
+        mkENT(Shared_Hugetlb),
+        mkENT(Private_Hugetlb),
+        mkENT(Swap),
+        mkENT(SwapPss),
+        mkENT(Locked)
+        /*    THPeligible          smaps only, not rollup */
+        /*    ProtectionKey                  "            */
+        /*    VmFlags                        "            */
+    };
+    char *head, *tail;
+    int i;
+
+    if (smaptab[0].slen < 0) {
+        for (i = 0; i < enMAX; i++)
+            smaptab[i].slen = (int)strlen(smaptab[i].item);
+    }
+    for (i = 0; i < enMAX; i++) {
+        if (!(head = strstr(s, smaptab[i].item)))
+            continue;
+        head += smaptab[i].slen;
+        *mkOBJ(i) = strtoul(head, &tail, 10);
+        // saves some overhead BUT makes us dependent on current order
+        s = tail;
+    }
+  #undef enMAX
+  #undef fZERO
+  #undef mkENT
+  #undef mkOBJ
+}
+
 static int file2str(const char *directory, const char *what, struct utlbuf_s *ub) {
  #define buffGRW 1024
     char path[PROCPATHLEN];
@@ -1054,6 +1120,11 @@ static proc_t* simple_readproc(PROCTAB *restrict const PT, proc_t *restrict cons
             io2proc(ub.buf, p);
     }
 
+    if (flags & PROC_FILLSMAPS) {               // read /proc/#/smaps_rollup
+        if (file2str(path, "smaps_rollup", &ub) != -1)
+            smaps2proc(ub.buf, p);
+    }
+
     if (flags & PROC_FILLMEM) {                 // read /proc/#/statm
         if (file2str(path, "statm", &ub) != -1)
             statm2proc(ub.buf, p);
@@ -1169,10 +1240,16 @@ static proc_t* simple_readtask(PROCTAB *restrict const PT, proc_t *restrict cons
             io2proc(ub.buf, t);
     }
 
+    if (flags & PROC_FILLSMAPS) {               // read /proc/#/task/#/smaps_rollup
+        if (file2str(path, "smaps_rollup", &ub) != -1)
+            smaps2proc(ub.buf, t);
+    }
+
     if (flags & PROC_FILLMEM) {                 // read /proc/#/task/#/statm
         if (file2str(path, "statm", &ub) != -1)
             statm2proc(ub.buf, t);
     }
+
     if (flags & PROC_FILLSTATUS) {              // read /proc/#/task/#/status
         if (file2str(path, "status", &ub) != -1) {
             rc += status2proc(ub.buf, t, 0);

diff --git a/proc/readproc.h b/proc/readproc.h
index 992eb53e0554ba7fa89a3bea284d72e6bbd30be8..21ca926df5fc182982dd0ec5578c28fab3fe8788 100644 (file)
--- a/proc/readproc.h
+++ b/proc/readproc.h
@@ -104,7 +104,27 @@ typedef struct proc_t {
         syscw,          // io              number of write I/O operations
         read_bytes,     // io              number of bytes fetched from the storage layer
         write_bytes,    // io              number of bytes sent to the storage layer
-        cancelled_write_bytes; // io       number of bytes truncating pagecache
+        cancelled_write_bytes, // io       number of bytes truncating pagecache
+        smap_Rss,              // smaps_rollup  mapping currently resident in RAM
+        smap_Pss,              //    "     Rss divided by total processes sharing it
+        smap_Pss_Anon,         //    "     proportional share of 'anonymous' memory
+        smap_Pss_File,         //    "     proportional share of 'file' memory
+        smap_Pss_Shmem,        //    "     proportional share of 'shmem' memory
+        smap_Shared_Clean,     //    "     unmodified shared memory
+        smap_Shared_Dirty,     //    "     altered shared memory
+        smap_Private_Clean,    //    "     unmodified private memory
+        smap_Private_Dirty,    //    "     altered private memory
+        smap_Referenced,       //    "     memory marked as referenced/accessed
+        smap_Anonymous,        //    "     memory not belonging to any file
+        smap_LazyFree,         //    "     memory marked by madvise(MADV_FREE)
+        smap_AnonHugePages,    //    "     memory backed by transparent huge pages
+        smap_ShmemPmdMapped,   //    "     shmem/tmpfs memory backed by huge pages
+        smap_FilePmdMapped,    //    "     file memory backed by huge pages
+        smap_Shared_Hugetlb,   //    "     hugetlbfs backed memory *not* counted in Rss/Pss
+        smap_Private_Hugetlb,  //    "     hugetlbfs backed memory *not* counted in Rss/Pss
+        smap_Swap,             //    "     swapped would-be-anonymous memory (includes swapped out shmem)
+        smap_SwapPss,          //    "     the proportional share of 'Swap' (excludes swapped out shmem)
+        smap_Locked;           //    "     memory amount locked to RAM
     char
         *environ,       // (special)       environment as string (/proc/#/environ)
         *cmdline,       // (special)       command line as string (/proc/#/cmdline)
@@ -217,6 +237,7 @@ typedef struct PROCTAB {
 #define PROC_FILL_LUID     0x400000 // fill in proc_t luid (login user id)
 #define PROC_FILL_EXE      0x200000 // fill in proc_t exe path + pgm name
 #define PROC_FILLIO      0x01000000 // fill in proc_t io information
+#define PROC_FILLSMAPS   0x02000000 // fill in proc_t smaps_rollup stuff
 
 // consider only processes with one of the passed:
 #define PROC_PID             0x1000  // process id numbers ( 0   terminated)
@@ -233,10 +254,10 @@ typedef struct PROCTAB {
 #define PROC_FILL_SUPGRP   ( 0x0400 | PROC_FILLSTATUS ) // obtain supplementary group names
 
 // it helps to give app code a few spare bits
-#define PROC_SPARE_1     0x02000000
-#define PROC_SPARE_2     0x04000000
-#define PROC_SPARE_3     0x08000000
-#define PROC_SPARE_4     0x10000000
+#define PROC_SPARE_1     0x04000000
+#define PROC_SPARE_2     0x08000000
+#define PROC_SPARE_3     0x10000000
+#define PROC_SPARE_4     0x20000000
 
 // Function definitions
 // Initialize a PROCTAB structure holding needed call-to-call persistent data