LCOV - code coverage report
Current view: top level - gcc - profile-count.h (source / functions) Hit Total Coverage
Test: gcc.info Lines: 470 499 94.2 %
Date: 2020-03-28 11:57:23 Functions: 43 44 97.7 %
Legend: Lines: hit not hit | Branches: + taken - not taken # not executed Branches: 0 0 -

           Branch data     Line data    Source code
       1                 :            : /* Profile counter container type.
       2                 :            :    Copyright (C) 2017-2020 Free Software Foundation, Inc.
       3                 :            :    Contributed by Jan Hubicka
       4                 :            : 
       5                 :            : This file is part of GCC.
       6                 :            : 
       7                 :            : GCC is free software; you can redistribute it and/or modify it under
       8                 :            : the terms of the GNU General Public License as published by the Free
       9                 :            : Software Foundation; either version 3, or (at your option) any later
      10                 :            : version.
      11                 :            : 
      12                 :            : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      13                 :            : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      14                 :            : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      15                 :            : for more details.
      16                 :            : 
      17                 :            : You should have received a copy of the GNU General Public License
      18                 :            : along with GCC; see the file COPYING3.  If not see
      19                 :            : <http://www.gnu.org/licenses/>.  */
      20                 :            : 
      21                 :            : #ifndef GCC_PROFILE_COUNT_H
      22                 :            : #define GCC_PROFILE_COUNT_H
      23                 :            : 
      24                 :            : struct function;
      25                 :            : struct profile_count;
      26                 :            : class sreal;
      27                 :            : 
      28                 :            : /* Quality of the profile count.  Because gengtype does not support enums
      29                 :            :    inside of classes, this is in global namespace.  */
      30                 :            : enum profile_quality {
      31                 :            :   /* Uninitialized value.  */
      32                 :            :   UNINITIALIZED_PROFILE,
      33                 :            : 
      34                 :            :   /* Profile is based on static branch prediction heuristics and may
      35                 :            :      or may not match reality.  It is local to function and cannot be compared
      36                 :            :      inter-procedurally.  Never used by probabilities (they are always local).
      37                 :            :    */
      38                 :            :   GUESSED_LOCAL,
      39                 :            : 
      40                 :            :   /* Profile was read by feedback and was 0, we used local heuristics to guess
      41                 :            :      better.  This is the case of functions not run in profile feedback.
      42                 :            :      Never used by probabilities.  */
      43                 :            :   GUESSED_GLOBAL0,
      44                 :            : 
      45                 :            :   /* Same as GUESSED_GLOBAL0 but global count is adjusted 0.  */
      46                 :            :   GUESSED_GLOBAL0_ADJUSTED,
      47                 :            : 
      48                 :            :   /* Profile is based on static branch prediction heuristics.  It may or may
      49                 :            :      not reflect the reality but it can be compared interprocedurally
      50                 :            :      (for example, we inlined function w/o profile feedback into function
      51                 :            :       with feedback and propagated from that).
      52                 :            :      Never used by probabilities.  */
      53                 :            :   GUESSED,
      54                 :            : 
      55                 :            :   /* Profile was determined by autofdo.  */
      56                 :            :   AFDO,
      57                 :            : 
      58                 :            :   /* Profile was originally based on feedback but it was adjusted
      59                 :            :      by code duplicating optimization.  It may not precisely reflect the
      60                 :            :      particular code path.  */
      61                 :            :   ADJUSTED,
      62                 :            : 
      63                 :            :   /* Profile was read from profile feedback or determined by accurate static
      64                 :            :      method.  */
      65                 :            :   PRECISE
      66                 :            : };
      67                 :            : 
      68                 :            : extern const char *profile_quality_as_string (enum profile_quality);
      69                 :            : extern bool parse_profile_quality (const char *value,
      70                 :            :                                    profile_quality *quality);
      71                 :            : 
      72                 :            : /* The base value for branch probability notes and edge probabilities.  */
      73                 :            : #define REG_BR_PROB_BASE  10000
      74                 :            : 
      75                 :            : #define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
      76                 :            : 
      77                 :            : bool slow_safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res);
      78                 :            : 
      79                 :            : /* Compute RES=(a*b + c/2)/c capping and return false if overflow happened.  */
      80                 :            : 
      81                 :            : inline bool
      82                 :  667396074 : safe_scale_64bit (uint64_t a, uint64_t b, uint64_t c, uint64_t *res)
      83                 :            : {
      84                 :            : #if (GCC_VERSION >= 5000)
      85                 :  667396074 :   uint64_t tmp;
      86                 :  667396074 :   if (!__builtin_mul_overflow (a, b, &tmp)
      87                 :  667072713 :       && !__builtin_add_overflow (tmp, c/2, &tmp))
      88                 :            :     {
      89                 :  666882333 :       *res = tmp / c;
      90                 :  666882333 :       return true;
      91                 :            :     }
      92                 :     322318 :   if (c == 1)
      93                 :            :     {
      94                 :      57781 :       *res = (uint64_t) -1;
      95                 :      57781 :       return false;
      96                 :            :     }
      97                 :            : #else
      98                 :            :   if (a < ((uint64_t)1 << 31)
      99                 :            :       && b < ((uint64_t)1 << 31)
     100                 :            :       && c < ((uint64_t)1 << 31))
     101                 :            :     {
     102                 :            :       *res = (a * b + (c / 2)) / c;
     103                 :            :       return true;
     104                 :            :     }
     105                 :            : #endif
     106                 :     455275 :   return slow_safe_scale_64bit (a, b, c, res);
     107                 :            : }
     108                 :            : 
     109                 :            : /* Data type to hold probabilities.  It implements fixed point arithmetics
     110                 :            :    with capping so probability is always in range [0,1] and scaling requiring
     111                 :            :    values greater than 1 needs to be represented otherwise.
     112                 :            : 
     113                 :            :    In addition to actual value the quality of profile is tracked and propagated
     114                 :            :    through all operations.  Special value UNINITIALIZED_PROFILE is used for probabilities
     115                 :            :    that has not been determined yet (for example because of
     116                 :            :    -fno-guess-branch-probability)
     117                 :            : 
     118                 :            :    Typically probabilities are derived from profile feedback (via
     119                 :            :    probability_in_gcov_type), autoFDO or guessed statically and then propagated
     120                 :            :    thorough the compilation.
     121                 :            : 
     122                 :            :    Named probabilities are available:
     123                 :            :      - never           (0 probability)
     124                 :            :      - guessed_never
     125                 :            :      - very_unlikely   (1/2000 probability)
     126                 :            :      - unlikely        (1/5 probability)
     127                 :            :      - even            (1/2 probability)
     128                 :            :      - likely          (4/5 probability)
     129                 :            :      - very_likely     (1999/2000 probability)
     130                 :            :      - guessed_always
     131                 :            :      - always
     132                 :            : 
     133                 :            :    Named probabilities except for never/always are assumed to be statically
     134                 :            :    guessed and thus not necessarily accurate.  The difference between never
     135                 :            :    and guessed_never is that the first one should be used only in case that
     136                 :            :    well behaving program will very likely not execute the "never" path.
     137                 :            :    For example if the path is going to abort () call or it exception handling.
     138                 :            : 
     139                 :            :    Always and guessed_always probabilities are symmetric.
     140                 :            : 
     141                 :            :    For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
     142                 :            :    integer arithmetics. Once the code is converted to branch probabilities,
     143                 :            :    these conversions will probably go away because they are lossy.
     144                 :            : */
     145                 :            : 
     146                 :            : class GTY((user)) profile_probability
     147                 :            : {
     148                 :            :   static const int n_bits = 29;
     149                 :            :   /* We can technically use ((uint32_t) 1 << (n_bits - 1)) - 2 but that
     150                 :            :      will lead to harder multiplication sequences.  */
     151                 :            :   static const uint32_t max_probability = (uint32_t) 1 << (n_bits - 2);
     152                 :            :   static const uint32_t uninitialized_probability
     153                 :            :                  = ((uint32_t) 1 << (n_bits - 1)) - 1;
     154                 :            : 
     155                 :            :   uint32_t m_val : 29;
     156                 :            :   enum profile_quality m_quality : 3;
     157                 :            : 
     158                 :            :   friend struct profile_count;
     159                 :            : public:
     160                 : 2189246018 :   profile_probability (): m_val (uninitialized_probability),
     161                 : 1054314208 :     m_quality (GUESSED)
     162                 :            :   {}
     163                 :            : 
     164                 :         90 :   profile_probability (uint32_t val, profile_quality quality):
     165                 :         90 :     m_val (val), m_quality (quality)
     166                 :            :   {}
     167                 :            : 
     168                 :            :   /* Named probabilities.  */
     169                 :  450522402 :   static profile_probability never ()
     170                 :            :     {
     171                 :  162182848 :       profile_probability ret;
     172                 :  450522402 :       ret.m_val = 0;
     173                 :  450522402 :       ret.m_quality = PRECISE;
     174                 :  183719493 :       return ret;
     175                 :            :     }
     176                 :            : 
     177                 :    3280229 :   static profile_probability guessed_never ()
     178                 :            :     {
     179                 :    3280229 :       profile_probability ret;
     180                 :    3280229 :       ret.m_val = 0;
     181                 :    3280229 :       ret.m_quality = GUESSED;
     182                 :    3280229 :       return ret;
     183                 :            :     }
     184                 :            : 
     185                 :    3274858 :   static profile_probability very_unlikely ()
     186                 :            :     {
     187                 :            :       /* Be consistent with PROB_VERY_UNLIKELY in predict.h.  */
     188                 :    6549706 :       profile_probability r = guessed_always ().apply_scale (1, 2000);
     189                 :    3274858 :       r.m_val--;
     190                 :    3274858 :       return r;
     191                 :            :     }
     192                 :            : 
     193                 :      36519 :   static profile_probability unlikely ()
     194                 :            :     {
     195                 :            :       /* Be consistent with PROB_VERY_LIKELY in predict.h.  */
     196                 :      73038 :       profile_probability r = guessed_always ().apply_scale (1, 5);
     197                 :      36519 :       r.m_val--;
     198                 :      36519 :       return r;
     199                 :            :     }
     200                 :            : 
     201                 :     494514 :   static profile_probability even ()
     202                 :            :     {
     203                 :     494509 :       return guessed_always ().apply_scale (1, 2);
     204                 :            :     }
     205                 :            : 
     206                 :     250215 :   static profile_probability very_likely ()
     207                 :            :     {
     208                 :     250215 :       return always () - very_unlikely ();
     209                 :            :     }
     210                 :            : 
     211                 :      14520 :   static profile_probability likely ()
     212                 :            :     {
     213                 :      14520 :       return always () - unlikely ();
     214                 :            :     }
     215                 :            : 
     216                 :    4016001 :   static profile_probability guessed_always ()
     217                 :            :     {
     218                 :    4016001 :       profile_probability ret;
     219                 :    4016001 :       ret.m_val = max_probability;
     220                 :    4016001 :       ret.m_quality = GUESSED;
     221                 :    4015500 :       return ret;
     222                 :            :     }
     223                 :            : 
     224                 :   91098597 :   static profile_probability always ()
     225                 :            :     {
     226                 :   91098597 :       profile_probability ret;
     227                 :   91098597 :       ret.m_val = max_probability;
     228                 :   91098597 :       ret.m_quality = PRECISE;
     229                 :   56582546 :       return ret;
     230                 :            :     }
     231                 :            : 
     232                 :            :   /* Probabilities which has not been initialized. Either because
     233                 :            :      initialization did not happen yet or because profile is unknown.  */
     234                 :  532253488 :   static profile_probability uninitialized ()
     235                 :            :     {
     236                 :  532253488 :       profile_probability c;
     237                 :  532253488 :       c.m_val = uninitialized_probability;
     238                 :  532253488 :       c.m_quality = GUESSED;
     239                 :  523617875 :       return c;
     240                 :            :     }
     241                 :            : 
     242                 :            :   /* Return true if value has been initialized.  */
     243                 : 1919984395 :   bool initialized_p () const
     244                 :            :     {
     245                 :  972611827 :       return m_val != uninitialized_probability;
     246                 :            :     }
     247                 :            : 
     248                 :            :   /* Return true if value can be trusted.  */
     249                 :        453 :   bool reliable_p () const
     250                 :            :     {
     251                 :        453 :       return m_quality >= ADJUSTED;
     252                 :            :     }
     253                 :            : 
     254                 :            :   /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
     255                 :            :      this is mostly to support legacy code and should go away.  */
     256                 :    2074040 :   static profile_probability from_reg_br_prob_base (int v)
     257                 :            :     {
     258                 :    2074040 :       profile_probability ret;
     259                 :    2074040 :       gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
     260                 :    2074040 :       ret.m_val = RDIV (v * (uint64_t) max_probability, REG_BR_PROB_BASE);
     261                 :    2074040 :       ret.m_quality = GUESSED;
     262                 :    2074040 :       return ret;
     263                 :            :     }
     264                 :            : 
     265                 :            :   /* Return THIS with quality set to ADJUSTED.  */
     266                 :         30 :   profile_probability adjusted () const
     267                 :            :     {
     268                 :         30 :       profile_probability ret = *this;
     269                 :         30 :       if (!initialized_p ())
     270                 :          0 :         return *this;
     271                 :         30 :       ret.m_quality = ADJUSTED;
     272                 :         30 :       return ret;
     273                 :            :     }
     274                 :            : 
     275                 :  597493285 :   int to_reg_br_prob_base () const
     276                 :            :     {
     277                 :  597493285 :       gcc_checking_assert (initialized_p ());
     278                 :  597493285 :       return RDIV (m_val * (uint64_t) REG_BR_PROB_BASE, max_probability);
     279                 :            :     }
     280                 :            : 
     281                 :            :   /* Conversion to and from RTL representation of profile probabilities.  */
     282                 :  279948724 :   static profile_probability from_reg_br_prob_note (int v)
     283                 :            :     {
     284                 :  279948724 :       profile_probability ret;
     285                 :  279948724 :       ret.m_val = ((unsigned int)v) / 8;
     286                 :  279948724 :       ret.m_quality = (enum profile_quality)(v & 7);
     287                 :   10056224 :       return ret;
     288                 :            :     }
     289                 :            : 
     290                 :  261649100 :   int to_reg_br_prob_note () const
     291                 :            :     {
     292                 :  261649100 :       gcc_checking_assert (initialized_p ());
     293                 :  261649100 :       int ret = m_val * 8 + m_quality;
     294                 :  523298200 :       gcc_checking_assert (from_reg_br_prob_note (ret) == *this);
     295                 :  261649100 :       return ret;
     296                 :            :     }
     297                 :            : 
     298                 :            :   /* Return VAL1/VAL2.  */
     299                 :       2876 :   static profile_probability probability_in_gcov_type
     300                 :            :                                  (gcov_type val1, gcov_type val2)
     301                 :            :     {
     302                 :       2876 :       profile_probability ret;
     303                 :       2876 :       gcc_checking_assert (val1 >= 0 && val2 > 0);
     304                 :       2876 :       if (val1 > val2)
     305                 :            :         ret.m_val = max_probability;
     306                 :            :       else
     307                 :            :         {
     308                 :       2876 :           uint64_t tmp;
     309                 :       2876 :           safe_scale_64bit (val1, max_probability, val2, &tmp);
     310                 :       2876 :           gcc_checking_assert (tmp <= max_probability);
     311                 :       2876 :           ret.m_val = tmp;
     312                 :            :         }
     313                 :       2876 :       ret.m_quality = PRECISE;
     314                 :       2876 :       return ret;
     315                 :            :     }
     316                 :            : 
     317                 :            :   /* Basic operations.  */
     318                 :  723442517 :   bool operator== (const profile_probability &other) const
     319                 :            :     {
     320                 :  457924900 :       return m_val == other.m_val && m_quality == other.m_quality;
     321                 :            :     }
     322                 :            : 
     323                 :    3871334 :   profile_probability operator+ (const profile_probability &other) const
     324                 :            :     {
     325                 :    3871334 :       if (other == never ())
     326                 :       8955 :         return *this;
     327                 :    3862379 :       if (*this == never ())
     328                 :       2373 :         return other;
     329                 :    3860006 :       if (!initialized_p () || !other.initialized_p ())
     330                 :    3277376 :         return uninitialized ();
     331                 :            : 
     332                 :     582632 :       profile_probability ret;
     333                 :     582632 :       ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
     334                 :     582632 :       ret.m_quality = MIN (m_quality, other.m_quality);
     335                 :     582632 :       return ret;
     336                 :            :     }
     337                 :            : 
     338                 :    1926720 :   profile_probability &operator+= (const profile_probability &other)
     339                 :            :     {
     340                 :    1926720 :       if (other == never ())
     341                 :     298621 :         return *this;
     342                 :    1628100 :       if (*this == never ())
     343                 :            :         {
     344                 :     465499 :           *this = other;
     345                 :     465499 :           return *this;
     346                 :            :         }
     347                 :    1162601 :       if (!initialized_p () || !other.initialized_p ())
     348                 :     598773 :         return *this = uninitialized ();
     349                 :            :       else
     350                 :            :         {
     351                 :     563828 :           m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
     352                 :     563828 :           m_quality = MIN (m_quality, other.m_quality);
     353                 :            :         }
     354                 :     563828 :       return *this;
     355                 :            :     }
     356                 :            : 
     357                 :   20366901 :   profile_probability operator- (const profile_probability &other) const
     358                 :            :     {
     359                 :   20464241 :       if (*this == never ()
     360                 :   21525379 :           || other == never ())
     361                 :    1353136 :         return *this;
     362                 :   19013776 :       if (!initialized_p () || !other.initialized_p ())
     363                 :    1163869 :         return uninitialized ();
     364                 :   17849893 :       profile_probability ret;
     365                 :   17849893 :       ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
     366                 :   17849893 :       ret.m_quality = MIN (m_quality, other.m_quality);
     367                 :   17849893 :       return ret;
     368                 :            :     }
     369                 :            : 
     370                 :    6253345 :   profile_probability &operator-= (const profile_probability &other)
     371                 :            :     {
     372                 :    6330990 :       if (*this == never ()
     373                 :    7550168 :           || other == never ())
     374                 :    1452103 :         return *this;
     375                 :    4801242 :       if (!initialized_p () || !other.initialized_p ())
     376                 :      11169 :         return *this = uninitialized ();
     377                 :            :       else
     378                 :            :         {
     379                 :    4790073 :           m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
     380                 :    4790073 :           m_quality = MIN (m_quality, other.m_quality);
     381                 :            :         }
     382                 :    4790073 :       return *this;
     383                 :            :     }
     384                 :            : 
     385                 :     680443 :   profile_probability operator* (const profile_probability &other) const
     386                 :            :     {
     387                 :     780433 :       if (*this == never ()
     388                 :     582802 :           || other == never ())
     389                 :     102339 :         return never ();
     390                 :     578104 :       if (!initialized_p () || !other.initialized_p ())
     391                 :      46362 :         return uninitialized ();
     392                 :     531742 :       profile_probability ret;
     393                 :     531742 :       ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
     394                 :     531742 :       ret.m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
     395                 :     531742 :       return ret;
     396                 :            :     }
     397                 :            : 
     398                 :      27430 :   profile_probability &operator*= (const profile_probability &other)
     399                 :            :     {
     400                 :      27440 :       if (*this == never ()
     401                 :      27420 :           || other == never ())
     402                 :         10 :         return *this = never ();
     403                 :      27420 :       if (!initialized_p () || !other.initialized_p ())
     404                 :          0 :         return *this = uninitialized ();
     405                 :            :       else
     406                 :            :         {
     407                 :      27420 :           m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
     408                 :      27420 :           m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
     409                 :            :         }
     410                 :      27420 :       return *this;
     411                 :            :     }
     412                 :            : 
     413                 :    1063196 :   profile_probability operator/ (const profile_probability &other) const
     414                 :            :     {
     415                 :    1063196 :       if (*this == never ())
     416                 :      99583 :         return never ();
     417                 :     963613 :       if (!initialized_p () || !other.initialized_p ())
     418                 :      56293 :         return uninitialized ();
     419                 :     907320 :       profile_probability ret;
     420                 :            :       /* If we get probability above 1, mark it as unreliable and return 1. */
     421                 :     907320 :       if (m_val >= other.m_val)
     422                 :            :         {
     423                 :       6572 :           ret.m_val = max_probability;
     424                 :       6572 :           ret.m_quality = MIN (MIN (m_quality, other.m_quality),
     425                 :            :                                GUESSED);
     426                 :       6572 :           return ret;
     427                 :            :         }
     428                 :     900748 :       else if (!m_val)
     429                 :            :         ret.m_val = 0;
     430                 :            :       else
     431                 :            :         {
     432                 :     886846 :           gcc_checking_assert (other.m_val);
     433                 :     886846 :           ret.m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
     434                 :            :                                  other.m_val),
     435                 :            :                            max_probability);
     436                 :            :         }
     437                 :     900748 :       ret.m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
     438                 :     900748 :       return ret;
     439                 :            :     }
     440                 :            : 
     441                 :    1051075 :   profile_probability &operator/= (const profile_probability &other)
     442                 :            :     {
     443                 :    1051075 :       if (*this == never ())
     444                 :       1878 :         return *this = never ();
     445                 :    1049197 :       if (!initialized_p () || !other.initialized_p ())
     446                 :         25 :         return *this = uninitialized ();
     447                 :            :       else
     448                 :            :         {
     449                 :            :           /* If we get probability above 1, mark it as unreliable
     450                 :            :              and return 1. */
     451                 :    1049172 :           if (m_val > other.m_val)
     452                 :            :             {
     453                 :       1384 :               m_val = max_probability;
     454                 :       1384 :               m_quality = MIN (MIN (m_quality, other.m_quality),
     455                 :            :                                GUESSED);
     456                 :       1384 :               return *this;
     457                 :            :             }
     458                 :    1047788 :           else if (!m_val)
     459                 :            :             ;
     460                 :            :           else
     461                 :            :             {
     462                 :    1040576 :               gcc_checking_assert (other.m_val);
     463                 :    1040576 :               m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
     464                 :            :                                  other.m_val),
     465                 :            :                            max_probability);
     466                 :            :             }
     467                 :    1047788 :           m_quality = MIN (MIN (m_quality, other.m_quality), ADJUSTED);
     468                 :            :         }
     469                 :    1047788 :       return *this;
     470                 :            :     }
     471                 :            : 
     472                 :            :   /* Split *THIS (ORIG) probability into 2 probabilities, such that
     473                 :            :      the returned one (FIRST) is *THIS * CPROB and *THIS is
     474                 :            :      adjusted (SECOND) so that FIRST + FIRST.invert () * SECOND
     475                 :            :      == ORIG.  This is useful e.g. when splitting a conditional
     476                 :            :      branch like:
     477                 :            :      if (cond)
     478                 :            :        goto lab; // ORIG probability
     479                 :            :      into
     480                 :            :      if (cond1)
     481                 :            :        goto lab; // FIRST = ORIG * CPROB probability
     482                 :            :      if (cond2)
     483                 :            :        goto lab; // SECOND probability
     484                 :            :      such that the overall probability of jumping to lab remains
     485                 :            :      the same.  CPROB gives the relative probability between the
     486                 :            :      branches.  */
     487                 :     234054 :   profile_probability split (const profile_probability &cprob)
     488                 :            :     {
     489                 :     234054 :       profile_probability ret = *this * cprob;
     490                 :            :       /* The following is equivalent to:
     491                 :            :          *this = cprob.invert () * *this / ret.invert ();
     492                 :            :          Avoid scaling when overall outcome is supposed to be always.
     493                 :            :          Without knowing that one is inverse of other, the result would be
     494                 :            :          conservative.  */
     495                 :     242718 :       if (!(*this == always ()))
     496                 :     225390 :         *this = (*this - ret) / ret.invert ();
     497                 :     234054 :       return ret;
     498                 :            :     }
     499                 :            : 
     500                 :     198792 :   gcov_type apply (gcov_type val) const
     501                 :            :     {
     502                 :     198792 :       if (*this == uninitialized ())
     503                 :        132 :         return val / 2;
     504                 :     198660 :       return RDIV (val * m_val, max_probability);
     505                 :            :     }
     506                 :            : 
     507                 :            :   /* Return 1-*THIS.  */
     508                 :   18567866 :   profile_probability invert () const
     509                 :            :     {
     510                 :   18562396 :       return always() - *this;
     511                 :            :     }
     512                 :            : 
     513                 :            :   /* Return THIS with quality dropped to GUESSED.  */
     514                 :      20566 :   profile_probability guessed () const
     515                 :            :     {
     516                 :      20566 :       profile_probability ret = *this;
     517                 :      20566 :       ret.m_quality = GUESSED;
     518                 :      20566 :       return ret;
     519                 :            :     }
     520                 :            : 
     521                 :            :   /* Return THIS with quality dropped to AFDO.  */
     522                 :            :   profile_probability afdo () const
     523                 :            :     {
     524                 :            :       profile_probability ret = *this;
     525                 :            :       ret.m_quality = AFDO;
     526                 :            :       return ret;
     527                 :            :     }
     528                 :            : 
     529                 :            :   /* Return *THIS * NUM / DEN.  */
     530                 :    8862714 :   profile_probability apply_scale (int64_t num, int64_t den) const
     531                 :            :     {
     532                 :    8862714 :       if (*this == never ())
     533                 :      14192 :         return *this;
     534                 :    8848520 :       if (!initialized_p ())
     535                 :    3538383 :         return uninitialized ();
     536                 :    5310137 :       profile_probability ret;
     537                 :    5310137 :       uint64_t tmp;
     538                 :    5310137 :       safe_scale_64bit (m_val, num, den, &tmp);
     539                 :    5310137 :       ret.m_val = MIN (tmp, max_probability);
     540                 :    5310137 :       ret.m_quality = MIN (m_quality, ADJUSTED);
     541                 :    5310137 :       return ret;
     542                 :            :     }
     543                 :            : 
     544                 :            :   /* Return true when the probability of edge is reliable.
     545                 :            : 
     546                 :            :      The profile guessing code is good at predicting branch outcome (i.e.
     547                 :            :      taken/not taken), that is predicted right slightly over 75% of time.
     548                 :            :      It is however notoriously poor on predicting the probability itself.
     549                 :            :      In general the profile appear a lot flatter (with probabilities closer
     550                 :            :      to 50%) than the reality so it is bad idea to use it to drive optimization
     551                 :            :      such as those disabling dynamic branch prediction for well predictable
     552                 :            :      branches.
     553                 :            : 
     554                 :            :      There are two exceptions - edges leading to noreturn edges and edges
     555                 :            :      predicted by number of iterations heuristics are predicted well.  This macro
     556                 :            :      should be able to distinguish those, but at the moment it simply check for
     557                 :            :      noreturn heuristic that is only one giving probability over 99% or bellow
     558                 :            :      1%.  In future we might want to propagate reliability information across the
     559                 :            :      CFG if we find this information useful on multiple places.   */
     560                 :          0 :   bool probably_reliable_p () const
     561                 :            :     {
     562                 :          0 :       if (m_quality >= ADJUSTED)
     563                 :            :         return true;
     564                 :          0 :       if (!initialized_p ())
     565                 :            :         return false;
     566                 :          0 :       return m_val < max_probability / 100
     567                 :          0 :              || m_val > max_probability - max_probability / 100;
     568                 :            :     }
     569                 :            : 
     570                 :            :   /* Return false if profile_probability is bogus.  */
     571                 : 2382450000 :   bool verify () const
     572                 :            :     {
     573                 : 2382450000 :       gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
     574                 : 2382450000 :       if (m_val == uninitialized_probability)
     575                 :  542773000 :         return m_quality == GUESSED;
     576                 : 1839680000 :       else if (m_quality < GUESSED)
     577                 :            :         return false;
     578                 : 1839680000 :       return m_val <= max_probability;
     579                 :            :     }
     580                 :            : 
     581                 :            :   /* Comparisons are three-state and conservative.  False is returned if
     582                 :            :      the inequality cannot be decided.  */
     583                 :    1850242 :   bool operator< (const profile_probability &other) const
     584                 :            :     {
     585                 :    1850242 :       return initialized_p () && other.initialized_p () && m_val < other.m_val;
     586                 :            :     }
     587                 :            : 
     588                 :   11219750 :   bool operator> (const profile_probability &other) const
     589                 :            :     {
     590                 :   11219750 :       return initialized_p () && other.initialized_p () && m_val > other.m_val;
     591                 :            :     }
     592                 :            : 
     593                 :    3263876 :   bool operator<= (const profile_probability &other) const
     594                 :            :     {
     595                 :    3263876 :       return initialized_p () && other.initialized_p () && m_val <= other.m_val;
     596                 :            :     }
     597                 :            : 
     598                 :          0 :   bool operator>= (const profile_probability &other) const
     599                 :            :     {
     600                 :          0 :       return initialized_p () && other.initialized_p () && m_val >= other.m_val;
     601                 :            :     }
     602                 :            : 
     603                 :            :   /* Get the value of the count.  */
     604                 :          0 :   uint32_t value () const { return m_val; }
     605                 :            : 
     606                 :            :   /* Get the quality of the count.  */
     607                 :          0 :   enum profile_quality quality () const { return m_quality; }
     608                 :            : 
     609                 :            :   /* Output THIS to F.  */
     610                 :            :   void dump (FILE *f) const;
     611                 :            : 
     612                 :            :   /* Print THIS to stderr.  */
     613                 :            :   void debug () const;
     614                 :            : 
     615                 :            :   /* Return true if THIS is known to differ significantly from OTHER.  */
     616                 :            :   bool differs_from_p (profile_probability other) const;
     617                 :            : 
     618                 :            :   /* Return if difference is greater than 50%.  */
     619                 :            :   bool differs_lot_from_p (profile_probability other) const;
     620                 :            : 
     621                 :            :   /* COUNT1 times event happens with *THIS probability, COUNT2 times OTHER
     622                 :            :      happens with COUNT2 probability. Return probability that either *THIS or
     623                 :            :      OTHER happens.  */
     624                 :            :   profile_probability combine_with_count (profile_count count1,
     625                 :            :                                           profile_probability other,
     626                 :            :                                           profile_count count2) const;
     627                 :            : 
     628                 :            :   /* Return probability as sreal.  */
     629                 :            :   sreal to_sreal () const;
     630                 :            :   /* LTO streaming support.  */
     631                 :            :   static profile_probability stream_in (class lto_input_block *);
     632                 :            :   void stream_out (struct output_block *);
     633                 :            :   void stream_out (struct lto_output_stream *);
     634                 :            : };
     635                 :            : 
     636                 :            : /* Main data type to hold profile counters in GCC. Profile counts originate
     637                 :            :    either from profile feedback, static profile estimation or both.  We do not
     638                 :            :    perform whole program profile propagation and thus profile estimation
     639                 :            :    counters are often local to function, while counters from profile feedback
     640                 :            :    (or special cases of profile estimation) can be used inter-procedurally.
     641                 :            : 
     642                 :            :    There are 3 basic types
     643                 :            :      1) local counters which are result of intra-procedural static profile
     644                 :            :         estimation.
     645                 :            :      2) ipa counters which are result of profile feedback or special case
     646                 :            :         of static profile estimation (such as in function main).
     647                 :            :      3) counters which counts as 0 inter-procedurally (because given function
     648                 :            :         was never run in train feedback) but they hold local static profile
     649                 :            :         estimate.
     650                 :            : 
     651                 :            :    Counters of type 1 and 3 cannot be mixed with counters of different type
     652                 :            :    within operation (because whole function should use one type of counter)
     653                 :            :    with exception that global zero mix in most operations where outcome is
     654                 :            :    well defined.
     655                 :            : 
     656                 :            :    To take local counter and use it inter-procedurally use ipa member function
     657                 :            :    which strips information irrelevant at the inter-procedural level.
     658                 :            : 
     659                 :            :    Counters are 61bit integers representing number of executions during the
     660                 :            :    train run or normalized frequency within the function.
     661                 :            : 
     662                 :            :    As the profile is maintained during the compilation, many adjustments are
     663                 :            :    made.  Not all transformations can be made precisely, most importantly
     664                 :            :    when code is being duplicated.  It also may happen that part of CFG has
     665                 :            :    profile counts known while other do not - for example when LTO optimizing
     666                 :            :    partly profiled program or when profile was lost due to COMDAT merging.
     667                 :            : 
     668                 :            :    For this reason profile_count tracks more information than
     669                 :            :    just unsigned integer and it is also ready for profile mismatches.
     670                 :            :    The API of this data type represent operations that are natural
     671                 :            :    on profile counts - sum, difference and operation with scales and
     672                 :            :    probabilities.  All operations are safe by never getting negative counts
     673                 :            :    and they do end up in uninitialized scale if any of the parameters is
     674                 :            :    uninitialized.
     675                 :            : 
     676                 :            :    All comparisons that are three state and handling of probabilities.  Thus
     677                 :            :    a < b is not equal to !(a >= b).
     678                 :            : 
     679                 :            :    The following pre-defined counts are available:
     680                 :            : 
     681                 :            :    profile_count::zero ()  for code that is known to execute zero times at
     682                 :            :       runtime (this can be detected statically i.e. for paths leading to
     683                 :            :       abort ();
     684                 :            :    profile_count::one () for code that is known to execute once (such as
     685                 :            :       main () function
     686                 :            :    profile_count::uninitialized ()  for unknown execution count.
     687                 :            : 
     688                 :            :  */
     689                 :            : 
     690                 :            : struct GTY(()) profile_count
     691                 :            : {
     692                 :            : public:
     693                 :            :   /* Use 62bit to hold basic block counters.  Should be at least
     694                 :            :      64bit.  Although a counter cannot be negative, we use a signed
     695                 :            :      type to hold various extra stages.  */
     696                 :            : 
     697                 :            :   static const int n_bits = 61;
     698                 :            :   static const uint64_t max_count = ((uint64_t) 1 << n_bits) - 2;
     699                 :            : private:
     700                 :            :   static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
     701                 :            : 
     702                 :            : #if defined (__arm__) && (__GNUC__ >= 6 && __GNUC__ <= 8)
     703                 :            :   /* Work-around for PR88469.  A bug in the gcc-6/7/8 PCS layout code
     704                 :            :      incorrectly detects the alignment of a structure where the only
     705                 :            :      64-bit aligned object is a bit-field.  We force the alignment of
     706                 :            :      the entire field to mitigate this.  */
     707                 :            : #define UINT64_BIT_FIELD_ALIGN __attribute__ ((aligned(8)))
     708                 :            : #else
     709                 :            : #define UINT64_BIT_FIELD_ALIGN
     710                 :            : #endif
     711                 :            :   uint64_t UINT64_BIT_FIELD_ALIGN m_val : n_bits;
     712                 :            : #undef UINT64_BIT_FIELD_ALIGN
     713                 :            :   enum profile_quality m_quality : 3;
     714                 :            : public:
     715                 :            : 
     716                 :            :   /* Return true if both values can meaningfully appear in single function
     717                 :            :      body.  We have either all counters in function local or global, otherwise
     718                 :            :      operations between them are not really defined well.  */
     719                 : 1351716000 :   bool compatible_p (const profile_count other) const
     720                 :            :     {
     721                 : 2671702000 :       if (!initialized_p () || !other.initialized_p ())
     722                 :   31786360 :         return true;
     723                 :    6143930 :       if (*this == zero ()
     724                 :          1 :           || other == zero ())
     725                 :    6143930 :         return true;
     726                 :            :       /* Do not allow nonzero global profile together with local guesses
     727                 :            :          that are globally0.  */
     728                 : 2627484000 :       if (ipa ().nonzero_p ()
     729                 :     184148 :           && !(other.ipa () == other))
     730                 :          0 :         return false;
     731                 : 2627468000 :       if (other.ipa ().nonzero_p ()
     732                 :     201124 :           && !(ipa () == *this))
     733                 :          0 :         return false;
     734                 :            :         
     735                 : 1313786000 :       return ipa_p () == other.ipa_p ();
     736                 :            :     }
     737                 :            : 
     738                 :            :   /* Used for counters which are expected to be never executed.  */
     739                 : 6186904946 :   static profile_count zero ()
     740                 :            :     {
     741                 : 4103351272 :       return from_gcov_type (0);
     742                 :            :     }
     743                 :            : 
     744                 :       2158 :   static profile_count adjusted_zero ()
     745                 :            :     {
     746                 :       2158 :       profile_count c;
     747                 :       2158 :       c.m_val = 0;
     748                 :       2158 :       c.m_quality = ADJUSTED;
     749                 :       2158 :       return c;
     750                 :            :     }
     751                 :            : 
     752                 :          0 :   static profile_count guessed_zero ()
     753                 :            :     {
     754                 :          0 :       profile_count c;
     755                 :          0 :       c.m_val = 0;
     756                 :          0 :       c.m_quality = GUESSED;
     757                 :          0 :       return c;
     758                 :            :     }
     759                 :            : 
     760                 :        730 :   static profile_count one ()
     761                 :            :     {
     762                 :        365 :       return from_gcov_type (1);
     763                 :            :     }
     764                 :            : 
     765                 :            :   /* Value of counters which has not been initialized. Either because
     766                 :            :      initialization did not happen yet or because profile is unknown.  */
     767                 : 3542638477 :   static profile_count uninitialized ()
     768                 :            :     {
     769                 : 3542638477 :       profile_count c;
     770                 : 3542638477 :       c.m_val = uninitialized_count;
     771                 : 3542638477 :       c.m_quality = GUESSED_LOCAL;
     772                 : 2226210790 :       return c;
     773                 :            :     }
     774                 :            : 
     775                 :            :   /* Conversion to gcov_type is lossy.  */
     776                 :     255165 :   gcov_type to_gcov_type () const
     777                 :            :     {
     778                 :          0 :       gcc_checking_assert (initialized_p ());
     779                 :     255165 :       return m_val;
     780                 :            :     }
     781                 :            : 
     782                 :            :   /* Return true if value has been initialized.  */
     783                 :14112085683 :   bool initialized_p () const
     784                 :            :     {
     785                 : 3476377330 :       return m_val != uninitialized_count;
     786                 :            :     }
     787                 :            : 
     788                 :            :   /* Return true if value can be trusted.  */
     789                 :    6184769 :   bool reliable_p () const
     790                 :            :     {
     791                 :    6184748 :       return m_quality >= ADJUSTED;
     792                 :            :     }
     793                 :            : 
     794                 :            :   /* Return true if value can be operated inter-procedurally.  */
     795                 : 3169154192 :   bool ipa_p () const
     796                 :            :     {
     797                 : 3168864269 :       return !initialized_p () || m_quality >= GUESSED_GLOBAL0;
     798                 :            :     }
     799                 :            : 
     800                 :            :   /* Return true if quality of profile is precise.  */
     801                 :   13041144 :   bool precise_p () const
     802                 :            :     {
     803                 :   13041144 :       return m_quality == PRECISE;
     804                 :            :     }
     805                 :            : 
     806                 :            :   /* Get the value of the count.  */
     807                 :          0 :   uint32_t value () const { return m_val; }
     808                 :            : 
     809                 :            :   /* Get the quality of the count.  */
     810                 :      11937 :   enum profile_quality quality () const { return m_quality; }
     811                 :            : 
     812                 :            :   /* When merging basic blocks, the two different profile counts are unified.
     813                 :            :      Return true if this can be done without losing info about profile.
     814                 :            :      The only case we care about here is when first BB contains something
     815                 :            :      that makes it terminate in a way not visible in CFG.  */
     816                 :    1946550 :   bool ok_for_merging (profile_count other) const
     817                 :            :     {
     818                 :    1946550 :       if (m_quality < ADJUSTED
     819                 :       7210 :           || other.m_quality < ADJUSTED)
     820                 :            :         return true;
     821                 :       6929 :       return !(other < *this);
     822                 :            :     }
     823                 :            : 
     824                 :            :   /* When merging two BBs with different counts, pick common count that looks
     825                 :            :      most representative.  */
     826                 :    9024900 :   profile_count merge (profile_count other) const
     827                 :            :     {
     828                 :    9024900 :       if (*this == other || !other.initialized_p ()
     829                 :     767085 :           || m_quality > other.m_quality)
     830                 :    8258420 :         return *this;
     831                 :     766485 :       if (other.m_quality > m_quality
     832                 :     766485 :           || other > *this)
     833                 :     287128 :         return other;
     834                 :     479357 :       return *this;
     835                 :            :     }
     836                 :            : 
     837                 :            :   /* Basic operations.  */
     838                 : 6869986506 :   bool operator== (const profile_count &other) const
     839                 :            :     {
     840                 : 4335820210 :       return m_val == other.m_val && m_quality == other.m_quality;
     841                 :            :     }
     842                 :            : 
     843                 :    1657661 :   profile_count operator+ (const profile_count &other) const
     844                 :            :     {
     845                 :    1657661 :       if (other == zero ())
     846                 :       1819 :         return *this;
     847                 :    1655845 :       if (*this == zero ())
     848                 :     110540 :         return other;
     849                 :    1545303 :       if (!initialized_p () || !other.initialized_p ())
     850                 :       3511 :         return uninitialized ();
     851                 :            : 
     852                 :    1541792 :       profile_count ret;
     853                 :    1541792 :       gcc_checking_assert (compatible_p (other));
     854                 :    1541792 :       ret.m_val = m_val + other.m_val;
     855                 :    1541792 :       ret.m_quality = MIN (m_quality, other.m_quality);
     856                 :    1541792 :       return ret;
     857                 :            :     }
     858                 :            : 
     859                 :   22027082 :   profile_count &operator+= (const profile_count &other)
     860                 :            :     {
     861                 :   22027082 :       if (other == zero ())
     862                 :     895446 :         return *this;
     863                 :   21131672 :       if (*this == zero ())
     864                 :            :         {
     865                 :    7939038 :           *this = other;
     866                 :    7939038 :           return *this;
     867                 :            :         }
     868                 :   13192657 :       if (!initialized_p () || !other.initialized_p ())
     869                 :     138206 :         return *this = uninitialized ();
     870                 :            :       else
     871                 :            :         {
     872                 :   13054451 :           gcc_checking_assert (compatible_p (other));
     873                 :   13054451 :           m_val += other.m_val;
     874                 :   13054451 :           m_quality = MIN (m_quality, other.m_quality);
     875                 :            :         }
     876                 :   13054451 :       return *this;
     877                 :            :     }
     878                 :            : 
     879                 :    5330103 :   profile_count operator- (const profile_count &other) const
     880                 :            :     {
     881                 :    5595596 :       if (*this == zero () || other == zero ())
     882                 :     238967 :         return *this;
     883                 :    5091140 :       if (!initialized_p () || !other.initialized_p ())
     884                 :    4023162 :         return uninitialized ();
     885                 :    1067981 :       gcc_checking_assert (compatible_p (other));
     886                 :    1067981 :       profile_count ret;
     887                 :    1067981 :       ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
     888                 :    1067981 :       ret.m_quality = MIN (m_quality, other.m_quality);
     889                 :    1067981 :       return ret;
     890                 :            :     }
     891                 :            : 
     892                 :    5234601 :   profile_count &operator-= (const profile_count &other)
     893                 :            :     {
     894                 :    5579697 :       if (*this == zero () || other == zero ())
     895                 :     187189 :         return *this;
     896                 :    5047414 :       if (!initialized_p () || !other.initialized_p ())
     897                 :     278289 :         return *this = uninitialized ();
     898                 :            :       else
     899                 :            :         {
     900                 :    4769128 :           gcc_checking_assert (compatible_p (other));
     901                 :    4769128 :           m_val = m_val >= other.m_val ? m_val - other.m_val: 0;
     902                 :    4769128 :           m_quality = MIN (m_quality, other.m_quality);
     903                 :            :         }
     904                 :    4769128 :       return *this;
     905                 :            :     }
     906                 :            : 
     907                 :            :   /* Return false if profile_count is bogus.  */
     908                 :            :   bool verify () const
     909                 :            :     {
     910                 :            :       gcc_checking_assert (m_quality != UNINITIALIZED_PROFILE);
     911                 :            :       return m_val != uninitialized_count || m_quality == GUESSED_LOCAL;
     912                 :            :     }
     913                 :            : 
     914                 :            :   /* Comparisons are three-state and conservative.  False is returned if
     915                 :            :      the inequality cannot be decided.  */
     916                 :  545710724 :   bool operator< (const profile_count &other) const
     917                 :            :     {
     918                 :  545710724 :       if (!initialized_p () || !other.initialized_p ())
     919                 :    2123631 :         return false;
     920                 :  543587589 :       if (*this == zero ())
     921                 :    2399912 :         return !(other == zero ());
     922                 :  542387356 :       if (other == zero ())
     923                 :        866 :         return false;
     924                 :  542386933 :       gcc_checking_assert (compatible_p (other));
     925                 :  542386933 :       return m_val < other.m_val;
     926                 :            :     }
     927                 :            : 
     928                 :   17771698 :   bool operator> (const profile_count &other) const
     929                 :            :     {
     930                 :   17771698 :       if (!initialized_p () || !other.initialized_p ())
     931                 :     534125 :         return false;
     932                 :   17237579 :       if (*this  == zero ())
     933                 :     135787 :         return false;
     934                 :   17101787 :       if (other == zero ())
     935                 :     997122 :         return !(*this == zero ());
     936                 :   16603230 :       gcc_checking_assert (compatible_p (other));
     937                 :   16603230 :       return initialized_p () && other.initialized_p () && m_val > other.m_val;
     938                 :            :     }
     939                 :            : 
     940                 :            :   bool operator< (const gcov_type other) const
     941                 :            :     {
     942                 :            :       gcc_checking_assert (ipa_p ());
     943                 :            :       gcc_checking_assert (other >= 0);
     944                 :            :       return ipa ().initialized_p () && ipa ().m_val < (uint64_t) other;
     945                 :            :     }
     946                 :            : 
     947                 :     190715 :   bool operator> (const gcov_type other) const
     948                 :            :     {
     949                 :     190715 :       gcc_checking_assert (ipa_p ());
     950                 :     190715 :       gcc_checking_assert (other >= 0);
     951                 :     190715 :       return ipa ().initialized_p () && ipa ().m_val > (uint64_t) other;
     952                 :            :     }
     953                 :            : 
     954                 :      17390 :   bool operator<= (const profile_count &other) const
     955                 :            :     {
     956                 :      17390 :       if (!initialized_p () || !other.initialized_p ())
     957                 :          0 :         return false;
     958                 :      17390 :       if (*this == zero ())
     959                 :       1119 :         return true;
     960                 :      16271 :       if (other == zero ())
     961                 :          0 :         return (*this == zero ());
     962                 :      16271 :       gcc_checking_assert (compatible_p (other));
     963                 :      16271 :       return m_val <= other.m_val;
     964                 :            :     }
     965                 :            : 
     966                 :    3502453 :   bool operator>= (const profile_count &other) const
     967                 :            :     {
     968                 :    3502453 :       if (!initialized_p () || !other.initialized_p ())
     969                 :       1285 :         return false;
     970                 :    3501162 :       if (other == zero ())
     971                 :      41496 :         return true;
     972                 :    3459674 :       if (*this == zero ())
     973                 :      15010 :         return (other == zero ());
     974                 :    3452162 :       gcc_checking_assert (compatible_p (other));
     975                 :    3452162 :       return m_val >= other.m_val;
     976                 :            :     }
     977                 :            : 
     978                 :      97293 :   bool operator<= (const gcov_type other) const
     979                 :            :     {
     980                 :      97293 :       gcc_checking_assert (ipa_p ());
     981                 :      97293 :       gcc_checking_assert (other >= 0);
     982                 :      97293 :       return ipa ().initialized_p () && ipa ().m_val <= (uint64_t) other;
     983                 :            :     }
     984                 :            : 
     985                 :      59569 :   bool operator>= (const gcov_type other) const
     986                 :            :     {
     987                 :      59569 :       gcc_checking_assert (ipa_p ());
     988                 :      59569 :       gcc_checking_assert (other >= 0);
     989                 :      59569 :       return ipa ().initialized_p () && ipa ().m_val >= (uint64_t) other;
     990                 :            :     }
     991                 :            : 
     992                 :            :   /* Return true when value is not zero and can be used for scaling. 
     993                 :            :      This is different from *this > 0 because that requires counter to
     994                 :            :      be IPA.  */
     995                 : 2839558443 :   bool nonzero_p () const
     996                 :            :     {
     997                 : 2743409557 :       return initialized_p () && m_val != 0;
     998                 :            :     }
     999                 :            : 
    1000                 :            :   /* Make counter forcibly nonzero.  */
    1001                 :    9979590 :   profile_count force_nonzero () const
    1002                 :            :     {
    1003                 :    9979590 :       if (!initialized_p ())
    1004                 :       3606 :         return *this;
    1005                 :    9975990 :       profile_count ret = *this;
    1006                 :    9975990 :       if (ret.m_val == 0)
    1007                 :            :         {
    1008                 :       7194 :           ret.m_val = 1;
    1009                 :       7194 :           ret.m_quality = MIN (m_quality, ADJUSTED);
    1010                 :            :         }
    1011                 :    9975990 :       return ret;
    1012                 :            :     }
    1013                 :            : 
    1014                 :   50582591 :   profile_count max (profile_count other) const
    1015                 :            :     {
    1016                 :   50582591 :       profile_count val = *this;
    1017                 :            : 
    1018                 :            :       /* Always prefer nonzero IPA counts over local counts.  */
    1019                 :   55593900 :       if (ipa ().nonzero_p () || other.ipa ().nonzero_p ())
    1020                 :            :         {
    1021                 :       8433 :           val = ipa ();
    1022                 :       8433 :           other = other.ipa ();
    1023                 :            :         }
    1024                 :   50582591 :       if (!initialized_p ())
    1025                 :   13387734 :         return other;
    1026                 :   37194779 :       if (!other.initialized_p ())
    1027                 :     313778 :         return *this;
    1028                 :   36881006 :       if (*this == zero ())
    1029                 :    1478937 :         return other;
    1030                 :   35402084 :       if (other == zero ())
    1031                 :    2585197 :         return *this;
    1032                 :   32816896 :       gcc_checking_assert (compatible_p (other));
    1033                 :   32816896 :       if (val.m_val < other.m_val || (m_val == other.m_val
    1034                 :    7684652 :                                       && val.m_quality < other.m_quality))
    1035                 :    4511043 :         return other;
    1036                 :   28305791 :       return *this;
    1037                 :            :     }
    1038                 :            : 
    1039                 :            :   /* PROB is a probability in scale 0...REG_BR_PROB_BASE.  Scale counter
    1040                 :            :      accordingly.  */
    1041                 :            :   profile_count apply_probability (int prob) const
    1042                 :            :     {
    1043                 :            :       gcc_checking_assert (prob >= 0 && prob <= REG_BR_PROB_BASE);
    1044                 :            :       if (m_val == 0)
    1045                 :            :         return *this;
    1046                 :            :       if (!initialized_p ())
    1047                 :            :         return uninitialized ();
    1048                 :            :       profile_count ret;
    1049                 :            :       ret.m_val = RDIV (m_val * prob, REG_BR_PROB_BASE);
    1050                 :            :       ret.m_quality = MIN (m_quality, ADJUSTED);
    1051                 :            :       return ret;
    1052                 :            :     }
    1053                 :            : 
    1054                 :            :   /* Scale counter according to PROB.  */
    1055                 :  211434555 :   profile_count apply_probability (profile_probability prob) const
    1056                 :            :     {
    1057                 :  211434555 :       if (*this == zero ())
    1058                 :    2747088 :         return *this;
    1059                 :  208687446 :       if (prob == profile_probability::never ())
    1060                 :   14451653 :         return zero ();
    1061                 :  194235826 :       if (!initialized_p ())
    1062                 :   59062210 :         return uninitialized ();
    1063                 :  135173558 :       profile_count ret;
    1064                 :  135173558 :       uint64_t tmp;
    1065                 :  135173558 :       safe_scale_64bit (m_val, prob.m_val, profile_probability::max_probability,
    1066                 :            :                         &tmp);
    1067                 :  135173558 :       ret.m_val = tmp;
    1068                 :  135173558 :       ret.m_quality = MIN (m_quality, prob.m_quality);
    1069                 :  135173558 :       return ret;
    1070                 :            :     }
    1071                 :            : 
    1072                 :            :   /* Return *THIS * NUM / DEN.  */
    1073                 :  527226438 :   profile_count apply_scale (int64_t num, int64_t den) const
    1074                 :            :     {
    1075                 :  527226438 :       if (m_val == 0)
    1076                 :    7069875 :         return *this;
    1077                 :  520157214 :       if (!initialized_p ())
    1078                 :       5819 :         return uninitialized ();
    1079                 :  520151407 :       profile_count ret;
    1080                 :  520151407 :       uint64_t tmp;
    1081                 :            : 
    1082                 :  520151407 :       gcc_checking_assert (num >= 0 && den > 0);
    1083                 :  520151407 :       safe_scale_64bit (m_val, num, den, &tmp);
    1084                 :  520151407 :       ret.m_val = MIN (tmp, max_count);
    1085                 :  520151407 :       ret.m_quality = MIN (m_quality, ADJUSTED);
    1086                 :  520151407 :       return ret;
    1087                 :            :     }
    1088                 :            : 
    1089                 :   13432340 :   profile_count apply_scale (profile_count num, profile_count den) const
    1090                 :            :     {
    1091                 :   13432340 :       if (*this == zero ())
    1092                 :     556611 :         return *this;
    1093                 :   12875670 :       if (num == zero ())
    1094                 :      30280 :         return num;
    1095                 :   12845390 :       if (!initialized_p () || !num.initialized_p () || !den.initialized_p ())
    1096                 :    4095097 :         return uninitialized ();
    1097                 :    8750322 :       if (num == den)
    1098                 :    2267050 :         return *this;
    1099                 :    6483272 :       gcc_checking_assert (den.m_val);
    1100                 :            : 
    1101                 :    6483272 :       profile_count ret;
    1102                 :    6483272 :       uint64_t val;
    1103                 :    6483272 :       safe_scale_64bit (m_val, num.m_val, den.m_val, &val);
    1104                 :    6483272 :       ret.m_val = MIN (val, max_count);
    1105                 :    6483272 :       ret.m_quality = MIN (MIN (MIN (m_quality, ADJUSTED),
    1106                 :            :                                 num.m_quality), den.m_quality);
    1107                 :            :       /* Be sure that ret is not local if num is global.
    1108                 :            :          Also ensure that ret is not global0 when num is global.  */
    1109                 :    6483272 :       if (num.ipa_p ())
    1110                 :       3092 :         ret.m_quality = MAX (ret.m_quality,
    1111                 :            :                              num == num.ipa () ? GUESSED : num.m_quality);
    1112                 :    6483272 :       return ret;
    1113                 :            :     }
    1114                 :            : 
    1115                 :            :   /* Return THIS with quality dropped to GUESSED_LOCAL.  */
    1116                 :   13581666 :   profile_count guessed_local () const
    1117                 :            :     {
    1118                 :   13581666 :       profile_count ret = *this;
    1119                 :   13581666 :       if (!initialized_p ())
    1120                 :          0 :         return *this;
    1121                 :   13581666 :       ret.m_quality = GUESSED_LOCAL;
    1122                 :   13581666 :       return ret;
    1123                 :            :     }
    1124                 :            : 
    1125                 :            :   /* We know that profile is globally 0 but keep local profile if present.  */
    1126                 :        658 :   profile_count global0 () const
    1127                 :            :     {
    1128                 :        658 :       profile_count ret = *this;
    1129                 :        658 :       if (!initialized_p ())
    1130                 :          0 :         return *this;
    1131                 :        658 :       ret.m_quality = GUESSED_GLOBAL0;
    1132                 :        658 :       return ret;
    1133                 :            :     }
    1134                 :            : 
    1135                 :            :   /* We know that profile is globally adjusted 0 but keep local profile
    1136                 :            :      if present.  */
    1137                 :         29 :   profile_count global0adjusted () const
    1138                 :            :     {
    1139                 :         29 :       profile_count ret = *this;
    1140                 :         29 :       if (!initialized_p ())
    1141                 :          0 :         return *this;
    1142                 :         29 :       ret.m_quality = GUESSED_GLOBAL0_ADJUSTED;
    1143                 :         29 :       return ret;
    1144                 :            :     }
    1145                 :            : 
    1146                 :            :   /* Return THIS with quality dropped to GUESSED.  */
    1147                 :            :   profile_count guessed () const
    1148                 :            :     {
    1149                 :            :       profile_count ret = *this;
    1150                 :            :       ret.m_quality = MIN (ret.m_quality, GUESSED);
    1151                 :            :       return ret;
    1152                 :            :     }
    1153                 :            : 
    1154                 :            :   /* Return variant of profile count which is always safe to compare
    1155                 :            :      across functions.  */
    1156                 : 3405759973 :   profile_count ipa () const
    1157                 :            :     {
    1158                 : 3405759973 :       if (m_quality > GUESSED_GLOBAL0_ADJUSTED)
    1159                 :   37027942 :         return *this;
    1160                 : 3368733328 :       if (m_quality == GUESSED_GLOBAL0)
    1161                 :      32884 :         return zero ();
    1162                 : 3368700223 :       if (m_quality == GUESSED_GLOBAL0_ADJUSTED)
    1163                 :       2158 :         return adjusted_zero ();
    1164                 : 3368698058 :       return uninitialized ();
    1165                 :            :     }
    1166                 :            : 
    1167                 :            :   /* Return THIS with quality dropped to AFDO.  */
    1168                 :          0 :   profile_count afdo () const
    1169                 :            :     {
    1170                 :          0 :       profile_count ret = *this;
    1171                 :          0 :       ret.m_quality = AFDO;
    1172                 :          0 :       return ret;
    1173                 :            :     }
    1174                 :            : 
    1175                 :            :   /* Return probability of event with counter THIS within event with counter
    1176                 :            :      OVERALL.  */
    1177                 :  575563276 :   profile_probability probability_in (const profile_count overall) const
    1178                 :            :     {
    1179                 :  581156696 :       if (*this == zero ()
    1180                 :      14831 :           && !(overall == zero ()))
    1181                 :        797 :         return profile_probability::never ();
    1182                 :  575550028 :       if (!initialized_p () || !overall.initialized_p ()
    1183                 : 1151100228 :           || !overall.m_val)
    1184                 :     118475 :         return profile_probability::uninitialized ();
    1185                 :  683133979 :       if (*this == overall && m_quality == PRECISE)
    1186                 :      20890 :         return profile_probability::always ();
    1187                 :  575422885 :       profile_probability ret;
    1188                 :  575422885 :       gcc_checking_assert (compatible_p (overall));
    1189                 :            : 
    1190                 :  575422885 :       if (overall.m_val < m_val)
    1191                 :            :         {
    1192                 :     954510 :           ret.m_val = profile_probability::max_probability;
    1193                 :     954510 :           ret.m_quality = GUESSED;
    1194                 :     954510 :           return ret;
    1195                 :            :         }
    1196                 :            :       else
    1197                 :  574468087 :         ret.m_val = RDIV (m_val * profile_probability::max_probability,
    1198                 :            :                           overall.m_val);
    1199                 :  574468087 :       ret.m_quality = MIN (MAX (MIN (m_quality, overall.m_quality),
    1200                 :            :                                 GUESSED), ADJUSTED);
    1201                 :  574468087 :       return ret;
    1202                 :            :     }
    1203                 :            : 
    1204                 :            :   int to_frequency (struct function *fun) const;
    1205                 :            :   int to_cgraph_frequency (profile_count entry_bb_count) const;
    1206                 :            :   sreal to_sreal_scale (profile_count in, bool *known = NULL) const;
    1207                 :            : 
    1208                 :            :   /* Output THIS to F.  */
    1209                 :            :   void dump (FILE *f) const;
    1210                 :            : 
    1211                 :            :   /* Print THIS to stderr.  */
    1212                 :            :   void debug () const;
    1213                 :            : 
    1214                 :            :   /* Return true if THIS is known to differ significantly from OTHER.  */
    1215                 :            :   bool differs_from_p (profile_count other) const;
    1216                 :            : 
    1217                 :            :   /* We want to scale profile across function boundary from NUM to DEN.
    1218                 :            :      Take care of the side case when NUM and DEN are zeros of incompatible
    1219                 :            :      kinds.  */
    1220                 :            :   static void adjust_for_ipa_scaling (profile_count *num, profile_count *den);
    1221                 :            : 
    1222                 :            :   /* THIS is a count of bb which is known to be executed IPA times.
    1223                 :            :      Combine this information into bb counter.  This means returning IPA
    1224                 :            :      if it is nonzero, not changing anything if IPA is uninitialized
    1225                 :            :      and if IPA is zero, turning THIS into corresponding local profile with
    1226                 :            :      global0.  */
    1227                 :            :   profile_count combine_with_ipa_count (profile_count ipa);
    1228                 :            : 
    1229                 :            :   /* Same as combine_with_ipa_count but inside function with count IPA2.  */
    1230                 :            :   profile_count combine_with_ipa_count_within
    1231                 :            :                  (profile_count ipa, profile_count ipa2);
    1232                 :            : 
    1233                 :            :   /* The profiling runtime uses gcov_type, which is usually 64bit integer.
    1234                 :            :      Conversions back and forth are used to read the coverage and get it
    1235                 :            :      into internal representation.  */
    1236                 :            :   static profile_count from_gcov_type (gcov_type v,
    1237                 :            :                                        profile_quality quality = PRECISE);
    1238                 :            : 
    1239                 :            :   /* LTO streaming support.  */
    1240                 :            :   static profile_count stream_in (class lto_input_block *);
    1241                 :            :   void stream_out (struct output_block *);
    1242                 :            :   void stream_out (struct lto_output_stream *);
    1243                 :            : };
    1244                 :            : #endif

Generated by: LCOV version 1.0

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto --enable-host-shared. GCC test suite is run with the built compiler.