LCOV - code coverage report
Current view: top level - gcc - haifa-sched.c (source / functions) Hit Total Coverage
Test: gcc.info Lines: 1826 3897 46.9 %
Date: 2020-03-28 11:57:23 Functions: 106 196 54.1 %
Legend: Lines: hit not hit | Branches: + taken - not taken # not executed Branches: 0 0 -

           Branch data     Line data    Source code
       1                 :            : /* Instruction scheduling pass.
       2                 :            :    Copyright (C) 1992-2020 Free Software Foundation, Inc.
       3                 :            :    Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
       4                 :            :    and currently maintained by, Jim Wilson (wilson@cygnus.com)
       5                 :            : 
       6                 :            : This file is part of GCC.
       7                 :            : 
       8                 :            : GCC is free software; you can redistribute it and/or modify it under
       9                 :            : the terms of the GNU General Public License as published by the Free
      10                 :            : Software Foundation; either version 3, or (at your option) any later
      11                 :            : version.
      12                 :            : 
      13                 :            : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      14                 :            : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      15                 :            : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      16                 :            : for more details.
      17                 :            : 
      18                 :            : You should have received a copy of the GNU General Public License
      19                 :            : along with GCC; see the file COPYING3.  If not see
      20                 :            : <http://www.gnu.org/licenses/>.  */
      21                 :            : 
      22                 :            : /* Instruction scheduling pass.  This file, along with sched-deps.c,
      23                 :            :    contains the generic parts.  The actual entry point for
      24                 :            :    the normal instruction scheduling pass is found in sched-rgn.c.
      25                 :            : 
      26                 :            :    We compute insn priorities based on data dependencies.  Flow
      27                 :            :    analysis only creates a fraction of the data-dependencies we must
      28                 :            :    observe: namely, only those dependencies which the combiner can be
      29                 :            :    expected to use.  For this pass, we must therefore create the
      30                 :            :    remaining dependencies we need to observe: register dependencies,
      31                 :            :    memory dependencies, dependencies to keep function calls in order,
      32                 :            :    and the dependence between a conditional branch and the setting of
      33                 :            :    condition codes are all dealt with here.
      34                 :            : 
      35                 :            :    The scheduler first traverses the data flow graph, starting with
      36                 :            :    the last instruction, and proceeding to the first, assigning values
      37                 :            :    to insn_priority as it goes.  This sorts the instructions
      38                 :            :    topologically by data dependence.
      39                 :            : 
      40                 :            :    Once priorities have been established, we order the insns using
      41                 :            :    list scheduling.  This works as follows: starting with a list of
      42                 :            :    all the ready insns, and sorted according to priority number, we
      43                 :            :    schedule the insn from the end of the list by placing its
      44                 :            :    predecessors in the list according to their priority order.  We
      45                 :            :    consider this insn scheduled by setting the pointer to the "end" of
      46                 :            :    the list to point to the previous insn.  When an insn has no
      47                 :            :    predecessors, we either queue it until sufficient time has elapsed
      48                 :            :    or add it to the ready list.  As the instructions are scheduled or
      49                 :            :    when stalls are introduced, the queue advances and dumps insns into
      50                 :            :    the ready list.  When all insns down to the lowest priority have
      51                 :            :    been scheduled, the critical path of the basic block has been made
      52                 :            :    as short as possible.  The remaining insns are then scheduled in
      53                 :            :    remaining slots.
      54                 :            : 
      55                 :            :    The following list shows the order in which we want to break ties
      56                 :            :    among insns in the ready list:
      57                 :            : 
      58                 :            :    1.  choose insn with the longest path to end of bb, ties
      59                 :            :    broken by
      60                 :            :    2.  choose insn with least contribution to register pressure,
      61                 :            :    ties broken by
      62                 :            :    3.  prefer in-block upon interblock motion, ties broken by
      63                 :            :    4.  prefer useful upon speculative motion, ties broken by
      64                 :            :    5.  choose insn with largest control flow probability, ties
      65                 :            :    broken by
      66                 :            :    6.  choose insn with the least dependences upon the previously
      67                 :            :    scheduled insn, or finally
      68                 :            :    7   choose the insn which has the most insns dependent on it.
      69                 :            :    8.  choose insn with lowest UID.
      70                 :            : 
      71                 :            :    Memory references complicate matters.  Only if we can be certain
      72                 :            :    that memory references are not part of the data dependency graph
      73                 :            :    (via true, anti, or output dependence), can we move operations past
      74                 :            :    memory references.  To first approximation, reads can be done
      75                 :            :    independently, while writes introduce dependencies.  Better
      76                 :            :    approximations will yield fewer dependencies.
      77                 :            : 
      78                 :            :    Before reload, an extended analysis of interblock data dependences
      79                 :            :    is required for interblock scheduling.  This is performed in
      80                 :            :    compute_block_dependences ().
      81                 :            : 
      82                 :            :    Dependencies set up by memory references are treated in exactly the
      83                 :            :    same way as other dependencies, by using insn backward dependences
      84                 :            :    INSN_BACK_DEPS.  INSN_BACK_DEPS are translated into forward dependences
      85                 :            :    INSN_FORW_DEPS for the purpose of forward list scheduling.
      86                 :            : 
      87                 :            :    Having optimized the critical path, we may have also unduly
      88                 :            :    extended the lifetimes of some registers.  If an operation requires
      89                 :            :    that constants be loaded into registers, it is certainly desirable
      90                 :            :    to load those constants as early as necessary, but no earlier.
      91                 :            :    I.e., it will not do to load up a bunch of registers at the
      92                 :            :    beginning of a basic block only to use them at the end, if they
      93                 :            :    could be loaded later, since this may result in excessive register
      94                 :            :    utilization.
      95                 :            : 
      96                 :            :    Note that since branches are never in basic blocks, but only end
      97                 :            :    basic blocks, this pass will not move branches.  But that is ok,
      98                 :            :    since we can use GNU's delayed branch scheduling pass to take care
      99                 :            :    of this case.
     100                 :            : 
     101                 :            :    Also note that no further optimizations based on algebraic
     102                 :            :    identities are performed, so this pass would be a good one to
     103                 :            :    perform instruction splitting, such as breaking up a multiply
     104                 :            :    instruction into shifts and adds where that is profitable.
     105                 :            : 
     106                 :            :    Given the memory aliasing analysis that this pass should perform,
     107                 :            :    it should be possible to remove redundant stores to memory, and to
     108                 :            :    load values from registers instead of hitting memory.
     109                 :            : 
     110                 :            :    Before reload, speculative insns are moved only if a 'proof' exists
     111                 :            :    that no exception will be caused by this, and if no live registers
     112                 :            :    exist that inhibit the motion (live registers constraints are not
     113                 :            :    represented by data dependence edges).
     114                 :            : 
     115                 :            :    This pass must update information that subsequent passes expect to
     116                 :            :    be correct.  Namely: reg_n_refs, reg_n_sets, reg_n_deaths,
     117                 :            :    reg_n_calls_crossed, and reg_live_length.  Also, BB_HEAD, BB_END.
     118                 :            : 
     119                 :            :    The information in the line number notes is carefully retained by
     120                 :            :    this pass.  Notes that refer to the starting and ending of
     121                 :            :    exception regions are also carefully retained by this pass.  All
     122                 :            :    other NOTE insns are grouped in their same relative order at the
     123                 :            :    beginning of basic blocks and regions that have been scheduled.  */
     124                 :            : 
     125                 :            : #include "config.h"
     126                 :            : #include "system.h"
     127                 :            : #include "coretypes.h"
     128                 :            : #include "backend.h"
     129                 :            : #include "target.h"
     130                 :            : #include "rtl.h"
     131                 :            : #include "cfghooks.h"
     132                 :            : #include "df.h"
     133                 :            : #include "memmodel.h"
     134                 :            : #include "tm_p.h"
     135                 :            : #include "insn-config.h"
     136                 :            : #include "regs.h"
     137                 :            : #include "ira.h"
     138                 :            : #include "recog.h"
     139                 :            : #include "insn-attr.h"
     140                 :            : #include "cfgrtl.h"
     141                 :            : #include "cfgbuild.h"
     142                 :            : #include "sched-int.h"
     143                 :            : #include "common/common-target.h"
     144                 :            : #include "dbgcnt.h"
     145                 :            : #include "cfgloop.h"
     146                 :            : #include "dumpfile.h"
     147                 :            : #include "print-rtl.h"
     148                 :            : #include "function-abi.h"
     149                 :            : 
     150                 :            : #ifdef INSN_SCHEDULING
     151                 :            : 
     152                 :            : /* True if we do register pressure relief through live-range
     153                 :            :    shrinkage.  */
     154                 :            : static bool live_range_shrinkage_p;
     155                 :            : 
     156                 :            : /* Switch on live range shrinkage.  */
     157                 :            : void
     158                 :         20 : initialize_live_range_shrinkage (void)
     159                 :            : {
     160                 :         20 :   live_range_shrinkage_p = true;
     161                 :         20 : }
     162                 :            : 
     163                 :            : /* Switch off live range shrinkage.  */
     164                 :            : void
     165                 :         20 : finish_live_range_shrinkage (void)
     166                 :            : {
     167                 :         20 :   live_range_shrinkage_p = false;
     168                 :         20 : }
     169                 :            : 
     170                 :            : /* issue_rate is the number of insns that can be scheduled in the same
     171                 :            :    machine cycle.  It can be defined in the config/mach/mach.h file,
     172                 :            :    otherwise we set it to 1.  */
     173                 :            : 
     174                 :            : int issue_rate;
     175                 :            : 
     176                 :            : /* This can be set to true by a backend if the scheduler should not
     177                 :            :    enable a DCE pass.  */
     178                 :            : bool sched_no_dce;
     179                 :            : 
     180                 :            : /* The current initiation interval used when modulo scheduling.  */
     181                 :            : static int modulo_ii;
     182                 :            : 
     183                 :            : /* The maximum number of stages we are prepared to handle.  */
     184                 :            : static int modulo_max_stages;
     185                 :            : 
     186                 :            : /* The number of insns that exist in each iteration of the loop.  We use this
     187                 :            :    to detect when we've scheduled all insns from the first iteration.  */
     188                 :            : static int modulo_n_insns;
     189                 :            : 
     190                 :            : /* The current count of insns in the first iteration of the loop that have
     191                 :            :    already been scheduled.  */
     192                 :            : static int modulo_insns_scheduled;
     193                 :            : 
     194                 :            : /* The maximum uid of insns from the first iteration of the loop.  */
     195                 :            : static int modulo_iter0_max_uid;
     196                 :            : 
     197                 :            : /* The number of times we should attempt to backtrack when modulo scheduling.
     198                 :            :    Decreased each time we have to backtrack.  */
     199                 :            : static int modulo_backtracks_left;
     200                 :            : 
     201                 :            : /* The stage in which the last insn from the original loop was
     202                 :            :    scheduled.  */
     203                 :            : static int modulo_last_stage;
     204                 :            : 
     205                 :            : /* sched-verbose controls the amount of debugging output the
     206                 :            :    scheduler prints.  It is controlled by -fsched-verbose=N:
     207                 :            :    N=0: no debugging output.
     208                 :            :    N=1: default value.
     209                 :            :    N=2: bb's probabilities, detailed ready list info, unit/insn info.
     210                 :            :    N=3: rtl at abort point, control-flow, regions info.
     211                 :            :    N=5: dependences info.  */
     212                 :            : int sched_verbose = 0;
     213                 :            : 
     214                 :            : /* Debugging file.  All printouts are sent to dump. */
     215                 :            : FILE *sched_dump = 0;
     216                 :            : 
     217                 :            : /* This is a placeholder for the scheduler parameters common
     218                 :            :    to all schedulers.  */
     219                 :            : struct common_sched_info_def *common_sched_info;
     220                 :            : 
     221                 :            : #define INSN_TICK(INSN) (HID (INSN)->tick)
     222                 :            : #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick)
     223                 :            : #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate)
     224                 :            : #define INTER_TICK(INSN) (HID (INSN)->inter_tick)
     225                 :            : #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn)
     226                 :            : #define SHADOW_P(INSN) (HID (INSN)->shadow_p)
     227                 :            : #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec)
     228                 :            : /* Cached cost of the instruction.  Use insn_sched_cost to get cost of the
     229                 :            :    insn.  -1 here means that the field is not initialized.  */
     230                 :            : #define INSN_COST(INSN) (HID (INSN)->cost)
     231                 :            : 
     232                 :            : /* If INSN_TICK of an instruction is equal to INVALID_TICK,
     233                 :            :    then it should be recalculated from scratch.  */
     234                 :            : #define INVALID_TICK (-(max_insn_queue_index + 1))
     235                 :            : /* The minimal value of the INSN_TICK of an instruction.  */
     236                 :            : #define MIN_TICK (-max_insn_queue_index)
     237                 :            : 
     238                 :            : /* Original order of insns in the ready list.
     239                 :            :    Used to keep order of normal insns while separating DEBUG_INSNs.  */
     240                 :            : #define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
     241                 :            : 
     242                 :            : /* The deciding reason for INSN's place in the ready list.  */
     243                 :            : #define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
     244                 :            : 
     245                 :            : /* List of important notes we must keep around.  This is a pointer to the
     246                 :            :    last element in the list.  */
     247                 :            : rtx_insn *note_list;
     248                 :            : 
     249                 :            : static struct spec_info_def spec_info_var;
     250                 :            : /* Description of the speculative part of the scheduling.
     251                 :            :    If NULL - no speculation.  */
     252                 :            : spec_info_t spec_info = NULL;
     253                 :            : 
     254                 :            : /* True, if recovery block was added during scheduling of current block.
     255                 :            :    Used to determine, if we need to fix INSN_TICKs.  */
     256                 :            : static bool haifa_recovery_bb_recently_added_p;
     257                 :            : 
     258                 :            : /* True, if recovery block was added during this scheduling pass.
     259                 :            :    Used to determine if we should have empty memory pools of dependencies
     260                 :            :    after finishing current region.  */
     261                 :            : bool haifa_recovery_bb_ever_added_p;
     262                 :            : 
     263                 :            : /* Counters of different types of speculative instructions.  */
     264                 :            : static int nr_begin_data, nr_be_in_data, nr_begin_control, nr_be_in_control;
     265                 :            : 
     266                 :            : /* Array used in {unlink, restore}_bb_notes.  */
     267                 :            : static rtx_insn **bb_header = 0;
     268                 :            : 
     269                 :            : /* Basic block after which recovery blocks will be created.  */
     270                 :            : static basic_block before_recovery;
     271                 :            : 
     272                 :            : /* Basic block just before the EXIT_BLOCK and after recovery, if we have
     273                 :            :    created it.  */
     274                 :            : basic_block after_recovery;
     275                 :            : 
     276                 :            : /* FALSE if we add bb to another region, so we don't need to initialize it.  */
     277                 :            : bool adding_bb_to_current_region_p = true;
     278                 :            : 
     279                 :            : /* Queues, etc.  */
     280                 :            : 
     281                 :            : /* An instruction is ready to be scheduled when all insns preceding it
     282                 :            :    have already been scheduled.  It is important to ensure that all
     283                 :            :    insns which use its result will not be executed until its result
     284                 :            :    has been computed.  An insn is maintained in one of four structures:
     285                 :            : 
     286                 :            :    (P) the "Pending" set of insns which cannot be scheduled until
     287                 :            :    their dependencies have been satisfied.
     288                 :            :    (Q) the "Queued" set of insns that can be scheduled when sufficient
     289                 :            :    time has passed.
     290                 :            :    (R) the "Ready" list of unscheduled, uncommitted insns.
     291                 :            :    (S) the "Scheduled" list of insns.
     292                 :            : 
     293                 :            :    Initially, all insns are either "Pending" or "Ready" depending on
     294                 :            :    whether their dependencies are satisfied.
     295                 :            : 
     296                 :            :    Insns move from the "Ready" list to the "Scheduled" list as they
     297                 :            :    are committed to the schedule.  As this occurs, the insns in the
     298                 :            :    "Pending" list have their dependencies satisfied and move to either
     299                 :            :    the "Ready" list or the "Queued" set depending on whether
     300                 :            :    sufficient time has passed to make them ready.  As time passes,
     301                 :            :    insns move from the "Queued" set to the "Ready" list.
     302                 :            : 
     303                 :            :    The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the
     304                 :            :    unscheduled insns, i.e., those that are ready, queued, and pending.
     305                 :            :    The "Queued" set (Q) is implemented by the variable `insn_queue'.
     306                 :            :    The "Ready" list (R) is implemented by the variables `ready' and
     307                 :            :    `n_ready'.
     308                 :            :    The "Scheduled" list (S) is the new insn chain built by this pass.
     309                 :            : 
     310                 :            :    The transition (R->S) is implemented in the scheduling loop in
     311                 :            :    `schedule_block' when the best insn to schedule is chosen.
     312                 :            :    The transitions (P->R and P->Q) are implemented in `schedule_insn' as
     313                 :            :    insns move from the ready list to the scheduled list.
     314                 :            :    The transition (Q->R) is implemented in 'queue_to_insn' as time
     315                 :            :    passes or stalls are introduced.  */
     316                 :            : 
     317                 :            : /* Implement a circular buffer to delay instructions until sufficient
     318                 :            :    time has passed.  For the new pipeline description interface,
     319                 :            :    MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less
     320                 :            :    than maximal time of instruction execution computed by genattr.c on
     321                 :            :    the base maximal time of functional unit reservations and getting a
     322                 :            :    result.  This is the longest time an insn may be queued.  */
     323                 :            : 
     324                 :            : static rtx_insn_list **insn_queue;
     325                 :            : static int q_ptr = 0;
     326                 :            : static int q_size = 0;
     327                 :            : #define NEXT_Q(X) (((X)+1) & max_insn_queue_index)
     328                 :            : #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index)
     329                 :            : 
     330                 :            : #define QUEUE_SCHEDULED (-3)
     331                 :            : #define QUEUE_NOWHERE   (-2)
     332                 :            : #define QUEUE_READY     (-1)
     333                 :            : /* QUEUE_SCHEDULED - INSN is scheduled.
     334                 :            :    QUEUE_NOWHERE   - INSN isn't scheduled yet and is neither in
     335                 :            :    queue or ready list.
     336                 :            :    QUEUE_READY     - INSN is in ready list.
     337                 :            :    N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles.  */
     338                 :            : 
     339                 :            : #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index)
     340                 :            : 
     341                 :            : /* The following variable value refers for all current and future
     342                 :            :    reservations of the processor units.  */
     343                 :            : state_t curr_state;
     344                 :            : 
     345                 :            : /* The following variable value is size of memory representing all
     346                 :            :    current and future reservations of the processor units.  */
     347                 :            : size_t dfa_state_size;
     348                 :            : 
     349                 :            : /* The following array is used to find the best insn from ready when
     350                 :            :    the automaton pipeline interface is used.  */
     351                 :            : signed char *ready_try = NULL;
     352                 :            : 
     353                 :            : /* The ready list.  */
     354                 :            : struct ready_list ready = {NULL, 0, 0, 0, 0};
     355                 :            : 
     356                 :            : /* The pointer to the ready list (to be removed).  */
     357                 :            : static struct ready_list *readyp = &ready;
     358                 :            : 
     359                 :            : /* Scheduling clock.  */
     360                 :            : static int clock_var;
     361                 :            : 
     362                 :            : /* Clock at which the previous instruction was issued.  */
     363                 :            : static int last_clock_var;
     364                 :            : 
     365                 :            : /* Set to true if, when queuing a shadow insn, we discover that it would be
     366                 :            :    scheduled too late.  */
     367                 :            : static bool must_backtrack;
     368                 :            : 
     369                 :            : /* The following variable value is number of essential insns issued on
     370                 :            :    the current cycle.  An insn is essential one if it changes the
     371                 :            :    processors state.  */
     372                 :            : int cycle_issued_insns;
     373                 :            : 
     374                 :            : /* This records the actual schedule.  It is built up during the main phase
     375                 :            :    of schedule_block, and afterwards used to reorder the insns in the RTL.  */
     376                 :            : static vec<rtx_insn *> scheduled_insns;
     377                 :            : 
     378                 :            : static int may_trap_exp (const_rtx, int);
     379                 :            : 
     380                 :            : /* Nonzero iff the address is comprised from at most 1 register.  */
     381                 :            : #define CONST_BASED_ADDRESS_P(x)                        \
     382                 :            :   (REG_P (x)                                    \
     383                 :            :    || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS   \
     384                 :            :         || (GET_CODE (x) == LO_SUM))                    \
     385                 :            :        && (CONSTANT_P (XEXP (x, 0))                     \
     386                 :            :            || CONSTANT_P (XEXP (x, 1)))))
     387                 :            : 
     388                 :            : /* Returns a class that insn with GET_DEST(insn)=x may belong to,
     389                 :            :    as found by analyzing insn's expression.  */
     390                 :            : 
     391                 :            : 
     392                 :            : static int haifa_luid_for_non_insn (rtx x);
     393                 :            : 
     394                 :            : /* Haifa version of sched_info hooks common to all headers.  */
     395                 :            : const struct common_sched_info_def haifa_common_sched_info =
     396                 :            :   {
     397                 :            :     NULL, /* fix_recovery_cfg */
     398                 :            :     NULL, /* add_block */
     399                 :            :     NULL, /* estimate_number_of_insns */
     400                 :            :     haifa_luid_for_non_insn, /* luid_for_non_insn */
     401                 :            :     SCHED_PASS_UNKNOWN /* sched_pass_id */
     402                 :            :   };
     403                 :            : 
     404                 :            : /* Mapping from instruction UID to its Logical UID.  */
     405                 :            : vec<int> sched_luids;
     406                 :            : 
     407                 :            : /* Next LUID to assign to an instruction.  */
     408                 :            : int sched_max_luid = 1;
     409                 :            : 
     410                 :            : /* Haifa Instruction Data.  */
     411                 :            : vec<haifa_insn_data_def> h_i_d;
     412                 :            : 
     413                 :            : void (* sched_init_only_bb) (basic_block, basic_block);
     414                 :            : 
     415                 :            : /* Split block function.  Different schedulers might use different functions
     416                 :            :    to handle their internal data consistent.  */
     417                 :            : basic_block (* sched_split_block) (basic_block, rtx);
     418                 :            : 
     419                 :            : /* Create empty basic block after the specified block.  */
     420                 :            : basic_block (* sched_create_empty_bb) (basic_block);
     421                 :            : 
     422                 :            : /* Return the number of cycles until INSN is expected to be ready.
     423                 :            :    Return zero if it already is.  */
     424                 :            : static int
     425                 :      70006 : insn_delay (rtx_insn *insn)
     426                 :            : {
     427                 :      70006 :   return MAX (INSN_TICK (insn) - clock_var, 0);
     428                 :            : }
     429                 :            : 
     430                 :            : static int
     431                 :      26519 : may_trap_exp (const_rtx x, int is_store)
     432                 :            : {
     433                 :      26519 :   enum rtx_code code;
     434                 :            : 
     435                 :      26519 :   if (x == 0)
     436                 :            :     return TRAP_FREE;
     437                 :      26519 :   code = GET_CODE (x);
     438                 :      26519 :   if (is_store)
     439                 :            :     {
     440                 :       5633 :       if (code == MEM && may_trap_p (x))
     441                 :            :         return TRAP_RISKY;
     442                 :            :       else
     443                 :       5403 :         return TRAP_FREE;
     444                 :            :     }
     445                 :      20886 :   if (code == MEM)
     446                 :            :     {
     447                 :            :       /* The insn uses memory:  a volatile load.  */
     448                 :       1702 :       if (MEM_VOLATILE_P (x))
     449                 :            :         return IRISKY;
     450                 :            :       /* An exception-free load.  */
     451                 :       1676 :       if (!may_trap_p (x))
     452                 :            :         return IFREE;
     453                 :            :       /* A load with 1 base register, to be further checked.  */
     454                 :        410 :       if (CONST_BASED_ADDRESS_P (XEXP (x, 0)))
     455                 :            :         return PFREE_CANDIDATE;
     456                 :            :       /* No info on the load, to be further checked.  */
     457                 :         40 :       return PRISKY_CANDIDATE;
     458                 :            :     }
     459                 :            :   else
     460                 :            :     {
     461                 :      19184 :       const char *fmt;
     462                 :      19184 :       int i, insn_class = TRAP_FREE;
     463                 :            : 
     464                 :            :       /* Neither store nor load, check if it may cause a trap.  */
     465                 :      19184 :       if (may_trap_p (x))
     466                 :            :         return TRAP_RISKY;
     467                 :            :       /* Recursive step: walk the insn...  */
     468                 :      18854 :       fmt = GET_RTX_FORMAT (code);
     469                 :      42894 :       for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
     470                 :            :         {
     471                 :      24042 :           if (fmt[i] == 'e')
     472                 :            :             {
     473                 :      11242 :               int tmp_class = may_trap_exp (XEXP (x, i), is_store);
     474                 :      11242 :               insn_class = WORST_CLASS (insn_class, tmp_class);
     475                 :            :             }
     476                 :      12800 :           else if (fmt[i] == 'E')
     477                 :            :             {
     478                 :            :               int j;
     479                 :        150 :               for (j = 0; j < XVECLEN (x, i); j++)
     480                 :            :                 {
     481                 :        122 :                   int tmp_class = may_trap_exp (XVECEXP (x, i, j), is_store);
     482                 :        122 :                   insn_class = WORST_CLASS (insn_class, tmp_class);
     483                 :        122 :                   if (insn_class == TRAP_RISKY || insn_class == IRISKY)
     484                 :            :                     break;
     485                 :            :                 }
     486                 :            :             }
     487                 :      24042 :           if (insn_class == TRAP_RISKY || insn_class == IRISKY)
     488                 :            :             break;
     489                 :            :         }
     490                 :      18854 :       return insn_class;
     491                 :            :     }
     492                 :            : }
     493                 :            : 
     494                 :            : /* Classifies rtx X of an insn for the purpose of verifying that X can be
     495                 :            :    executed speculatively (and consequently the insn can be moved
     496                 :            :    speculatively), by examining X, returning:
     497                 :            :    TRAP_RISKY: store, or risky non-load insn (e.g. division by variable).
     498                 :            :    TRAP_FREE: non-load insn.
     499                 :            :    IFREE: load from a globally safe location.
     500                 :            :    IRISKY: volatile load.
     501                 :            :    PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for
     502                 :            :    being either PFREE or PRISKY.  */
     503                 :            : 
     504                 :            : static int
     505                 :       6767 : haifa_classify_rtx (const_rtx x)
     506                 :            : {
     507                 :       6767 :   int tmp_class = TRAP_FREE;
     508                 :       6767 :   int insn_class = TRAP_FREE;
     509                 :       6767 :   enum rtx_code code;
     510                 :            : 
     511                 :       6767 :   if (GET_CODE (x) == PARALLEL)
     512                 :            :     {
     513                 :        770 :       int i, len = XVECLEN (x, 0);
     514                 :            : 
     515                 :       2134 :       for (i = len - 1; i >= 0; i--)
     516                 :            :         {
     517                 :       1525 :           tmp_class = haifa_classify_rtx (XVECEXP (x, 0, i));
     518                 :       1525 :           insn_class = WORST_CLASS (insn_class, tmp_class);
     519                 :       1525 :           if (insn_class == TRAP_RISKY || insn_class == IRISKY)
     520                 :            :             break;
     521                 :            :         }
     522                 :            :     }
     523                 :            :   else
     524                 :            :     {
     525                 :       5997 :       code = GET_CODE (x);
     526                 :       5997 :       switch (code)
     527                 :            :         {
     528                 :        715 :         case CLOBBER:
     529                 :            :           /* Test if it is a 'store'.  */
     530                 :        715 :           tmp_class = may_trap_exp (XEXP (x, 0), 1);
     531                 :        715 :           break;
     532                 :       4918 :         case SET:
     533                 :            :           /* Test if it is a store.  */
     534                 :       4918 :           tmp_class = may_trap_exp (SET_DEST (x), 1);
     535                 :       4918 :           if (tmp_class == TRAP_RISKY)
     536                 :            :             break;
     537                 :            :           /* Test if it is a load.  */
     538                 :       4761 :           tmp_class =
     539                 :       4761 :             WORST_CLASS (tmp_class,
     540                 :            :                          may_trap_exp (SET_SRC (x), 0));
     541                 :            :           break;
     542                 :          0 :         case COND_EXEC:
     543                 :          0 :           tmp_class = haifa_classify_rtx (COND_EXEC_CODE (x));
     544                 :          0 :           if (tmp_class == TRAP_RISKY)
     545                 :            :             break;
     546                 :          0 :           tmp_class = WORST_CLASS (tmp_class,
     547                 :            :                                    may_trap_exp (COND_EXEC_TEST (x), 0));
     548                 :            :           break;
     549                 :          6 :         case TRAP_IF:
     550                 :          6 :           tmp_class = TRAP_RISKY;
     551                 :          6 :           break;
     552                 :        157 :         default:;
     553                 :            :         }
     554                 :            :       insn_class = tmp_class;
     555                 :            :     }
     556                 :            : 
     557                 :       6767 :   return insn_class;
     558                 :            : }
     559                 :            : 
     560                 :            : int
     561                 :       5242 : haifa_classify_insn (const_rtx insn)
     562                 :            : {
     563                 :       5242 :   return haifa_classify_rtx (PATTERN (insn));
     564                 :            : }
     565                 :            : 
     566                 :            : /* After the scheduler initialization function has been called, this function
     567                 :            :    can be called to enable modulo scheduling.  II is the initiation interval
     568                 :            :    we should use, it affects the delays for delay_pairs that were recorded as
     569                 :            :    separated by a given number of stages.
     570                 :            : 
     571                 :            :    MAX_STAGES provides us with a limit
     572                 :            :    after which we give up scheduling; the caller must have unrolled at least
     573                 :            :    as many copies of the loop body and recorded delay_pairs for them.
     574                 :            :    
     575                 :            :    INSNS is the number of real (non-debug) insns in one iteration of
     576                 :            :    the loop.  MAX_UID can be used to test whether an insn belongs to
     577                 :            :    the first iteration of the loop; all of them have a uid lower than
     578                 :            :    MAX_UID.  */
     579                 :            : void
     580                 :          0 : set_modulo_params (int ii, int max_stages, int insns, int max_uid)
     581                 :            : {
     582                 :          0 :   modulo_ii = ii;
     583                 :          0 :   modulo_max_stages = max_stages;
     584                 :          0 :   modulo_n_insns = insns;
     585                 :          0 :   modulo_iter0_max_uid = max_uid;
     586                 :          0 :   modulo_backtracks_left = param_max_modulo_backtrack_attempts;
     587                 :          0 : }
     588                 :            : 
     589                 :            : /* A structure to record a pair of insns where the first one is a real
     590                 :            :    insn that has delay slots, and the second is its delayed shadow.
     591                 :            :    I1 is scheduled normally and will emit an assembly instruction,
     592                 :            :    while I2 describes the side effect that takes place at the
     593                 :            :    transition between cycles CYCLES and (CYCLES + 1) after I1.  */
     594                 :            : struct delay_pair
     595                 :            : {
     596                 :            :   struct delay_pair *next_same_i1;
     597                 :            :   rtx_insn *i1, *i2;
     598                 :            :   int cycles;
     599                 :            :   /* When doing modulo scheduling, we a delay_pair can also be used to
     600                 :            :      show that I1 and I2 are the same insn in a different stage.  If that
     601                 :            :      is the case, STAGES will be nonzero.  */
     602                 :            :   int stages;
     603                 :            : };
     604                 :            : 
     605                 :            : /* Helpers for delay hashing.  */
     606                 :            : 
     607                 :            : struct delay_i1_hasher : nofree_ptr_hash <delay_pair>
     608                 :            : {
     609                 :            :   typedef void *compare_type;
     610                 :            :   static inline hashval_t hash (const delay_pair *);
     611                 :            :   static inline bool equal (const delay_pair *, const void *);
     612                 :            : };
     613                 :            : 
     614                 :            : /* Returns a hash value for X, based on hashing just I1.  */
     615                 :            : 
     616                 :            : inline hashval_t
     617                 :          0 : delay_i1_hasher::hash (const delay_pair *x)
     618                 :            : {
     619                 :          0 :   return htab_hash_pointer (x->i1);
     620                 :            : }
     621                 :            : 
     622                 :            : /* Return true if I1 of pair X is the same as that of pair Y.  */
     623                 :            : 
     624                 :            : inline bool
     625                 :          0 : delay_i1_hasher::equal (const delay_pair *x, const void *y)
     626                 :            : {
     627                 :          0 :   return x->i1 == y;
     628                 :            : }
     629                 :            : 
     630                 :            : struct delay_i2_hasher : free_ptr_hash <delay_pair>
     631                 :            : {
     632                 :            :   typedef void *compare_type;
     633                 :            :   static inline hashval_t hash (const delay_pair *);
     634                 :            :   static inline bool equal (const delay_pair *, const void *);
     635                 :            : };
     636                 :            : 
     637                 :            : /* Returns a hash value for X, based on hashing just I2.  */
     638                 :            : 
     639                 :            : inline hashval_t
     640                 :          0 : delay_i2_hasher::hash (const delay_pair *x)
     641                 :            : {
     642                 :          0 :   return htab_hash_pointer (x->i2);
     643                 :            : }
     644                 :            : 
     645                 :            : /* Return true if I2 of pair X is the same as that of pair Y.  */
     646                 :            : 
     647                 :            : inline bool
     648                 :          0 : delay_i2_hasher::equal (const delay_pair *x, const void *y)
     649                 :            : {
     650                 :          0 :   return x->i2 == y;
     651                 :            : }
     652                 :            : 
     653                 :            : /* Two hash tables to record delay_pairs, one indexed by I1 and the other
     654                 :            :    indexed by I2.  */
     655                 :            : static hash_table<delay_i1_hasher> *delay_htab;
     656                 :            : static hash_table<delay_i2_hasher> *delay_htab_i2;
     657                 :            : 
     658                 :            : /* Called through htab_traverse.  Walk the hashtable using I2 as
     659                 :            :    index, and delete all elements involving an UID higher than
     660                 :            :    that pointed to by *DATA.  */
     661                 :            : int
     662                 :          0 : haifa_htab_i2_traverse (delay_pair **slot, int *data)
     663                 :            : {
     664                 :          0 :   int maxuid = *data;
     665                 :          0 :   struct delay_pair *p = *slot;
     666                 :          0 :   if (INSN_UID (p->i2) >= maxuid || INSN_UID (p->i1) >= maxuid)
     667                 :            :     {
     668                 :          0 :       delay_htab_i2->clear_slot (slot);
     669                 :            :     }
     670                 :          0 :   return 1;
     671                 :            : }
     672                 :            : 
     673                 :            : /* Called through htab_traverse.  Walk the hashtable using I2 as
     674                 :            :    index, and delete all elements involving an UID higher than
     675                 :            :    that pointed to by *DATA.  */
     676                 :            : int
     677                 :          0 : haifa_htab_i1_traverse (delay_pair **pslot, int *data)
     678                 :            : {
     679                 :          0 :   int maxuid = *data;
     680                 :          0 :   struct delay_pair *p, *first, **pprev;
     681                 :            : 
     682                 :          0 :   if (INSN_UID ((*pslot)->i1) >= maxuid)
     683                 :            :     {
     684                 :          0 :       delay_htab->clear_slot (pslot);
     685                 :          0 :       return 1;
     686                 :            :     }
     687                 :            :   pprev = &first;
     688                 :          0 :   for (p = *pslot; p; p = p->next_same_i1)
     689                 :            :     {
     690                 :          0 :       if (INSN_UID (p->i2) < maxuid)
     691                 :            :         {
     692                 :          0 :           *pprev = p;
     693                 :          0 :           pprev = &p->next_same_i1;
     694                 :            :         }
     695                 :            :     }
     696                 :          0 :   *pprev = NULL;
     697                 :          0 :   if (first == NULL)
     698                 :          0 :     delay_htab->clear_slot (pslot);
     699                 :            :   else
     700                 :          0 :     *pslot = first;
     701                 :            :   return 1;
     702                 :            : }
     703                 :            : 
     704                 :            : /* Discard all delay pairs which involve an insn with an UID higher
     705                 :            :    than MAX_UID.  */
     706                 :            : void
     707                 :          0 : discard_delay_pairs_above (int max_uid)
     708                 :            : {
     709                 :          0 :   delay_htab->traverse <int *, haifa_htab_i1_traverse> (&max_uid);
     710                 :          0 :   delay_htab_i2->traverse <int *, haifa_htab_i2_traverse> (&max_uid);
     711                 :          0 : }
     712                 :            : 
     713                 :            : /* This function can be called by a port just before it starts the final
     714                 :            :    scheduling pass.  It records the fact that an instruction with delay
     715                 :            :    slots has been split into two insns, I1 and I2.  The first one will be
     716                 :            :    scheduled normally and initiates the operation.  The second one is a
     717                 :            :    shadow which must follow a specific number of cycles after I1; its only
     718                 :            :    purpose is to show the side effect that occurs at that cycle in the RTL.
     719                 :            :    If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN,
     720                 :            :    while I2 retains the original insn type.
     721                 :            : 
     722                 :            :    There are two ways in which the number of cycles can be specified,
     723                 :            :    involving the CYCLES and STAGES arguments to this function.  If STAGES
     724                 :            :    is zero, we just use the value of CYCLES.  Otherwise, STAGES is a factor
     725                 :            :    which is multiplied by MODULO_II to give the number of cycles.  This is
     726                 :            :    only useful if the caller also calls set_modulo_params to enable modulo
     727                 :            :    scheduling.  */
     728                 :            : 
     729                 :            : void
     730                 :          0 : record_delay_slot_pair (rtx_insn *i1, rtx_insn *i2, int cycles, int stages)
     731                 :            : {
     732                 :          0 :   struct delay_pair *p = XNEW (struct delay_pair);
     733                 :          0 :   struct delay_pair **slot;
     734                 :            : 
     735                 :          0 :   p->i1 = i1;
     736                 :          0 :   p->i2 = i2;
     737                 :          0 :   p->cycles = cycles;
     738                 :          0 :   p->stages = stages;
     739                 :            : 
     740                 :          0 :   if (!delay_htab)
     741                 :            :     {
     742                 :          0 :       delay_htab = new hash_table<delay_i1_hasher> (10);
     743                 :          0 :       delay_htab_i2 = new hash_table<delay_i2_hasher> (10);
     744                 :            :     }
     745                 :          0 :   slot = delay_htab->find_slot_with_hash (i1, htab_hash_pointer (i1), INSERT);
     746                 :          0 :   p->next_same_i1 = *slot;
     747                 :          0 :   *slot = p;
     748                 :          0 :   slot = delay_htab_i2->find_slot (p, INSERT);
     749                 :          0 :   *slot = p;
     750                 :          0 : }
     751                 :            : 
     752                 :            : /* Examine the delay pair hashtable to see if INSN is a shadow for another,
     753                 :            :    and return the other insn if so.  Return NULL otherwise.  */
     754                 :            : rtx_insn *
     755                 :          0 : real_insn_for_shadow (rtx_insn *insn)
     756                 :            : {
     757                 :          0 :   struct delay_pair *pair;
     758                 :            : 
     759                 :          0 :   if (!delay_htab)
     760                 :            :     return NULL;
     761                 :            : 
     762                 :          0 :   pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn));
     763                 :          0 :   if (!pair || pair->stages > 0)
     764                 :            :     return NULL;
     765                 :          0 :   return pair->i1;
     766                 :            : }
     767                 :            : 
     768                 :            : /* For a pair P of insns, return the fixed distance in cycles from the first
     769                 :            :    insn after which the second must be scheduled.  */
     770                 :            : static int
     771                 :          0 : pair_delay (struct delay_pair *p)
     772                 :            : {
     773                 :          0 :   if (p->stages == 0)
     774                 :          0 :     return p->cycles;
     775                 :            :   else
     776                 :          0 :     return p->stages * modulo_ii;
     777                 :            : }
     778                 :            : 
     779                 :            : /* Given an insn INSN, add a dependence on its delayed shadow if it
     780                 :            :    has one.  Also try to find situations where shadows depend on each other
     781                 :            :    and add dependencies to the real insns to limit the amount of backtracking
     782                 :            :    needed.  */
     783                 :            : void
     784                 :       1590 : add_delay_dependencies (rtx_insn *insn)
     785                 :            : {
     786                 :       1590 :   struct delay_pair *pair;
     787                 :       1590 :   sd_iterator_def sd_it;
     788                 :       1590 :   dep_t dep;
     789                 :            : 
     790                 :       1590 :   if (!delay_htab)
     791                 :       1590 :     return;
     792                 :            : 
     793                 :          0 :   pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn));
     794                 :          0 :   if (!pair)
     795                 :            :     return;
     796                 :          0 :   add_dependence (insn, pair->i1, REG_DEP_ANTI);
     797                 :          0 :   if (pair->stages)
     798                 :            :     return;
     799                 :            : 
     800                 :          0 :   FOR_EACH_DEP (pair->i2, SD_LIST_BACK, sd_it, dep)
     801                 :            :     {
     802                 :          0 :       rtx_insn *pro = DEP_PRO (dep);
     803                 :          0 :       struct delay_pair *other_pair
     804                 :          0 :         = delay_htab_i2->find_with_hash (pro, htab_hash_pointer (pro));
     805                 :          0 :       if (!other_pair || other_pair->stages)
     806                 :          0 :         continue;
     807                 :          0 :       if (pair_delay (other_pair) >= pair_delay (pair))
     808                 :            :         {
     809                 :          0 :           if (sched_verbose >= 4)
     810                 :            :             {
     811                 :          0 :               fprintf (sched_dump, ";;\tadding dependence %d <- %d\n",
     812                 :          0 :                        INSN_UID (other_pair->i1),
     813                 :          0 :                        INSN_UID (pair->i1));
     814                 :          0 :               fprintf (sched_dump, ";;\tpair1 %d <- %d, cost %d\n",
     815                 :          0 :                        INSN_UID (pair->i1),
     816                 :          0 :                        INSN_UID (pair->i2),
     817                 :            :                        pair_delay (pair));
     818                 :          0 :               fprintf (sched_dump, ";;\tpair2 %d <- %d, cost %d\n",
     819                 :          0 :                        INSN_UID (other_pair->i1),
     820                 :          0 :                        INSN_UID (other_pair->i2),
     821                 :            :                        pair_delay (other_pair));
     822                 :            :             }
     823                 :          0 :           add_dependence (pair->i1, other_pair->i1, REG_DEP_ANTI);
     824                 :            :         }
     825                 :            :     }
     826                 :            : }
     827                 :            : 
     828                 :            : /* Forward declarations.  */
     829                 :            : 
     830                 :            : static int priority (rtx_insn *, bool force_recompute = false);
     831                 :            : static int autopref_rank_for_schedule (const rtx_insn *, const rtx_insn *);
     832                 :            : static int rank_for_schedule (const void *, const void *);
     833                 :            : static void swap_sort (rtx_insn **, int);
     834                 :            : static void queue_insn (rtx_insn *, int, const char *);
     835                 :            : static int schedule_insn (rtx_insn *);
     836                 :            : static void adjust_priority (rtx_insn *);
     837                 :            : static void advance_one_cycle (void);
     838                 :            : static void extend_h_i_d (void);
     839                 :            : 
     840                 :            : 
     841                 :            : /* Notes handling mechanism:
     842                 :            :    =========================
     843                 :            :    Generally, NOTES are saved before scheduling and restored after scheduling.
     844                 :            :    The scheduler distinguishes between two types of notes:
     845                 :            : 
     846                 :            :    (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes:
     847                 :            :    Before scheduling a region, a pointer to the note is added to the insn
     848                 :            :    that follows or precedes it.  (This happens as part of the data dependence
     849                 :            :    computation).  After scheduling an insn, the pointer contained in it is
     850                 :            :    used for regenerating the corresponding note (in reemit_notes).
     851                 :            : 
     852                 :            :    (2) All other notes (e.g. INSN_DELETED):  Before scheduling a block,
     853                 :            :    these notes are put in a list (in rm_other_notes() and
     854                 :            :    unlink_other_notes ()).  After scheduling the block, these notes are
     855                 :            :    inserted at the beginning of the block (in schedule_block()).  */
     856                 :            : 
     857                 :            : static void ready_add (struct ready_list *, rtx_insn *, bool);
     858                 :            : static rtx_insn *ready_remove_first (struct ready_list *);
     859                 :            : static rtx_insn *ready_remove_first_dispatch (struct ready_list *ready);
     860                 :            : 
     861                 :            : static void queue_to_ready (struct ready_list *);
     862                 :            : static int early_queue_to_ready (state_t, struct ready_list *);
     863                 :            : 
     864                 :            : /* The following functions are used to implement multi-pass scheduling
     865                 :            :    on the first cycle.  */
     866                 :            : static rtx_insn *ready_remove (struct ready_list *, int);
     867                 :            : static void ready_remove_insn (rtx_insn *);
     868                 :            : 
     869                 :            : static void fix_inter_tick (rtx_insn *, rtx_insn *);
     870                 :            : static int fix_tick_ready (rtx_insn *);
     871                 :            : static void change_queue_index (rtx_insn *, int);
     872                 :            : 
     873                 :            : /* The following functions are used to implement scheduling of data/control
     874                 :            :    speculative instructions.  */
     875                 :            : 
     876                 :            : static void extend_h_i_d (void);
     877                 :            : static void init_h_i_d (rtx_insn *);
     878                 :            : static int haifa_speculate_insn (rtx_insn *, ds_t, rtx *);
     879                 :            : static void generate_recovery_code (rtx_insn *);
     880                 :            : static void process_insn_forw_deps_be_in_spec (rtx_insn *, rtx_insn *, ds_t);
     881                 :            : static void begin_speculative_block (rtx_insn *);
     882                 :            : static void add_to_speculative_block (rtx_insn *);
     883                 :            : static void init_before_recovery (basic_block *);
     884                 :            : static void create_check_block_twin (rtx_insn *, bool);
     885                 :            : static void fix_recovery_deps (basic_block);
     886                 :            : static bool haifa_change_pattern (rtx_insn *, rtx);
     887                 :            : static void dump_new_block_header (int, basic_block, rtx_insn *, rtx_insn *);
     888                 :            : static void restore_bb_notes (basic_block);
     889                 :            : static void fix_jump_move (rtx_insn *);
     890                 :            : static void move_block_after_check (rtx_insn *);
     891                 :            : static void move_succs (vec<edge, va_gc> **, basic_block);
     892                 :            : static void sched_remove_insn (rtx_insn *);
     893                 :            : static void clear_priorities (rtx_insn *, rtx_vec_t *);
     894                 :            : static void calc_priorities (rtx_vec_t);
     895                 :            : static void add_jump_dependencies (rtx_insn *, rtx_insn *);
     896                 :            : 
     897                 :            : #endif /* INSN_SCHEDULING */
     898                 :            : 
     899                 :            : /* Point to state used for the current scheduling pass.  */
     900                 :            : struct haifa_sched_info *current_sched_info;
     901                 :            : 
     902                 :            : #ifndef INSN_SCHEDULING
     903                 :            : void
     904                 :            : schedule_insns (void)
     905                 :            : {
     906                 :            : }
     907                 :            : #else
     908                 :            : 
     909                 :            : /* Do register pressure sensitive insn scheduling if the flag is set
     910                 :            :    up.  */
     911                 :            : enum sched_pressure_algorithm sched_pressure;
     912                 :            : 
     913                 :            : /* Map regno -> its pressure class.  The map defined only when
     914                 :            :    SCHED_PRESSURE != SCHED_PRESSURE_NONE.  */
     915                 :            : enum reg_class *sched_regno_pressure_class;
     916                 :            : 
     917                 :            : /* The current register pressure.  Only elements corresponding pressure
     918                 :            :    classes are defined.  */
     919                 :            : static int curr_reg_pressure[N_REG_CLASSES];
     920                 :            : 
     921                 :            : /* Saved value of the previous array.  */
     922                 :            : static int saved_reg_pressure[N_REG_CLASSES];
     923                 :            : 
     924                 :            : /* Register living at given scheduling point.  */
     925                 :            : static bitmap curr_reg_live;
     926                 :            : 
     927                 :            : /* Saved value of the previous array.  */
     928                 :            : static bitmap saved_reg_live;
     929                 :            : 
     930                 :            : /* Registers mentioned in the current region.  */
     931                 :            : static bitmap region_ref_regs;
     932                 :            : 
     933                 :            : /* Temporary bitmap used for SCHED_PRESSURE_MODEL.  */
     934                 :            : static bitmap tmp_bitmap;
     935                 :            : 
     936                 :            : /* Effective number of available registers of a given class (see comment
     937                 :            :    in sched_pressure_start_bb).  */
     938                 :            : static int sched_class_regs_num[N_REG_CLASSES];
     939                 :            : /* The number of registers that the function would need to save before it
     940                 :            :    uses them, and the number of fixed_regs.  Helpers for calculating of
     941                 :            :    sched_class_regs_num.  */
     942                 :            : static int call_saved_regs_num[N_REG_CLASSES];
     943                 :            : static int fixed_regs_num[N_REG_CLASSES];
     944                 :            : 
     945                 :            : /* Initiate register pressure relative info for scheduling the current
     946                 :            :    region.  Currently it is only clearing register mentioned in the
     947                 :            :    current region.  */
     948                 :            : void
     949                 :        928 : sched_init_region_reg_pressure_info (void)
     950                 :            : {
     951                 :        928 :   bitmap_clear (region_ref_regs);
     952                 :        928 : }
     953                 :            : 
     954                 :            : /* PRESSURE[CL] describes the pressure on register class CL.  Update it
     955                 :            :    for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO.
     956                 :            :    LIVE tracks the set of live registers; if it is null, assume that
     957                 :            :    every birth or death is genuine.  */
     958                 :            : static inline void
     959                 :      43222 : mark_regno_birth_or_death (bitmap live, int *pressure, int regno, bool birth_p)
     960                 :            : {
     961                 :      43222 :   enum reg_class pressure_class;
     962                 :            : 
     963                 :      43222 :   pressure_class = sched_regno_pressure_class[regno];
     964                 :      43222 :   if (regno >= FIRST_PSEUDO_REGISTER)
     965                 :            :     {
     966                 :      24159 :       if (pressure_class != NO_REGS)
     967                 :            :         {
     968                 :      23803 :           if (birth_p)
     969                 :            :             {
     970                 :      18344 :               if (!live || bitmap_set_bit (live, regno))
     971                 :      17584 :                 pressure[pressure_class]
     972                 :      17584 :                   += (ira_reg_class_max_nregs
     973                 :      17584 :                       [pressure_class][PSEUDO_REGNO_MODE (regno)]);
     974                 :            :             }
     975                 :            :           else
     976                 :            :             {
     977                 :       5459 :               if (!live || bitmap_clear_bit (live, regno))
     978                 :       5254 :                 pressure[pressure_class]
     979                 :       5254 :                   -= (ira_reg_class_max_nregs
     980                 :       5254 :                       [pressure_class][PSEUDO_REGNO_MODE (regno)]);
     981                 :            :             }
     982                 :            :         }
     983                 :            :     }
     984                 :      19063 :   else if (pressure_class != NO_REGS
     985                 :      19063 :            && ! TEST_HARD_REG_BIT (ira_no_alloc_regs, regno))
     986                 :            :     {
     987                 :      16306 :       if (birth_p)
     988                 :            :         {
     989                 :      10842 :           if (!live || bitmap_set_bit (live, regno))
     990                 :      10836 :             pressure[pressure_class]++;
     991                 :            :         }
     992                 :            :       else
     993                 :            :         {
     994                 :       5464 :           if (!live || bitmap_clear_bit (live, regno))
     995                 :       5464 :             pressure[pressure_class]--;
     996                 :            :         }
     997                 :            :     }
     998                 :      43222 : }
     999                 :            : 
    1000                 :            : /* Initiate current register pressure related info from living
    1001                 :            :    registers given by LIVE.  */
    1002                 :            : static void
    1003                 :        987 : initiate_reg_pressure_info (bitmap live)
    1004                 :            : {
    1005                 :        987 :   int i;
    1006                 :        987 :   unsigned int j;
    1007                 :        987 :   bitmap_iterator bi;
    1008                 :            : 
    1009                 :       4935 :   for (i = 0; i < ira_pressure_classes_num; i++)
    1010                 :       3948 :     curr_reg_pressure[ira_pressure_classes[i]] = 0;
    1011                 :        987 :   bitmap_clear (curr_reg_live);
    1012                 :      11832 :   EXECUTE_IF_SET_IN_BITMAP (live, 0, j, bi)
    1013                 :      10845 :     if (sched_pressure == SCHED_PRESSURE_MODEL
    1014                 :      10845 :         || current_nr_blocks == 1
    1015                 :      12465 :         || bitmap_bit_p (region_ref_regs, j))
    1016                 :      10161 :       mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure, j, true);
    1017                 :        987 : }
    1018                 :            : 
    1019                 :            : /* Mark registers in X as mentioned in the current region.  */
    1020                 :            : static void
    1021                 :       8172 : setup_ref_regs (rtx x)
    1022                 :            : {
    1023                 :       8172 :   int i, j;
    1024                 :       8172 :   const RTX_CODE code = GET_CODE (x);
    1025                 :       8172 :   const char *fmt;
    1026                 :            : 
    1027                 :       8172 :   if (REG_P (x))
    1028                 :            :     {
    1029                 :       2916 :       bitmap_set_range (region_ref_regs, REGNO (x), REG_NREGS (x));
    1030                 :       2916 :       return;
    1031                 :            :     }
    1032                 :       5256 :   fmt = GET_RTX_FORMAT (code);
    1033                 :      14148 :   for (i = GET_RTX_LENGTH (code) - 1; i >= 0; i--)
    1034                 :       8892 :     if (fmt[i] == 'e')
    1035                 :       6300 :       setup_ref_regs (XEXP (x, i));
    1036                 :       2592 :     else if (fmt[i] == 'E')
    1037                 :            :       {
    1038                 :        108 :         for (j = 0; j < XVECLEN (x, i); j++)
    1039                 :         72 :           setup_ref_regs (XVECEXP (x, i, j));
    1040                 :            :       }
    1041                 :            : }
    1042                 :            : 
    1043                 :            : /* Initiate current register pressure related info at the start of
    1044                 :            :    basic block BB.  */
    1045                 :            : static void
    1046                 :        987 : initiate_bb_reg_pressure_info (basic_block bb)
    1047                 :            : {
    1048                 :        987 :   unsigned int i ATTRIBUTE_UNUSED;
    1049                 :        987 :   rtx_insn *insn;
    1050                 :            : 
    1051                 :        987 :   if (current_nr_blocks > 1)
    1052                 :       4452 :     FOR_BB_INSNS (bb, insn)
    1053                 :       2172 :       if (NONDEBUG_INSN_P (insn))
    1054                 :       1800 :         setup_ref_regs (PATTERN (insn));
    1055                 :        987 :   initiate_reg_pressure_info (df_get_live_in (bb));
    1056                 :        987 :   if (bb_has_eh_pred (bb))
    1057                 :         24 :     for (i = 0; ; ++i)
    1058                 :            :       {
    1059                 :         36 :         unsigned int regno = EH_RETURN_DATA_REGNO (i);
    1060                 :            : 
    1061                 :         24 :         if (regno == INVALID_REGNUM)
    1062                 :            :           break;
    1063                 :         24 :         if (! bitmap_bit_p (df_get_live_in (bb), regno))
    1064                 :         24 :           mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
    1065                 :            :                                      regno, true);
    1066                 :         24 :       }
    1067                 :        987 : }
    1068                 :            : 
    1069                 :            : /* Save current register pressure related info.  */
    1070                 :            : static void
    1071                 :       4856 : save_reg_pressure (void)
    1072                 :            : {
    1073                 :       4856 :   int i;
    1074                 :            : 
    1075                 :      24280 :   for (i = 0; i < ira_pressure_classes_num; i++)
    1076                 :      19424 :     saved_reg_pressure[ira_pressure_classes[i]]
    1077                 :      19424 :       = curr_reg_pressure[ira_pressure_classes[i]];
    1078                 :       4856 :   bitmap_copy (saved_reg_live, curr_reg_live);
    1079                 :       4856 : }
    1080                 :            : 
    1081                 :            : /* Restore saved register pressure related info.  */
    1082                 :            : static void
    1083                 :       4856 : restore_reg_pressure (void)
    1084                 :            : {
    1085                 :       4856 :   int i;
    1086                 :            : 
    1087                 :      24280 :   for (i = 0; i < ira_pressure_classes_num; i++)
    1088                 :      19424 :     curr_reg_pressure[ira_pressure_classes[i]]
    1089                 :      19424 :       = saved_reg_pressure[ira_pressure_classes[i]];
    1090                 :       4856 :   bitmap_copy (curr_reg_live, saved_reg_live);
    1091                 :       4856 : }
    1092                 :            : 
    1093                 :            : /* Return TRUE if the register is dying after its USE.  */
    1094                 :            : static bool
    1095                 :      21554 : dying_use_p (struct reg_use_data *use)
    1096                 :            : {
    1097                 :      21554 :   struct reg_use_data *next;
    1098                 :            : 
    1099                 :      24367 :   for (next = use->next_regno_use; next != use; next = next->next_regno_use)
    1100                 :       4390 :     if (NONDEBUG_INSN_P (next->insn)
    1101                 :       4390 :         && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED)
    1102                 :            :       return false;
    1103                 :            :   return true;
    1104                 :            : }
    1105                 :            : 
    1106                 :            : /* Print info about the current register pressure and its excess for
    1107                 :            :    each pressure class.  */
    1108                 :            : static void
    1109                 :          0 : print_curr_reg_pressure (void)
    1110                 :            : {
    1111                 :          0 :   int i;
    1112                 :          0 :   enum reg_class cl;
    1113                 :            : 
    1114                 :          0 :   fprintf (sched_dump, ";;\t");
    1115                 :          0 :   for (i = 0; i < ira_pressure_classes_num; i++)
    1116                 :            :     {
    1117                 :          0 :       cl = ira_pressure_classes[i];
    1118                 :          0 :       gcc_assert (curr_reg_pressure[cl] >= 0);
    1119                 :          0 :       fprintf (sched_dump, "  %s:%d(%d)", reg_class_names[cl],
    1120                 :            :                curr_reg_pressure[cl],
    1121                 :          0 :                curr_reg_pressure[cl] - sched_class_regs_num[cl]);
    1122                 :            :     }
    1123                 :          0 :   fprintf (sched_dump, "\n");
    1124                 :          0 : }
    1125                 :            : 
    1126                 :            : /* Determine if INSN has a condition that is clobbered if a register
    1127                 :            :    in SET_REGS is modified.  */
    1128                 :            : static bool
    1129                 :          0 : cond_clobbered_p (rtx_insn *insn, HARD_REG_SET set_regs)
    1130                 :            : {
    1131                 :          0 :   rtx pat = PATTERN (insn);
    1132                 :          0 :   gcc_assert (GET_CODE (pat) == COND_EXEC);
    1133                 :          0 :   if (TEST_HARD_REG_BIT (set_regs, REGNO (XEXP (COND_EXEC_TEST (pat), 0))))
    1134                 :            :     {
    1135                 :          0 :       sd_iterator_def sd_it;
    1136                 :          0 :       dep_t dep;
    1137                 :          0 :       haifa_change_pattern (insn, ORIG_PAT (insn));
    1138                 :          0 :       FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
    1139                 :          0 :         DEP_STATUS (dep) &= ~DEP_CANCELLED;
    1140                 :          0 :       TODO_SPEC (insn) = HARD_DEP;
    1141                 :          0 :       if (sched_verbose >= 2)
    1142                 :          0 :         fprintf (sched_dump,
    1143                 :            :                  ";;\t\tdequeue insn %s because of clobbered condition\n",
    1144                 :          0 :                  (*current_sched_info->print_insn) (insn, 0));
    1145                 :          0 :       return true;
    1146                 :            :     }
    1147                 :            : 
    1148                 :            :   return false;
    1149                 :            : }
    1150                 :            : 
    1151                 :            : /* This function should be called after modifying the pattern of INSN,
    1152                 :            :    to update scheduler data structures as needed.  */
    1153                 :            : static void
    1154                 :     395778 : update_insn_after_change (rtx_insn *insn)
    1155                 :            : {
    1156                 :     395778 :   sd_iterator_def sd_it;
    1157                 :     395778 :   dep_t dep;
    1158                 :            : 
    1159                 :     395778 :   dfa_clear_single_insn_cache (insn);
    1160                 :            : 
    1161                 :     395778 :   sd_it = sd_iterator_start (insn,
    1162                 :     395778 :                              SD_LIST_FORW | SD_LIST_BACK | SD_LIST_RES_BACK);
    1163                 :    4414720 :   while (sd_iterator_cond (&sd_it, &dep))
    1164                 :            :     {
    1165                 :    4018940 :       DEP_COST (dep) = UNKNOWN_DEP_COST;
    1166                 :    4018940 :       sd_iterator_next (&sd_it);
    1167                 :            :     }
    1168                 :            : 
    1169                 :            :   /* Invalidate INSN_COST, so it'll be recalculated.  */
    1170                 :     395778 :   INSN_COST (insn) = -1;
    1171                 :            :   /* Invalidate INSN_TICK, so it'll be recalculated.  */
    1172                 :     395778 :   INSN_TICK (insn) = INVALID_TICK;
    1173                 :            : 
    1174                 :            :   /* Invalidate autoprefetch data entry.  */
    1175                 :     395778 :   INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status
    1176                 :     395778 :     = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
    1177                 :     395778 :   INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status
    1178                 :     395778 :     = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
    1179                 :     395778 : }
    1180                 :            : 
    1181                 :            : 
    1182                 :            : /* Two VECs, one to hold dependencies for which pattern replacements
    1183                 :            :    need to be applied or restored at the start of the next cycle, and
    1184                 :            :    another to hold an integer that is either one, to apply the
    1185                 :            :    corresponding replacement, or zero to restore it.  */
    1186                 :            : static vec<dep_t> next_cycle_replace_deps;
    1187                 :            : static vec<int> next_cycle_apply;
    1188                 :            : 
    1189                 :            : static void apply_replacement (dep_t, bool);
    1190                 :            : static void restore_pattern (dep_t, bool);
    1191                 :            : 
    1192                 :            : /* Look at the remaining dependencies for insn NEXT, and compute and return
    1193                 :            :    the TODO_SPEC value we should use for it.  This is called after one of
    1194                 :            :    NEXT's dependencies has been resolved.
    1195                 :            :    We also perform pattern replacements for predication, and for broken
    1196                 :            :    replacement dependencies.  The latter is only done if FOR_BACKTRACK is
    1197                 :            :    false.  */
    1198                 :            : 
    1199                 :            : static ds_t
    1200                 :  198017000 : recompute_todo_spec (rtx_insn *next, bool for_backtrack)
    1201                 :            : {
    1202                 :  198017000 :   ds_t new_ds;
    1203                 :  198017000 :   sd_iterator_def sd_it;
    1204                 :  198017000 :   dep_t dep, modify_dep = NULL;
    1205                 :  198017000 :   int n_spec = 0;
    1206                 :  198017000 :   int n_control = 0;
    1207                 :  198017000 :   int n_replace = 0;
    1208                 :  198017000 :   bool first_p = true;
    1209                 :            : 
    1210                 :  198017000 :   if (sd_lists_empty_p (next, SD_LIST_BACK))
    1211                 :            :     /* NEXT has all its dependencies resolved.  */
    1212                 :            :     return 0;
    1213                 :            : 
    1214                 :  131224000 :   if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK))
    1215                 :            :     return HARD_DEP;
    1216                 :            : 
    1217                 :            :   /* If NEXT is intended to sit adjacent to this instruction, we don't
    1218                 :            :      want to try to break any dependencies.  Treat it as a HARD_DEP.  */
    1219                 :     200031 :   if (SCHED_GROUP_P (next))
    1220                 :            :     return HARD_DEP;
    1221                 :            : 
    1222                 :            :   /* Now we've got NEXT with speculative deps only.
    1223                 :            :      1. Look at the deps to see what we have to do.
    1224                 :            :      2. Check if we can do 'todo'.  */
    1225                 :     200031 :   new_ds = 0;
    1226                 :            : 
    1227                 :     721799 :   FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep)
    1228                 :            :     {
    1229                 :     521768 :       rtx_insn *pro = DEP_PRO (dep);
    1230                 :     521768 :       ds_t ds = DEP_STATUS (dep) & SPECULATIVE;
    1231                 :            : 
    1232                 :     521768 :       if (DEBUG_INSN_P (pro) && !DEBUG_INSN_P (next))
    1233                 :      62612 :         continue;
    1234                 :            : 
    1235                 :     459156 :       if (ds)
    1236                 :            :         {
    1237                 :          0 :           n_spec++;
    1238                 :          0 :           if (first_p)
    1239                 :            :             {
    1240                 :            :               first_p = false;
    1241                 :            : 
    1242                 :            :               new_ds = ds;
    1243                 :            :             }
    1244                 :            :           else
    1245                 :          0 :             new_ds = ds_merge (new_ds, ds);
    1246                 :            :         }
    1247                 :     459156 :       else if (DEP_TYPE (dep) == REG_DEP_CONTROL)
    1248                 :            :         {
    1249                 :          0 :           if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED)
    1250                 :            :             {
    1251                 :          0 :               n_control++;
    1252                 :          0 :               modify_dep = dep;
    1253                 :            :             }
    1254                 :          0 :           DEP_STATUS (dep) &= ~DEP_CANCELLED;
    1255                 :            :         }
    1256                 :     459156 :       else if (DEP_REPLACE (dep) != NULL)
    1257                 :            :         {
    1258                 :     459156 :           if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED)
    1259                 :            :             {
    1260                 :     459156 :               n_replace++;
    1261                 :     459156 :               modify_dep = dep;
    1262                 :            :             }
    1263                 :     459156 :           DEP_STATUS (dep) &= ~DEP_CANCELLED;
    1264                 :            :         }
    1265                 :            :     }
    1266                 :            : 
    1267                 :     200031 :   if (n_replace > 0 && n_control == 0 && n_spec == 0)
    1268                 :            :     {
    1269                 :     200031 :       if (!dbg_cnt (sched_breakdep))
    1270                 :            :         return HARD_DEP;
    1271                 :     721799 :       FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep)
    1272                 :            :         {
    1273                 :     521768 :           struct dep_replacement *desc = DEP_REPLACE (dep);
    1274                 :     521768 :           if (desc != NULL)
    1275                 :            :             {
    1276                 :     459156 :               if (desc->insn == next && !for_backtrack)
    1277                 :            :                 {
    1278                 :      56171 :                   gcc_assert (n_replace == 1);
    1279                 :      56171 :                   apply_replacement (dep, true);
    1280                 :            :                 }
    1281                 :     459156 :               DEP_STATUS (dep) |= DEP_CANCELLED;
    1282                 :            :             }
    1283                 :            :         }
    1284                 :            :       return 0;
    1285                 :            :     }
    1286                 :            :   
    1287                 :          0 :   else if (n_control == 1 && n_replace == 0 && n_spec == 0)
    1288                 :            :     {
    1289                 :          0 :       rtx_insn *pro, *other;
    1290                 :          0 :       rtx new_pat;
    1291                 :          0 :       rtx cond = NULL_RTX;
    1292                 :          0 :       bool success;
    1293                 :          0 :       rtx_insn *prev = NULL;
    1294                 :          0 :       int i;
    1295                 :          0 :       unsigned regno;
    1296                 :            :   
    1297                 :          0 :       if ((current_sched_info->flags & DO_PREDICATION) == 0
    1298                 :          0 :           || (ORIG_PAT (next) != NULL_RTX
    1299                 :          0 :               && PREDICATED_PAT (next) == NULL_RTX))
    1300                 :            :         return HARD_DEP;
    1301                 :            : 
    1302                 :          0 :       pro = DEP_PRO (modify_dep);
    1303                 :          0 :       other = real_insn_for_shadow (pro);
    1304                 :          0 :       if (other != NULL_RTX)
    1305                 :          0 :         pro = other;
    1306                 :            : 
    1307                 :          0 :       cond = sched_get_reverse_condition_uncached (pro);
    1308                 :          0 :       regno = REGNO (XEXP (cond, 0));
    1309                 :            : 
    1310                 :            :       /* Find the last scheduled insn that modifies the condition register.
    1311                 :            :          We can stop looking once we find the insn we depend on through the
    1312                 :            :          REG_DEP_CONTROL; if the condition register isn't modified after it,
    1313                 :            :          we know that it still has the right value.  */
    1314                 :          0 :       if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED)
    1315                 :          0 :         FOR_EACH_VEC_ELT_REVERSE (scheduled_insns, i, prev)
    1316                 :            :           {
    1317                 :          0 :             HARD_REG_SET t;
    1318                 :            : 
    1319                 :          0 :             find_all_hard_reg_sets (prev, &t, true);
    1320                 :          0 :             if (TEST_HARD_REG_BIT (t, regno))
    1321                 :          0 :               return HARD_DEP;
    1322                 :          0 :             if (prev == pro)
    1323                 :            :               break;
    1324                 :            :           }
    1325                 :          0 :       if (ORIG_PAT (next) == NULL_RTX)
    1326                 :            :         {
    1327                 :          0 :           ORIG_PAT (next) = PATTERN (next);
    1328                 :            : 
    1329                 :          0 :           new_pat = gen_rtx_COND_EXEC (VOIDmode, cond, PATTERN (next));
    1330                 :          0 :           success = haifa_change_pattern (next, new_pat);
    1331                 :          0 :           if (!success)
    1332                 :            :             return HARD_DEP;
    1333                 :          0 :           PREDICATED_PAT (next) = new_pat;
    1334                 :            :         }
    1335                 :          0 :       else if (PATTERN (next) != PREDICATED_PAT (next))
    1336                 :            :         {
    1337                 :          0 :           bool success = haifa_change_pattern (next,
    1338                 :          0 :                                                PREDICATED_PAT (next));
    1339                 :          0 :           gcc_assert (success);
    1340                 :            :         }
    1341                 :          0 :       DEP_STATUS (modify_dep) |= DEP_CANCELLED;
    1342                 :          0 :       return DEP_CONTROL;
    1343                 :            :     }
    1344                 :            : 
    1345                 :          0 :   if (PREDICATED_PAT (next) != NULL_RTX)
    1346                 :            :     {
    1347                 :          0 :       int tick = INSN_TICK (next);
    1348                 :          0 :       bool success = haifa_change_pattern (next,
    1349                 :          0 :                                            ORIG_PAT (next));
    1350                 :          0 :       INSN_TICK (next) = tick;
    1351                 :          0 :       gcc_assert (success);
    1352                 :            :     }
    1353                 :            : 
    1354                 :            :   /* We can't handle the case where there are both speculative and control
    1355                 :            :      dependencies, so we return HARD_DEP in such a case.  Also fail if
    1356                 :            :      we have speculative dependencies with not enough points, or more than
    1357                 :            :      one control dependency.  */
    1358                 :          0 :   if ((n_spec > 0 && (n_control > 0 || n_replace > 0))
    1359                 :          0 :       || (n_spec > 0
    1360                 :            :           /* Too few points?  */
    1361                 :          0 :           && ds_weak (new_ds) < spec_info->data_weakness_cutoff)
    1362                 :          0 :       || n_control > 0
    1363                 :          0 :       || n_replace > 0)
    1364                 :          0 :     return HARD_DEP;
    1365                 :            : 
    1366                 :            :   return new_ds;
    1367                 :            : }
    1368                 :            : 
    1369                 :            : /* Pointer to the last instruction scheduled.  */
    1370                 :            : static rtx_insn *last_scheduled_insn;
    1371                 :            : 
    1372                 :            : /* Pointer to the last nondebug instruction scheduled within the
    1373                 :            :    block, or the prev_head of the scheduling block.  Used by
    1374                 :            :    rank_for_schedule, so that insns independent of the last scheduled
    1375                 :            :    insn will be preferred over dependent instructions.  */
    1376                 :            : static rtx_insn *last_nondebug_scheduled_insn;
    1377                 :            : 
    1378                 :            : /* Pointer that iterates through the list of unscheduled insns if we
    1379                 :            :    have a dbg_cnt enabled.  It always points at an insn prior to the
    1380                 :            :    first unscheduled one.  */
    1381                 :            : static rtx_insn *nonscheduled_insns_begin;
    1382                 :            : 
    1383                 :            : /* Compute cost of executing INSN.
    1384                 :            :    This is the number of cycles between instruction issue and
    1385                 :            :    instruction results.  */
    1386                 :            : int
    1387                 :  136276000 : insn_sched_cost (rtx_insn *insn)
    1388                 :            : {
    1389                 :  136276000 :   int cost;
    1390                 :            : 
    1391                 :  136276000 :   if (sched_fusion)
    1392                 :            :     return 0;
    1393                 :            : 
    1394                 :  136276000 :   if (sel_sched_p ())
    1395                 :            :     {
    1396                 :      32585 :       if (recog_memoized (insn) < 0)
    1397                 :            :         return 0;
    1398                 :            : 
    1399                 :      31015 :       cost = insn_default_latency (insn);
    1400                 :      31015 :       if (cost < 0)
    1401                 :            :         cost = 0;
    1402                 :            : 
    1403                 :      31015 :       return cost;
    1404                 :            :     }
    1405                 :            : 
    1406                 :  136244000 :   cost = INSN_COST (insn);
    1407                 :            : 
    1408                 :  136244000 :   if (cost < 0)
    1409                 :            :     {
    1410                 :            :       /* A USE insn, or something else we don't need to
    1411                 :            :          understand.  We can't pass these directly to
    1412                 :            :          result_ready_cost or insn_default_latency because it will
    1413                 :            :          trigger a fatal error for unrecognizable insns.  */
    1414                 :   96662500 :       if (recog_memoized (insn) < 0)
    1415                 :            :         {
    1416                 :   28297000 :           INSN_COST (insn) = 0;
    1417                 :   28297000 :           return 0;
    1418                 :            :         }
    1419                 :            :       else
    1420                 :            :         {
    1421                 :   39038200 :           cost = insn_default_latency (insn);
    1422                 :   39038200 :           if (cost < 0)
    1423                 :            :             cost = 0;
    1424                 :            : 
    1425                 :   39038200 :           INSN_COST (insn) = cost;
    1426                 :            :         }
    1427                 :            :     }
    1428                 :            : 
    1429                 :            :   return cost;
    1430                 :            : }
    1431                 :            : 
    1432                 :            : /* Compute cost of dependence LINK.
    1433                 :            :    This is the number of cycles between instruction issue and
    1434                 :            :    instruction results.
    1435                 :            :    ??? We also use this function to call recog_memoized on all insns.  */
    1436                 :            : int
    1437                 :  226486000 : dep_cost_1 (dep_t link, dw_t dw)
    1438                 :            : {
    1439                 :  226486000 :   rtx_insn *insn = DEP_PRO (link);
    1440                 :  226486000 :   rtx_insn *used = DEP_CON (link);
    1441                 :  226486000 :   int cost;
    1442                 :            : 
    1443                 :  226486000 :   if (DEP_COST (link) != UNKNOWN_DEP_COST)
    1444                 :  124010000 :     return DEP_COST (link);
    1445                 :            : 
    1446                 :  102476000 :   if (delay_htab)
    1447                 :            :     {
    1448                 :          0 :       struct delay_pair *delay_entry;
    1449                 :          0 :       delay_entry
    1450                 :          0 :         = delay_htab_i2->find_with_hash (used, htab_hash_pointer (used));
    1451                 :          0 :       if (delay_entry)
    1452                 :            :         {
    1453                 :          0 :           if (delay_entry->i1 == insn)
    1454                 :            :             {
    1455                 :          0 :               DEP_COST (link) = pair_delay (delay_entry);
    1456                 :          0 :               return DEP_COST (link);
    1457                 :            :             }
    1458                 :            :         }
    1459                 :            :     }
    1460                 :            : 
    1461                 :            :   /* A USE insn should never require the value used to be computed.
    1462                 :            :      This allows the computation of a function's result and parameter
    1463                 :            :      values to overlap the return and call.  We don't care about the
    1464                 :            :      dependence cost when only decreasing register pressure.  */
    1465                 :  103467000 :   if (recog_memoized (used) < 0)
    1466                 :            :     {
    1467                 :     990935 :       cost = 0;
    1468                 :     990935 :       recog_memoized (insn);
    1469                 :            :     }
    1470                 :            :   else
    1471                 :            :     {
    1472                 :  101485000 :       enum reg_note dep_type = DEP_TYPE (link);
    1473                 :            : 
    1474                 :  101485000 :       cost = insn_sched_cost (insn);
    1475                 :            : 
    1476                 :  101485000 :       if (INSN_CODE (insn) >= 0)
    1477                 :            :         {
    1478                 :   98475200 :           if (dep_type == REG_DEP_ANTI)
    1479                 :            :             cost = 0;
    1480                 :   55678400 :           else if (dep_type == REG_DEP_OUTPUT)
    1481                 :            :             {
    1482                 :   15933600 :               cost = (insn_default_latency (insn)
    1483                 :   15933600 :                       - insn_default_latency (used));
    1484                 :   15933600 :               if (cost <= 0)
    1485                 :            :                 cost = 1;
    1486                 :            :             }
    1487                 :   39744800 :           else if (bypass_p (insn))
    1488                 :       4988 :             cost = insn_latency (insn, used);
    1489                 :            :         }
    1490                 :            : 
    1491                 :            : 
    1492                 :  101485000 :       if (targetm.sched.adjust_cost)
    1493                 :  101485000 :         cost = targetm.sched.adjust_cost (used, (int) dep_type, insn, cost,
    1494                 :            :                                           dw);
    1495                 :            : 
    1496                 :  101485000 :       if (cost < 0)
    1497                 :            :         cost = 0;
    1498                 :            :     }
    1499                 :            : 
    1500                 :  102476000 :   DEP_COST (link) = cost;
    1501                 :  102476000 :   return cost;
    1502                 :            : }
    1503                 :            : 
    1504                 :            : /* Compute cost of dependence LINK.
    1505                 :            :    This is the number of cycles between instruction issue and
    1506                 :            :    instruction results.  */
    1507                 :            : int
    1508                 :  226461000 : dep_cost (dep_t link)
    1509                 :            : {
    1510                 :  226461000 :   return dep_cost_1 (link, 0);
    1511                 :            : }
    1512                 :            : 
    1513                 :            : /* Use this sel-sched.c friendly function in reorder2 instead of increasing
    1514                 :            :    INSN_PRIORITY explicitly.  */
    1515                 :            : void
    1516                 :          0 : increase_insn_priority (rtx_insn *insn, int amount)
    1517                 :            : {
    1518                 :          0 :   if (!sel_sched_p ())
    1519                 :            :     {
    1520                 :            :       /* We're dealing with haifa-sched.c INSN_PRIORITY.  */
    1521                 :          0 :       if (INSN_PRIORITY_KNOWN (insn))
    1522                 :          0 :           INSN_PRIORITY (insn) += amount;
    1523                 :            :     }
    1524                 :            :   else
    1525                 :            :     {
    1526                 :            :       /* In sel-sched.c INSN_PRIORITY is not kept up to date.
    1527                 :            :          Use EXPR_PRIORITY instead. */
    1528                 :          0 :       sel_add_to_insn_priority (insn, amount);
    1529                 :            :     }
    1530                 :          0 : }
    1531                 :            : 
    1532                 :            : /* Return 'true' if DEP should be included in priority calculations.  */
    1533                 :            : static bool
    1534                 :  104308000 : contributes_to_priority_p (dep_t dep)
    1535                 :            : {
    1536                 :  104308000 :   if (DEBUG_INSN_P (DEP_CON (dep))
    1537                 :   96508600 :       || DEBUG_INSN_P (DEP_PRO (dep)))
    1538                 :            :     return false;
    1539                 :            : 
    1540                 :            :   /* Critical path is meaningful in block boundaries only.  */
    1541                 :   96508600 :   if (!current_sched_info->contributes_to_priority (DEP_CON (dep),
    1542                 :            :                                                     DEP_PRO (dep)))
    1543                 :            :     return false;
    1544                 :            : 
    1545                 :   96502200 :   if (DEP_REPLACE (dep) != NULL)
    1546                 :            :     return false;
    1547                 :            : 
    1548                 :            :   /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set,
    1549                 :            :      then speculative instructions will less likely be
    1550                 :            :      scheduled.  That is because the priority of
    1551                 :            :      their producers will increase, and, thus, the
    1552                 :            :      producers will more likely be scheduled, thus,
    1553                 :            :      resolving the dependence.  */
    1554                 :   96093800 :   if (sched_deps_info->generate_spec_deps
    1555                 :   96093800 :       && !(spec_info->flags & COUNT_SPEC_IN_CRITICAL_PATH)
    1556                 :          0 :       && (DEP_STATUS (dep) & SPECULATIVE))
    1557                 :          0 :     return false;
    1558                 :            : 
    1559                 :            :   return true;
    1560                 :            : }
    1561                 :            : 
    1562                 :            : /* Compute the number of nondebug deps in list LIST for INSN.  */
    1563                 :            : 
    1564                 :            : static int
    1565                 :  471686000 : dep_list_size (rtx_insn *insn, sd_list_types_def list)
    1566                 :            : {
    1567                 :  471686000 :   sd_iterator_def sd_it;
    1568                 :  471686000 :   dep_t dep;
    1569                 :  471686000 :   int dbgcount = 0, nodbgcount = 0;
    1570                 :            : 
    1571                 :  471686000 :   if (!MAY_HAVE_DEBUG_INSNS)
    1572                 :  164197000 :     return sd_lists_size (insn, list);
    1573                 :            : 
    1574                 : 1076200000 :   FOR_EACH_DEP (insn, list, sd_it, dep)
    1575                 :            :     {
    1576                 :  768709000 :       if (DEBUG_INSN_P (DEP_CON (dep)))
    1577                 :   54423100 :         dbgcount++;
    1578                 :  714286000 :       else if (!DEBUG_INSN_P (DEP_PRO (dep)))
    1579                 :  711676000 :         nodbgcount++;
    1580                 :            :     }
    1581                 :            : 
    1582                 :  307489000 :   gcc_assert (dbgcount + nodbgcount == sd_lists_size (insn, list));
    1583                 :            : 
    1584                 :            :   return nodbgcount;
    1585                 :            : }
    1586                 :            : 
    1587                 :            : bool sched_fusion;
    1588                 :            : 
    1589                 :            : /* Compute the priority number for INSN.  */
    1590                 :            : static int
    1591                 :  163448000 : priority (rtx_insn *insn, bool force_recompute)
    1592                 :            : {
    1593                 :  163448000 :   if (! INSN_P (insn))
    1594                 :            :     return 0;
    1595                 :            : 
    1596                 :            :   /* We should not be interested in priority of an already scheduled insn.  */
    1597                 :  163448000 :   gcc_assert (QUEUE_INDEX (insn) != QUEUE_SCHEDULED);
    1598                 :            : 
    1599                 :  163448000 :   if (force_recompute || !INSN_PRIORITY_KNOWN (insn))
    1600                 :            :     {
    1601                 :   67354400 :       int this_priority = -1;
    1602                 :            : 
    1603                 :   67354400 :       if (sched_fusion)
    1604                 :            :         {
    1605                 :          0 :           int this_fusion_priority;
    1606                 :            : 
    1607                 :          0 :           targetm.sched.fusion_priority (insn, FUSION_MAX_PRIORITY,
    1608                 :            :                                          &this_fusion_priority, &this_priority);
    1609                 :          0 :           INSN_FUSION_PRIORITY (insn) = this_fusion_priority;
    1610                 :            :         }
    1611                 :   67354400 :       else if (dep_list_size (insn, SD_LIST_FORW) == 0)
    1612                 :            :         /* ??? We should set INSN_PRIORITY to insn_sched_cost when and insn
    1613                 :            :            has some forward deps but all of them are ignored by
    1614                 :            :            contributes_to_priority hook.  At the moment we set priority of
    1615                 :            :            such insn to 0.  */
    1616                 :   34783400 :         this_priority = insn_sched_cost (insn);
    1617                 :            :       else
    1618                 :            :         {
    1619                 :   32571000 :           rtx_insn *prev_first, *twin;
    1620                 :   32571000 :           basic_block rec;
    1621                 :            : 
    1622                 :            :           /* For recovery check instructions we calculate priority slightly
    1623                 :            :              different than that of normal instructions.  Instead of walking
    1624                 :            :              through INSN_FORW_DEPS (check) list, we walk through
    1625                 :            :              INSN_FORW_DEPS list of each instruction in the corresponding
    1626                 :            :              recovery block.  */
    1627                 :            : 
    1628                 :            :           /* Selective scheduling does not define RECOVERY_BLOCK macro.  */
    1629                 :   65138800 :           rec = sel_sched_p () ? NULL : RECOVERY_BLOCK (insn);
    1630                 :   32567800 :           if (!rec || rec == EXIT_BLOCK_PTR_FOR_FN (cfun))
    1631                 :            :             {
    1632                 :   32571000 :               prev_first = PREV_INSN (insn);
    1633                 :   32571000 :               twin = insn;
    1634                 :            :             }
    1635                 :            :           else
    1636                 :            :             {
    1637                 :          0 :               prev_first = NEXT_INSN (BB_HEAD (rec));
    1638                 :          0 :               twin = PREV_INSN (BB_END (rec));
    1639                 :            :             }
    1640                 :            : 
    1641                 :   32571000 :           do
    1642                 :            :             {
    1643                 :   32571000 :               sd_iterator_def sd_it;
    1644                 :   32571000 :               dep_t dep;
    1645                 :            : 
    1646                 :  136879000 :               FOR_EACH_DEP (twin, SD_LIST_FORW, sd_it, dep)
    1647                 :            :                 {
    1648                 :  104308000 :                   rtx_insn *next;
    1649                 :  104308000 :                   int next_priority;
    1650                 :            : 
    1651                 :  104308000 :                   next = DEP_CON (dep);
    1652                 :            : 
    1653                 :  104308000 :                   if (BLOCK_FOR_INSN (next) != rec)
    1654                 :            :                     {
    1655                 :  104308000 :                       int cost;
    1656                 :            : 
    1657                 :  104308000 :                       if (!contributes_to_priority_p (dep))
    1658                 :    8214120 :                         continue;
    1659                 :            : 
    1660                 :   96093800 :                       if (twin == insn)
    1661                 :   96093800 :                         cost = dep_cost (dep);
    1662                 :            :                       else
    1663                 :            :                         {
    1664                 :          0 :                           struct _dep _dep1, *dep1 = &_dep1;
    1665                 :            : 
    1666                 :          0 :                           init_dep (dep1, insn, next, REG_DEP_ANTI);
    1667                 :            : 
    1668                 :          0 :                           cost = dep_cost (dep1);
    1669                 :            :                         }
    1670                 :            : 
    1671                 :   96093800 :                       next_priority = cost + priority (next);
    1672                 :            : 
    1673                 :   96093800 :                       if (next_priority > this_priority)
    1674                 :   67840900 :                         this_priority = next_priority;
    1675                 :            :                     }
    1676                 :            :                 }
    1677                 :            : 
    1678                 :   32571000 :               twin = PREV_INSN (twin);
    1679                 :            :             }
    1680                 :   32571000 :           while (twin != prev_first);
    1681                 :            :         }
    1682                 :            : 
    1683                 :   67354400 :       if (this_priority < 0)
    1684                 :            :         {
    1685                 :       7505 :           gcc_assert (this_priority == -1);
    1686                 :            : 
    1687                 :       7505 :           this_priority = insn_sched_cost (insn);
    1688                 :            :         }
    1689                 :            : 
    1690                 :   67354400 :       INSN_PRIORITY (insn) = this_priority;
    1691                 :   67354400 :       INSN_PRIORITY_STATUS (insn) = 1;
    1692                 :            :     }
    1693                 :            : 
    1694                 :  163448000 :   return INSN_PRIORITY (insn);
    1695                 :            : }
    1696                 :            : 
    1697                 :            : /* Macros and functions for keeping the priority queue sorted, and
    1698                 :            :    dealing with queuing and dequeuing of instructions.  */
    1699                 :            : 
    1700                 :            : /* For each pressure class CL, set DEATH[CL] to the number of registers
    1701                 :            :    in that class that die in INSN.  */
    1702                 :            : 
    1703                 :            : static void
    1704                 :      14386 : calculate_reg_deaths (rtx_insn *insn, int *death)
    1705                 :            : {
    1706                 :      14386 :   int i;
    1707                 :      14386 :   struct reg_use_data *use;
    1708                 :            : 
    1709                 :      71930 :   for (i = 0; i < ira_pressure_classes_num; i++)
    1710                 :      57544 :     death[ira_pressure_classes[i]] = 0;
    1711                 :      24280 :   for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
    1712                 :       9894 :     if (dying_use_p (use))
    1713                 :       8925 :       mark_regno_birth_or_death (0, death, use->regno, true);
    1714                 :      14386 : }
    1715                 :            : 
    1716                 :            : /* Setup info about the current register pressure impact of scheduling
    1717                 :            :    INSN at the current scheduling point.  */
    1718                 :            : static void
    1719                 :      14386 : setup_insn_reg_pressure_info (rtx_insn *insn)
    1720                 :            : {
    1721                 :      14386 :   int i, change, before, after, hard_regno;
    1722                 :      14386 :   int excess_cost_change;
    1723                 :      14386 :   machine_mode mode;
    1724                 :      14386 :   enum reg_class cl;
    1725                 :      14386 :   struct reg_pressure_data *pressure_info;
    1726                 :      14386 :   int *max_reg_pressure;
    1727                 :      14386 :   static int death[N_REG_CLASSES];
    1728                 :            : 
    1729                 :      14386 :   gcc_checking_assert (!DEBUG_INSN_P (insn));
    1730                 :            : 
    1731                 :      14386 :   excess_cost_change = 0;
    1732                 :      14386 :   calculate_reg_deaths (insn, death);
    1733                 :      14386 :   pressure_info = INSN_REG_PRESSURE (insn);
    1734                 :      14386 :   max_reg_pressure = INSN_MAX_REG_PRESSURE (insn);
    1735                 :      14386 :   gcc_assert (pressure_info != NULL && max_reg_pressure != NULL);
    1736                 :      71930 :   for (i = 0; i < ira_pressure_classes_num; i++)
    1737                 :            :     {
    1738                 :      57544 :       cl = ira_pressure_classes[i];
    1739                 :      57544 :       gcc_assert (curr_reg_pressure[cl] >= 0);
    1740                 :      57544 :       change = (int) pressure_info[i].set_increase - death[cl];
    1741                 :      57544 :       before = MAX (0, max_reg_pressure[i] - sched_class_regs_num[cl]);
    1742                 :      57544 :       after = MAX (0, max_reg_pressure[i] + change
    1743                 :            :                    - sched_class_regs_num[cl]);
    1744                 :      57544 :       hard_regno = ira_class_hard_regs[cl][0];
    1745                 :      57544 :       gcc_assert (hard_regno >= 0);
    1746                 :      57544 :       mode = reg_raw_mode[hard_regno];
    1747                 :      57544 :       excess_cost_change += ((after - before)
    1748                 :      57544 :                              * (ira_memory_move_cost[mode][cl][0]
    1749                 :      57544 :                                 + ira_memory_move_cost[mode][cl][1]));
    1750                 :            :     }
    1751                 :      14386 :   INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn) = excess_cost_change;
    1752                 :      14386 : }
    1753                 :            : 
    1754                 :            : /* This is the first page of code related to SCHED_PRESSURE_MODEL.
    1755                 :            :    It tries to make the scheduler take register pressure into account
    1756                 :            :    without introducing too many unnecessary stalls.  It hooks into the
    1757                 :            :    main scheduling algorithm at several points:
    1758                 :            : 
    1759                 :            :     - Before scheduling starts, model_start_schedule constructs a
    1760                 :            :       "model schedule" for the current block.  This model schedule is
    1761                 :            :       chosen solely to keep register pressure down.  It does not take the
    1762                 :            :       target's pipeline or the original instruction order into account,
    1763                 :            :       except as a tie-breaker.  It also doesn't work to a particular
    1764                 :            :       pressure limit.
    1765                 :            : 
    1766                 :            :       This model schedule gives us an idea of what pressure can be
    1767                 :            :       achieved for the block and gives us an example of a schedule that
    1768                 :            :       keeps to that pressure.  It also makes the final schedule less
    1769                 :            :       dependent on the original instruction order.  This is important
    1770                 :            :       because the original order can either be "wide" (many values live
    1771                 :            :       at once, such as in user-scheduled code) or "narrow" (few values
    1772                 :            :       live at once, such as after loop unrolling, where several
    1773                 :            :       iterations are executed sequentially).
    1774                 :            : 
    1775                 :            :       We do not apply this model schedule to the rtx stream.  We simply
    1776                 :            :       record it in model_schedule.  We also compute the maximum pressure,
    1777                 :            :       MP, that was seen during this schedule.
    1778                 :            : 
    1779                 :            :     - Instructions are added to the ready queue even if they require
    1780                 :            :       a stall.  The length of the stall is instead computed as:
    1781                 :            : 
    1782                 :            :          MAX (INSN_TICK (INSN) - clock_var, 0)
    1783                 :            : 
    1784                 :            :       (= insn_delay).  This allows rank_for_schedule to choose between
    1785                 :            :       introducing a deliberate stall or increasing pressure.
    1786                 :            : 
    1787                 :            :     - Before sorting the ready queue, model_set_excess_costs assigns
    1788                 :            :       a pressure-based cost to each ready instruction in the queue.
    1789                 :            :       This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE
    1790                 :            :       (ECC for short) and is effectively measured in cycles.
    1791                 :            : 
    1792                 :            :     - rank_for_schedule ranks instructions based on:
    1793                 :            : 
    1794                 :            :         ECC (insn) + insn_delay (insn)
    1795                 :            : 
    1796                 :            :       then as:
    1797                 :            : 
    1798                 :            :         insn_delay (insn)
    1799                 :            : 
    1800                 :            :       So, for example, an instruction X1 with an ECC of 1 that can issue
    1801                 :            :       now will win over an instruction X0 with an ECC of zero that would
    1802                 :            :       introduce a stall of one cycle.  However, an instruction X2 with an
    1803                 :            :       ECC of 2 that can issue now will lose to both X0 and X1.
    1804                 :            : 
    1805                 :            :     - When an instruction is scheduled, model_recompute updates the model
    1806                 :            :       schedule with the new pressures (some of which might now exceed the
    1807                 :            :       original maximum pressure MP).  model_update_limit_points then searches
    1808                 :            :       for the new point of maximum pressure, if not already known.  */
    1809                 :            : 
    1810                 :            : /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL
    1811                 :            :    from surrounding debug information.  */
    1812                 :            : #define MODEL_BAR \
    1813                 :            :   ";;\t\t+------------------------------------------------------\n"
    1814                 :            : 
    1815                 :            : /* Information about the pressure on a particular register class at a
    1816                 :            :    particular point of the model schedule.  */
    1817                 :            : struct model_pressure_data {
    1818                 :            :   /* The pressure at this point of the model schedule, or -1 if the
    1819                 :            :      point is associated with an instruction that has already been
    1820                 :            :      scheduled.  */
    1821                 :            :   int ref_pressure;
    1822                 :            : 
    1823                 :            :   /* The maximum pressure during or after this point of the model schedule.  */
    1824                 :            :   int max_pressure;
    1825                 :            : };
    1826                 :            : 
    1827                 :            : /* Per-instruction information that is used while building the model
    1828                 :            :    schedule.  Here, "schedule" refers to the model schedule rather
    1829                 :            :    than the main schedule.  */
    1830                 :            : struct model_insn_info {
    1831                 :            :   /* The instruction itself.  */
    1832                 :            :   rtx_insn *insn;
    1833                 :            : 
    1834                 :            :   /* If this instruction is in model_worklist, these fields link to the
    1835                 :            :      previous (higher-priority) and next (lower-priority) instructions
    1836                 :            :      in the list.  */
    1837                 :            :   struct model_insn_info *prev;
    1838                 :            :   struct model_insn_info *next;
    1839                 :            : 
    1840                 :            :   /* While constructing the schedule, QUEUE_INDEX describes whether an
    1841                 :            :      instruction has already been added to the schedule (QUEUE_SCHEDULED),
    1842                 :            :      is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE).
    1843                 :            :      old_queue records the value that QUEUE_INDEX had before scheduling
    1844                 :            :      started, so that we can restore it once the schedule is complete.  */
    1845                 :            :   int old_queue;
    1846                 :            : 
    1847                 :            :   /* The relative importance of an unscheduled instruction.  Higher
    1848                 :            :      values indicate greater importance.  */
    1849                 :            :   unsigned int model_priority;
    1850                 :            : 
    1851                 :            :   /* The length of the longest path of satisfied true dependencies
    1852                 :            :      that leads to this instruction.  */
    1853                 :            :   unsigned int depth;
    1854                 :            : 
    1855                 :            :   /* The length of the longest path of dependencies of any kind
    1856                 :            :      that leads from this instruction.  */
    1857                 :            :   unsigned int alap;
    1858                 :            : 
    1859                 :            :   /* The number of predecessor nodes that must still be scheduled.  */
    1860                 :            :   int unscheduled_preds;
    1861                 :            : };
    1862                 :            : 
    1863                 :            : /* Information about the pressure limit for a particular register class.
    1864                 :            :    This structure is used when applying a model schedule to the main
    1865                 :            :    schedule.  */
    1866                 :            : struct model_pressure_limit {
    1867                 :            :   /* The maximum register pressure seen in the original model schedule.  */
    1868                 :            :   int orig_pressure;
    1869                 :            : 
    1870                 :            :   /* The maximum register pressure seen in the current model schedule
    1871                 :            :      (which excludes instructions that have already been scheduled).  */
    1872                 :            :   int pressure;
    1873                 :            : 
    1874                 :            :   /* The point of the current model schedule at which PRESSURE is first
    1875                 :            :      reached.  It is set to -1 if the value needs to be recomputed.  */
    1876                 :            :   int point;
    1877                 :            : };
    1878                 :            : 
    1879                 :            : /* Describes a particular way of measuring register pressure.  */
    1880                 :            : struct model_pressure_group {
    1881                 :            :   /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI].  */
    1882                 :            :   struct model_pressure_limit limits[N_REG_CLASSES];
    1883                 :            : 
    1884                 :            :   /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure
    1885                 :            :      on register class ira_pressure_classes[PCI] at point POINT of the
    1886                 :            :      current model schedule.  A POINT of model_num_insns describes the
    1887                 :            :      pressure at the end of the schedule.  */
    1888                 :            :   struct model_pressure_data *model;
    1889                 :            : };
    1890                 :            : 
    1891                 :            : /* Index POINT gives the instruction at point POINT of the model schedule.
    1892                 :            :    This array doesn't change during main scheduling.  */
    1893                 :            : static vec<rtx_insn *> model_schedule;
    1894                 :            : 
    1895                 :            : /* The list of instructions in the model worklist, sorted in order of
    1896                 :            :    decreasing priority.  */
    1897                 :            : static struct model_insn_info *model_worklist;
    1898                 :            : 
    1899                 :            : /* Index I describes the instruction with INSN_LUID I.  */
    1900                 :            : static struct model_insn_info *model_insns;
    1901                 :            : 
    1902                 :            : /* The number of instructions in the model schedule.  */
    1903                 :            : static int model_num_insns;
    1904                 :            : 
    1905                 :            : /* The index of the first instruction in model_schedule that hasn't yet been
    1906                 :            :    added to the main schedule, or model_num_insns if all of them have.  */
    1907                 :            : static int model_curr_point;
    1908                 :            : 
    1909                 :            : /* Describes the pressure before each instruction in the model schedule.  */
    1910                 :            : static struct model_pressure_group model_before_pressure;
    1911                 :            : 
    1912                 :            : /* The first unused model_priority value (as used in model_insn_info).  */
    1913                 :            : static unsigned int model_next_priority;
    1914                 :            : 
    1915                 :            : 
    1916                 :            : /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP
    1917                 :            :    at point POINT of the model schedule.  */
    1918                 :            : #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \
    1919                 :            :   (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)])
    1920                 :            : 
    1921                 :            : /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or
    1922                 :            :    after point POINT of the model schedule.  */
    1923                 :            : #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \
    1924                 :            :   (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure)
    1925                 :            : 
    1926                 :            : /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT
    1927                 :            :    of the model schedule.  */
    1928                 :            : #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \
    1929                 :            :   (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure)
    1930                 :            : 
    1931                 :            : /* Information about INSN that is used when creating the model schedule.  */
    1932                 :            : #define MODEL_INSN_INFO(INSN) \
    1933                 :            :   (&model_insns[INSN_LUID (INSN)])
    1934                 :            : 
    1935                 :            : /* The instruction at point POINT of the model schedule.  */
    1936                 :            : #define MODEL_INSN(POINT) \
    1937                 :            :   (model_schedule[POINT])
    1938                 :            : 
    1939                 :            : 
    1940                 :            : /* Return INSN's index in the model schedule, or model_num_insns if it
    1941                 :            :    doesn't belong to that schedule.  */
    1942                 :            : 
    1943                 :            : static int
    1944                 :          0 : model_index (rtx_insn *insn)
    1945                 :            : {
    1946                 :          0 :   if (INSN_MODEL_INDEX (insn) == 0)
    1947                 :          0 :     return model_num_insns;
    1948                 :          0 :   return INSN_MODEL_INDEX (insn) - 1;
    1949                 :            : }
    1950                 :            : 
    1951                 :            : /* Make sure that GROUP->limits is up-to-date for the current point
    1952                 :            :    of the model schedule.  */
    1953                 :            : 
    1954                 :            : static void
    1955                 :          0 : model_update_limit_points_in_group (struct model_pressure_group *group)
    1956                 :            : {
    1957                 :          0 :   int pci, max_pressure, point;
    1958                 :            : 
    1959                 :          0 :   for (pci = 0; pci < ira_pressure_classes_num; pci++)
    1960                 :            :     {
    1961                 :            :       /* We may have passed the final point at which the pressure in
    1962                 :            :          group->limits[pci].pressure was reached.  Update the limit if so.  */
    1963                 :          0 :       max_pressure = MODEL_MAX_PRESSURE (group, model_curr_point, pci);
    1964                 :          0 :       group->limits[pci].pressure = max_pressure;
    1965                 :            : 
    1966                 :            :       /* Find the point at which MAX_PRESSURE is first reached.  We need
    1967                 :            :          to search in three cases:
    1968                 :            : 
    1969                 :            :          - We've already moved past the previous pressure point.
    1970                 :            :            In this case we search forward from model_curr_point.
    1971                 :            : 
    1972                 :            :          - We scheduled the previous point of maximum pressure ahead of
    1973                 :            :            its position in the model schedule, but doing so didn't bring
    1974                 :            :            the pressure point earlier.  In this case we search forward
    1975                 :            :            from that previous pressure point.
    1976                 :            : 
    1977                 :            :          - Scheduling an instruction early caused the maximum pressure
    1978                 :            :            to decrease.  In this case we will have set the pressure
    1979                 :            :            point to -1, and we search forward from model_curr_point.  */
    1980                 :          0 :       point = MAX (group->limits[pci].point, model_curr_point);
    1981                 :          0 :       while (point < model_num_insns
    1982                 :          0 :              && MODEL_REF_PRESSURE (group, point, pci) < max_pressure)
    1983                 :          0 :         point++;
    1984                 :          0 :       group->limits[pci].point = point;
    1985                 :            : 
    1986                 :          0 :       gcc_assert (MODEL_REF_PRESSURE (group, point, pci) == max_pressure);
    1987                 :          0 :       gcc_assert (MODEL_MAX_PRESSURE (group, point, pci) == max_pressure);
    1988                 :            :     }
    1989                 :          0 : }
    1990                 :            : 
    1991                 :            : /* Make sure that all register-pressure limits are up-to-date for the
    1992                 :            :    current position in the model schedule.  */
    1993                 :            : 
    1994                 :            : static void
    1995                 :          0 : model_update_limit_points (void)
    1996                 :            : {
    1997                 :          0 :   model_update_limit_points_in_group (&model_before_pressure);
    1998                 :          0 : }
    1999                 :            : 
    2000                 :            : /* Return the model_index of the last unscheduled use in chain USE
    2001                 :            :    outside of USE's instruction.  Return -1 if there are no other uses,
    2002                 :            :    or model_num_insns if the register is live at the end of the block.  */
    2003                 :            : 
    2004                 :            : static int
    2005                 :          0 : model_last_use_except (struct reg_use_data *use)
    2006                 :            : {
    2007                 :          0 :   struct reg_use_data *next;
    2008                 :          0 :   int last, index;
    2009                 :            : 
    2010                 :          0 :   last = -1;
    2011                 :          0 :   for (next = use->next_regno_use; next != use; next = next->next_regno_use)
    2012                 :          0 :     if (NONDEBUG_INSN_P (next->insn)
    2013                 :          0 :         && QUEUE_INDEX (next->insn) != QUEUE_SCHEDULED)
    2014                 :            :       {
    2015                 :          0 :         index = model_index (next->insn);
    2016                 :          0 :         if (index == model_num_insns)
    2017                 :          0 :           return model_num_insns;
    2018                 :          0 :         if (last < index)
    2019                 :            :           last = index;
    2020                 :            :       }
    2021                 :            :   return last;
    2022                 :            : }
    2023                 :            : 
    2024                 :            : /* An instruction with model_index POINT has just been scheduled, and it
    2025                 :            :    adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1.
    2026                 :            :    Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and
    2027                 :            :    MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly.  */
    2028                 :            : 
    2029                 :            : static void
    2030                 :          0 : model_start_update_pressure (struct model_pressure_group *group,
    2031                 :            :                              int point, int pci, int delta)
    2032                 :            : {
    2033                 :          0 :   int next_max_pressure;
    2034                 :            : 
    2035                 :          0 :   if (point == model_num_insns)
    2036                 :            :     {
    2037                 :            :       /* The instruction wasn't part of the model schedule; it was moved
    2038                 :            :          from a different block.  Update the pressure for the end of
    2039                 :            :          the model schedule.  */
    2040                 :          0 :       MODEL_REF_PRESSURE (group, point, pci) += delta;
    2041                 :          0 :       MODEL_MAX_PRESSURE (group, point, pci) += delta;
    2042                 :            :     }
    2043                 :            :   else
    2044                 :            :     {
    2045                 :            :       /* Record that this instruction has been scheduled.  Nothing now
    2046                 :            :          changes between POINT and POINT + 1, so get the maximum pressure
    2047                 :            :          from the latter.  If the maximum pressure decreases, the new
    2048                 :            :          pressure point may be before POINT.  */
    2049                 :          0 :       MODEL_REF_PRESSURE (group, point, pci) = -1;
    2050                 :          0 :       next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci);
    2051                 :          0 :       if (MODEL_MAX_PRESSURE (group, point, pci) > next_max_pressure)
    2052                 :            :         {
    2053                 :          0 :           MODEL_MAX_PRESSURE (group, point, pci) = next_max_pressure;
    2054                 :          0 :           if (group->limits[pci].point == point)
    2055                 :          0 :             group->limits[pci].point = -1;
    2056                 :            :         }
    2057                 :            :     }
    2058                 :          0 : }
    2059                 :            : 
    2060                 :            : /* Record that scheduling a later instruction has changed the pressure
    2061                 :            :    at point POINT of the model schedule by DELTA (which might be 0).
    2062                 :            :    Update GROUP accordingly.  Return nonzero if these changes might
    2063                 :            :    trigger changes to previous points as well.  */
    2064                 :            : 
    2065                 :            : static int
    2066                 :          0 : model_update_pressure (struct model_pressure_group *group,
    2067                 :            :                        int point, int pci, int delta)
    2068                 :            : {
    2069                 :          0 :   int ref_pressure, max_pressure, next_max_pressure;
    2070                 :            : 
    2071                 :            :   /* If POINT hasn't yet been scheduled, update its pressure.  */
    2072                 :          0 :   ref_pressure = MODEL_REF_PRESSURE (group, point, pci);
    2073                 :          0 :   if (ref_pressure >= 0 && delta != 0)
    2074                 :            :     {
    2075                 :          0 :       ref_pressure += delta;
    2076                 :          0 :       MODEL_REF_PRESSURE (group, point, pci) = ref_pressure;
    2077                 :            : 
    2078                 :            :       /* Check whether the maximum pressure in the overall schedule
    2079                 :            :          has increased.  (This means that the MODEL_MAX_PRESSURE of
    2080                 :            :          every point <= POINT will need to increase too; see below.)  */
    2081                 :          0 :       if (group->limits[pci].pressure < ref_pressure)
    2082                 :          0 :         group->limits[pci].pressure = ref_pressure;
    2083                 :            : 
    2084                 :            :       /* If we are at maximum pressure, and the maximum pressure
    2085                 :            :          point was previously unknown or later than POINT,
    2086                 :            :          bring it forward.  */
    2087                 :          0 :       if (group->limits[pci].pressure == ref_pressure
    2088                 :          0 :           && !IN_RANGE (group->limits[pci].point, 0, point))
    2089                 :          0 :         group->limits[pci].point = point;
    2090                 :            : 
    2091                 :            :       /* If POINT used to be the point of maximum pressure, but isn't
    2092                 :            :          any longer, we need to recalculate it using a forward walk.  */
    2093                 :          0 :       if (group->limits[pci].pressure > ref_pressure
    2094                 :          0 :           && group->limits[pci].point == point)
    2095                 :          0 :         group->limits[pci].point = -1;
    2096                 :            :     }
    2097                 :            : 
    2098                 :            :   /* Update the maximum pressure at POINT.  Changes here might also
    2099                 :            :      affect the maximum pressure at POINT - 1.  */
    2100                 :          0 :   next_max_pressure = MODEL_MAX_PRESSURE (group, point + 1, pci);
    2101                 :          0 :   max_pressure = MAX (ref_pressure, next_max_pressure);
    2102                 :          0 :   if (MODEL_MAX_PRESSURE (group, point, pci) != max_pressure)
    2103                 :            :     {
    2104                 :          0 :       MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
    2105                 :          0 :       return 1;
    2106                 :            :     }
    2107                 :            :   return 0;
    2108                 :            : }
    2109                 :            : 
    2110                 :            : /* INSN has just been scheduled.  Update the model schedule accordingly.  */
    2111                 :            : 
    2112                 :            : static void
    2113                 :          0 : model_recompute (rtx_insn *insn)
    2114                 :            : {
    2115                 :          0 :   struct {
    2116                 :            :     int last_use;
    2117                 :            :     int regno;
    2118                 :            :   } uses[FIRST_PSEUDO_REGISTER + MAX_RECOG_OPERANDS];
    2119                 :          0 :   struct reg_use_data *use;
    2120                 :          0 :   struct reg_pressure_data *reg_pressure;
    2121                 :          0 :   int delta[N_REG_CLASSES];
    2122                 :          0 :   int pci, point, mix, new_last, cl, ref_pressure, queue;
    2123                 :          0 :   unsigned int i, num_uses, num_pending_births;
    2124                 :          0 :   bool print_p;
    2125                 :            : 
    2126                 :            :   /* The destinations of INSN were previously live from POINT onwards, but are
    2127                 :            :      now live from model_curr_point onwards.  Set up DELTA accordingly.  */
    2128                 :          0 :   point = model_index (insn);
    2129                 :          0 :   reg_pressure = INSN_REG_PRESSURE (insn);
    2130                 :          0 :   for (pci = 0; pci < ira_pressure_classes_num; pci++)
    2131                 :            :     {
    2132                 :          0 :       cl = ira_pressure_classes[pci];
    2133                 :          0 :       delta[cl] = reg_pressure[pci].set_increase;
    2134                 :            :     }
    2135                 :            : 
    2136                 :            :   /* Record which registers previously died at POINT, but which now die
    2137                 :            :      before POINT.  Adjust DELTA so that it represents the effect of
    2138                 :            :      this change after POINT - 1.  Set NUM_PENDING_BIRTHS to the number of
    2139                 :            :      registers that will be born in the range [model_curr_point, POINT).  */
    2140                 :          0 :   num_uses = 0;
    2141                 :          0 :   num_pending_births = 0;
    2142                 :          0 :   bitmap_clear (tmp_bitmap);
    2143                 :          0 :   for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
    2144                 :            :     {
    2145                 :          0 :       new_last = model_last_use_except (use);
    2146                 :          0 :       if (new_last < point && bitmap_set_bit (tmp_bitmap, use->regno))
    2147                 :            :         {
    2148                 :          0 :           gcc_assert (num_uses < ARRAY_SIZE (uses));
    2149                 :          0 :           uses[num_uses].last_use = new_last;
    2150                 :          0 :           uses[num_uses].regno = use->regno;
    2151                 :            :           /* This register is no longer live after POINT - 1.  */
    2152                 :          0 :           mark_regno_birth_or_death (NULL, delta, use->regno, false);
    2153                 :          0 :           num_uses++;
    2154                 :          0 :           if (new_last >= 0)
    2155                 :          0 :             num_pending_births++;
    2156                 :            :         }
    2157                 :            :     }
    2158                 :            : 
    2159                 :            :   /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT.
    2160                 :            :      Also set each group pressure limit for POINT.  */
    2161                 :          0 :   for (pci = 0; pci < ira_pressure_classes_num; pci++)
    2162                 :            :     {
    2163                 :          0 :       cl = ira_pressure_classes[pci];
    2164                 :          0 :       model_start_update_pressure (&model_before_pressure,
    2165                 :            :                                    point, pci, delta[cl]);
    2166                 :            :     }
    2167                 :            : 
    2168                 :            :   /* Walk the model schedule backwards, starting immediately before POINT.  */
    2169                 :          0 :   print_p = false;
    2170                 :          0 :   if (point != model_curr_point)
    2171                 :          0 :     do
    2172                 :            :       {
    2173                 :          0 :         point--;
    2174                 :          0 :         insn = MODEL_INSN (point);
    2175                 :          0 :         queue = QUEUE_INDEX (insn);
    2176                 :            : 
    2177                 :          0 :         if (queue != QUEUE_SCHEDULED)
    2178                 :            :           {
    2179                 :            :             /* DELTA describes the effect of the move on the register pressure
    2180                 :            :                after POINT.  Make it describe the effect on the pressure
    2181                 :            :                before POINT.  */
    2182                 :            :             i = 0;
    2183                 :          0 :             while (i < num_uses)
    2184                 :            :               {
    2185                 :          0 :                 if (uses[i].last_use == point)
    2186                 :            :                   {
    2187                 :            :                     /* This register is now live again.  */
    2188                 :          0 :                     mark_regno_birth_or_death (NULL, delta,
    2189                 :            :                                                uses[i].regno, true);
    2190                 :            : 
    2191                 :            :                     /* Remove this use from the array.  */
    2192                 :          0 :                     uses[i] = uses[num_uses - 1];
    2193                 :          0 :                     num_uses--;
    2194                 :          0 :                     num_pending_births--;
    2195                 :            :                   }
    2196                 :            :                 else
    2197                 :          0 :                   i++;
    2198                 :            :               }
    2199                 :            : 
    2200                 :          0 :             if (sched_verbose >= 5)
    2201                 :            :               {
    2202                 :          0 :                 if (!print_p)
    2203                 :            :                   {
    2204                 :          0 :                     fprintf (sched_dump, MODEL_BAR);
    2205                 :          0 :                     fprintf (sched_dump, ";;\t\t| New pressure for model"
    2206                 :            :                              " schedule\n");
    2207                 :          0 :                     fprintf (sched_dump, MODEL_BAR);
    2208                 :          0 :                     print_p = true;
    2209                 :            :                   }
    2210                 :            : 
    2211                 :          0 :                 fprintf (sched_dump, ";;\t\t| %3d %4d %-30s ",
    2212                 :          0 :                          point, INSN_UID (insn),
    2213                 :          0 :                          str_pattern_slim (PATTERN (insn)));
    2214                 :          0 :                 for (pci = 0; pci < ira_pressure_classes_num; pci++)
    2215                 :            :                   {
    2216                 :          0 :                     cl = ira_pressure_classes[pci];
    2217                 :          0 :                     ref_pressure = MODEL_REF_PRESSURE (&model_before_pressure,
    2218                 :            :                                                        point, pci);
    2219                 :          0 :                     fprintf (sched_dump, " %s:[%d->%d]",
    2220                 :            :                              reg_class_names[ira_pressure_classes[pci]],
    2221                 :          0 :                              ref_pressure, ref_pressure + delta[cl]);
    2222                 :            :                   }
    2223                 :          0 :                 fprintf (sched_dump, "\n");
    2224                 :            :               }
    2225                 :            :           }
    2226                 :            : 
    2227                 :            :         /* Adjust the pressure at POINT.  Set MIX to nonzero if POINT - 1
    2228                 :            :            might have changed as well.  */
    2229                 :          0 :         mix = num_pending_births;
    2230                 :          0 :         for (pci = 0; pci < ira_pressure_classes_num; pci++)
    2231                 :            :           {
    2232                 :          0 :             cl = ira_pressure_classes[pci];
    2233                 :          0 :             mix |= delta[cl];
    2234                 :          0 :             mix |= model_update_pressure (&model_before_pressure,
    2235                 :            :                                           point, pci, delta[cl]);
    2236                 :            :           }
    2237                 :            :       }
    2238                 :          0 :     while (mix && point > model_curr_point);
    2239                 :            : 
    2240                 :          0 :   if (print_p)
    2241                 :          0 :     fprintf (sched_dump, MODEL_BAR);
    2242                 :          0 : }
    2243                 :            : 
    2244                 :            : /* After DEP, which was cancelled, has been resolved for insn NEXT,
    2245                 :            :    check whether the insn's pattern needs restoring.  */
    2246                 :            : static bool
    2247                 :     459156 : must_restore_pattern_p (rtx_insn *next, dep_t dep)
    2248                 :            : {
    2249                 :     459156 :   if (QUEUE_INDEX (next) == QUEUE_SCHEDULED)
    2250                 :            :     return false;
    2251                 :            : 
    2252                 :     140168 :   if (DEP_TYPE (dep) == REG_DEP_CONTROL)
    2253                 :            :     {
    2254                 :          0 :       gcc_assert (ORIG_PAT (next) != NULL_RTX);
    2255                 :          0 :       gcc_assert (next == DEP_CON (dep));
    2256                 :            :     }
    2257                 :            :   else
    2258                 :            :     {
    2259                 :     140168 :       struct dep_replacement *desc = DEP_REPLACE (dep);
    2260                 :     140168 :       if (desc->insn != next)
    2261                 :            :         {
    2262                 :     101773 :           gcc_assert (*desc->loc == desc->orig);
    2263                 :            :           return false;
    2264                 :            :         }
    2265                 :            :     }
    2266                 :            :   return true;
    2267                 :            : }
    2268                 :            : 
    2269                 :            : /* model_spill_cost (CL, P, P') returns the cost of increasing the
    2270                 :            :    pressure on CL from P to P'.  We use this to calculate a "base ECC",
    2271                 :            :    baseECC (CL, X), for each pressure class CL and each instruction X.
    2272                 :            :    Supposing X changes the pressure on CL from P to P', and that the
    2273                 :            :    maximum pressure on CL in the current model schedule is MP', then:
    2274                 :            : 
    2275                 :            :    * if X occurs before or at the next point of maximum pressure in
    2276                 :            :      the model schedule and P' > MP', then:
    2277                 :            : 
    2278                 :            :        baseECC (CL, X) = model_spill_cost (CL, MP, P')
    2279                 :            : 
    2280                 :            :      The idea is that the pressure after scheduling a fixed set of
    2281                 :            :      instructions -- in this case, the set up to and including the
    2282                 :            :      next maximum pressure point -- is going to be the same regardless
    2283                 :            :      of the order; we simply want to keep the intermediate pressure
    2284                 :            :      under control.  Thus X has a cost of zero unless scheduling it
    2285                 :            :      now would exceed MP'.
    2286                 :            : 
    2287                 :            :      If all increases in the set are by the same amount, no zero-cost
    2288                 :            :      instruction will ever cause the pressure to exceed MP'.  However,
    2289                 :            :      if X is instead moved past an instruction X' with pressure in the
    2290                 :            :      range (MP' - (P' - P), MP'), the pressure at X' will increase
    2291                 :            :      beyond MP'.  Since baseECC is very much a heuristic anyway,
    2292                 :            :      it doesn't seem worth the overhead of tracking cases like these.
    2293                 :            : 
    2294                 :            :      The cost of exceeding MP' is always based on the original maximum
    2295                 :            :      pressure MP.  This is so that going 2 registers over the original
    2296                 :            :      limit has the same cost regardless of whether it comes from two
    2297                 :            :      separate +1 deltas or from a single +2 delta.
    2298                 :            : 
    2299                 :            :    * if X occurs after the next point of maximum pressure in the model
    2300                 :            :      schedule and P' > P, then:
    2301                 :            : 
    2302                 :            :        baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P))
    2303                 :            : 
    2304                 :            :      That is, if we move X forward across a point of maximum pressure,
    2305                 :            :      and if X increases the pressure by P' - P, then we conservatively
    2306                 :            :      assume that scheduling X next would increase the maximum pressure
    2307                 :            :      by P' - P.  Again, the cost of doing this is based on the original
    2308                 :            :      maximum pressure MP, for the same reason as above.
    2309                 :            : 
    2310                 :            :    * if P' < P, P > MP, and X occurs at or after the next point of
    2311                 :            :      maximum pressure, then:
    2312                 :            : 
    2313                 :            :        baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P)
    2314                 :            : 
    2315                 :            :      That is, if we have already exceeded the original maximum pressure MP,
    2316                 :            :      and if X might reduce the maximum pressure again -- or at least push
    2317                 :            :      it further back, and thus allow more scheduling freedom -- it is given
    2318                 :            :      a negative cost to reflect the improvement.
    2319                 :            : 
    2320                 :            :    * otherwise,
    2321                 :            : 
    2322                 :            :        baseECC (CL, X) = 0
    2323                 :            : 
    2324                 :            :      In this case, X is not expected to affect the maximum pressure MP',
    2325                 :            :      so it has zero cost.
    2326                 :            : 
    2327                 :            :    We then create a combined value baseECC (X) that is the sum of
    2328                 :            :    baseECC (CL, X) for each pressure class CL.
    2329                 :            : 
    2330                 :            :    baseECC (X) could itself be used as the ECC value described above.
    2331                 :            :    However, this is often too conservative, in the sense that it
    2332                 :            :    tends to make high-priority instructions that increase pressure
    2333                 :            :    wait too long in cases where introducing a spill would be better.
    2334                 :            :    For this reason the final ECC is a priority-adjusted form of
    2335                 :            :    baseECC (X).  Specifically, we calculate:
    2336                 :            : 
    2337                 :            :      P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X)
    2338                 :            :      baseP = MAX { P (X) | baseECC (X) <= 0 }
    2339                 :            : 
    2340                 :            :    Then:
    2341                 :            : 
    2342                 :            :      ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0)
    2343                 :            : 
    2344                 :            :    Thus an instruction's effect on pressure is ignored if it has a high
    2345                 :            :    enough priority relative to the ones that don't increase pressure.
    2346                 :            :    Negative values of baseECC (X) do not increase the priority of X
    2347                 :            :    itself, but they do make it harder for other instructions to
    2348                 :            :    increase the pressure further.
    2349                 :            : 
    2350                 :            :    This pressure cost is deliberately timid.  The intention has been
    2351                 :            :    to choose a heuristic that rarely interferes with the normal list
    2352                 :            :    scheduler in cases where that scheduler would produce good code.
    2353                 :            :    We simply want to curb some of its worst excesses.  */
    2354                 :            : 
    2355                 :            : /* Return the cost of increasing the pressure in class CL from FROM to TO.
    2356                 :            : 
    2357                 :            :    Here we use the very simplistic cost model that every register above
    2358                 :            :    sched_class_regs_num[CL] has a spill cost of 1.  We could use other
    2359                 :            :    measures instead, such as one based on MEMORY_MOVE_COST.  However:
    2360                 :            : 
    2361                 :            :       (1) In order for an instruction to be scheduled, the higher cost
    2362                 :            :           would need to be justified in a single saving of that many stalls.
    2363                 :            :           This is overly pessimistic, because the benefit of spilling is
    2364                 :            :           often to avoid a sequence of several short stalls rather than
    2365                 :            :           a single long one.
    2366                 :            : 
    2367                 :            :       (2) The cost is still arbitrary.  Because we are not allocating
    2368                 :            :           registers during scheduling, we have no way of knowing for
    2369                 :            :           sure how many memory accesses will be required by each spill,
    2370                 :            :           where the spills will be placed within the block, or even
    2371                 :            :           which block(s) will contain the spills.
    2372                 :            : 
    2373                 :            :    So a higher cost than 1 is often too conservative in practice,
    2374                 :            :    forcing blocks to contain unnecessary stalls instead of spill code.
    2375                 :            :    The simple cost below seems to be the best compromise.  It reduces
    2376                 :            :    the interference with the normal list scheduler, which helps make
    2377                 :            :    it more suitable for a default-on option.  */
    2378                 :            : 
    2379                 :            : static int
    2380                 :          0 : model_spill_cost (int cl, int from, int to)
    2381                 :            : {
    2382                 :          0 :   from = MAX (from, sched_class_regs_num[cl]);
    2383                 :          0 :   return MAX (to, from) - from;
    2384                 :            : }
    2385                 :            : 
    2386                 :            : /* Return baseECC (ira_pressure_classes[PCI], POINT), given that
    2387                 :            :    P = curr_reg_pressure[ira_pressure_classes[PCI]] and that
    2388                 :            :    P' = P + DELTA.  */
    2389                 :            : 
    2390                 :            : static int
    2391                 :          0 : model_excess_group_cost (struct model_pressure_group *group,
    2392                 :            :                          int point, int pci, int delta)
    2393                 :            : {
    2394                 :          0 :   int pressure, cl;
    2395                 :            : 
    2396                 :          0 :   cl = ira_pressure_classes[pci];
    2397                 :          0 :   if (delta < 0 && point >= group->limits[pci].point)
    2398                 :            :     {
    2399                 :          0 :       pressure = MAX (group->limits[pci].orig_pressure,
    2400                 :            :                       curr_reg_pressure[cl] + delta);
    2401                 :          0 :       return -model_spill_cost (cl, pressure, curr_reg_pressure[cl]);
    2402                 :            :     }
    2403                 :            : 
    2404                 :          0 :   if (delta > 0)
    2405                 :            :     {
    2406                 :          0 :       if (point > group->limits[pci].point)
    2407                 :          0 :         pressure = group->limits[pci].pressure + delta;
    2408                 :            :       else
    2409                 :          0 :         pressure = curr_reg_pressure[cl] + delta;
    2410                 :            : 
    2411                 :          0 :       if (pressure > group->limits[pci].pressure)
    2412                 :          0 :         return model_spill_cost (cl, group->limits[pci].orig_pressure,
    2413                 :            :                                  pressure);
    2414                 :            :     }
    2415                 :            : 
    2416                 :            :   return 0;
    2417                 :            : }
    2418                 :            : 
    2419                 :            : /* Return baseECC (MODEL_INSN (INSN)).  Dump the costs to sched_dump
    2420                 :            :    if PRINT_P.  */
    2421                 :            : 
    2422                 :            : static int
    2423                 :          0 : model_excess_cost (rtx_insn *insn, bool print_p)
    2424                 :            : {
    2425                 :          0 :   int point, pci, cl, cost, this_cost, delta;
    2426                 :          0 :   struct reg_pressure_data *insn_reg_pressure;
    2427                 :          0 :   int insn_death[N_REG_CLASSES];
    2428                 :            : 
    2429                 :          0 :   calculate_reg_deaths (insn, insn_death);
    2430                 :          0 :   point = model_index (insn);
    2431                 :          0 :   insn_reg_pressure = INSN_REG_PRESSURE (insn);
    2432                 :          0 :   cost = 0;
    2433                 :            : 
    2434                 :          0 :   if (print_p)
    2435                 :          0 :     fprintf (sched_dump, ";;\t\t| %3d %4d | %4d %+3d |", point,
    2436                 :          0 :              INSN_UID (insn), INSN_PRIORITY (insn), insn_delay (insn));
    2437                 :            : 
    2438                 :            :   /* Sum up the individual costs for each register class.  */
    2439                 :          0 :   for (pci = 0; pci < ira_pressure_classes_num; pci++)
    2440                 :            :     {
    2441                 :          0 :       cl = ira_pressure_classes[pci];
    2442                 :          0 :       delta = insn_reg_pressure[pci].set_increase - insn_death[cl];
    2443                 :          0 :       this_cost = model_excess_group_cost (&model_before_pressure,
    2444                 :            :                                            point, pci, delta);
    2445                 :          0 :       cost += this_cost;
    2446                 :          0 :       if (print_p)
    2447                 :          0 :         fprintf (sched_dump, " %s:[%d base cost %d]",
    2448                 :            :                  reg_class_names[cl], delta, this_cost);
    2449                 :            :     }
    2450                 :            : 
    2451                 :          0 :   if (print_p)
    2452                 :          0 :     fprintf (sched_dump, "\n");
    2453                 :            : 
    2454                 :          0 :   return cost;
    2455                 :            : }
    2456                 :            : 
    2457                 :            : /* Dump the next points of maximum pressure for GROUP.  */
    2458                 :            : 
    2459                 :            : static void
    2460                 :          0 : model_dump_pressure_points (struct model_pressure_group *group)
    2461                 :            : {
    2462                 :          0 :   int pci, cl;
    2463                 :            : 
    2464                 :          0 :   fprintf (sched_dump, ";;\t\t|  pressure points");
    2465                 :          0 :   for (pci = 0; pci < ira_pressure_classes_num; pci++)
    2466                 :            :     {
    2467                 :          0 :       cl = ira_pressure_classes[pci];
    2468                 :          0 :       fprintf (sched_dump, " %s:[%d->%d at ", reg_class_names[cl],
    2469                 :            :                curr_reg_pressure[cl], group->limits[pci].pressure);
    2470                 :          0 :       if (group->limits[pci].point < model_num_insns)
    2471                 :          0 :         fprintf (sched_dump, "%d:%d]", group->limits[pci].point,
    2472                 :          0 :                  INSN_UID (MODEL_INSN (group->limits[pci].point)));
    2473                 :            :       else
    2474                 :          0 :         fprintf (sched_dump, "end]");
    2475                 :            :     }
    2476                 :          0 :   fprintf (sched_dump, "\n");
    2477                 :          0 : }
    2478                 :            : 
    2479                 :            : /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1].  */
    2480                 :            : 
    2481                 :            : static void
    2482                 :          0 : model_set_excess_costs (rtx_insn **insns, int count)
    2483                 :            : {
    2484                 :          0 :   int i, cost, priority_base, priority;
    2485                 :          0 :   bool print_p;
    2486                 :            : 
    2487                 :            :   /* Record the baseECC value for each instruction in the model schedule,
    2488                 :            :      except that negative costs are converted to zero ones now rather than
    2489                 :            :      later.  Do not assign a cost to debug instructions, since they must
    2490                 :            :      not change code-generation decisions.  Experiments suggest we also
    2491                 :            :      get better results by not assigning a cost to instructions from
    2492                 :            :      a different block.
    2493                 :            : 
    2494                 :            :      Set PRIORITY_BASE to baseP in the block comment above.  This is the
    2495                 :            :      maximum priority of the "cheap" instructions, which should always
    2496                 :            :      include the next model instruction.  */
    2497                 :          0 :   priority_base = 0;
    2498                 :          0 :   print_p = false;
    2499                 :          0 :   for (i = 0; i < count; i++)
    2500                 :          0 :     if (INSN_MODEL_INDEX (insns[i]))
    2501                 :            :       {
    2502                 :          0 :         if (sched_verbose >= 6 && !print_p)
    2503                 :            :           {
    2504                 :          0 :             fprintf (sched_dump, MODEL_BAR);
    2505                 :          0 :             fprintf (sched_dump, ";;\t\t| Pressure costs for ready queue\n");
    2506                 :          0 :             model_dump_pressure_points (&model_before_pressure);
    2507                 :          0 :             fprintf (sched_dump, MODEL_BAR);
    2508                 :          0 :             print_p = true;
    2509                 :            :           }
    2510                 :          0 :         cost = model_excess_cost (insns[i], print_p);
    2511                 :          0 :         if (cost <= 0)
    2512                 :            :           {
    2513                 :          0 :             priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]) - cost;
    2514                 :          0 :             priority_base = MAX (priority_base, priority);
    2515                 :            :             cost = 0;
    2516                 :            :           }
    2517                 :          0 :         INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = cost;
    2518                 :            :       }
    2519                 :          0 :   if (print_p)
    2520                 :          0 :     fprintf (sched_dump, MODEL_BAR);
    2521                 :            : 
    2522                 :            :   /* Use MAX (baseECC, 0) and baseP to calculcate ECC for each
    2523                 :            :      instruction.  */
    2524                 :          0 :   for (i = 0; i < count; i++)
    2525                 :            :     {
    2526                 :          0 :       cost = INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]);
    2527                 :          0 :       priority = INSN_PRIORITY (insns[i]) - insn_delay (insns[i]);
    2528                 :          0 :       if (cost > 0 && priority > priority_base)
    2529                 :            :         {
    2530                 :          0 :           cost += priority_base - priority;
    2531                 :          0 :           INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns[i]) = MAX (cost, 0);
    2532                 :            :         }
    2533                 :            :     }
    2534                 :          0 : }
    2535                 :            : 
    2536                 :            : 
    2537                 :            : /* Enum of rank_for_schedule heuristic decisions.  */
    2538                 :            : enum rfs_decision {
    2539                 :            :   RFS_LIVE_RANGE_SHRINK1, RFS_LIVE_RANGE_SHRINK2,
    2540                 :            :   RFS_SCHED_GROUP, RFS_PRESSURE_DELAY, RFS_PRESSURE_TICK,
    2541                 :            :   RFS_FEEDS_BACKTRACK_INSN, RFS_PRIORITY, RFS_SPECULATION,
    2542                 :            :   RFS_SCHED_RANK, RFS_LAST_INSN, RFS_PRESSURE_INDEX,
    2543                 :            :   RFS_DEP_COUNT, RFS_TIE, RFS_FUSION, RFS_COST, RFS_N };
    2544                 :            : 
    2545                 :            : /* Corresponding strings for print outs.  */
    2546                 :            : static const char *rfs_str[RFS_N] = {
    2547                 :            :   "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
    2548                 :            :   "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
    2549                 :            :   "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_SPECULATION",
    2550                 :            :   "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
    2551                 :            :   "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION", "RFS_COST" };
    2552                 :            : 
    2553                 :            : /* Statistical breakdown of rank_for_schedule decisions.  */
    2554                 :            : struct rank_for_schedule_stats_t { unsigned stats[RFS_N]; };
    2555                 :            : static rank_for_schedule_stats_t rank_for_schedule_stats;
    2556                 :            : 
    2557                 :            : /* Return the result of comparing insns TMP and TMP2 and update
    2558                 :            :    Rank_For_Schedule statistics.  */
    2559                 :            : static int
    2560                 :  255008000 : rfs_result (enum rfs_decision decision, int result, rtx tmp, rtx tmp2)
    2561                 :            : {
    2562                 :  255008000 :   ++rank_for_schedule_stats.stats[decision];
    2563                 :  255008000 :   if (result < 0)
    2564                 :  141516000 :     INSN_LAST_RFS_WIN (tmp) = decision;
    2565                 :  113492000 :   else if (result > 0)
    2566                 :  113492000 :     INSN_LAST_RFS_WIN (tmp2) = decision;
    2567                 :            :   else
    2568                 :          0 :     gcc_unreachable ();
    2569                 :  255008000 :   return result;
    2570                 :            : }
    2571                 :            : 
    2572                 :            : /* Sorting predicate to move DEBUG_INSNs to the top of ready list, while
    2573                 :            :    keeping normal insns in original order.  */
    2574                 :            : 
    2575                 :            : static int
    2576                 :  247185000 : rank_for_schedule_debug (const void *x, const void *y)
    2577                 :            : {
    2578                 :  247185000 :   rtx_insn *tmp = *(rtx_insn * const *) y;
    2579                 :  247185000 :   rtx_insn *tmp2 = *(rtx_insn * const *) x;
    2580                 :            : 
    2581                 :            :   /* Schedule debug insns as early as possible.  */
    2582                 :  247185000 :   if (DEBUG_INSN_P (tmp) && !DEBUG_INSN_P (tmp2))
    2583                 :            :     return -1;
    2584                 :  160665000 :   else if (!DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
    2585                 :            :     return 1;
    2586                 :  122917000 :   else if (DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
    2587                 :          0 :     return INSN_LUID (tmp) - INSN_LUID (tmp2);
    2588                 :            :   else
    2589                 :  122917000 :     return INSN_RFS_DEBUG_ORIG_ORDER (tmp2) - INSN_RFS_DEBUG_ORIG_ORDER (tmp);
    2590                 :            : }
    2591                 :            : 
    2592                 :            : /* Returns a positive value if x is preferred; returns a negative value if
    2593                 :            :    y is preferred.  Should never return 0, since that will make the sort
    2594                 :            :    unstable.  */
    2595                 :            : 
    2596                 :            : static int
    2597                 :  255008000 : rank_for_schedule (const void *x, const void *y)
    2598                 :            : {
    2599                 :  255008000 :   rtx_insn *tmp = *(rtx_insn * const *) y;
    2600                 :  255008000 :   rtx_insn *tmp2 = *(rtx_insn * const *) x;
    2601                 :  255008000 :   int tmp_class, tmp2_class;
    2602                 :  255008000 :   int val, priority_val, info_val, diff;
    2603                 :            : 
    2604                 :  255008000 :   if (live_range_shrinkage_p)
    2605                 :            :     {
    2606                 :            :       /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse
    2607                 :            :          code.  */
    2608                 :       3763 :       gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED);
    2609                 :       3763 :       if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp) < 0
    2610                 :       3653 :            || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2) < 0)
    2611                 :       3852 :           && (diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp)
    2612                 :        199 :                       - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2))) != 0)
    2613                 :        199 :         return rfs_result (RFS_LIVE_RANGE_SHRINK1, diff, tmp, tmp2);
    2614                 :            :       /* Sort by INSN_LUID (original insn order), so that we make the
    2615                 :            :          sort stable.  This minimizes instruction movement, thus
    2616                 :            :          minimizing sched's effect on debugging and cross-jumping.  */
    2617                 :       3564 :       return rfs_result (RFS_LIVE_RANGE_SHRINK2,
    2618                 :       3564 :                          INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
    2619                 :            :     }
    2620                 :            : 
    2621                 :            :   /* The insn in a schedule group should be issued the first.  */
    2622                 :  255004000 :   if (flag_sched_group_heuristic &&
    2623                 :  255004000 :       SCHED_GROUP_P (tmp) != SCHED_GROUP_P (tmp2))
    2624                 :        256 :     return rfs_result (RFS_SCHED_GROUP, SCHED_GROUP_P (tmp2) ? 1 : -1,
    2625                 :        176 :                        tmp, tmp2);
    2626                 :            : 
    2627                 :            :   /* Make sure that priority of TMP and TMP2 are initialized.  */
    2628                 :  510009000 :   gcc_assert (INSN_PRIORITY_KNOWN (tmp) && INSN_PRIORITY_KNOWN (tmp2));
    2629                 :            : 
    2630                 :  255004000 :   if (sched_fusion)
    2631                 :            :     {
    2632                 :            :       /* The instruction that has the same fusion priority as the last
    2633                 :            :          instruction is the instruction we picked next.  If that is not
    2634                 :            :          the case, we sort ready list firstly by fusion priority, then
    2635                 :            :          by priority, and at last by INSN_LUID.  */
    2636                 :          0 :       int a = INSN_FUSION_PRIORITY (tmp);
    2637                 :          0 :       int b = INSN_FUSION_PRIORITY (tmp2);
    2638                 :          0 :       int last = -1;
    2639                 :            : 
    2640                 :          0 :       if (last_nondebug_scheduled_insn
    2641                 :          0 :           && !NOTE_P (last_nondebug_scheduled_insn)
    2642                 :          0 :           && BLOCK_FOR_INSN (tmp)
    2643                 :          0 :                == BLOCK_FOR_INSN (last_nondebug_scheduled_insn))
    2644                 :          0 :         last = INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn);
    2645                 :            : 
    2646                 :          0 :       if (a != last && b != last)
    2647                 :            :         {
    2648                 :          0 :           if (a == b)
    2649                 :            :             {
    2650                 :          0 :               a = INSN_PRIORITY (tmp);
    2651                 :          0 :               b = INSN_PRIORITY (tmp2);
    2652                 :            :             }
    2653                 :          0 :           if (a != b)
    2654                 :          0 :             return rfs_result (RFS_FUSION, b - a, tmp, tmp2);
    2655                 :            :           else
    2656                 :          0 :             return rfs_result (RFS_FUSION,
    2657                 :          0 :                                INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
    2658                 :            :         }
    2659                 :          0 :       else if (a == b)
    2660                 :            :         {
    2661                 :          0 :           gcc_assert (last_nondebug_scheduled_insn
    2662                 :            :                       && !NOTE_P (last_nondebug_scheduled_insn));
    2663                 :          0 :           last = INSN_PRIORITY (last_nondebug_scheduled_insn);
    2664                 :            : 
    2665                 :          0 :           a = abs (INSN_PRIORITY (tmp) - last);
    2666                 :          0 :           b = abs (INSN_PRIORITY (tmp2) - last);
    2667                 :          0 :           if (a != b)
    2668                 :          0 :             return rfs_result (RFS_FUSION, a - b, tmp, tmp2);
    2669                 :            :           else
    2670                 :          0 :             return rfs_result (RFS_FUSION,
    2671                 :          0 :                                INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
    2672                 :            :         }
    2673                 :          0 :       else if (a == last)
    2674                 :          0 :         return rfs_result (RFS_FUSION, -1, tmp, tmp2);
    2675                 :            :       else
    2676                 :          0 :         return rfs_result (RFS_FUSION, 1, tmp, tmp2);
    2677                 :            :     }
    2678                 :            : 
    2679                 :  255004000 :   if (sched_pressure != SCHED_PRESSURE_NONE)
    2680                 :            :     {
    2681                 :            :       /* Prefer insn whose scheduling results in the smallest register
    2682                 :            :          pressure excess.  */
    2683                 :      70006 :       if ((diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp)
    2684                 :      35003 :                    + insn_delay (tmp)
    2685                 :      35003 :                    - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2)
    2686                 :      35003 :                    - insn_delay (tmp2))))
    2687                 :      11426 :         return rfs_result (RFS_PRESSURE_DELAY, diff, tmp, tmp2);
    2688                 :            :     }
    2689                 :            : 
    2690                 :  254993000 :   if (sched_pressure != SCHED_PRESSURE_NONE
    2691                 :      23577 :       && (INSN_TICK (tmp2) > clock_var || INSN_TICK (tmp) > clock_var)
    2692                 :  254995000 :       && INSN_TICK (tmp2) != INSN_TICK (tmp))
    2693                 :            :     {
    2694                 :          0 :       diff = INSN_TICK (tmp) - INSN_TICK (tmp2);
    2695                 :          0 :       return rfs_result (RFS_PRESSURE_TICK, diff, tmp, tmp2);
    2696                 :            :     }
    2697                 :            : 
    2698                 :            :   /* If we are doing backtracking in this schedule, prefer insns that
    2699                 :            :      have forward dependencies with negative cost against an insn that
    2700                 :            :      was already scheduled.  */
    2701                 :  254993000 :   if (current_sched_info->flags & DO_BACKTRACKING)
    2702                 :            :     {
    2703                 :          0 :       priority_val = FEEDS_BACKTRACK_INSN (tmp2) - FEEDS_BACKTRACK_INSN (tmp);
    2704                 :          0 :       if (priority_val)
    2705                 :          0 :         return rfs_result (RFS_FEEDS_BACKTRACK_INSN, priority_val, tmp, tmp2);
    2706                 :            :     }
    2707                 :            : 
    2708                 :            :   /* Prefer insn with higher priority.  */
    2709                 :  254993000 :   priority_val = INSN_PRIORITY (tmp2) - INSN_PRIORITY (tmp);
    2710                 :            : 
    2711                 :  254993000 :   if (flag_sched_critical_path_heuristic && priority_val)
    2712                 :   50991600 :     return rfs_result (RFS_PRIORITY, priority_val, tmp, tmp2);
    2713                 :            : 
    2714                 :  204001000 :   if (param_sched_autopref_queue_depth >= 0)
    2715                 :            :     {
    2716                 :          0 :       int autopref = autopref_rank_for_schedule (tmp, tmp2);
    2717                 :          0 :       if (autopref != 0)
    2718                 :            :         return autopref;
    2719                 :            :     }
    2720                 :            : 
    2721                 :            :   /* Prefer speculative insn with greater dependencies weakness.  */
    2722                 :  204001000 :   if (flag_sched_spec_insn_heuristic && spec_info)
    2723                 :            :     {
    2724                 :          0 :       ds_t ds1, ds2;
    2725                 :          0 :       dw_t dw1, dw2;
    2726                 :          0 :       int dw;
    2727                 :            : 
    2728                 :          0 :       ds1 = TODO_SPEC (tmp) & SPECULATIVE;
    2729                 :          0 :       if (ds1)
    2730                 :          0 :         dw1 = ds_weak (ds1);
    2731                 :            :       else
    2732                 :            :         dw1 = NO_DEP_WEAK;
    2733                 :            : 
    2734                 :          0 :       ds2 = TODO_SPEC (tmp2) & SPECULATIVE;
    2735                 :          0 :       if (ds2)
    2736                 :          0 :         dw2 = ds_weak (ds2);
    2737                 :            :       else
    2738                 :            :         dw2 = NO_DEP_WEAK;
    2739                 :            : 
    2740                 :          0 :       dw = dw2 - dw1;
    2741                 :          0 :       if (dw > (NO_DEP_WEAK / 8) || dw < -(NO_DEP_WEAK / 8))
    2742                 :          0 :         return rfs_result (RFS_SPECULATION, dw, tmp, tmp2);
    2743                 :            :     }
    2744                 :            : 
    2745                 :  204001000 :   info_val = (*current_sched_info->rank) (tmp, tmp2);
    2746                 :  204001000 :   if (flag_sched_rank_heuristic && info_val)
    2747                 :          1 :     return rfs_result (RFS_SCHED_RANK, info_val, tmp, tmp2);
    2748                 :            : 
    2749                 :            :   /* Compare insns based on their relation to the last scheduled
    2750                 :            :      non-debug insn.  */
    2751                 :  204001000 :   if (flag_sched_last_insn_heuristic && last_nondebug_scheduled_insn)
    2752                 :            :     {
    2753                 :  187685000 :       dep_t dep1;
    2754                 :  187685000 :       dep_t dep2;
    2755                 :  187685000 :       rtx_insn *last = last_nondebug_scheduled_insn;
    2756                 :            : 
    2757                 :            :       /* Classify the instructions into three classes:
    2758                 :            :          1) Data dependent on last schedule insn.
    2759                 :            :          2) Anti/Output dependent on last scheduled insn.
    2760                 :            :          3) Independent of last scheduled insn, or has latency of one.
    2761                 :            :          Choose the insn from the highest numbered class if different.  */
    2762                 :  187685000 :       dep1 = sd_find_dep_between (last, tmp, true);
    2763                 :            : 
    2764                 :  187685000 :       if (dep1 == NULL || dep_cost (dep1) == 1)
    2765                 :            :         tmp_class = 3;
    2766                 :   13597400 :       else if (/* Data dependence.  */
    2767                 :   13597400 :                DEP_TYPE (dep1) == REG_DEP_TRUE)
    2768                 :            :         tmp_class = 1;
    2769                 :            :       else
    2770                 :   12694000 :         tmp_class = 2;
    2771                 :            : 
    2772                 :  187685000 :       dep2 = sd_find_dep_between (last, tmp2, true);
    2773                 :            : 
    2774                 :  187685000 :       if (dep2 == NULL || dep_cost (dep2)  == 1)
    2775                 :            :         tmp2_class = 3;
    2776                 :   13789400 :       else if (/* Data dependence.  */
    2777                 :   13789400 :                DEP_TYPE (dep2) == REG_DEP_TRUE)
    2778                 :            :         tmp2_class = 1;
    2779                 :            :       else
    2780                 :   12786700 :         tmp2_class = 2;
    2781                 :            : 
    2782                 :  187685000 :       if ((val = tmp2_class - tmp_class))
    2783                 :    1835470 :         return rfs_result (RFS_LAST_INSN, val, tmp, tmp2);
    2784                 :            :     }
    2785                 :            : 
    2786                 :            :   /* Prefer instructions that occur earlier in the model schedule.  */
    2787                 :  202166000 :   if (sched_pressure == SCHED_PRESSURE_MODEL)
    2788                 :            :     {
    2789                 :          0 :       diff = model_index (tmp) - model_index (tmp2);
    2790                 :          0 :       if (diff != 0)
    2791                 :          0 :         return rfs_result (RFS_PRESSURE_INDEX, diff, tmp, tmp2);
    2792                 :            :     }
    2793                 :            : 
    2794                 :            :   /* Prefer the insn which has more later insns that depend on it.
    2795                 :            :      This gives the scheduler more freedom when scheduling later
    2796                 :            :      instructions at the expense of added register pressure.  */
    2797                 :            : 
    2798                 :  202166000 :   val = (dep_list_size (tmp2, SD_LIST_FORW)
    2799                 :  202166000 :          - dep_list_size (tmp, SD_LIST_FORW));
    2800                 :            : 
    2801                 :  202166000 :   if (flag_sched_dep_count_heuristic && val != 0)
    2802                 :   17376500 :     return rfs_result (RFS_DEP_COUNT, val, tmp, tmp2);
    2803                 :            : 
    2804                 :            :   /* Sort by INSN_COST rather than INSN_LUID.  This means that instructions
    2805                 :            :      which take longer to execute are prioritised and it leads to more
    2806                 :            :      dual-issue opportunities on in-order cores which have this feature.  */
    2807                 :            : 
    2808                 :  184789000 :   if (INSN_COST (tmp) != INSN_COST (tmp2))
    2809                 :    6506950 :     return rfs_result (RFS_COST, INSN_COST (tmp2) - INSN_COST (tmp),
    2810                 :    6506950 :                        tmp, tmp2);
    2811                 :            : 
    2812                 :            :   /* If insns are equally good, sort by INSN_LUID (original insn order),
    2813                 :            :      so that we make the sort stable.  This minimizes instruction movement,
    2814                 :            :      thus minimizing sched's effect on debugging and cross-jumping.  */
    2815                 :  178282000 :   return rfs_result (RFS_TIE, INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
    2816                 :            : }
    2817                 :            : 
    2818                 :            : /* Resort the array A in which only element at index N may be out of order.  */
    2819                 :            : 
    2820                 :            : HAIFA_INLINE static void
    2821                 :    4550100 : swap_sort (rtx_insn **a, int n)
    2822                 :            : {
    2823                 :    4550100 :   rtx_insn *insn = a[n - 1];
    2824                 :    4550100 :   int i = n - 2;
    2825                 :            : 
    2826                 :    6240750 :   while (i >= 0 && rank_for_schedule (a + i, &insn) >= 0)
    2827                 :            :     {
    2828                 :    1690650 :       a[i + 1] = a[i];
    2829                 :    1690650 :       i -= 1;
    2830                 :            :     }
    2831                 :    4550100 :   a[i + 1] = insn;
    2832                 :    4550100 : }
    2833                 :            : 
    2834                 :            : /* Add INSN to the insn queue so that it can be executed at least
    2835                 :            :    N_CYCLES after the currently executing insn.  Preserve insns
    2836                 :            :    chain for debugging purposes.  REASON will be printed in debugging
    2837                 :            :    output.  */
    2838                 :            : 
    2839                 :            : HAIFA_INLINE static void
    2840                 :   28803500 : queue_insn (rtx_insn *insn, int n_cycles, const char *reason)
    2841                 :            : {
    2842                 :   28803500 :   int next_q = NEXT_Q_AFTER (q_ptr, n_cycles);
    2843                 :   28803500 :   rtx_insn_list *link = alloc_INSN_LIST (insn, insn_queue[next_q]);
    2844                 :   28803500 :   int new_tick;
    2845                 :            : 
    2846                 :   28803500 :   gcc_assert (n_cycles <= max_insn_queue_index);
    2847                 :   28803500 :   gcc_assert (!DEBUG_INSN_P (insn));
    2848                 :            : 
    2849                 :   28803500 :   insn_queue[next_q] = link;
    2850                 :   28803500 :   q_size += 1;
    2851                 :            : 
    2852                 :   28803500 :   if (sched_verbose >= 2)
    2853                 :            :     {
    2854                 :          0 :       fprintf (sched_dump, ";;\t\tReady-->Q: insn %s: ",
    2855                 :          0 :                (*current_sched_info->print_insn) (insn, 0));
    2856                 :            : 
    2857                 :          0 :       fprintf (sched_dump, "queued for %d cycles (%s).\n", n_cycles, reason);
    2858                 :            :     }
    2859                 :            : 
    2860                 :   28803500 :   QUEUE_INDEX (insn) = next_q;
    2861                 :            : 
    2862                 :   28803500 :   if (current_sched_info->flags & DO_BACKTRACKING)
    2863                 :            :     {
    2864                 :          0 :       new_tick = clock_var + n_cycles;
    2865                 :          0 :       if (INSN_TICK (insn) == INVALID_TICK || INSN_TICK (insn) < new_tick)
    2866                 :          0 :         INSN_TICK (insn) = new_tick;
    2867                 :            : 
    2868                 :          0 :       if (INSN_EXACT_TICK (insn) != INVALID_TICK
    2869                 :          0 :           && INSN_EXACT_TICK (insn) < clock_var + n_cycles)
    2870                 :            :         {
    2871                 :          0 :           must_backtrack = true;
    2872                 :          0 :           if (sched_verbose >= 2)
    2873                 :          0 :             fprintf (sched_dump, ";;\t\tcausing a backtrack.\n");
    2874                 :            :         }
    2875                 :            :     }
    2876                 :   28803500 : }
    2877                 :            : 
    2878                 :            : /* Remove INSN from queue.  */
    2879                 :            : static void
    2880                 :     115734 : queue_remove (rtx_insn *insn)
    2881                 :            : {
    2882                 :     115734 :   gcc_assert (QUEUE_INDEX (insn) >= 0);
    2883                 :     115734 :   remove_free_INSN_LIST_elem (insn, &insn_queue[QUEUE_INDEX (insn)]);
    2884                 :     115734 :   q_size--;
    2885                 :     115734 :   QUEUE_INDEX (insn) = QUEUE_NOWHERE;
    2886                 :     115734 : }
    2887                 :            : 
    2888                 :            : /* Return a pointer to the bottom of the ready list, i.e. the insn
    2889                 :            :    with the lowest priority.  */
    2890                 :            : 
    2891                 :            : rtx_insn **
    2892                 :   90952100 : ready_lastpos (struct ready_list *ready)
    2893                 :            : {
    2894                 :   90952100 :   gcc_assert (ready->n_ready >= 1);
    2895                 :   90952100 :   return ready->vec + ready->first - ready->n_ready + 1;
    2896                 :            : }
    2897                 :            : 
    2898                 :            : /* Add an element INSN to the ready list so that it ends up with the
    2899                 :            :    lowest/highest priority depending on FIRST_P.  */
    2900                 :            : 
    2901                 :            : HAIFA_INLINE static void
    2902                 :   82895200 : ready_add (struct ready_list *ready, rtx_insn *insn, bool first_p)
    2903                 :            : {
    2904                 :   82895200 :   if (!first_p)
    2905                 :            :     {
    2906                 :   82893100 :       if (ready->first == ready->n_ready)
    2907                 :            :         {
    2908                 :        108 :           memmove (ready->vec + ready->veclen - ready->n_ready,
    2909                 :         54 :                    ready_lastpos (ready),
    2910                 :         54 :                    ready->n_ready * sizeof (rtx));
    2911                 :         54 :           ready->first = ready->veclen - 1;
    2912                 :            :         }
    2913                 :   82893100 :       ready->vec[ready->first - ready->n_ready] = insn;
    2914                 :            :     }
    2915                 :            :   else
    2916                 :            :     {
    2917                 :       2086 :       if (ready->first == ready->veclen - 1)
    2918                 :            :         {
    2919                 :       1576 :           if (ready->n_ready)
    2920                 :            :             /* ready_lastpos() fails when called with (ready->n_ready == 0).  */
    2921                 :          0 :             memmove (ready->vec + ready->veclen - ready->n_ready - 1,
    2922                 :          0 :                      ready_lastpos (ready),
    2923                 :          0 :                      ready->n_ready * sizeof (rtx));
    2924                 :       1576 :           ready->first = ready->veclen - 2;
    2925                 :            :         }
    2926                 :       2086 :       ready->vec[++(ready->first)] = insn;
    2927                 :            :     }
    2928                 :            : 
    2929                 :   82895200 :   ready->n_ready++;
    2930                 :   82895200 :   if (DEBUG_INSN_P (insn))
    2931                 :   27754400 :     ready->n_debug++;
    2932                 :            : 
    2933                 :   82895200 :   gcc_assert (QUEUE_INDEX (insn) != QUEUE_READY);
    2934                 :   82895200 :   QUEUE_INDEX (insn) = QUEUE_READY;
    2935                 :            : 
    2936                 :   82895200 :   if (INSN_EXACT_TICK (insn) != INVALID_TICK
    2937                 :   82895200 :       && INSN_EXACT_TICK (insn) < clock_var)
    2938                 :            :     {
    2939                 :          0 :       must_backtrack = true;
    2940                 :            :     }
    2941                 :   82895200 : }
    2942                 :            : 
    2943                 :            : /* Remove the element with the highest priority from the ready list and
    2944                 :            :    return it.  */
    2945                 :            : 
    2946                 :            : HAIFA_INLINE static rtx_insn *
    2947                 :   81551700 : ready_remove_first (struct ready_list *ready)
    2948                 :            : {
    2949                 :   81551700 :   rtx_insn *t;
    2950                 :            : 
    2951                 :   81551700 :   gcc_assert (ready->n_ready);
    2952                 :   81551700 :   t = ready->vec[ready->first--];
    2953                 :   81551700 :   ready->n_ready--;
    2954                 :   81551700 :   if (DEBUG_INSN_P (t))
    2955                 :   27754400 :     ready->n_debug--;
    2956                 :            :   /* If the queue becomes empty, reset it.  */
    2957                 :   81551700 :   if (ready->n_ready == 0)
    2958                 :   43013500 :     ready->first = ready->veclen - 1;
    2959                 :            : 
    2960                 :   81551700 :   gcc_assert (QUEUE_INDEX (t) == QUEUE_READY);
    2961                 :   81551700 :   QUEUE_INDEX (t) = QUEUE_NOWHERE;
    2962                 :            : 
    2963                 :   81551700 :   return t;
    2964                 :            : }
    2965                 :            : 
    2966                 :            : /* The following code implements multi-pass scheduling for the first
    2967                 :            :    cycle.  In other words, we will try to choose ready insn which
    2968                 :            :    permits to start maximum number of insns on the same cycle.  */
    2969                 :            : 
    2970                 :            : /* Return a pointer to the element INDEX from the ready.  INDEX for
    2971                 :            :    insn with the highest priority is 0, and the lowest priority has
    2972                 :            :    N_READY - 1.  */
    2973                 :            : 
    2974                 :            : rtx_insn *
    2975                 :  710463000 : ready_element (struct ready_list *ready, int index)
    2976                 :            : {
    2977                 :  710463000 :   gcc_assert (ready->n_ready && index < ready->n_ready);
    2978                 :            : 
    2979                 :  710463000 :   return ready->vec[ready->first - index];
    2980                 :            : }
    2981                 :            : 
    2982                 :            : /* Remove the element INDEX from the ready list and return it.  INDEX
    2983                 :            :    for insn with the highest priority is 0, and the lowest priority
    2984                 :            :    has N_READY - 1.  */
    2985                 :            : 
    2986                 :            : HAIFA_INLINE static rtx_insn *
    2987                 :   51026900 : ready_remove (struct ready_list *ready, int index)
    2988                 :            : {
    2989                 :   51026900 :   rtx_insn *t;
    2990                 :   51026900 :   int i;
    2991                 :            : 
    2992                 :   51026900 :   if (index == 0)
    2993                 :   49683500 :     return ready_remove_first (ready);
    2994                 :    1343430 :   gcc_assert (ready->n_ready && index < ready->n_ready);
    2995                 :    1343430 :   t = ready->vec[ready->first - index];
    2996                 :    1343430 :   ready->n_ready--;
    2997                 :    1343430 :   if (DEBUG_INSN_P (t))
    2998                 :          0 :     ready->n_debug--;
    2999                 :    4400460 :   for (i = index; i < ready->n_ready; i++)
    3000                 :    3057030 :     ready->vec[ready->first - i] = ready->vec[ready->first - i - 1];
    3001                 :    1343430 :   QUEUE_INDEX (t) = QUEUE_NOWHERE;
    3002                 :    1343430 :   return t;
    3003                 :            : }
    3004                 :            : 
    3005                 :            : /* Remove INSN from the ready list.  */
    3006                 :            : static void
    3007                 :          0 : ready_remove_insn (rtx_insn *insn)
    3008                 :            : {
    3009                 :          0 :   int i;
    3010                 :            : 
    3011                 :          0 :   for (i = 0; i < readyp->n_ready; i++)
    3012                 :          0 :     if (ready_element (readyp, i) == insn)
    3013                 :            :       {
    3014                 :          0 :         ready_remove (readyp, i);
    3015                 :          0 :         return;
    3016                 :            :       }
    3017                 :          0 :   gcc_unreachable ();
    3018                 :            : }
    3019                 :            : 
    3020                 :            : /* Calculate difference of two statistics set WAS and NOW.
    3021                 :            :    Result returned in WAS.  */
    3022                 :            : static void
    3023                 :          0 : rank_for_schedule_stats_diff (rank_for_schedule_stats_t *was,
    3024                 :            :                               const rank_for_schedule_stats_t *now)
    3025                 :            : {
    3026                 :          0 :   for (int i = 0; i < RFS_N; ++i)
    3027                 :          0 :     was->stats[i] = now->stats[i] - was->stats[i];
    3028                 :          0 : }
    3029                 :            : 
    3030                 :            : /* Print rank_for_schedule statistics.  */
    3031                 :            : static void
    3032                 :          0 : print_rank_for_schedule_stats (const char *prefix,
    3033                 :            :                                const rank_for_schedule_stats_t *stats,
    3034                 :            :                                struct ready_list *ready)
    3035                 :            : {
    3036                 :          0 :   for (int i = 0; i < RFS_N; ++i)
    3037                 :          0 :     if (stats->stats[i])
    3038                 :            :       {
    3039                 :          0 :         fprintf (sched_dump, "%s%20s: %u", prefix, rfs_str[i], stats->stats[i]);
    3040                 :            : 
    3041                 :          0 :         if (ready != NULL)
    3042                 :            :           /* Print out insns that won due to RFS_<I>.  */
    3043                 :            :           {
    3044                 :          0 :             rtx_insn **p = ready_lastpos (ready);
    3045                 :            : 
    3046                 :          0 :             fprintf (sched_dump, ":");
    3047                 :            :             /* Start with 1 since least-priority insn didn't have any wins.  */
    3048                 :          0 :             for (int j = 1; j < ready->n_ready; ++j)
    3049                 :          0 :               if (INSN_LAST_RFS_WIN (p[j]) == i)
    3050                 :          0 :                 fprintf (sched_dump, " %s",
    3051                 :          0 :                          (*current_sched_info->print_insn) (p[j], 0));
    3052                 :            :           }
    3053                 :          0 :         fprintf (sched_dump, "\n");
    3054                 :            :       }
    3055                 :          0 : }
    3056                 :            : 
    3057                 :            : /* Separate DEBUG_INSNS from normal insns.  DEBUG_INSNs go to the end
    3058                 :            :    of array.  */
    3059                 :            : static void
    3060                 :   27754500 : ready_sort_debug (struct ready_list *ready)
    3061                 :            : {
    3062                 :   27754500 :   int i;
    3063                 :   27754500 :   rtx_insn **first = ready_lastpos (ready);
    3064                 :            : 
    3065                 :   91407700 :   for (i = 0; i < ready->n_ready; ++i)
    3066                 :   63653200 :     if (!DEBUG_INSN_P (first[i]))
    3067                 :   35898800 :       INSN_RFS_DEBUG_ORIG_ORDER (first[i]) = i;
    3068                 :            : 
    3069                 :   27754500 :   qsort (first, ready->n_ready, sizeof (rtx), rank_for_schedule_debug);
    3070                 :   27754500 : }
    3071                 :            : 
    3072                 :            : /* Sort non-debug insns in the ready list READY by ascending priority.
    3073                 :            :    Assumes that all debug insns are separated from the real insns.  */
    3074                 :            : static void
    3075                 :   39267000 : ready_sort_real (struct ready_list *ready)
    3076                 :            : {
    3077                 :   39267000 :   int i;
    3078                 :   39267000 :   rtx_insn **first = ready_lastpos (ready);
    3079                 :   39267000 :   int n_ready_real = ready->n_ready - ready->n_debug;
    3080                 :            : 
    3081                 :   39267000 :   if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
    3082                 :      23270 :     for (i = 0; i < n_ready_real; ++i)
    3083                 :      14386 :       setup_insn_reg_pressure_info (first[i]);
    3084                 :   39258200 :   else if (sched_pressure == SCHED_PRESSURE_MODEL
    3085                 :          0 :            && model_curr_point < model_num_insns)
    3086                 :          0 :     model_set_excess_costs (first, n_ready_real);
    3087                 :            : 
    3088                 :   39267000 :   rank_for_schedule_stats_t stats1;
    3089                 :   39267000 :   if (sched_verbose >= 4)
    3090                 :          0 :     stats1 = rank_for_schedule_stats;
    3091                 :            : 
    3092                 :   39267000 :   if (n_ready_real == 2)
    3093                 :    4550100 :     swap_sort (first, n_ready_real);
    3094                 :   34716900 :   else if (n_ready_real > 2)
    3095                 :    4698020 :     qsort (first, n_ready_real, sizeof (rtx), rank_for_schedule);
    3096                 :            : 
    3097                 :   39267000 :   if (sched_verbose >= 4)
    3098                 :            :     {
    3099                 :          0 :       rank_for_schedule_stats_diff (&stats1, &rank_for_schedule_stats);
    3100                 :          0 :       print_rank_for_schedule_stats (";;\t\t", &stats1, ready);
    3101                 :            :     }
    3102                 :   39267000 : }
    3103                 :            : 
    3104                 :            : /* Sort the ready list READY by ascending priority.  */
    3105                 :            : static void
    3106                 :   67021500 : ready_sort (struct ready_list *ready)
    3107                 :            : {
    3108                 :   67021500 :   if (ready->n_debug > 0)
    3109                 :   27754400 :     ready_sort_debug (ready);
    3110                 :            :   else
    3111                 :   39267000 :     ready_sort_real (ready);
    3112                 :   67021500 : }
    3113                 :            : 
    3114                 :            : /* PREV is an insn that is ready to execute.  Adjust its priority if that
    3115                 :            :    will help shorten or lengthen register lifetimes as appropriate.  Also
    3116                 :            :    provide a hook for the target to tweak itself.  */
    3117                 :            : 
    3118                 :            : HAIFA_INLINE static void
    3119                 :   66992900 : adjust_priority (rtx_insn *prev)
    3120                 :            : {
    3121                 :            :   /* ??? There used to be code here to try and estimate how an insn
    3122                 :            :      affected register lifetimes, but it did it by looking at REG_DEAD
    3123                 :            :      notes, which we removed in schedule_region.  Nor did it try to
    3124                 :            :      take into account register pressure or anything useful like that.
    3125                 :            : 
    3126                 :            :      Revisit when we have a machine model to work with and not before.  */
    3127                 :            : 
    3128                 :   66992900 :   if (targetm.sched.adjust_priority)
    3129                 :   66992900 :     INSN_PRIORITY (prev) =
    3130                 :   66992900 :       targetm.sched.adjust_priority (prev, INSN_PRIORITY (prev));
    3131                 :   66992900 : }
    3132                 :            : 
    3133                 :            : /* Advance DFA state STATE on one cycle.  */
    3134                 :            : void
    3135                 :   32025400 : advance_state (state_t state)
    3136                 :            : {
    3137                 :   32025400 :   if (targetm.sched.dfa_pre_advance_cycle)
    3138                 :          0 :     targetm.sched.dfa_pre_advance_cycle ();
    3139                 :            : 
    3140                 :   32025400 :   if (targetm.sched.dfa_pre_cycle_insn)
    3141                 :          0 :     state_transition (state,
    3142                 :            :                       targetm.sched.dfa_pre_cycle_insn ());
    3143                 :            : 
    3144                 :   32025400 :   state_transition (state, NULL);
    3145                 :            : 
    3146                 :   32025400 :   if (targetm.sched.dfa_post_cycle_insn)
    3147                 :          0 :     state_transition (state,
    3148                 :          0 :                       targetm.sched.dfa_post_cycle_insn ());
    3149                 :            : 
    3150                 :   32025400 :   if (targetm.sched.dfa_post_advance_cycle)
    3151                 :   31895100 :     targetm.sched.dfa_post_advance_cycle ();
    3152                 :   32025400 : }
    3153                 :            : 
    3154                 :            : /* Advance time on one cycle.  */
    3155                 :            : HAIFA_INLINE static void
    3156                 :   32020600 : advance_one_cycle (void)
    3157                 :            : {
    3158                 :   32020600 :   advance_state (curr_state);
    3159                 :   32020600 :   if (sched_verbose >= 4)
    3160                 :          0 :     fprintf (sched_dump, ";;\tAdvance the current state.\n");
    3161                 :   32020600 : }
    3162                 :            : 
    3163                 :            : /* Update register pressure after scheduling INSN.  */
    3164                 :            : static void
    3165                 :      21218 : update_register_pressure (rtx_insn *insn)
    3166                 :            : {
    3167                 :      21218 :   struct reg_use_data *use;
    3168                 :      21218 :   struct reg_set_data *set;
    3169                 :            : 
    3170                 :      21218 :   gcc_checking_assert (!DEBUG_INSN_P (insn));
    3171                 :            : 
    3172                 :      32878 :   for (use = INSN_REG_USE_LIST (insn); use != NULL; use = use->next_insn_use)
    3173                 :      11660 :     if (dying_use_p (use))
    3174                 :      11052 :       mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
    3175                 :            :                                  use->regno, false);
    3176                 :      34278 :   for (set = INSN_REG_SET_LIST (insn); set != NULL; set = set->next_insn_set)
    3177                 :      13060 :     mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
    3178                 :            :                                set->regno, true);
    3179                 :      21218 : }
    3180                 :            : 
    3181                 :            : /* Set up or update (if UPDATE_P) max register pressure (see its
    3182                 :            :    meaning in sched-int.h::_haifa_insn_data) for all current BB insns
    3183                 :            :    after insn AFTER.  */
    3184                 :            : static void
    3185                 :       4856 : setup_insn_max_reg_pressure (rtx_insn *after, bool update_p)
    3186                 :            : {
    3187                 :       4856 :   int i, p;
    3188                 :       4856 :   bool eq_p;
    3189                 :       4856 :   rtx_insn *insn;
    3190                 :       4856 :   static int max_reg_pressure[N_REG_CLASSES];
    3191                 :            : 
    3192                 :       4856 :   save_reg_pressure ();
    3193                 :      24280 :   for (i = 0; i < ira_pressure_classes_num; i++)
    3194                 :      19424 :     max_reg_pressure[ira_pressure_classes[i]]
    3195                 :      19424 :       = curr_reg_pressure[ira_pressure_classes[i]];
    3196                 :       4856 :   for (insn = NEXT_INSN (after);
    3197                 :      24649 :        insn != NULL_RTX && ! BARRIER_P (insn)
    3198                 :      48922 :          && BLOCK_FOR_INSN (insn) == BLOCK_FOR_INSN (after);
    3199                 :      19793 :        insn = NEXT_INSN (insn))
    3200                 :      22419 :     if (NONDEBUG_INSN_P (insn))
    3201                 :            :       {
    3202                 :            :         eq_p = true;
    3203                 :      87255 :         for (i = 0; i < ira_pressure_classes_num; i++)
    3204                 :            :           {
    3205                 :      69804 :             p = max_reg_pressure[ira_pressure_classes[i]];
    3206                 :      69804 :             if (INSN_MAX_REG_PRESSURE (insn)[i] != p)
    3207                 :            :               {
    3208                 :      15670 :                 eq_p = false;
    3209                 :      15670 :                 INSN_MAX_REG_PRESSURE (insn)[i]
    3210                 :      15670 :                   = max_reg_pressure[ira_pressure_classes[i]];
    3211                 :            :               }
    3212                 :            :           }
    3213                 :      17451 :         if (update_p && eq_p)
    3214                 :            :           break;
    3215                 :      14825 :         update_register_pressure (insn);
    3216                 :      74125 :         for (i = 0; i < ira_pressure_classes_num; i++)
    3217                 :      59300 :           if (max_reg_pressure[ira_pressure_classes[i]]
    3218                 :      59300 :               < curr_reg_pressure[ira_pressure_classes[i]])
    3219                 :       3377 :             max_reg_pressure[ira_pressure_classes[i]]
    3220                 :       3377 :               = curr_reg_pressure[ira_pressure_classes[i]];
    3221                 :            :       }
    3222                 :       4856 :   restore_reg_pressure ();
    3223                 :       4856 : }
    3224                 :            : 
    3225                 :            : /* Update the current register pressure after scheduling INSN.  Update
    3226                 :            :    also max register pressure for unscheduled insns of the current
    3227                 :            :    BB.  */
    3228                 :            : static void
    3229                 :       6393 : update_reg_and_insn_max_reg_pressure (rtx_insn *insn)
    3230                 :            : {
    3231                 :       6393 :   int i;
    3232                 :       6393 :   int before[N_REG_CLASSES];
    3233                 :            : 
    3234                 :      31965 :   for (i = 0; i < ira_pressure_classes_num; i++)
    3235                 :      25572 :     before[i] = curr_reg_pressure[ira_pressure_classes[i]];
    3236                 :       6393 :   update_register_pressure (insn);
    3237                 :      16662 :   for (i = 0; i < ira_pressure_classes_num; i++)
    3238                 :      14138 :     if (curr_reg_pressure[ira_pressure_classes[i]] != before[i])
    3239                 :            :       break;
    3240                 :       6393 :   if (i < ira_pressure_classes_num)
    3241                 :       3869 :     setup_insn_max_reg_pressure (insn, true);
    3242                 :       6393 : }
    3243                 :            : 
    3244                 :            : /* Set up register pressure at the beginning of basic block BB whose
    3245                 :            :    insns starting after insn AFTER.  Set up also max register pressure
    3246                 :            :    for all insns of the basic block.  */
    3247                 :            : void
    3248                 :        987 : sched_setup_bb_reg_pressure_info (basic_block bb, rtx_insn *after)
    3249                 :            : {
    3250                 :        987 :   gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED);
    3251                 :        987 :   initiate_bb_reg_pressure_info (bb);
    3252                 :        987 :   setup_insn_max_reg_pressure (after, false);
    3253                 :        987 : }
    3254                 :            : 
    3255                 :            : /* If doing predication while scheduling, verify whether INSN, which
    3256                 :            :    has just been scheduled, clobbers the conditions of any
    3257                 :            :    instructions that must be predicated in order to break their
    3258                 :            :    dependencies.  If so, remove them from the queues so that they will
    3259                 :            :    only be scheduled once their control dependency is resolved.  */
    3260                 :            : 
    3261                 :            : static void
    3262                 :   66992900 : check_clobbered_conditions (rtx_insn *insn)
    3263                 :            : {
    3264                 :   66992900 :   HARD_REG_SET t;
    3265                 :   66992900 :   int i;
    3266                 :            : 
    3267                 :   66992900 :   if ((current_sched_info->flags & DO_PREDICATION) == 0)
    3268                 :   66992900 :     return;
    3269                 :            : 
    3270                 :          0 :   find_all_hard_reg_sets (insn, &t, true);
    3271                 :            : 
    3272                 :          0 :  restart:
    3273                 :          0 :   for (i = 0; i < ready.n_ready; i++)
    3274                 :            :     {
    3275                 :          0 :       rtx_insn *x = ready_element (&ready, i);
    3276                 :          0 :       if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t))
    3277                 :            :         {
    3278                 :          0 :           ready_remove_insn (x);
    3279                 :          0 :           goto restart;
    3280                 :            :         }
    3281                 :            :     }
    3282                 :          0 :   for (i = 0; i <= max_insn_queue_index; i++)
    3283                 :            :     {
    3284                 :          0 :       rtx_insn_list *link;
    3285                 :          0 :       int q = NEXT_Q_AFTER (q_ptr, i);
    3286                 :            : 
    3287                 :          0 :     restart_queue:
    3288                 :          0 :       for (link = insn_queue[q]; link; link = link->next ())
    3289                 :            :         {
    3290                 :          0 :           rtx_insn *x = link->insn ();
    3291                 :          0 :           if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t))
    3292                 :            :             {
    3293                 :          0 :               queue_remove (x);
    3294                 :          0 :               goto restart_queue;
    3295                 :            :             }
    3296                 :            :         }
    3297                 :            :     }
    3298                 :            : }
    3299                 :            : 
    3300                 :            : /* Return (in order):
    3301                 :            : 
    3302                 :            :    - positive if INSN adversely affects the pressure on one
    3303                 :            :      register class
    3304                 :            : 
    3305                 :            :    - negative if INSN reduces the pressure on one register class
    3306                 :            : 
    3307                 :            :    - 0 if INSN doesn't affect the pressure on any register class.  */
    3308                 :            : 
    3309                 :            : static int
    3310                 :          0 : model_classify_pressure (struct model_insn_info *insn)
    3311                 :            : {
    3312                 :          0 :   struct reg_pressure_data *reg_pressure;
    3313                 :          0 :   int death[N_REG_CLASSES];
    3314                 :          0 :   int pci, cl, sum;
    3315                 :            : 
    3316                 :          0 :   calculate_reg_deaths (insn->insn, death);
    3317                 :          0 :   reg_pressure = INSN_REG_PRESSURE (insn->insn);
    3318                 :          0 :   sum = 0;
    3319                 :          0 :   for (pci = 0; pci < ira_pressure_classes_num; pci++)
    3320                 :            :     {
    3321                 :          0 :       cl = ira_pressure_classes[pci];
    3322                 :          0 :       if (death[cl] < reg_pressure[pci].set_increase)
    3323                 :            :         return 1;
    3324                 :          0 :       sum += reg_pressure[pci].set_increase - death[cl];
    3325                 :            :     }
    3326                 :            :   return sum;
    3327                 :            : }
    3328                 :            : 
    3329                 :            : /* Return true if INSN1 should come before INSN2 in the model schedule.  */
    3330                 :            : 
    3331                 :            : static int
    3332                 :          0 : model_order_p (struct model_insn_info *insn1, struct model_insn_info *insn2)
    3333                 :            : {
    3334                 :          0 :   unsigned int height1, height2;
    3335                 :          0 :   unsigned int priority1, priority2;
    3336                 :            : 
    3337                 :            :   /* Prefer instructions with a higher model priority.  */
    3338                 :          0 :   if (insn1->model_priority != insn2->model_priority)
    3339                 :          0 :     return insn1->model_priority > insn2->model_priority;
    3340                 :            : 
    3341                 :            :   /* Combine the length of the longest path of satisfied true dependencies
    3342                 :            :      that leads to each instruction (depth) with the length of the longest
    3343                 :            :      path of any dependencies that leads from the instruction (alap).
    3344                 :            :      Prefer instructions with the greatest combined length.  If the combined
    3345                 :            :      lengths are equal, prefer instructions with the greatest depth.
    3346                 :            : 
    3347                 :            :      The idea is that, if we have a set S of "equal" instructions that each
    3348                 :            :      have ALAP value X, and we pick one such instruction I, any true-dependent
    3349                 :            :      successors of I that have ALAP value X - 1 should be preferred over S.
    3350                 :            :      This encourages the schedule to be "narrow" rather than "wide".
    3351                 :            :      However, if I is a low-priority instruction that we decided to
    3352                 :            :      schedule because of its model_classify_pressure, and if there
    3353                 :            :      is a set of higher-priority instructions T, the aforementioned
    3354                 :            :      successors of I should not have the edge over T.  */
    3355                 :          0 :   height1 = insn1->depth + insn1->alap;
    3356                 :          0 :   height2 = insn2->depth + insn2->alap;
    3357                 :          0 :   if (height1 != height2)
    3358                 :          0 :     return height1 > height2;
    3359                 :          0 :   if (insn1->depth != insn2->depth)
    3360                 :          0 :     return insn1->depth > insn2->depth;
    3361                 :            : 
    3362                 :            :   /* We have no real preference between INSN1 an INSN2 as far as attempts
    3363                 :            :      to reduce pressure go.  Prefer instructions with higher priorities.  */
    3364                 :          0 :   priority1 = INSN_PRIORITY (insn1->insn);
    3365                 :          0 :   priority2 = INSN_PRIORITY (insn2->insn);
    3366                 :          0 :   if (priority1 != priority2)
    3367                 :          0 :     return priority1 > priority2;
    3368                 :            : 
    3369                 :            :   /* Use the original rtl sequence as a tie-breaker.  */
    3370                 :          0 :   return insn1 < insn2;
    3371                 :            : }
    3372                 :            : 
    3373                 :            : /* Add INSN to the model worklist immediately after PREV.  Add it to the
    3374                 :            :    beginning of the list if PREV is null.  */
    3375                 :            : 
    3376                 :            : static void
    3377                 :          0 : model_add_to_worklist_at (struct model_insn_info *insn,
    3378                 :            :                           struct model_insn_info *prev)
    3379                 :            : {
    3380                 :          0 :   gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_NOWHERE);
    3381                 :          0 :   QUEUE_INDEX (insn->insn) = QUEUE_READY;
    3382                 :            : 
    3383                 :          0 :   insn->prev = prev;
    3384                 :          0 :   if (prev)
    3385                 :            :     {
    3386                 :          0 :       insn->next = prev->next;
    3387                 :          0 :       prev->next = insn;
    3388                 :            :     }
    3389                 :            :   else
    3390                 :            :     {
    3391                 :          0 :       insn->next = model_worklist;
    3392                 :          0 :       model_worklist = insn;
    3393                 :            :     }
    3394                 :          0 :   if (insn->next)
    3395                 :          0 :     insn->next->prev = insn;
    3396                 :          0 : }
    3397                 :            : 
    3398                 :            : /* Remove INSN from the model worklist.  */
    3399                 :            : 
    3400                 :            : static void
    3401                 :          0 : model_remove_from_worklist (struct model_insn_info *insn)
    3402                 :            : {
    3403                 :          0 :   gcc_assert (QUEUE_INDEX (insn->insn) == QUEUE_READY);
    3404                 :          0 :   QUEUE_INDEX (insn->insn) = QUEUE_NOWHERE;
    3405                 :            : 
    3406                 :          0 :   if (insn->prev)
    3407                 :          0 :     insn->prev->next = insn->next;
    3408                 :            :   else
    3409                 :          0 :     model_worklist = insn->next;
    3410                 :          0 :   if (insn->next)
    3411                 :          0 :     insn->next->prev = insn->prev;
    3412                 :          0 : }
    3413                 :            : 
    3414                 :            : /* Add INSN to the model worklist.  Start looking for a suitable position
    3415                 :            :    between neighbors PREV and NEXT, testing at most param_max_sched_ready_insns
    3416                 :            :    insns either side.  A null PREV indicates the beginning of the list and
    3417                 :            :    a null NEXT indicates the end.  */
    3418                 :            : 
    3419                 :            : static void
    3420                 :          0 : model_add_to_worklist (struct model_insn_info *insn,
    3421                 :            :                        struct model_insn_info *prev,
    3422                 :            :                        struct model_insn_info *next)
    3423                 :            : {
    3424                 :          0 :   int count;
    3425                 :            : 
    3426                 :          0 :   count = param_max_sched_ready_insns;
    3427                 :          0 :   if (count > 0 && prev && model_order_p (insn, prev))
    3428                 :          0 :     do
    3429                 :            :       {
    3430                 :          0 :         count--;
    3431                 :          0 :         prev = prev->prev;
    3432                 :            :       }
    3433                 :          0 :     while (count > 0 && prev && model_order_p (insn, prev));
    3434                 :            :   else
    3435                 :          0 :     while (count > 0 && next && model_order_p (next, insn))
    3436                 :            :       {
    3437                 :          0 :         count--;
    3438                 :          0 :         prev = next;
    3439                 :          0 :         next = next->next;
    3440                 :            :       }
    3441                 :          0 :   model_add_to_worklist_at (insn, prev);
    3442                 :          0 : }
    3443                 :            : 
    3444                 :            : /* INSN may now have a higher priority (in the model_order_p sense)
    3445                 :            :    than before.  Move it up the worklist if necessary.  */
    3446                 :            : 
    3447                 :            : static void
    3448                 :          0 : model_promote_insn (struct model_insn_info *insn)
    3449                 :            : {
    3450                 :          0 :   struct model_insn_info *prev;
    3451                 :          0 :   int count;
    3452                 :            : 
    3453                 :          0 :   prev = insn->prev;
    3454                 :          0 :   count = param_max_sched_ready_insns;
    3455                 :          0 :   while (count > 0 && prev && model_order_p (insn, prev))
    3456                 :            :     {
    3457                 :          0 :       count--;
    3458                 :          0 :       prev = prev->prev;
    3459                 :            :     }
    3460                 :          0 :   if (prev != insn->prev)
    3461                 :            :     {
    3462                 :          0 :       model_remove_from_worklist (insn);
    3463                 :          0 :       model_add_to_worklist_at (insn, prev);
    3464                 :            :     }
    3465                 :          0 : }
    3466                 :            : 
    3467                 :            : /* Add INSN to the end of the model schedule.  */
    3468                 :            : 
    3469                 :            : static void
    3470                 :          0 : model_add_to_schedule (rtx_insn *insn)
    3471                 :            : {
    3472                 :          0 :   unsigned int point;
    3473                 :            : 
    3474                 :          0 :   gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE);
    3475                 :          0 :   QUEUE_INDEX (insn) = QUEUE_SCHEDULED;
    3476                 :            : 
    3477                 :          0 :   point = model_schedule.length ();
    3478                 :          0 :   model_schedule.quick_push (insn);
    3479                 :          0 :   INSN_MODEL_INDEX (insn) = point + 1;
    3480                 :          0 : }
    3481                 :            : 
    3482                 :            : /* Analyze the instructions that are to be scheduled, setting up
    3483                 :            :    MODEL_INSN_INFO (...) and model_num_insns accordingly.  Add ready
    3484                 :            :    instructions to model_worklist.  */
    3485                 :            : 
    3486                 :            : static void
    3487                 :          0 : model_analyze_insns (void)
    3488                 :            : {
    3489                 :          0 :   rtx_insn *start, *end, *iter;
    3490                 :          0 :   sd_iterator_def sd_it;
    3491                 :          0 :   dep_t dep;
    3492                 :          0 :   struct model_insn_info *insn, *con;
    3493                 :            : 
    3494                 :          0 :   model_num_insns = 0;
    3495                 :          0 :   start = PREV_INSN (current_sched_info->next_tail);
    3496                 :          0 :   end = current_sched_info->prev_head;
    3497                 :          0 :   for (iter = start; iter != end; iter = PREV_INSN (iter))
    3498                 :          0 :     if (NONDEBUG_INSN_P (iter))
    3499                 :            :       {
    3500                 :          0 :         insn = MODEL_INSN_INFO (iter);
    3501                 :          0 :         insn->insn = iter;
    3502                 :          0 :         FOR_EACH_DEP (iter, SD_LIST_FORW, sd_it, dep)
    3503                 :            :           {
    3504                 :          0 :             con = MODEL_INSN_INFO (DEP_CON (dep));
    3505                 :          0 :             if (con->insn && insn->alap < con->alap + 1)
    3506                 :          0 :               insn->alap = con->alap + 1;
    3507                 :            :           }
    3508                 :            : 
    3509                 :          0 :         insn->old_queue = QUEUE_INDEX (iter);
    3510                 :          0 :         QUEUE_INDEX (iter) = QUEUE_NOWHERE;
    3511                 :            : 
    3512                 :          0 :         insn->unscheduled_preds = dep_list_size (iter, SD_LIST_HARD_BACK);
    3513                 :          0 :         if (insn->unscheduled_preds == 0)
    3514                 :          0 :           model_add_to_worklist (insn, NULL, model_worklist);
    3515                 :            : 
    3516                 :          0 :         model_num_insns++;
    3517                 :            :       }
    3518                 :          0 : }
    3519                 :            : 
    3520                 :            : /* The global state describes the register pressure at the start of the
    3521                 :            :    model schedule.  Initialize GROUP accordingly.  */
    3522                 :            : 
    3523                 :            : static void
    3524                 :          0 : model_init_pressure_group (struct model_pressure_group *group)
    3525                 :            : {
    3526                 :          0 :   int pci, cl;
    3527                 :            : 
    3528                 :          0 :   for (pci = 0; pci < ira_pressure_classes_num; pci++)
    3529                 :            :     {
    3530                 :          0 :       cl = ira_pressure_classes[pci];
    3531                 :          0 :       group->limits[pci].pressure = curr_reg_pressure[cl];
    3532                 :          0 :       group->limits[pci].point = 0;
    3533                 :            :     }
    3534                 :            :   /* Use index model_num_insns to record the state after the last
    3535                 :            :      instruction in the model schedule.  */
    3536                 :          0 :   group->model = XNEWVEC (struct model_pressure_data,
    3537                 :            :                           (model_num_insns + 1) * ira_pressure_classes_num);
    3538                 :          0 : }
    3539                 :            : 
    3540                 :            : /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE.
    3541                 :            :    Update the maximum pressure for the whole schedule.  */
    3542                 :            : 
    3543                 :            : static void
    3544                 :          0 : model_record_pressure (struct model_pressure_group *group,
    3545                 :            :                        int point, int pci, int pressure)
    3546                 :            : {
    3547                 :          0 :   MODEL_REF_PRESSURE (group, point, pci) = pressure;
    3548                 :          0 :   if (group->limits[pci].pressure < pressure)
    3549                 :            :     {
    3550                 :          0 :       group->limits[pci].pressure = pressure;
    3551                 :          0 :       group->limits[pci].point = point;
    3552                 :            :     }
    3553                 :          0 : }
    3554                 :            : 
    3555                 :            : /* INSN has just been added to the end of the model schedule.  Record its
    3556                 :            :    register-pressure information.  */
    3557                 :            : 
    3558                 :            : static void
    3559                 :          0 : model_record_pressures (struct model_insn_info *insn)
    3560                 :            : {
    3561                 :          0 :   struct reg_pressure_data *reg_pressure;
    3562                 :          0 :   int point, pci, cl, delta;
    3563                 :          0 :   int death[N_REG_CLASSES];
    3564                 :            : 
    3565                 :          0 :   point = model_index (insn->insn);
    3566                 :          0 :   if (sched_verbose >= 2)
    3567                 :            :     {
    3568                 :          0 :       if (point == 0)
    3569                 :            :         {
    3570                 :          0 :           fprintf (sched_dump, "\n;;\tModel schedule:\n;;\n");
    3571                 :          0 :           fprintf (sched_dump, ";;\t| idx insn | mpri hght dpth prio |\n");
    3572                 :            :         }
    3573                 :          0 :       fprintf (sched_dump, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ",
    3574                 :          0 :                point, INSN_UID (insn->insn), insn->model_priority,
    3575                 :          0 :                insn->depth + insn->alap, insn->depth,
    3576                 :          0 :                INSN_PRIORITY (insn->insn),
    3577                 :          0 :                str_pattern_slim (PATTERN (insn->insn)));
    3578                 :            :     }
    3579                 :          0 :   calculate_reg_deaths (insn->insn, death);
    3580                 :          0 :   reg_pressure = INSN_REG_PRESSURE (insn->insn);
    3581                 :          0 :   for (pci = 0; pci < ira_pressure_classes_num; pci++)
    3582                 :            :     {
    3583                 :          0 :       cl = ira_pressure_classes[pci];
    3584                 :          0 :       delta = reg_pressure[pci].set_increase - death[cl];
    3585                 :          0 :       if (sched_verbose >= 2)
    3586                 :          0 :         fprintf (sched_dump, " %s:[%d,%+d]", reg_class_names[cl],
    3587                 :            :                  curr_reg_pressure[cl], delta);
    3588                 :          0 :       model_record_pressure (&model_before_pressure, point, pci,
    3589                 :            :                              curr_reg_pressure[cl]);
    3590                 :            :     }
    3591                 :          0 :   if (sched_verbose >= 2)
    3592                 :          0 :     fprintf (sched_dump, "\n");
    3593                 :          0 : }
    3594                 :            : 
    3595                 :            : /* All instructions have been added to the model schedule.  Record the
    3596                 :            :    final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs.  */
    3597                 :            : 
    3598                 :            : static void
    3599                 :          0 : model_record_final_pressures (struct model_pressure_group *group)
    3600                 :            : {
    3601                 :          0 :   int point, pci, max_pressure, ref_pressure, cl;
    3602                 :            : 
    3603                 :          0 :   for (pci = 0; pci < ira_pressure_classes_num; pci++)
    3604                 :            :     {
    3605                 :            :       /* Record the final pressure for this class.  */
    3606                 :          0 :       cl = ira_pressure_classes[pci];
    3607                 :          0 :       point = model_num_insns;
    3608                 :          0 :       ref_pressure = curr_reg_pressure[cl];
    3609                 :          0 :       model_record_pressure (group, point, pci, ref_pressure);
    3610                 :            : 
    3611                 :            :       /* Record the original maximum pressure.  */
    3612                 :          0 :       group->limits[pci].orig_pressure = group->limits[pci].pressure;
    3613                 :            : 
    3614                 :            :       /* Update the MODEL_MAX_PRESSURE for every point of the schedule.  */
    3615                 :          0 :       max_pressure = ref_pressure;
    3616                 :          0 :       MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
    3617                 :          0 :       while (point > 0)
    3618                 :            :         {
    3619                 :          0 :           point--;
    3620                 :          0 :           ref_pressure = MODEL_REF_PRESSURE (group, point, pci);
    3621                 :          0 :           max_pressure = MAX (max_pressure, ref_pressure);
    3622                 :          0 :           MODEL_MAX_PRESSURE (group, point, pci) = max_pressure;
    3623                 :            :         }
    3624                 :            :     }
    3625                 :          0 : }
    3626                 :            : 
    3627                 :            : /* Update all successors of INSN, given that INSN has just been scheduled.  */
    3628                 :            : 
    3629                 :            : static void
    3630                 :          0 : model_add_successors_to_worklist (struct model_insn_info *insn)
    3631                 :            : {
    3632                 :          0 :   sd_iterator_def sd_it;
    3633                 :          0 :   struct model_insn_info *con;
    3634                 :          0 :   dep_t dep;
    3635                 :            : 
    3636                 :          0 :   FOR_EACH_DEP (insn->insn, SD_LIST_FORW, sd_it, dep)
    3637                 :            :     {
    3638                 :          0 :       con = MODEL_INSN_INFO (DEP_CON (dep));
    3639                 :            :       /* Ignore debug instructions, and instructions from other blocks.  */
    3640                 :          0 :       if (con->insn)
    3641                 :            :         {
    3642                 :          0 :           con->unscheduled_preds--;
    3643                 :            : 
    3644                 :            :           /* Update the depth field of each true-dependent successor.
    3645                 :            :              Increasing the depth gives them a higher priority than
    3646                 :            :              before.  */
    3647                 :          0 :           if (DEP_TYPE (dep) == REG_DEP_TRUE && con->depth < insn->depth + 1)
    3648                 :            :             {
    3649                 :          0 :               con->depth = insn->depth + 1;
    3650                 :          0 :               if (QUEUE_INDEX (con->insn) == QUEUE_READY)
    3651                 :          0 :                 model_promote_insn (con);
    3652                 :            :             }
    3653                 :            : 
    3654                 :            :           /* If this is a true dependency, or if there are no remaining
    3655                 :            :              dependencies for CON (meaning that CON only had non-true
    3656                 :            :              dependencies), make sure that CON is on the worklist.
    3657                 :            :              We don't bother otherwise because it would tend to fill the
    3658                 :            :              worklist with a lot of low-priority instructions that are not
    3659                 :            :              yet ready to issue.  */
    3660                 :          0 :           if ((con->depth > 0 || con->unscheduled_preds == 0)
    3661                 :          0 :               && QUEUE_INDEX (con->insn) == QUEUE_NOWHERE)
    3662                 :          0 :             model_add_to_worklist (con, insn, insn->next);
    3663                 :            :         }
    3664                 :            :     }
    3665                 :          0 : }
    3666                 :            : 
    3667                 :            : /* Give INSN a higher priority than any current instruction, then give
    3668                 :            :    unscheduled predecessors of INSN a higher priority still.  If any of
    3669                 :            :    those predecessors are not on the model worklist, do the same for its
    3670                 :            :    predecessors, and so on.  */
    3671                 :            : 
    3672                 :            : static void
    3673                 :          0 : model_promote_predecessors (struct model_insn_info *insn)
    3674                 :            : {
    3675                 :          0 :   struct model_insn_info *pro, *first;
    3676                 :          0 :   sd_iterator_def sd_it;
    3677                 :          0 :   dep_t dep;
    3678                 :            : 
    3679                 :          0 :   if (sched_verbose >= 7)
    3680                 :          0 :     fprintf (sched_dump, ";;\t+--- priority of %d = %d, priority of",
    3681                 :          0 :              INSN_UID (insn->insn), model_next_priority);
    3682                 :          0 :   insn->model_priority = model_next_priority++;
    3683                 :          0 :   model_remove_from_worklist (insn);
    3684                 :          0 :   model_add_to_worklist_at (insn, NULL);
    3685                 :            : 
    3686                 :          0 :   first = NULL;
    3687                 :          0 :   for (;;)
    3688                 :            :     {
    3689                 :          0 :       FOR_EACH_DEP (insn->insn, SD_LIST_HARD_BACK, sd_it, dep)
    3690                 :            :         {
    3691                 :          0 :           pro = MODEL_INSN_INFO (DEP_PRO (dep));
    3692                 :            :           /* The first test is to ignore debug instructions, and instructions
    3693                 :            :              from other blocks.  */
    3694                 :          0 :           if (pro->insn
    3695                 :          0 :               && pro->model_priority != model_next_priority
    3696                 :          0 :               && QUEUE_INDEX (pro->insn) != QUEUE_SCHEDULED)
    3697                 :            :             {
    3698                 :          0 :               pro->model_priority = model_next_priority;
    3699                 :          0 :               if (sched_verbose >= 7)
    3700                 :          0 :                 fprintf (sched_dump, " %d", INSN_UID (pro->insn));
    3701                 :          0 :               if (QUEUE_INDEX (pro->insn) == QUEUE_READY)
    3702                 :            :                 {
    3703                 :            :                   /* PRO is already in the worklist, but it now has
    3704                 :            :                      a higher priority than before.  Move it at the
    3705                 :            :                      appropriate place.  */
    3706                 :          0 :                   model_remove_from_worklist (pro);
    3707                 :          0 :                   model_add_to_worklist (pro, NULL, model_worklist);
    3708                 :            :                 }
    3709                 :            :               else
    3710                 :            :                 {
    3711                 :            :                   /* PRO isn't in the worklist.  Recursively process
    3712                 :            :                      its predecessors until we find one that is.  */
    3713                 :          0 :                   pro->next = first;
    3714                 :          0 :                   first = pro;
    3715                 :            :                 }
    3716                 :            :             }
    3717                 :            :         }
    3718                 :          0 :       if (!first)
    3719                 :            :         break;
    3720                 :          0 :       insn = first;
    3721                 :          0 :       first = insn->next;
    3722                 :            :     }
    3723                 :          0 :   if (sched_verbose >= 7)
    3724                 :          0 :     fprintf (sched_dump, " = %d\n", model_next_priority);
    3725                 :          0 :   model_next_priority++;
    3726                 :          0 : }
    3727                 :            : 
    3728                 :            : /* Pick one instruction from model_worklist and process it.  */
    3729                 :            : 
    3730                 :            : static void
    3731                 :          0 : model_choose_insn (void)
    3732                 :            : {
    3733                 :          0 :   struct model_insn_info *insn, *fallback;
    3734                 :          0 :   int count;
    3735                 :            : 
    3736                 :          0 :   if (sched_verbose >= 7)
    3737                 :            :     {
    3738                 :          0 :       fprintf (sched_dump, ";;\t+--- worklist:\n");
    3739                 :          0 :       insn = model_worklist;
    3740                 :          0 :       count = param_max_sched_ready_insns;
    3741                 :          0 :       while (count > 0 && insn)
    3742                 :            :         {
    3743                 :          0 :           fprintf (sched_dump, ";;\t+---   %d [%d, %d, %d, %d]\n",
    3744                 :          0 :                    INSN_UID (insn->insn), insn->model_priority,
    3745                 :          0 :                    insn->depth + insn->alap, insn->depth,
    3746                 :          0 :                    INSN_PRIORITY (insn->insn));
    3747                 :          0 :           count--;
    3748                 :          0 :           insn = insn->next;
    3749                 :            :         }
    3750                 :            :     }
    3751                 :            : 
    3752                 :            :   /* Look for a ready instruction whose model_classify_priority is zero
    3753                 :            :      or negative, picking the highest-priority one.  Adding such an
    3754                 :            :      instruction to the schedule now should do no harm, and may actually
    3755                 :            :      do some good.
    3756                 :            : 
    3757                 :            :      Failing that, see whether there is an instruction with the highest
    3758                 :            :      extant model_priority that is not yet ready, but which would reduce
    3759                 :            :      pressure if it became ready.  This is designed to catch cases like:
    3760                 :            : 
    3761                 :            :        (set (mem (reg R1)) (reg R2))
    3762                 :            : 
    3763                 :            :      where the instruction is the last remaining use of R1 and where the
    3764                 :            :      value of R2 is not yet available (or vice versa).  The death of R1
    3765                 :            :      means that this instruction already reduces pressure.  It is of
    3766                 :            :      course possible that the computation of R2 involves other registers
    3767                 :            :      that are hard to kill, but such cases are rare enough for this
    3768                 :            :      heuristic to be a win in general.
    3769                 :            : 
    3770                 :            :      Failing that, just pick the highest-priority instruction in the
    3771                 :            :      worklist.  */
    3772                 :          0 :   count = param_max_sched_ready_insns;
    3773                 :          0 :   insn = model_worklist;
    3774                 :          0 :   fallback = 0;
    3775                 :          0 :   for (;;)
    3776                 :            :     {
    3777                 :          0 :       if (count == 0 || !insn)
    3778                 :            :         {
    3779                 :          0 :           insn = fallback ? fallback : model_worklist;
    3780                 :            :           break;
    3781                 :            :         }
    3782                 :          0 :       if (insn->unscheduled_preds)
    3783                 :            :         {
    3784                 :          0 :           if (model_worklist->model_priority == insn->model_priority
    3785                 :          0 :               && !fallback
    3786                 :          0 :               && model_classify_pressure (insn) < 0)
    3787                 :            :             fallback = insn;
    3788                 :            :         }
    3789                 :            :       else
    3790                 :            :         {
    3791                 :          0 :           if (model_classify_pressure (insn) <= 0)
    3792                 :            :             break;
    3793                 :            :         }
    3794                 :          0 :       count--;
    3795                 :          0 :       insn = insn->next;
    3796                 :            :     }
    3797                 :            : 
    3798                 :          0 :   if (sched_verbose >= 7 && insn != model_worklist)
    3799                 :            :     {
    3800                 :          0 :       if (insn->unscheduled_preds)
    3801                 :          0 :         fprintf (sched_dump, ";;\t+--- promoting insn %d, with dependencies\n",
    3802                 :          0 :                  INSN_UID (insn->insn));
    3803                 :            :       else
    3804                 :          0 :         fprintf (sched_dump, ";;\t+--- promoting insn %d, which is ready\n",
    3805                 :          0 :                  INSN_UID (insn->insn));
    3806                 :            :     }
    3807                 :          0 :   if (insn->unscheduled_preds)
    3808                 :            :     /* INSN isn't yet ready to issue.  Give all its predecessors the
    3809                 :            :        highest priority.  */
    3810                 :          0 :     model_promote_predecessors (insn);
    3811                 :            :   else
    3812                 :            :     {
    3813                 :            :       /* INSN is ready.  Add it to the end of model_schedule and
    3814                 :            :          process its successors.  */
    3815                 :          0 :       model_add_successors_to_worklist (insn);
    3816                 :          0 :       model_remove_from_worklist (insn);
    3817                 :          0 :       model_add_to_schedule (insn->insn);
    3818                 :          0 :       model_record_pressures (insn);
    3819                 :          0 :       update_register_pressure (insn->insn);
    3820                 :            :     }
    3821                 :          0 : }
    3822                 :            : 
    3823                 :            : /* Restore all QUEUE_INDEXs to the values that they had before
    3824                 :            :    model_start_schedule was called.  */
    3825                 :            : 
    3826                 :            : static void
    3827                 :          0 : model_reset_queue_indices (void)
    3828                 :            : {
    3829                 :          0 :   unsigned int i;
    3830                 :          0 :   rtx_insn *insn;
    3831                 :            : 
    3832                 :          0 :   FOR_EACH_VEC_ELT (model_schedule, i, insn)
    3833                 :          0 :     QUEUE_INDEX (insn) = MODEL_INSN_INFO (insn)->old_queue;
    3834                 :          0 : }
    3835                 :            : 
    3836                 :            : /* We have calculated the model schedule and spill costs.  Print a summary
    3837                 :            :    to sched_dump.  */
    3838                 :            : 
    3839                 :            : static void
    3840                 :          0 : model_dump_pressure_summary (void)
    3841                 :            : {
    3842                 :          0 :   int pci, cl;
    3843                 :            : 
    3844                 :          0 :   fprintf (sched_dump, ";; Pressure summary:");
    3845                 :          0 :   for (pci = 0; pci < ira_pressure_classes_num; pci++)
    3846                 :            :     {
    3847                 :          0 :       cl = ira_pressure_classes[pci];
    3848                 :          0 :       fprintf (sched_dump, " %s:%d", reg_class_names[cl],
    3849                 :            :                model_before_pressure.limits[pci].pressure);
    3850                 :            :     }
    3851                 :          0 :   fprintf (sched_dump, "\n\n");
    3852                 :          0 : }
    3853                 :            : 
    3854                 :            : /* Initialize the SCHED_PRESSURE_MODEL information for the current
    3855                 :            :    scheduling region.  */
    3856                 :            : 
    3857                 :            : static void
    3858                 :          0 : model_start_schedule (basic_block bb)
    3859                 :            : {
    3860                 :          0 :   model_next_priority = 1;
    3861                 :          0 :   model_schedule.create (sched_max_luid);
    3862                 :          0 :   model_insns = XCNEWVEC (struct model_insn_info, sched_max_luid);
    3863                 :            : 
    3864                 :          0 :   gcc_assert (bb == BLOCK_FOR_INSN (NEXT_INSN (current_sched_info->prev_head)));
    3865                 :          0 :   initiate_reg_pressure_info (df_get_live_in (bb));
    3866                 :            : 
    3867                 :          0 :   model_analyze_insns ();
    3868                 :          0 :   model_init_pressure_group (&model_before_pressure);
    3869                 :          0 :   while (model_worklist)
    3870                 :          0 :     model_choose_insn ();
    3871                 :          0 :   gcc_assert (model_num_insns == (int) model_schedule.length ());
    3872                 :          0 :   if (sched_verbose >= 2)
    3873                 :          0 :     fprintf (sched_dump, "\n");
    3874                 :            : 
    3875                 :          0 :   model_record_final_pressures (&model_before_pressure);
    3876                 :          0 :   model_reset_queue_indices ();
    3877                 :            : 
    3878                 :          0 :   XDELETEVEC (model_insns);
    3879                 :            : 
    3880                 :          0 :   model_curr_point = 0;
    3881                 :          0 :   initiate_reg_pressure_info (df_get_live_in (bb));
    3882                 :          0 :   if (sched_verbose >= 1)
    3883                 :          0 :     model_dump_pressure_summary ();
    3884                 :          0 : }
    3885                 :            : 
    3886                 :            : /* Free the information associated with GROUP.  */
    3887                 :            : 
    3888                 :            : static void
    3889                 :          0 : model_finalize_pressure_group (struct model_pressure_group *group)
    3890                 :            : {
    3891                 :          0 :   XDELETEVEC (group->model);
    3892                 :          0 : }
    3893                 :            : 
    3894                 :            : /* Free the information created by model_start_schedule.  */
    3895                 :            : 
    3896                 :            : static void
    3897                 :          0 : model_end_schedule (void)
    3898                 :            : {
    3899                 :          0 :   model_finalize_pressure_group (&model_before_pressure);
    3900                 :          0 :   model_schedule.release ();
    3901                 :          0 : }
    3902                 :            : 
    3903                 :            : /* Prepare reg pressure scheduling for basic block BB.  */
    3904                 :            : static void
    3905                 :        987 : sched_pressure_start_bb (basic_block bb)
    3906                 :            : {
    3907                 :            :   /* Set the number of available registers for each class taking into account
    3908                 :            :      relative probability of current basic block versus function prologue and
    3909                 :            :      epilogue.
    3910                 :            :      * If the basic block executes much more often than the prologue/epilogue
    3911                 :            :      (e.g., inside a hot loop), then cost of spill in the prologue is close to
    3912                 :            :      nil, so the effective number of available registers is
    3913                 :            :      (ira_class_hard_regs_num[cl] - fixed_regs_num[cl] - 0).
    3914                 :            :      * If the basic block executes as often as the prologue/epilogue,
    3915                 :            :      then spill in the block is as costly as in the prologue, so the effective
    3916                 :            :      number of available registers is
    3917                 :            :      (ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
    3918                 :            :       - call_saved_regs_num[cl]).
    3919                 :            :      Note that all-else-equal, we prefer to spill in the prologue, since that
    3920                 :            :      allows "extra" registers for other basic blocks of the function.
    3921                 :            :      * If the basic block is on the cold path of the function and executes
    3922                 :            :      rarely, then we should always prefer to spill in the block, rather than
    3923                 :            :      in the prologue/epilogue.  The effective number of available register is
    3924                 :            :      (ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
    3925                 :            :       - call_saved_regs_num[cl]).  */
    3926                 :        987 :   {
    3927                 :        987 :     int i;
    3928                 :        987 :     int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.to_frequency (cfun);
    3929                 :        987 :     int bb_freq = bb->count.to_frequency (cfun);
    3930                 :            : 
    3931                 :        987 :     if (bb_freq == 0)
    3932                 :            :       {
    3933                 :         64 :         if (entry_freq == 0)
    3934                 :          0 :           entry_freq = bb_freq = 1;
    3935                 :            :       }
    3936                 :        987 :     if (bb_freq < entry_freq)
    3937                 :            :       bb_freq = entry_freq;
    3938                 :            : 
    3939                 :       4935 :     for (i = 0; i < ira_pressure_classes_num; ++i)
    3940                 :            :       {
    3941                 :       3948 :         enum reg_class cl = ira_pressure_classes[i];
    3942                 :       3948 :         sched_class_regs_num[cl] = ira_class_hard_regs_num[cl]
    3943                 :       3948 :                                    - fixed_regs_num[cl];
    3944                 :       3948 :         sched_class_regs_num[cl]
    3945                 :       3948 :           -= (call_saved_regs_num[cl] * entry_freq) / bb_freq;
    3946                 :            :       }
    3947                 :            :   }
    3948                 :            : 
    3949                 :        987 :   if (sched_pressure == SCHED_PRESSURE_MODEL)
    3950                 :          0 :     model_start_schedule (bb);
    3951                 :        987 : }
    3952                 :            : 
    3953                 :            : /* A structure that holds local state for the loop in schedule_block.  */
    3954                 :            : struct sched_block_state
    3955                 :            : {
    3956                 :            :   /* True if no real insns have been scheduled in the current cycle.  */
    3957                 :            :   bool first_cycle_insn_p;
    3958                 :            :   /* True if a shadow insn has been scheduled in the current cycle, which
    3959                 :            :      means that no more normal insns can be issued.  */
    3960                 :            :   bool shadows_only_p;
    3961                 :            :   /* True if we're winding down a modulo schedule, which means that we only
    3962                 :            :      issue insns with INSN_EXACT_TICK set.  */
    3963                 :            :   bool modulo_epilogue;
    3964                 :            :   /* Initialized with the machine's issue rate every cycle, and updated
    3965                 :            :      by calls to the variable_issue hook.  */
    3966                 :            :   int can_issue_more;
    3967                 :            : };
    3968                 :            : 
    3969                 :            : /* INSN is the "currently executing insn".  Launch each insn which was
    3970                 :            :    waiting on INSN.  READY is the ready list which contains the insns
    3971                 :            :    that are ready to fire.  CLOCK is the current cycle.  The function
    3972                 :            :    returns necessary cycle advance after issuing the insn (it is not
    3973                 :            :    zero for insns in a schedule group).  */
    3974                 :            : 
    3975                 :            : static int
    3976                 :   66992900 : schedule_insn (rtx_insn *insn)
    3977                 :            : {
    3978                 :   66992900 :   sd_iterator_def sd_it;
    3979                 :   66992900 :   dep_t dep;
    3980                 :   66992900 :   int i;
    3981                 :   66992900 :   int advance = 0;
    3982                 :            : 
    3983                 :   66992900 :   if (sched_verbose >= 1)
    3984                 :            :     {
    3985                 :       1034 :       struct reg_pressure_data *pressure_info;
    3986                 :       2068 :       fprintf (sched_dump, ";;\t%3i--> %s %-40s:",
    3987                 :       1034 :                clock_var, (*current_sched_info->print_insn) (insn, 1),
    3988                 :       1034 :                str_pattern_slim (PATTERN (insn)));
    3989                 :            : 
    3990                 :       1172 :       if (recog_memoized (insn) < 0)
    3991                 :        138 :         fprintf (sched_dump, "nothing");
    3992                 :            :       else
    3993                 :        896 :         print_reservation (sched_dump, insn);
    3994                 :       1034 :       pressure_info = INSN_REG_PRESSURE (insn);
    3995                 :       1034 :       if (pressure_info != NULL)
    3996                 :            :         {
    3997                 :          0 :           fputc (':', sched_dump);
    3998                 :          0 :           for (i = 0; i < ira_pressure_classes_num; i++)
    3999                 :          0 :             fprintf (sched_dump, "%s%s%+d(%d)",
    4000                 :          0 :                      scheduled_insns.length () > 1
    4001                 :          0 :                      && INSN_LUID (insn)
    4002                 :          0 :                      < INSN_LUID (scheduled_insns[scheduled_insns.length () - 2]) ? "@" : "",
    4003                 :          0 :                      reg_class_names[ira_pressure_classes[i]],
    4004                 :          0 :                      pressure_info[i].set_increase, pressure_info[i].change);
    4005                 :            :         }
    4006                 :       1034 :       if (sched_pressure == SCHED_PRESSURE_MODEL
    4007                 :          0 :           && model_curr_point < model_num_insns
    4008                 :       1034 :           && model_index (insn) == model_curr_point)
    4009                 :          0 :         fprintf (sched_dump, ":model %d", model_curr_point);
    4010                 :       1034 :       fputc ('\n', sched_dump);
    4011                 :            :     }
    4012                 :            : 
    4013                 :   66992900 :   if (sched_pressure == SCHED_PRESSURE_WEIGHTED && !DEBUG_INSN_P (insn))
    4014                 :       6393 :     update_reg_and_insn_max_reg_pressure (insn);
    4015                 :            : 
    4016                 :            :   /* Scheduling instruction should have all its dependencies resolved and
    4017                 :            :      should have been removed from the ready list.  */
    4018                 :   66992900 :   gcc_assert (sd_lists_empty_p (insn, SD_LIST_HARD_BACK));
    4019                 :            : 
    4020                 :            :   /* Reset debug insns invalidated by moving this insn.  */
    4021                 :   66992900 :   if (MAY_HAVE_DEBUG_BIND_INSNS && !DEBUG_INSN_P (insn))
    4022                 :   28169800 :     for (sd_it = sd_iterator_start (insn, SD_LIST_BACK);
    4023                 :   28480800 :          sd_iterator_cond (&sd_it, &dep);)
    4024                 :            :       {
    4025                 :     311034 :         rtx_insn *dbg = DEP_PRO (dep);
    4026                 :     311034 :         struct reg_use_data *use, *next;
    4027                 :            : 
    4028                 :     311034 :         if (DEP_STATUS (dep) & DEP_CANCELLED)
    4029                 :            :           {
    4030                 :     260345 :             sd_iterator_next (&sd_it);
    4031                 :     260345 :             continue;
    4032                 :            :           }
    4033                 :            : 
    4034                 :      50689 :         gcc_assert (DEBUG_BIND_INSN_P (dbg));
    4035                 :            : 
    4036                 :      50689 :         if (sched_verbose >= 6)
    4037                 :          0 :           fprintf (sched_dump, ";;\t\tresetting: debug insn %d\n",
    4038                 :          0 :                    INSN_UID (dbg));
    4039                 :            : 
    4040                 :            :         /* ??? Rather than resetting the debug insn, we might be able
    4041                 :            :            to emit a debug temp before the just-scheduled insn, but
    4042                 :            :            this would involve checking that the expression at the
    4043                 :            :            point of the debug insn is equivalent to the expression
    4044                 :            :            before the just-scheduled insn.  They might not be: the
    4045                 :            :            expression in the debug insn may depend on other insns not
    4046                 :            :            yet scheduled that set MEMs, REGs or even other debug
    4047                 :            :            insns.  It's not clear that attempting to preserve debug
    4048                 :            :            information in these cases is worth the effort, given how
    4049                 :            :            uncommon these resets are and the likelihood that the debug
    4050                 :            :            temps introduced won't survive the schedule change.  */
    4051                 :      50689 :         INSN_VAR_LOCATION_LOC (dbg) = gen_rtx_UNKNOWN_VAR_LOC ();
    4052                 :      50689 :         df_insn_rescan (dbg);
    4053                 :            : 
    4054                 :            :         /* Unknown location doesn't use any registers.  */
    4055                 :      50696 :         for (use = INSN_REG_USE_LIST (dbg); use != NULL; use = next)
    4056                 :            :           {
    4057                 :            :             struct reg_use_data *prev = use;
    4058                 :            : 
    4059                 :            :             /* Remove use from the cyclic next_regno_use chain first.  */
    4060                 :         14 :             while (prev->next_regno_use != use)
    4061                 :            :               prev = prev->next_regno_use;
    4062                 :          7 :             prev->next_regno_use = use->next_regno_use;
    4063                 :          7 :             next = use->next_insn_use;
    4064                 :          7 :             free (use);
    4065                 :            :           }
    4066                 :      50689 :         INSN_REG_USE_LIST (dbg) = NULL;
    4067                 :            : 
    4068                 :            :         /* We delete rather than resolve these deps, otherwise we
    4069                 :            :            crash in sched_free_deps(), because forward deps are
    4070                 :            :            expected to be released before backward deps.  */
    4071                 :      50689 :         sd_delete_dep (sd_it);
    4072                 :            :       }
    4073                 :            : 
    4074                 :   66992900 :   gcc_assert (QUEUE_INDEX (insn) == QUEUE_NOWHERE);
    4075                 :   66992900 :   QUEUE_INDEX (insn) = QUEUE_SCHEDULED;
    4076                 :            : 
    4077                 :   66992900 :   if (sched_pressure == SCHED_PRESSURE_MODEL
    4078                 :          0 :       && model_curr_point < model_num_insns
    4079                 :          0 :       && NONDEBUG_INSN_P (insn))
    4080                 :            :     {
    4081                 :          0 :       if (model_index (insn) == model_curr_point)
    4082                 :          0 :         do
    4083                 :          0 :           model_curr_point++;
    4084                 :            :         while (model_curr_point < model_num_insns
    4085                 :          0 :                && (QUEUE_INDEX (MODEL_INSN (model_curr_point))
    4086                 :            :                    == QUEUE_SCHEDULED));
    4087                 :            :       else
    4088                 :          0 :         model_recompute (insn);
    4089                 :          0 :       model_update_limit_points ();
    4090                 :          0 :       update_register_pressure (insn);
    4091                 :          0 :       if (sched_verbose >= 2)
    4092                 :          0 :         print_curr_reg_pressure ();
    4093                 :            :     }
    4094                 :            : 
    4095                 :   66992900 :   gcc_assert (INSN_TICK (insn) >= MIN_TICK);
    4096                 :   66992900 :   if (INSN_TICK (insn) > clock_var)
    4097                 :            :     /* INSN has been prematurely moved from the queue to the ready list.
    4098                 :            :        This is possible only if following flags are set.  */
    4099                 :          5 :     gcc_assert (flag_sched_stalled_insns || sched_fusion);
    4100                 :            : 
    4101                 :            :   /* ??? Probably, if INSN is scheduled prematurely, we should leave
    4102                 :            :      INSN_TICK untouched.  This is a machine-dependent issue, actually.  */
    4103                 :   66992900 :   INSN_TICK (insn) = clock_var;
    4104                 :            : 
    4105                 :   66992900 :   check_clobbered_conditions (insn);
    4106                 :            : 
    4107                 :            :   /* Update dependent instructions.  First, see if by scheduling this insn
    4108                 :            :      now we broke a dependence in a way that requires us to change another
    4109                 :            :      insn.  */
    4110                 :   66992900 :   for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
    4111                 :   67311900 :        sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it))
    4112                 :            :     {
    4113                 :     318988 :       struct dep_replacement *desc = DEP_REPLACE (dep);
    4114                 :     318988 :       rtx_insn *pro = DEP_PRO (dep);
    4115                 :     318988 :       if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED
    4116                 :     318988 :           && desc != NULL && desc->insn == pro)
    4117                 :     301212 :         apply_replacement (dep, false);
    4118                 :            :     }
    4119                 :            : 
    4120                 :            :   /* Go through and resolve forward dependencies.  */
    4121                 :   66992900 :   for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
    4122                 :  201092000 :        sd_iterator_cond (&sd_it, &dep);)
    4123                 :            :     {
    4124                 :  134099000 :       rtx_insn *next = DEP_CON (dep);
    4125                 :  134099000 :       bool cancelled = (DEP_STATUS (dep) & DEP_CANCELLED) != 0;
    4126                 :            : 
    4127                 :            :       /* Resolve the dependence between INSN and NEXT.
    4128                 :            :          sd_resolve_dep () moves current dep to another list thus
    4129                 :            :          advancing the iterator.  */
    4130                 :  134099000 :       sd_resolve_dep (sd_it);
    4131                 :            : 
    4132                 :  134099000 :       if (cancelled)
    4133                 :            :         {
    4134                 :     459156 :           if (must_restore_pattern_p (next, dep))
    4135                 :      38395 :             restore_pattern (dep, false);
    4136                 :     459156 :           continue;
    4137                 :            :         }
    4138                 :            : 
    4139                 :            :       /* Don't bother trying to mark next as ready if insn is a debug
    4140                 :            :          insn.  If insn is the last hard dependency, it will have
    4141                 :            :          already been discounted.  */
    4142                 :  133640000 :       if (DEBUG_INSN_P (insn) && !DEBUG_INSN_P (next))
    4143                 :    2617950 :         continue;
    4144                 :            : 
    4145                 :  131022000 :       if (!IS_SPECULATION_BRANCHY_CHECK_P (insn))
    4146                 :            :         {
    4147                 :  131022000 :           int effective_cost;
    4148                 :            : 
    4149                 :  131022000 :           effective_cost = try_ready (next);
    4150                 :            : 
    4151                 :  131022000 :           if (effective_cost >= 0
    4152                 :   12897100 :               && SCHED_GROUP_P (next)
    4153                 :  131022000 :               && advance < effective_cost)
    4154                 :            :             advance = effective_cost;
    4155                 :            :         }
    4156                 :            :       else
    4157                 :            :         /* Check always has only one forward dependence (to the first insn in
    4158                 :            :            the recovery block), therefore, this will be executed only once.  */
    4159                 :            :         {
    4160                 :          0 :           gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW));
    4161                 :          0 :           fix_recovery_deps (RECOVERY_BLOCK (insn));
    4162                 :            :         }
    4163                 :            :     }
    4164                 :            : 
    4165                 :            :   /* Annotate the instruction with issue information -- TImode
    4166                 :            :      indicates that the instruction is expected not to be able
    4167                 :            :      to issue on the same cycle as the previous insn.  A machine
    4168                 :            :      may use this information to decide how the instruction should
    4169                 :            :      be aligned.  */
    4170                 :   66992900 :   if (issue_rate > 1
    4171                 :   66976800 :       && GET_CODE (PATTERN (insn)) != USE
    4172                 :   66550000 :       && GET_CODE (PATTERN (insn)) != CLOBBER
    4173                 :  133480000 :       && !DEBUG_INSN_P (insn))
    4174                 :            :     {
    4175                 :   38733100 :       if (reload_completed)
    4176                 :   38723700 :         PUT_MODE (insn, clock_var > last_clock_var ? TImode : VOIDmode);
    4177                 :   38733100 :       last_clock_var = clock_var;
    4178                 :            :     }
    4179                 :            : 
    4180                 :   66992900 :   if (nonscheduled_insns_begin != NULL_RTX)
    4181                 :            :     /* Indicate to debug counters that INSN is scheduled.  */
    4182                 :          0 :     nonscheduled_insns_begin = insn;
    4183                 :            : 
    4184                 :   66992900 :   return advance;
    4185                 :            : }
    4186                 :            : 
    4187                 :            : /* Functions for handling of notes.  */
    4188                 :            : 
    4189                 :            : /* Add note list that ends on FROM_END to the end of TO_ENDP.  */
    4190                 :            : void
    4191                 :         52 : concat_note_lists (rtx_insn *from_end, rtx_insn **to_endp)
    4192                 :            : {
    4193                 :         52 :   rtx_insn *from_start;
    4194                 :            : 
    4195                 :            :   /* It's easy when have nothing to concat.  */
    4196                 :         52 :   if (from_end == NULL)
    4197                 :            :     return;
    4198                 :            : 
    4199                 :            :   /* It's also easy when destination is empty.  */
    4200                 :          0 :   if (*to_endp == NULL)
    4201                 :            :     {
    4202                 :          0 :       *to_endp = from_end;
    4203                 :          0 :       return;
    4204                 :            :     }
    4205                 :            : 
    4206                 :            :   from_start = from_end;
    4207                 :          0 :   while (PREV_INSN (from_start) != NULL)
    4208                 :          0 :     from_start = PREV_INSN (from_start);
    4209                 :            : 
    4210                 :          0 :   SET_PREV_INSN (from_start) = *to_endp;
    4211                 :          0 :   SET_NEXT_INSN (*to_endp) = from_start;
    4212                 :          0 :   *to_endp = from_end;
    4213                 :            : }
    4214                 :            : 
    4215                 :            : /* Delete notes between HEAD and TAIL and put them in the chain
    4216                 :            :    of notes ended by NOTE_LIST.  */
    4217                 :            : void
    4218                 :    6520150 : remove_notes (rtx_insn *head, rtx_insn *tail)
    4219                 :            : {
    4220                 :    6520150 :   rtx_insn *next_tail, *insn, *next;
    4221                 :            : 
    4222                 :    6520150 :   note_list = 0;
    4223                 :    6520150 :   if (head == tail && !INSN_P (head))
    4224                 :            :     return;
    4225                 :            : 
    4226                 :    6520120 :   next_tail = NEXT_INSN (tail);
    4227                 :   77400000 :   for (insn = head; insn != next_tail; insn = next)
    4228                 :            :     {
    4229                 :   70879900 :       next = NEXT_INSN (insn);
    4230                 :   70879900 :       if (!NOTE_P (insn))
    4231                 :   66997300 :         continue;
    4232                 :            : 
    4233                 :    3882530 :       switch (NOTE_KIND (insn))
    4234                 :            :         {
    4235                 :       1067 :         case NOTE_INSN_BASIC_BLOCK:
    4236                 :       1067 :           continue;
    4237                 :            : 
    4238                 :     577993 :         case NOTE_INSN_EPILOGUE_BEG:
    4239                 :     577993 :           if (insn != tail)
    4240                 :            :             {
    4241                 :     577993 :               remove_insn (insn);
    4242                 :            :               /* If an insn was split just before the EPILOGUE_BEG note and
    4243                 :            :                  that split created new basic blocks, we could have a
    4244                 :            :                  BASIC_BLOCK note here.  Safely advance over it in that case
    4245                 :            :                  and assert that we land on a real insn.  */
    4246                 :     577993 :               if (NOTE_P (next)
    4247                 :          0 :                   && NOTE_KIND (next) == NOTE_INSN_BASIC_BLOCK
    4248                 :          0 :                   && next != next_tail)
    4249                 :          0 :                 next = NEXT_INSN (next);
    4250                 :     577993 :               gcc_assert (INSN_P (next));
    4251                 :     577993 :               add_reg_note (next, REG_SAVE_NOTE,
    4252                 :            :                             GEN_INT (NOTE_INSN_EPILOGUE_BEG));
    4253                 :     577993 :               break;
    4254                 :            :             }
    4255                 :            :           /* FALLTHRU */
    4256                 :            : 
    4257                 :    3303470 :         default:
    4258                 :    3303470 :           remove_insn (insn);
    4259                 :            : 
    4260                 :            :           /* Add the note to list that ends at NOTE_LIST.  */
    4261                 :    3303470 :           SET_PREV_INSN (insn) = note_list;
    4262                 :    3303470 :           SET_NEXT_INSN (insn) = NULL_RTX;
    4263                 :    3303470 :           if (note_list)
    4264                 :    1948780 :             SET_NEXT_INSN (note_list) = insn;
    4265                 :    3303470 :           note_list = insn;
    4266                 :    3303470 :           break;
    4267                 :            :         }
    4268                 :            : 
    4269                 :    3881460 :       gcc_assert ((sel_sched_p () || insn != tail) && insn != head);
    4270                 :            :     }
    4271                 :            : }
    4272                 :            : 
    4273                 :            : /* A structure to record enough data to allow us to backtrack the scheduler to
    4274                 :            :    a previous state.  */
    4275                 :            : struct haifa_saved_data
    4276                 :            : {
    4277                 :            :   /* Next entry on the list.  */
    4278                 :            :   struct haifa_saved_data *next;
    4279                 :            : 
    4280                 :            :   /* Backtracking is associated with scheduling insns that have delay slots.
    4281                 :            :      DELAY_PAIR points to the structure that contains the insns involved, and
    4282                 :            :      the number of cycles between them.  */
    4283                 :            :   struct delay_pair *delay_pair;
    4284                 :            : 
    4285                 :            :   /* Data used by the frontend (e.g. sched-ebb or sched-rgn).  */
    4286                 :            :   void *fe_saved_data;
    4287                 :            :   /* Data used by the backend.  */
    4288                 :            :   void *be_saved_data;
    4289                 :            : 
    4290                 :            :   /* Copies of global state.  */
    4291                 :            :   int clock_var, last_clock_var;
    4292                 :            :   struct ready_list ready;
    4293                 :            :   state_t curr_state;
    4294                 :            : 
    4295                 :            :   rtx_insn *last_scheduled_insn;
    4296                 :            :   rtx_insn *last_nondebug_scheduled_insn;
    4297                 :            :   rtx_insn *nonscheduled_insns_begin;
    4298                 :            :   int cycle_issued_insns;
    4299                 :            : 
    4300                 :            :   /* Copies of state used in the inner loop of schedule_block.  */
    4301                 :            :   struct sched_block_state sched_block;
    4302                 :            : 
    4303                 :            :   /* We don't need to save q_ptr, as its value is arbitrary and we can set it
    4304                 :            :      to 0 when restoring.  */
    4305                 :            :   int q_size;
    4306                 :            :   rtx_insn_list **insn_queue;
    4307                 :            : 
    4308                 :            :   /* Describe pattern replacements that occurred since this backtrack point
    4309                 :            :      was queued.  */
    4310                 :            :   vec<dep_t> replacement_deps;
    4311                 :            :   vec<int> replace_apply;
    4312                 :            : 
    4313                 :            :   /* A copy of the next-cycle replacement vectors at the time of the backtrack
    4314                 :            :      point.  */
    4315                 :            :   vec<dep_t> next_cycle_deps;
    4316                 :            :   vec<int> next_cycle_apply;
    4317                 :            : };
    4318                 :            : 
    4319                 :            : /* A record, in reverse order, of all scheduled insns which have delay slots
    4320                 :            :    and may require backtracking.  */
    4321                 :            : static struct haifa_saved_data *backtrack_queue;
    4322                 :            : 
    4323                 :            : /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according
    4324                 :            :    to SET_P.  */
    4325                 :            : static void
    4326                 :          0 : mark_backtrack_feeds (rtx_insn *insn, int set_p)
    4327                 :            : {
    4328                 :          0 :   sd_iterator_def sd_it;
    4329                 :          0 :   dep_t dep;
    4330                 :          0 :   FOR_EACH_DEP (insn, SD_LIST_HARD_BACK, sd_it, dep)
    4331                 :            :     {
    4332                 :          0 :       FEEDS_BACKTRACK_INSN (DEP_PRO (dep)) = set_p;
    4333                 :            :     }
    4334                 :          0 : }
    4335                 :            : 
    4336                 :            : /* Save the current scheduler state so that we can backtrack to it
    4337                 :            :    later if necessary.  PAIR gives the insns that make it necessary to
    4338                 :            :    save this point.  SCHED_BLOCK is the local state of schedule_block
    4339                 :            :    that need to be saved.  */
    4340                 :            : static void
    4341                 :          0 : save_backtrack_point (struct delay_pair *pair,
    4342                 :            :                       struct sched_block_state sched_block)
    4343                 :            : {
    4344                 :          0 :   int i;
    4345                 :          0 :   struct haifa_saved_data *save = XNEW (struct haifa_saved_data);
    4346                 :            : 
    4347                 :          0 :   save->curr_state = xmalloc (dfa_state_size);
    4348                 :          0 :   memcpy (save->curr_state, curr_state, dfa_state_size);
    4349                 :            : 
    4350                 :          0 :   save->ready.first = ready.first;
    4351                 :          0 :   save->ready.n_ready = ready.n_ready;
    4352                 :          0 :   save->ready.n_debug = ready.n_debug;
    4353                 :          0 :   save->ready.veclen = ready.veclen;
    4354                 :          0 :   save->ready.vec = XNEWVEC (rtx_insn *, ready.veclen);
    4355                 :          0 :   memcpy (save->ready.vec, ready.vec, ready.veclen * sizeof (rtx));
    4356                 :            : 
    4357                 :          0 :   save->insn_queue = XNEWVEC (rtx_insn_list *, max_insn_queue_index + 1);
    4358                 :          0 :   save->q_size = q_size;
    4359                 :          0 :   for (i = 0; i <= max_insn_queue_index; i++)
    4360                 :            :     {
    4361                 :          0 :       int q = NEXT_Q_AFTER (q_ptr, i);
    4362                 :          0 :       save->insn_queue[i] = copy_INSN_LIST (insn_queue[q]);
    4363                 :            :     }
    4364                 :            : 
    4365                 :          0 :   save->clock_var = clock_var;
    4366                 :          0 :   save->last_clock_var = last_clock_var;
    4367                 :          0 :   save->cycle_issued_insns = cycle_issued_insns;
    4368                 :          0 :   save->last_scheduled_insn = last_scheduled_insn;
    4369                 :          0 :   save->last_nondebug_scheduled_insn = last_nondebug_scheduled_insn;
    4370                 :          0 :   save->nonscheduled_insns_begin = nonscheduled_insns_begin;
    4371                 :            : 
    4372                 :          0 :   save->sched_block = sched_block;
    4373                 :            : 
    4374                 :          0 :   save->replacement_deps.create (0);
    4375                 :          0 :   save->replace_apply.create (0);
    4376                 :          0 :   save->next_cycle_deps = next_cycle_replace_deps.copy ();
    4377                 :          0 :   save->next_cycle_apply = next_cycle_apply.copy ();
    4378                 :            : 
    4379                 :          0 :   if (current_sched_info->save_state)
    4380                 :          0 :     save->fe_saved_data = (*current_sched_info->save_state) ();
    4381                 :            : 
    4382                 :          0 :   if (targetm.sched.alloc_sched_context)
    4383                 :            :     {
    4384                 :          0 :       save->be_saved_data = targetm.sched.alloc_sched_context ();
    4385                 :          0 :       targetm.sched.init_sched_context (save->be_saved_data, false);
    4386                 :            :     }
    4387                 :            :   else
    4388                 :          0 :     save->be_saved_data = NULL;
    4389                 :            : 
    4390                 :          0 :   save->delay_pair = pair;
    4391                 :            : 
    4392                 :          0 :   save->next = backtrack_queue;
    4393                 :          0 :   backtrack_queue = save;
    4394                 :            : 
    4395                 :          0 :   while (pair)
    4396                 :            :     {
    4397                 :          0 :       mark_backtrack_feeds (pair->i2, 1);
    4398                 :          0 :       INSN_TICK (pair->i2) = INVALID_TICK;
    4399                 :          0 :       INSN_EXACT_TICK (pair->i2) = clock_var + pair_delay (pair);
    4400                 :          0 :       SHADOW_P (pair->i2) = pair->stages == 0;
    4401                 :          0 :       pair = pair->next_same_i1;
    4402                 :            :     }
    4403                 :          0 : }
    4404                 :            : 
    4405                 :            : /* Walk the ready list and all queues. If any insns have unresolved backwards
    4406                 :            :    dependencies, these must be cancelled deps, broken by predication.  Set or
    4407                 :            :    clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS.  */
    4408                 :            : 
    4409                 :            : static void
    4410                 :          0 : toggle_cancelled_flags (bool set)
    4411                 :            : {
    4412                 :          0 :   int i;
    4413                 :          0 :   sd_iterator_def sd_it;
    4414                 :          0 :   dep_t dep;
    4415                 :            : 
    4416                 :          0 :   if (ready.n_ready > 0)
    4417                 :            :     {
    4418                 :          0 :       rtx_insn **first = ready_lastpos (&ready);
    4419                 :          0 :       for (i = 0; i < ready.n_ready; i++)
    4420                 :          0 :         FOR_EACH_DEP (first[i], SD_LIST_BACK, sd_it, dep)
    4421                 :          0 :           if (!DEBUG_INSN_P (DEP_PRO (dep)))
    4422                 :            :             {
    4423                 :          0 :               if (set)
    4424                 :          0 :                 DEP_STATUS (dep) |= DEP_CANCELLED;
    4425                 :            :               else
    4426                 :          0 :                 DEP_STATUS (dep) &= ~DEP_CANCELLED;
    4427                 :            :             }
    4428                 :            :     }
    4429                 :          0 :   for (i = 0; i <= max_insn_queue_index; i++)
    4430                 :            :     {
    4431                 :          0 :       int q = NEXT_Q_AFTER (q_ptr, i);
    4432                 :          0 :       rtx_insn_list *link;
    4433                 :          0 :       for (link = insn_queue[q]; link; link = link->next ())
    4434                 :            :         {
    4435                 :          0 :           rtx_insn *insn = link->insn ();
    4436                 :          0 :           FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
    4437                 :          0 :             if (!DEBUG_INSN_P (DEP_PRO (dep)))
    4438                 :            :               {
    4439                 :          0 :                 if (set)
    4440                 :          0 :                   DEP_STATUS (dep) |= DEP_CANCELLED;
    4441                 :            :                 else
    4442                 :          0 :                   DEP_STATUS (dep) &= ~DEP_CANCELLED;
    4443                 :            :               }
    4444                 :            :         }
    4445                 :            :     }
    4446                 :          0 : }
    4447                 :            : 
    4448                 :            : /* Undo the replacements that have occurred after backtrack point SAVE
    4449                 :            :    was placed.  */
    4450                 :            : static void
    4451                 :          0 : undo_replacements_for_backtrack (struct haifa_saved_data *save)
    4452                 :            : {
    4453                 :          0 :   while (!save->replacement_deps.is_empty ())
    4454                 :            :     {
    4455                 :          0 :       dep_t dep = save->replacement_deps.pop ();
    4456                 :          0 :       int apply_p = save->replace_apply.pop ();
    4457                 :            : 
    4458                 :          0 :       if (apply_p)
    4459                 :          0 :         restore_pattern (dep, true);
    4460                 :            :       else
    4461                 :          0 :         apply_replacement (dep, true);
    4462                 :            :     }
    4463                 :          0 :   save->replacement_deps.release ();
    4464                 :          0 :   save->replace_apply.release ();
    4465                 :          0 : }
    4466                 :            : 
    4467                 :            : /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN.
    4468                 :            :    Restore their dependencies to an unresolved state, and mark them as
    4469                 :            :    queued nowhere.  */
    4470                 :            : 
    4471                 :            : static void
    4472                 :          0 : unschedule_insns_until (rtx_insn *insn)
    4473                 :            : {
    4474                 :          0 :   auto_vec<rtx_insn *> recompute_vec;
    4475                 :            : 
    4476                 :            :   /* Make two passes over the insns to be unscheduled.  First, we clear out
    4477                 :            :      dependencies and other trivial bookkeeping.  */
    4478                 :          0 :   for (;;)
    4479                 :            :     {
    4480                 :          0 :       rtx_insn *last;
    4481                 :          0 :       sd_iterator_def sd_it;
    4482                 :          0 :       dep_t dep;
    4483                 :            : 
    4484                 :          0 :       last = scheduled_insns.pop ();
    4485                 :            : 
    4486                 :            :       /* This will be changed by restore_backtrack_point if the insn is in
    4487                 :            :          any queue.  */
    4488                 :          0 :       QUEUE_INDEX (last) = QUEUE_NOWHERE;
    4489                 :          0 :       if (last != insn)
    4490                 :          0 :         INSN_TICK (last) = INVALID_TICK;
    4491                 :            : 
    4492                 :          0 :       if (modulo_ii > 0 && INSN_UID (last) < modulo_iter0_max_uid)
    4493                 :          0 :         modulo_insns_scheduled--;
    4494                 :            : 
    4495                 :          0 :       for (sd_it = sd_iterator_start (last, SD_LIST_RES_FORW);
    4496                 :          0 :            sd_iterator_cond (&sd_it, &dep);)
    4497                 :            :         {
    4498                 :          0 :           rtx_insn *con = DEP_CON (dep);
    4499                 :          0 :           sd_unresolve_dep (sd_it);
    4500                 :          0 :           if (!MUST_RECOMPUTE_SPEC_P (con))
    4501                 :            :             {
    4502                 :          0 :               MUST_RECOMPUTE_SPEC_P (con) = 1;
    4503                 :          0 :               recompute_vec.safe_push (con);
    4504                 :            :             }
    4505                 :            :         }
    4506                 :            : 
    4507                 :          0 :       if (last == insn)
    4508                 :            :         break;
    4509                 :          0 :     }
    4510                 :            : 
    4511                 :            :   /* A second pass, to update ready and speculation status for insns
    4512                 :            :      depending on the unscheduled ones.  The first pass must have
    4513                 :            :      popped the scheduled_insns vector up to the point where we
    4514                 :            :      restart scheduling, as recompute_todo_spec requires it to be
    4515                 :            :      up-to-date.  */
    4516                 :          0 :   while (!recompute_vec.is_empty ())
    4517                 :            :     {
    4518                 :          0 :       rtx_insn *con;
    4519                 :            : 
    4520                 :          0 :       con = recompute_vec.pop ();
    4521                 :          0 :       MUST_RECOMPUTE_SPEC_P (con) = 0;
    4522                 :          0 :       if (!sd_lists_empty_p (con, SD_LIST_HARD_BACK))
    4523                 :            :         {
    4524                 :          0 :           TODO_SPEC (con) = HARD_DEP;
    4525                 :          0 :           INSN_TICK (con) = INVALID_TICK;
    4526                 :          0 :           if (PREDICATED_PAT (con) != NULL_RTX)
    4527                 :          0 :             haifa_change_pattern (con, ORIG_PAT (con));
    4528                 :            :         }
    4529                 :          0 :       else if (QUEUE_INDEX (con) != QUEUE_SCHEDULED)
    4530                 :          0 :         TODO_SPEC (con) = recompute_todo_spec (con, true);
    4531                 :            :     }
    4532                 :          0 : }
    4533                 :            : 
    4534                 :            : /* Restore scheduler state from the topmost entry on the backtracking queue.
    4535                 :            :    PSCHED_BLOCK_P points to the local data of schedule_block that we must
    4536                 :            :    overwrite with the saved data.
    4537                 :            :    The caller must already have called unschedule_insns_until.  */
    4538                 :            : 
    4539                 :            : static void
    4540                 :          0 : restore_last_backtrack_point (struct sched_block_state *psched_block)
    4541                 :            : {
    4542                 :          0 :   int i;
    4543                 :          0 :   struct haifa_saved_data *save = backtrack_queue;
    4544                 :            : 
    4545                 :          0 :   backtrack_queue = save->next;
    4546                 :            : 
    4547                 :          0 :   if (current_sched_info->restore_state)
    4548                 :          0 :     (*current_sched_info->restore_state) (save->fe_saved_data);
    4549                 :            : 
    4550                 :          0 :   if (targetm.sched.alloc_sched_context)
    4551                 :            :     {
    4552                 :          0 :       targetm.sched.set_sched_context (save->be_saved_data);
    4553                 :          0 :       targetm.sched.free_sched_context (save->be_saved_data);
    4554                 :            :     }
    4555                 :            : 
    4556                 :            :   /* Do this first since it clobbers INSN_TICK of the involved
    4557                 :            :      instructions.  */
    4558                 :          0 :   undo_replacements_for_backtrack (save);
    4559                 :            : 
    4560                 :            :   /* Clear the QUEUE_INDEX of everything in the ready list or one
    4561                 :            :      of the queues.  */
    4562                 :          0 :   if (ready.n_ready > 0)
    4563                 :            :     {
    4564                 :          0 :       rtx_insn **first = ready_lastpos (&ready);
    4565                 :          0 :       for (i = 0; i < ready.n_ready; i++)
    4566                 :            :         {
    4567                 :          0 :           rtx_insn *insn = first[i];
    4568                 :          0 :           QUEUE_INDEX (insn) = QUEUE_NOWHERE;
    4569                 :          0 :           INSN_TICK (insn) = INVALID_TICK;
    4570                 :            :         }
    4571                 :            :     }
    4572                 :          0 :   for (i = 0; i <= max_insn_queue_index; i++)
    4573                 :            :     {
    4574                 :          0 :       int q = NEXT_Q_AFTER (q_ptr, i);
    4575                 :            : 
    4576                 :          0 :       for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ())
    4577                 :            :         {
    4578                 :          0 :           rtx_insn *x = link->insn ();
    4579                 :          0 :           QUEUE_INDEX (x) = QUEUE_NOWHERE;
    4580                 :          0 :           INSN_TICK (x) = INVALID_TICK;
    4581                 :            :         }
    4582                 :          0 :       free_INSN_LIST_list (&insn_queue[q]);
    4583                 :            :     }
    4584                 :            : 
    4585                 :          0 :   free (ready.vec);
    4586                 :          0 :   ready = save->ready;
    4587                 :            : 
    4588                 :          0 :   if (ready.n_ready > 0)
    4589                 :            :     {
    4590                 :          0 :       rtx_insn **first = ready_lastpos (&ready);
    4591                 :          0 :       for (i = 0; i < ready.n_ready; i++)
    4592                 :            :         {
    4593                 :          0 :           rtx_insn *insn = first[i];
    4594                 :          0 :           QUEUE_INDEX (insn) = QUEUE_READY;
    4595                 :          0 :           TODO_SPEC (insn) = recompute_todo_spec (insn, true);
    4596                 :          0 :           INSN_TICK (insn) = save->clock_var;
    4597                 :            :         }
    4598                 :            :     }
    4599                 :            : 
    4600                 :          0 :   q_ptr = 0;
    4601                 :          0 :   q_size = save->q_size;
    4602                 :          0 :   for (i = 0; i <= max_insn_queue_index; i++)
    4603                 :            :     {
    4604                 :          0 :       int q = NEXT_Q_AFTER (q_ptr, i);
    4605                 :            : 
    4606                 :          0 :       insn_queue[q] = save->insn_queue[q];
    4607                 :            : 
    4608                 :          0 :       for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ())
    4609                 :            :         {
    4610                 :          0 :           rtx_insn *x = link->insn ();
    4611                 :          0 :           QUEUE_INDEX (x) = i;
    4612                 :          0 :           TODO_SPEC (x) = recompute_todo_spec (x, true);
    4613                 :          0 :           INSN_TICK (x) = save->clock_var + i;
    4614                 :            :         }
    4615                 :            :     }
    4616                 :          0 :   free (save->insn_queue);
    4617                 :            : 
    4618                 :          0 :   toggle_cancelled_flags (true);
    4619                 :            : 
    4620                 :          0 :   clock_var = save->clock_var;
    4621                 :          0 :   last_clock_var = save->last_clock_var;
    4622                 :          0 :   cycle_issued_insns = save->cycle_issued_insns;
    4623                 :          0 :   last_scheduled_insn = save->last_scheduled_insn;
    4624                 :          0 :   last_nondebug_scheduled_insn = save->last_nondebug_scheduled_insn;
    4625                 :          0 :   nonscheduled_insns_begin = save->nonscheduled_insns_begin;
    4626                 :            : 
    4627                 :          0 :   *psched_block = save->sched_block;
    4628                 :            : 
    4629                 :          0 :   memcpy (curr_state, save->curr_state, dfa_state_size);
    4630                 :          0 :   free (save->curr_state);
    4631                 :            : 
    4632                 :          0 :   mark_backtrack_feeds (save->delay_pair->i2, 0);
    4633                 :            : 
    4634                 :          0 :   gcc_assert (next_cycle_replace_deps.is_empty ());
    4635                 :          0 :   next_cycle_replace_deps = save->next_cycle_deps.copy ();
    4636                 :          0 :   next_cycle_apply = save->next_cycle_apply.copy ();
    4637                 :            : 
    4638                 :          0 :   free (save);
    4639                 :            : 
    4640                 :          0 :   for (save = backtrack_queue; save; save = save->next)
    4641                 :            :     {
    4642                 :          0 :       mark_backtrack_feeds (save->delay_pair->i2, 1);
    4643                 :            :     }
    4644                 :          0 : }
    4645                 :            : 
    4646                 :            : /* Discard all data associated with the topmost entry in the backtrack
    4647                 :            :    queue.  If RESET_TICK is false, we just want to free the data.  If true,
    4648                 :            :    we are doing this because we discovered a reason to backtrack.  In the
    4649                 :            :    latter case, also reset the INSN_TICK for the shadow insn.  */
    4650                 :            : static void
    4651                 :          0 : free_topmost_backtrack_point (bool reset_tick)
    4652                 :            : {
    4653                 :          0 :   struct haifa_saved_data *save = backtrack_queue;
    4654                 :          0 :   int i;
    4655                 :            : 
    4656                 :          0 :   backtrack_queue = save->next;
    4657                 :            : 
    4658                 :          0 :   if (reset_tick)
    4659                 :            :     {
    4660                 :          0 :       struct delay_pair *pair = save->delay_pair;
    4661                 :          0 :       while (pair)
    4662                 :            :         {
    4663                 :          0 :           INSN_TICK (pair->i2) = INVALID_TICK;
    4664                 :          0 :           INSN_EXACT_TICK (pair->i2) = INVALID_TICK;
    4665                 :          0 :           pair = pair->next_same_i1;
    4666                 :            :         }
    4667                 :          0 :       undo_replacements_for_backtrack (save);
    4668                 :            :     }
    4669                 :            :   else
    4670                 :            :     {
    4671                 :          0 :       save->replacement_deps.release ();
    4672                 :          0 :       save->replace_apply.release ();
    4673                 :            :     }
    4674                 :            : 
    4675                 :          0 :   if (targetm.sched.free_sched_context)
    4676                 :          0 :     targetm.sched.free_sched_context (save->be_saved_data);
    4677                 :          0 :   if (current_sched_info->restore_state)
    4678                 :          0 :     free (save->fe_saved_data);
    4679                 :          0 :   for (i = 0; i <= max_insn_queue_index; i++)
    4680                 :          0 :     free_INSN_LIST_list (&save->insn_queue[i]);
    4681                 :          0 :   free (save->insn_queue);
    4682                 :          0 :   free (save->curr_state);
    4683                 :          0 :   free (save->ready.vec);
    4684                 :          0 :   free (save);
    4685                 :          0 : }
    4686                 :            : 
    4687                 :            : /* Free the entire backtrack queue.  */
    4688                 :            : static void
    4689                 :          0 : free_backtrack_queue (void)
    4690                 :            : {
    4691                 :    6519130 :   while (backtrack_queue)
    4692                 :          0 :     free_topmost_backtrack_point (false);
    4693                 :          0 : }
    4694                 :            : 
    4695                 :            : /* Apply a replacement described by DESC.  If IMMEDIATELY is false, we
    4696                 :            :    may have to postpone the replacement until the start of the next cycle,
    4697                 :            :    at which point we will be called again with IMMEDIATELY true.  This is
    4698                 :            :    only done for machines which have instruction packets with explicit
    4699                 :            :    parallelism however.  */
    4700                 :            : static void
    4701                 :     357383 : apply_replacement (dep_t dep, bool immediately)
    4702                 :            : {
    4703                 :     357383 :   struct dep_replacement *desc = DEP_REPLACE (dep);
    4704                 :     357383 :   if (!immediately && targetm.sched.exposed_pipeline && reload_completed)
    4705                 :            :     {
    4706                 :          0 :       next_cycle_replace_deps.safe_push (dep);
    4707                 :          0 :       next_cycle_apply.safe_push (1);
    4708                 :            :     }
    4709                 :            :   else
    4710                 :            :     {
    4711                 :     357383 :       bool success;
    4712                 :            : 
    4713                 :     357383 :       if (QUEUE_INDEX (desc->insn) == QUEUE_SCHEDULED)
    4714                 :            :         return;
    4715                 :            : 
    4716                 :     357383 :       if (sched_verbose >= 5)
    4717                 :          0 :         fprintf (sched_dump, "applying replacement for insn %d\n",
    4718                 :          0 :                  INSN_UID (desc->insn));
    4719                 :            : 
    4720                 :     357383 :       success = validate_change (desc->insn, desc->loc, desc->newval, 0);
    4721                 :     357383 :       gcc_assert (success);
    4722                 :            : 
    4723                 :     357383 :       rtx_insn *insn = DEP_PRO (dep);
    4724                 :            : 
    4725                 :            :       /* Recompute priority since dependent priorities may have changed.  */
    4726                 :     357383 :       priority (insn, true);
    4727                 :     357383 :       update_insn_after_change (desc->insn);
    4728                 :            : 
    4729                 :     357383 :       if ((TODO_SPEC (desc->insn) & (HARD_DEP | DEP_POSTPONED)) == 0)
    4730                 :     199456 :         fix_tick_ready (desc->insn);
    4731                 :            : 
    4732                 :     357383 :       if (backtrack_queue != NULL)
    4733                 :            :         {
    4734                 :          0 :           backtrack_queue->replacement_deps.safe_push (dep);
    4735                 :          0 :           backtrack_queue->replace_apply.safe_push (1);
    4736                 :            :         }
    4737                 :            :     }
    4738                 :            : }
    4739                 :            : 
    4740                 :            : /* We have determined that a pattern involved in DEP must be restored.
    4741                 :            :    If IMMEDIATELY is false, we may have to postpone the replacement
    4742                 :            :    until the start of the next cycle, at which point we will be called
    4743                 :            :    again with IMMEDIATELY true.  */
    4744                 :            : static void
    4745                 :      38395 : restore_pattern (dep_t dep, bool immediately)
    4746                 :            : {
    4747                 :      38395 :   rtx_insn *next = DEP_CON (dep);
    4748                 :      38395 :   int tick = INSN_TICK (next);
    4749                 :            : 
    4750                 :            :   /* If we already scheduled the insn, the modified version is
    4751                 :            :      correct.  */
    4752                 :      38395 :   if (QUEUE_INDEX (next) == QUEUE_SCHEDULED)
    4753                 :            :     return;
    4754                 :            : 
    4755                 :      38395 :   if (!immediately && targetm.sched.exposed_pipeline && reload_completed)
    4756                 :            :     {
    4757                 :          0 :       next_cycle_replace_deps.safe_push (dep);
    4758                 :          0 :       next_cycle_apply.safe_push (0);
    4759                 :          0 :       return;
    4760                 :            :     }
    4761                 :            : 
    4762                 :            : 
    4763                 :      38395 :   if (DEP_TYPE (dep) == REG_DEP_CONTROL)
    4764                 :            :     {
    4765                 :          0 :       if (sched_verbose >= 5)
    4766                 :          0 :         fprintf (sched_dump, "restoring pattern for insn %d\n",
    4767                 :          0 :                  INSN_UID (next));
    4768                 :          0 :       haifa_change_pattern (next, ORIG_PAT (next));
    4769                 :            :     }
    4770                 :            :   else
    4771                 :            :     {
    4772                 :      38395 :       struct dep_replacement *desc = DEP_REPLACE (dep);
    4773                 :      38395 :       bool success;
    4774                 :            : 
    4775                 :      38395 :       if (sched_verbose >= 5)
    4776                 :          0 :         fprintf (sched_dump, "restoring pattern for insn %d\n",
    4777                 :          0 :                  INSN_UID (desc->insn));
    4778                 :      38395 :       tick = INSN_TICK (desc->insn);
    4779                 :            : 
    4780                 :      38395 :       success = validate_change (desc->insn, desc->loc, desc->orig, 0);
    4781                 :      38395 :       gcc_assert (success);
    4782                 :            : 
    4783                 :      38395 :       rtx_insn *insn = DEP_PRO (dep);
    4784                 :            : 
    4785                 :      38395 :       if (QUEUE_INDEX (insn) != QUEUE_SCHEDULED)
    4786                 :            :         {
    4787                 :            :           /* Recompute priority since dependent priorities may have changed.  */
    4788                 :          0 :           priority (insn, true);
    4789                 :            :         }
    4790                 :            : 
    4791                 :      38395 :       update_insn_after_change (desc->insn);
    4792                 :            : 
    4793                 :      38395 :       if (backtrack_queue != NULL)
    4794                 :            :         {
    4795                 :          0 :           backtrack_queue->replacement_deps.safe_push (dep);
    4796                 :          0 :           backtrack_queue->replace_apply.safe_push (0);
    4797                 :            :         }
    4798                 :            :     }
    4799                 :      38395 :   INSN_TICK (next) = tick;
    4800                 :      38395 :   if (TODO_SPEC (next) == DEP_POSTPONED)
    4801                 :            :     return;
    4802                 :            : 
    4803                 :      38395 :   if (sd_lists_empty_p (next, SD_LIST_BACK))
    4804                 :      38395 :     TODO_SPEC (next) = 0;
    4805                 :          0 :   else if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK))
    4806                 :          0 :     TODO_SPEC (next) = HARD_DEP;
    4807                 :            : }
    4808                 :            : 
    4809                 :            : /* Perform pattern replacements that were queued up until the next
    4810                 :            :    cycle.  */
    4811                 :            : static void
    4812                 :   30463800 : perform_replacements_new_cycle (void)
    4813                 :            : {
    4814                 :   30463800 :   int i;
    4815                 :   30463800 :   dep_t dep;
    4816                 :   30463800 :   FOR_EACH_VEC_ELT (next_cycle_replace_deps, i, dep)
    4817                 :            :     {
    4818                 :          0 :       int apply_p = next_cycle_apply[i];
    4819                 :          0 :       if (apply_p)
    4820                 :          0 :         apply_replacement (dep, true);
    4821                 :            :       else
    4822                 :          0 :         restore_pattern (dep, true);
    4823                 :            :     }
    4824                 :   30463800 :   next_cycle_replace_deps.truncate (0);
    4825                 :   30463800 :   next_cycle_apply.truncate (0);
    4826                 :   30463800 : }
    4827                 :            : 
    4828                 :            : /* Compute INSN_TICK_ESTIMATE for INSN.  PROCESSED is a bitmap of
    4829                 :            :    instructions we've previously encountered, a set bit prevents
    4830                 :            :    recursion.  BUDGET is a limit on how far ahead we look, it is
    4831                 :            :    reduced on recursive calls.  Return true if we produced a good
    4832                 :            :    estimate, or false if we exceeded the budget.  */
    4833                 :            : static bool
    4834                 :          0 : estimate_insn_tick (bitmap processed, rtx_insn *insn, int budget)
    4835                 :            : {
    4836                 :          0 :   sd_iterator_def sd_it;
    4837                 :          0 :   dep_t dep;
    4838                 :          0 :   int earliest = INSN_TICK (insn);
    4839                 :            : 
    4840                 :          0 :   FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
    4841                 :            :     {
    4842                 :          0 :       rtx_insn *pro = DEP_PRO (dep);
    4843                 :          0 :       int t;
    4844                 :            : 
    4845                 :          0 :       if (DEP_STATUS (dep) & DEP_CANCELLED)
    4846                 :          0 :         continue;
    4847                 :            : 
    4848                 :          0 :       if (QUEUE_INDEX (pro) == QUEUE_SCHEDULED)
    4849                 :          0 :         gcc_assert (INSN_TICK (pro) + dep_cost (dep) <= INSN_TICK (insn));
    4850                 :            :       else
    4851                 :            :         {
    4852                 :          0 :           int cost = dep_cost (dep);
    4853                 :          0 :           if (cost >= budget)
    4854                 :            :             return false;
    4855                 :          0 :           if (!bitmap_bit_p (processed, INSN_LUID (pro)))
    4856                 :            :             {
    4857                 :          0 :               if (!estimate_insn_tick (processed, pro, budget - cost))
    4858                 :            :                 return false;
    4859                 :            :             }
    4860                 :          0 :           gcc_assert (INSN_TICK_ESTIMATE (pro) != INVALID_TICK);
    4861                 :          0 :           t = INSN_TICK_ESTIMATE (pro) + cost;
    4862                 :          0 :           if (earliest == INVALID_TICK || t > earliest)
    4863                 :          0 :             earliest = t;
    4864                 :            :         }
    4865                 :            :     }
    4866                 :          0 :   bitmap_set_bit (processed, INSN_LUID (insn));
    4867                 :          0 :   INSN_TICK_ESTIMATE (insn) = earliest;
    4868                 :          0 :   return true;
    4869                 :            : }
    4870                 :            : 
    4871                 :            : /* Examine the pair of insns in P, and estimate (optimistically, assuming
    4872                 :            :    infinite resources) the cycle in which the delayed shadow can be issued.
    4873                 :            :    Return the number of cycles that must pass before the real insn can be
    4874                 :            :    issued in order to meet this constraint.  */
    4875                 :            : static int
    4876                 :          0 : estimate_shadow_tick (struct delay_pair *p)
    4877                 :            : {
    4878                 :          0 :   auto_bitmap processed;
    4879                 :          0 :   int t;
    4880                 :          0 :   bool cutoff;
    4881                 :            : 
    4882                 :          0 :   cutoff = !estimate_insn_tick (processed, p->i2,
    4883                 :          0 :                                 max_insn_queue_index + pair_delay (p));
    4884                 :          0 :   if (cutoff)
    4885                 :            :     return max_insn_queue_index;
    4886                 :          0 :   t = INSN_TICK_ESTIMATE (p->i2) - (clock_var + pair_delay (p) + 1);
    4887                 :          0 :   if (t > 0)
    4888                 :            :     return t;
    4889                 :            :   return 0;
    4890                 :            : }
    4891                 :            : 
    4892                 :            : /* If INSN has no unresolved backwards dependencies, add it to the schedule and
    4893                 :            :    recursively resolve all its forward dependencies.  */
    4894                 :            : static void
    4895                 :          0 : resolve_dependencies (rtx_insn *insn)
    4896                 :            : {
    4897                 :          0 :   sd_iterator_def sd_it;
    4898                 :          0 :   dep_t dep;
    4899                 :            : 
    4900                 :            :   /* Don't use sd_lists_empty_p; it ignores debug insns.  */
    4901                 :          0 :   if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn)) != NULL
    4902                 :          0 :       || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn)) != NULL)
    4903                 :          0 :     return;
    4904                 :            : 
    4905                 :          0 :   if (sched_verbose >= 4)
    4906                 :          0 :     fprintf (sched_dump, ";;\tquickly resolving %d\n", INSN_UID (insn));
    4907                 :            : 
    4908                 :          0 :   if (QUEUE_INDEX (insn) >= 0)
    4909                 :          0 :     queue_remove (insn);
    4910                 :            : 
    4911                 :          0 :   scheduled_insns.safe_push (insn);
    4912                 :            : 
    4913                 :            :   /* Update dependent instructions.  */
    4914                 :          0 :   for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
    4915                 :          0 :        sd_iterator_cond (&sd_it, &dep);)
    4916                 :            :     {
    4917                 :          0 :       rtx_insn *next = DEP_CON (dep);
    4918                 :            : 
    4919                 :          0 :       if (sched_verbose >= 4)
    4920                 :          0 :         fprintf (sched_dump, ";;\t\tdep %d against %d\n", INSN_UID (insn),
    4921                 :          0 :                  INSN_UID (next));
    4922                 :            : 
    4923                 :            :       /* Resolve the dependence between INSN and NEXT.
    4924                 :            :          sd_resolve_dep () moves current dep to another list thus
    4925                 :            :          advancing the iterator.  */
    4926                 :          0 :       sd_resolve_dep (sd_it);
    4927                 :            : 
    4928                 :          0 :       if (!IS_SPECULATION_BRANCHY_CHECK_P (insn))
    4929                 :            :         {
    4930                 :          0 :           resolve_dependencies (next);
    4931                 :            :         }
    4932                 :            :       else
    4933                 :            :         /* Check always has only one forward dependence (to the first insn in
    4934                 :            :            the recovery block), therefore, this will be executed only once.  */
    4935                 :            :         {
    4936                 :          0 :           gcc_assert (sd_lists_empty_p (insn, SD_LIST_FORW));
    4937                 :            :         }
    4938                 :            :     }
    4939                 :            : }
    4940                 :            : 
    4941                 :            : 
    4942                 :            : /* Return the head and tail pointers of ebb starting at BEG and ending
    4943                 :            :    at END.  */
    4944                 :            : void
    4945                 :   26102700 : get_ebb_head_tail (basic_block beg, basic_block end,
    4946                 :            :                    rtx_insn **headp, rtx_insn **tailp)
    4947                 :            : {
    4948                 :   26102700 :   rtx_insn *beg_head = BB_HEAD (beg);
    4949                 :   26102700 :   rtx_insn * beg_tail = BB_END (beg);
    4950                 :   26102700 :   rtx_insn * end_head = BB_HEAD (end);
    4951                 :   26102700 :   rtx_insn * end_tail = BB_END (end);
    4952                 :            : 
    4953                 :            :   /* Don't include any notes or labels at the beginning of the BEG
    4954                 :            :      basic block, or notes at the end of the END basic blocks.  */
    4955                 :            : 
    4956                 :   26102700 :   if (LABEL_P (beg_head))
    4957                 :   12591100 :     beg_head = NEXT_INSN (beg_head);
    4958                 :            : 
    4959                 :   62340100 :   while (beg_head != beg_tail)
    4960                 :   60114900 :     if (NOTE_P (beg_head))
    4961                 :   36237300 :       beg_head = NEXT_INSN (beg_head);
    4962                 :   23877600 :     else if (DEBUG_INSN_P (beg_head))
    4963                 :            :       {
    4964                 :    6426950 :         rtx_insn * note, *next;
    4965                 :            : 
    4966                 :    6426950 :         for (note = NEXT_INSN (beg_head);
    4967                 :   50416700 :              note != beg_tail;
    4968                 :            :              note = next)
    4969                 :            :           {
    4970                 :   50201200 :             next = NEXT_INSN (note);
    4971                 :   50201200 :             if (NOTE_P (note))
    4972                 :            :               {
    4973                 :     303110 :                 if (sched_verbose >= 9)
    4974                 :          0 :                   fprintf (sched_dump, "reorder %i\n", INSN_UID (note));
    4975                 :            : 
    4976                 :     303110 :                 reorder_insns_nobb (note, note, PREV_INSN (beg_head));
    4977                 :            : 
    4978                 :     303110 :                 if (BLOCK_FOR_INSN (note) != beg)
    4979                 :          0 :                   df_insn_change_bb (note, beg);
    4980                 :            :               }
    4981                 :   49898100 :             else if (!DEBUG_INSN_P (note))
    4982                 :            :               break;
    4983                 :            :           }
    4984                 :            : 
    4985                 :            :         break;
    4986                 :            :       }
    4987                 :            :     else
    4988                 :            :       break;
    4989                 :            : 
    4990                 :   26102700 :   *headp = beg_head;
    4991                 :            : 
    4992                 :   26102700 :   if (beg == end)
    4993                 :   26102700 :     end_head = beg_head;
    4994                 :         56 :   else if (LABEL_P (end_head))
    4995                 :          1 :     end_head = NEXT_INSN (end_head);
    4996                 :            : 
    4997                 :   26247300 :   while (end_head != end_tail)
    4998                 :   24005600 :     if (NOTE_P (end_tail))
    4999                 :     144621 :       end_tail = PREV_INSN (end_tail);
    5000                 :   23861000 :     else if (DEBUG_INSN_P (end_tail))
    5001                 :            :       {
    5002                 :     484239 :         rtx_insn * note, *prev;
    5003                 :            : 
    5004                 :     484239 :         for (note = PREV_INSN (end_tail);
    5005                 :    3217360 :              note != end_head;
    5006                 :            :              note = prev)
    5007                 :            :           {
    5008                 :    3189940 :             prev = PREV_INSN (note);
    5009                 :    3189940 :             if (NOTE_P (note))
    5010                 :            :               {
    5011                 :       2353 :                 if (sched_verbose >= 9)
    5012                 :          0 :                   fprintf (sched_dump, "reorder %i\n", INSN_UID (note));
    5013                 :            : 
    5014                 :       2353 :                 reorder_insns_nobb (note, note, end_tail);
    5015                 :            : 
    5016                 :       2353 :                 if (end_tail == BB_END (end))
    5017                 :       1291 :                   BB_END (end) = note;
    5018                 :            : 
    5019                 :       2353 :                 if (BLOCK_FOR_INSN (note) != end)
    5020                 :          0 :                   df_insn_change_bb (note, end);
    5021                 :            :               }
    5022                 :    3187580 :             else if (!DEBUG_INSN_P (note))
    5023                 :            :               break;
    5024                 :            :           }
    5025                 :            : 
    5026                 :            :         break;
    5027                 :            :       }
    5028                 :            :     else
    5029                 :            :       break;
    5030                 :            : 
    5031                 :   26102700 :   *tailp = end_tail;
    5032                 :   26102700 : }
    5033                 :            : 
    5034                 :            : /* Return nonzero if there are no real insns in the range [ HEAD, TAIL ].  */
    5035                 :            : 
    5036                 :            : int
    5037                 :   19576300 : no_real_insns_p (const rtx_insn *head, const rtx_insn *tail)
    5038                 :            : {
    5039                 :   19593300 :   while (head != NEXT_INSN (tail))
    5040                 :            :     {
    5041                 :   19576300 :       if (!NOTE_P (head) && !LABEL_P (head))
    5042                 :            :         return 0;
    5043                 :      17042 :       head = NEXT_INSN (head);
    5044                 :            :     }
    5045                 :            :   return 1;
    5046                 :            : }
    5047                 :            : 
    5048                 :            : /* Restore-other-notes: NOTE_LIST is the end of a chain of notes
    5049                 :            :    previously found among the insns.  Insert them just before HEAD.  */
    5050                 :            : rtx_insn *
    5051                 :    6520100 : restore_other_notes (rtx_insn *head, basic_block head_bb)
    5052                 :            : {
    5053                 :    6520100 :   if (note_list != 0)
    5054                 :            :     {
    5055                 :    1354680 :       rtx_insn *note_head = note_list;
    5056                 :            : 
    5057                 :    1354680 :       if (head)
    5058                 :    1354360 :         head_bb = BLOCK_FOR_INSN (head);
    5059                 :            :       else
    5060                 :        325 :         head = NEXT_INSN (bb_note (head_bb));
    5061                 :            : 
    5062                 :    3303470 :       while (PREV_INSN (note_head))
    5063                 :            :         {
    5064                 :    1948780 :           set_block_for_insn (note_head, head_bb);
    5065                 :    1948780 :           note_head = PREV_INSN (note_head);
    5066                 :            :         }
    5067                 :            :       /* In the above cycle we've missed this note.  */
    5068                 :    1354680 :       set_block_for_insn (note_head, head_bb);
    5069                 :            : 
    5070                 :    1354680 :       SET_PREV_INSN (note_head) = PREV_INSN (head);
    5071                 :    1354680 :       SET_NEXT_INSN (PREV_INSN (head)) = note_head;
    5072                 :    1354680 :       SET_PREV_INSN (head) = note_list;
    5073                 :    1354680 :       SET_NEXT_INSN (note_list) = head;
    5074                 :            : 
    5075                 :    1354680 :       if (BLOCK_FOR_INSN (head) != head_bb)
    5076                 :          0 :         BB_END (head_bb) = note_list;
    5077                 :            : 
    5078                 :            :       head = note_head;
    5079                 :            :     }
    5080                 :            : 
    5081                 :    6520100 :   return head;
    5082                 :            : }
    5083                 :            : 
    5084                 :            : /* When we know we are going to discard the schedule due to a failed attempt
    5085                 :            :    at modulo scheduling, undo all replacements.  */
    5086                 :            : static void
    5087                 :          0 : undo_all_replacements (void)
    5088                 :            : {
    5089                 :          0 :   rtx_insn *insn;
    5090                 :          0 :   int i;
    5091                 :            : 
    5092                 :          0 :   FOR_EACH_VEC_ELT (scheduled_insns, i, insn)
    5093                 :            :     {
    5094                 :          0 :       sd_iterator_def sd_it;
    5095                 :          0 :       dep_t dep;
    5096                 :            : 
    5097                 :            :       /* See if we must undo a replacement.  */
    5098                 :          0 :       for (sd_it = sd_iterator_start (insn, SD_LIST_RES_FORW);
    5099                 :          0 :            sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it))
    5100                 :            :         {
    5101                 :          0 :           struct dep_replacement *desc = DEP_REPLACE (dep);
    5102                 :          0 :           if (desc != NULL)
    5103                 :          0 :             validate_change (desc->insn, desc->loc, desc->orig, 0);
    5104                 :            :         }
    5105                 :            :     }
    5106                 :          0 : }
    5107                 :            : 
    5108                 :            : /* Return first non-scheduled insn in the current scheduling block.
    5109                 :            :    This is mostly used for debug-counter purposes.  */
    5110                 :            : static rtx_insn *
    5111                 :          0 : first_nonscheduled_insn (void)
    5112                 :            : {
    5113                 :          0 :   rtx_insn *insn = (nonscheduled_insns_begin != NULL_RTX
    5114                 :          0 :                     ? nonscheduled_insns_begin
    5115                 :          0 :                     : current_sched_info->prev_head);
    5116                 :            : 
    5117                 :          0 :   do
    5118                 :            :     {
    5119                 :          0 :       insn = next_nonnote_nondebug_insn (insn);
    5120                 :            :     }
    5121                 :          0 :   while (QUEUE_INDEX (insn) == QUEUE_SCHEDULED);
    5122                 :            : 
    5123                 :          0 :   return insn;
    5124                 :            : }
    5125                 :            : 
    5126                 :            : /* Move insns that became ready to fire from queue to ready list.  */
    5127                 :            : 
    5128                 :            : static void
    5129                 :   23944700 : queue_to_ready (struct ready_list *ready)
    5130                 :            : {
    5131                 :   23944700 :   rtx_insn *insn;
    5132                 :   23944700 :   rtx_insn_list *link;
    5133                 :   23944700 :   rtx_insn *skip_insn;
    5134                 :            : 
    5135                 :   23944700 :   q_ptr = NEXT_Q (q_ptr);
    5136                 :            : 
    5137                 :   23944700 :   if (dbg_cnt (sched_insn) == false)
    5138                 :            :     /* If debug counter is activated do not requeue the first
    5139                 :            :        nonscheduled insn.  */
    5140                 :          0 :     skip_insn = first_nonscheduled_insn ();
    5141                 :            :   else
    5142                 :            :     skip_insn = NULL;
    5143                 :            : 
    5144                 :            :   /* Add all pending insns that can be scheduled without stalls to the
    5145                 :            :      ready list.  */
    5146                 :   59105200 :   for (link = insn_queue[q_ptr]; link; link = link->next ())
    5147                 :            :     {
    5148                 :   24080900 :       insn = link->insn ();
    5149                 :   24080900 :       q_size -= 1;
    5150                 :            : 
    5151                 :   24080900 :       if (sched_verbose >= 2)
    5152                 :          0 :         fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ",
    5153                 :          0 :                  (*current_sched_info->print_insn) (insn, 0));
    5154                 :            : 
    5155                 :            :       /* If the ready list is full, delay the insn for 1 cycle.
    5156                 :            :          See the comment in schedule_block for the rationale.  */
    5157                 :   24080900 :       if (!reload_completed
    5158                 :      16262 :           && (ready->n_ready - ready->n_debug > param_max_sched_ready_insns
    5159                 :       9896 :               || (sched_pressure == SCHED_PRESSURE_MODEL
    5160                 :            :                   /* Limit pressure recalculations to
    5161                 :            :                      param_max_sched_ready_insns instructions too.  */
    5162                 :          0 :                   && model_index (insn) > (model_curr_point
    5163                 :          0 :                                            + param_max_sched_ready_insns)))
    5164                 :       6366 :           && !(sched_pressure == SCHED_PRESSURE_MODEL
    5165                 :          0 :                && model_curr_point < model_num_insns
    5166                 :            :                /* Always allow the next model instruction to issue.  */
    5167                 :          0 :                && model_index (insn) == model_curr_point)
    5168                 :       6366 :           && !SCHED_GROUP_P (insn)
    5169                 :   24087300 :           && insn != skip_insn)
    5170                 :            :         {
    5171                 :       6366 :           if (sched_verbose >= 2)
    5172                 :          0 :             fprintf (sched_dump, "keeping in queue, ready full\n");
    5173                 :       6366 :           queue_insn (insn, 1, "ready full");
    5174                 :            :         }
    5175                 :            :       else
    5176                 :            :         {
    5177                 :   24074600 :           ready_add (ready, insn, false);
    5178                 :   24074600 :           if (sched_verbose >= 2)
    5179                 :          0 :             fprintf (sched_dump, "moving to ready without stalls\n");
    5180                 :            :         }
    5181                 :            :     }
    5182                 :   23944700 :   free_INSN_LIST_list (&insn_queue[q_ptr]);
    5183                 :            : 
    5184                 :            :   /* If there are no ready insns, stall until one is ready and add all
    5185                 :            :      of the pending insns at that point to the ready list.  */
    5186                 :   23944700 :   if (ready->n_ready == 0)
    5187                 :            :     {
    5188                 :            :       int stalls;
    5189                 :            : 
    5190                 :    8075390 :       for (stalls = 1; stalls <= max_insn_queue_index; stalls++)
    5191                 :            :         {
    5192                 :    8075390 :           if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]))
    5193                 :            :             {
    5194                 :    9213780 :               for (; link; link = link->next ())
    5195                 :            :                 {
    5196                 :    4606890 :                   insn = link->insn ();
    5197                 :    4606890 :                   q_size -= 1;
    5198                 :            : 
    5199                 :    4606890 :                   if (sched_verbose >= 2)
    5200                 :          0 :                     fprintf (sched_dump, ";;\t\tQ-->Ready: insn %s: ",
    5201                 :          0 :                              (*current_sched_info->print_insn) (insn, 0));
    5202                 :            : 
    5203                 :    4606890 :                   ready_add (ready, insn, false);
    5204                 :    4606890 :                   if (sched_verbose >= 2)
    5205                 :          0 :                     fprintf (sched_dump, "moving to ready with %d stalls\n", stalls);
    5206                 :            :                 }
    5207                 :    4353350 :               free_INSN_LIST_list (&insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]);
    5208                 :            : 
    5209                 :    4353350 :               advance_one_cycle ();
    5210                 :            : 
    5211                 :    4353350 :               break;
    5212                 :            :             }
    5213                 :            : 
    5214                 :    3722040 :           advance_one_cycle ();
    5215                 :            :         }
    5216                 :            : 
    5217                 :    4353350 :       q_ptr = NEXT_Q_AFTER (q_ptr, stalls);
    5218                 :    4353350 :       clock_var += stalls;
    5219                 :    4353350 :       if (sched_verbose >= 2)
    5220                 :          0 :         fprintf (sched_dump, ";;\tAdvancing clock by %d cycle[s] to %d\n",
    5221                 :            :                  stalls, clock_var);
    5222                 :            :     }
    5223                 :   23944700 : }
    5224                 :            : 
    5225                 :            : /* Used by early_queue_to_ready.  Determines whether it is "ok" to
    5226                 :            :    prematurely move INSN from the queue to the ready list.  Currently,
    5227                 :            :    if a target defines the hook 'is_costly_dependence', this function
    5228                 :            :    uses the hook to check whether there exist any dependences which are
    5229                 :            :    considered costly by the target, between INSN and other insns that
    5230                 :            :    have already been scheduled.  Dependences are checked up to Y cycles
    5231                 :            :    back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows
    5232                 :            :    controlling this value.
    5233                 :            :    (Other considerations could be taken into account instead (or in
    5234                 :            :    addition) depending on user flags and target hooks.  */
    5235                 :            : 
    5236                 :            : static bool
    5237                 :          5 : ok_for_early_queue_removal (rtx_insn *insn)
    5238                 :            : {
    5239                 :          5 :   if (targetm.sched.is_costly_dependence)
    5240                 :            :     {
    5241                 :          0 :       int n_cycles;
    5242                 :          0 :       int i = scheduled_insns.length ();
    5243                 :          0 :       for (n_cycles = flag_sched_stalled_insns_dep; n_cycles; n_cycles--)
    5244                 :            :         {
    5245                 :          0 :           while (i-- > 0)
    5246                 :            :             {
    5247                 :          0 :               int cost;
    5248                 :            : 
    5249                 :          0 :               rtx_insn *prev_insn = scheduled_insns[i];
    5250                 :            : 
    5251                 :          0 :               if (!NOTE_P (prev_insn))
    5252                 :            :                 {
    5253                 :          0 :                   dep_t dep;
    5254                 :            : 
    5255                 :          0 :                   dep = sd_find_dep_between (prev_insn, insn, true);
    5256                 :            : 
    5257                 :          0 :                   if (dep != NULL)
    5258                 :            :                     {
    5259                 :          0 :                       cost = dep_cost (dep);
    5260                 :            : 
    5261                 :          0 :                       if (targetm.sched.is_costly_dependence (dep, cost,
    5262                 :          0 :                                 flag_sched_stalled_insns_dep - n_cycles))
    5263                 :            :                         return false;
    5264                 :            :                     }
    5265                 :            :                 }
    5266                 :            : 
    5267                 :          0 :               if (GET_MODE (prev_insn) == TImode) /* end of dispatch group */
    5268                 :            :                 break;
    5269                 :            :             }
    5270                 :            : 
    5271                 :          0 :           if (i == 0)
    5272                 :            :             break;
    5273                 :            :         }
    5274                 :            :     }
    5275                 :            : 
    5276                 :            :   return true;
    5277                 :            : }
    5278                 :            : 
    5279                 :            : 
    5280                 :            : /* Remove insns from the queue, before they become "ready" with respect
    5281                 :            :    to FU latency considerations.  */
    5282                 :            : 
    5283                 :            : static int
    5284                 :   22845300 : early_queue_to_ready (state_t state, struct ready_list *ready)
    5285                 :            : {
    5286                 :   22845300 :   rtx_insn *insn;
    5287                 :   22845300 :   rtx_insn_list *link;
    5288                 :   22845300 :   rtx_insn_list *next_link;
    5289                 :   22845300 :   rtx_insn_list *prev_link;
    5290                 :   22845300 :   bool move_to_ready;
    5291                 :   22845300 :   int cost;
    5292                 :   22845300 :   state_t temp_state = alloca (dfa_state_size);
    5293                 :   22845300 :   int stalls;
    5294                 :   22845300 :   int insns_removed = 0;
    5295                 :            : 
    5296                 :            :   /*
    5297                 :            :      Flag '-fsched-stalled-insns=X' determines the aggressiveness of this
    5298                 :            :      function:
    5299                 :            : 
    5300                 :            :      X == 0: There is no limit on how many queued insns can be removed
    5301                 :            :              prematurely.  (flag_sched_stalled_insns = -1).
    5302                 :            : 
    5303                 :            :      X >= 1: Only X queued insns can be removed prematurely in each
    5304                 :            :              invocation.  (flag_sched_stalled_insns = X).
    5305                 :            : 
    5306                 :            :      Otherwise: Early queue removal is disabled.
    5307                 :            :          (flag_sched_stalled_insns = 0)
    5308                 :            :   */
    5309                 :            : 
    5310                 :   22845300 :   if (! flag_sched_stalled_insns)
    5311                 :            :     return 0;
    5312                 :            : 
    5313                 :       1799 :   for (stalls = 0; stalls <= max_insn_queue_index; stalls++)
    5314                 :            :     {
    5315                 :       1792 :       if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]))
    5316                 :            :         {
    5317                 :          4 :           if (sched_verbose > 6)
    5318                 :          0 :             fprintf (sched_dump, ";; look at index %d + %d\n", q_ptr, stalls);
    5319                 :            : 
    5320                 :            :           prev_link = 0;
    5321                 :          9 :           while (link)
    5322                 :            :             {
    5323                 :          5 :               next_link = link->next ();
    5324                 :          5 :               insn = link->insn ();
    5325                 :          5 :               if (insn && sched_verbose > 6)
    5326                 :          0 :                 print_rtl_single (sched_dump, insn);
    5327                 :            : 
    5328                 :          5 :               memcpy (temp_state, state, dfa_state_size);
    5329                 :          5 :               if (recog_memoized (insn) < 0)
    5330                 :            :                 /* non-negative to indicate that it's not ready
    5331                 :            :                    to avoid infinite Q->R->Q->R... */
    5332                 :            :                 cost = 0;
    5333                 :            :               else
    5334                 :          5 :                 cost = state_transition (temp_state, insn);
    5335                 :            : 
    5336                 :          5 :               if (sched_verbose >= 6)
    5337                 :          0 :                 fprintf (sched_dump, "transition cost = %d\n", cost);
    5338                 :            : 
    5339                 :          5 :               move_to_ready = false;
    5340                 :          5 :               if (cost < 0)
    5341                 :            :                 {
    5342                 :          5 :                   move_to_ready = ok_for_early_queue_removal (insn);
    5343                 :          5 :                   if (move_to_ready == true)
    5344                 :            :                     {
    5345                 :            :                       /* move from Q to R */
    5346                 :          5 :                       q_size -= 1;
    5347                 :          5 :                       ready_add (ready, insn, false);
    5348                 :            : 
    5349                 :          5 :                       if (prev_link)
    5350                 :          0 :                         XEXP (prev_link, 1) = next_link;
    5351                 :            :                       else
    5352                 :          5 :                         insn_queue[NEXT_Q_AFTER (q_ptr, stalls)] = next_link;
    5353                 :            : 
    5354                 :          5 :                       free_INSN_LIST_node (link);
    5355                 :            : 
    5356                 :          5 :                       if (sched_verbose >= 2)
    5357                 :          0 :                         fprintf (sched_dump, ";;\t\tEarly Q-->Ready: insn %s\n",
    5358                 :          0 :                                  (*current_sched_info->print_insn) (insn, 0));
    5359                 :            : 
    5360                 :          5 :                       insns_removed++;
    5361                 :          5 :                       if (insns_removed == flag_sched_stalled_insns)
    5362                 :            :                         /* Remove no more than flag_sched_stalled_insns insns
    5363                 :            :                            from Q at a time.  */
    5364                 :          0 :                         return insns_removed;
    5365                 :            :                     }
    5366                 :            :                 }
    5367                 :            : 
    5368                 :          5 :               if (move_to_ready == false)
    5369                 :            :                 prev_link = link;
    5370                 :            : 
    5371                 :            :               link = next_link;
    5372                 :            :             } /* while link */
    5373                 :            :         } /* if link */
    5374                 :            : 
    5375                 :            :     } /* for stalls.. */
    5376                 :            : 
    5377                 :            :   return insns_removed;
    5378                 :            : }
    5379                 :            : 
    5380                 :            : 
    5381                 :            : /* Print the ready list for debugging purposes.
    5382                 :            :    If READY_TRY is non-zero then only print insns that max_issue
    5383                 :            :    will consider.  */
    5384                 :            : static void
    5385                 :        184 : debug_ready_list_1 (struct ready_list *ready, signed char *ready_try)
    5386                 :            : {
    5387                 :        184 :   rtx_insn **p;
    5388                 :        184 :   int i;
    5389                 :            : 
    5390                 :        184 :   if (ready->n_ready == 0)
    5391                 :            :     {
    5392                 :        184 :       fprintf (sched_dump, "\n");
    5393                 :        184 :       return;
    5394                 :            :     }
    5395                 :            : 
    5396                 :          0 :   p = ready_lastpos (ready);
    5397                 :          0 :   for (i = 0; i < ready->n_ready; i++)
    5398                 :            :     {
    5399                 :          0 :       if (ready_try != NULL && ready_try[ready->n_ready - i - 1])
    5400                 :          0 :         continue;
    5401                 :            : 
    5402                 :          0 :       fprintf (sched_dump, "  %s:%d",
    5403                 :          0 :                (*current_sched_info->print_insn) (p[i], 0),
    5404                 :          0 :                INSN_LUID (p[i]));
    5405                 :          0 :       if (sched_pressure != SCHED_PRESSURE_NONE)
    5406                 :          0 :         fprintf (sched_dump, "(cost=%d",
    5407                 :          0 :                  INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p[i]));
    5408                 :          0 :       fprintf (sched_dump, ":prio=%d", INSN_PRIORITY (p[i]));
    5409                 :          0 :       if (INSN_TICK (p[i]) > clock_var)
    5410                 :          0 :         fprintf (sched_dump, ":delay=%d", INSN_TICK (p[i]) - clock_var);
    5411                 :          0 :       if (sched_pressure == SCHED_PRESSURE_MODEL)
    5412                 :          0 :         fprintf (sched_dump, ":idx=%d",
    5413                 :            :                  model_index (p[i]));
    5414                 :          0 :       if (sched_pressure != SCHED_PRESSURE_NONE)
    5415                 :          0 :         fprintf (sched_dump, ")");
    5416                 :            :     }
    5417                 :          0 :   fprintf (sched_dump, "\n");
    5418                 :            : }
    5419                 :            : 
    5420                 :            : /* Print the ready list.  Callable from debugger.  */
    5421                 :            : static void
    5422                 :        184 : debug_ready_list (struct ready_list *ready)
    5423                 :            : {
    5424                 :          0 :   debug_ready_list_1 (ready, NULL);
    5425                 :        184 : }
    5426                 :            : 
    5427                 :            : /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn
    5428                 :            :    NOTEs.  This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb
    5429                 :            :    replaces the epilogue note in the correct basic block.  */
    5430                 :            : void
    5431                 :   39242800 : reemit_notes (rtx_insn *insn)
    5432                 :            : {
    5433                 :   39242800 :   rtx note;
    5434                 :   39242800 :   rtx_insn *last = insn;
    5435                 :            : 
    5436                 :   78130700 :   for (note = REG_NOTES (insn); note; note = XEXP (note, 1))
    5437                 :            :     {
    5438                 :   38887900 :       if (REG_NOTE_KIND (note) == REG_SAVE_NOTE)
    5439                 :            :         {
    5440                 :     577993 :           enum insn_note note_type = (enum insn_note) INTVAL (XEXP (note, 0));
    5441                 :            : 
    5442                 :     577993 :           last = emit_note_before (note_type, last);
    5443                 :     577993 :           remove_note (insn, note);
    5444                 :     577993 :           df_insn_create_insn_record (last);
    5445                 :            :         }
    5446                 :            :     }
    5447                 :   39242800 : }
    5448                 :            : 
    5449                 :            : /* Move INSN.  Reemit notes if needed.  Update CFG, if needed.  */
    5450                 :            : static void
    5451                 :   66992900 : move_insn (rtx_insn *insn, rtx_insn *last, rtx nt)
    5452                 :            : {
    5453                 :   66992900 :   if (PREV_INSN (insn) != last)
    5454                 :            :     {
    5455                 :    8062540 :       basic_block bb;
    5456                 :    8062540 :       rtx_insn *note;
    5457                 :    8062540 :       int jump_p = 0;
    5458                 :            : 
    5459                 :    8062540 :       bb = BLOCK_FOR_INSN (insn);
    5460                 :            : 
    5461                 :            :       /* BB_HEAD is either LABEL or NOTE.  */
    5462                 :    8062540 :       gcc_assert (BB_HEAD (bb) != insn);
    5463                 :            : 
    5464                 :    8062540 :       if (BB_END (bb) == insn)
    5465                 :            :         /* If this is last instruction in BB, move end marker one
    5466                 :            :            instruction up.  */
    5467                 :            :         {
    5468                 :            :           /* Jumps are always placed at the end of basic block.  */
    5469                 :     111518 :           jump_p = control_flow_insn_p (insn);
    5470                 :            : 
    5471                 :     111518 :           gcc_assert (!jump_p
    5472                 :            :                       || ((common_sched_info->sched_pass_id == SCHED_RGN_PASS)
    5473                 :            :                           && IS_SPECULATION_BRANCHY_CHECK_P (insn))
    5474                 :            :                       || (common_sched_info->sched_pass_id
    5475                 :            :                           == SCHED_EBB_PASS));
    5476                 :            : 
    5477                 :     111518 :           gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn)) == bb);
    5478                 :            : 
    5479                 :     111518 :           BB_END (bb) = PREV_INSN (insn);
    5480                 :            :         }
    5481                 :            : 
    5482                 :    8062540 :       gcc_assert (BB_END (bb) != last);
    5483                 :            : 
    5484                 :    8062540 :       if (jump_p)
    5485                 :            :         /* We move the block note along with jump.  */
    5486                 :            :         {
    5487                 :          2 :           gcc_assert (nt);
    5488                 :            : 
    5489                 :          2 :           note = NEXT_INSN (insn);
    5490                 :          2 :           while (NOTE_NOT_BB_P (note) && note != nt)
    5491                 :          0 :             note = NEXT_INSN (note);
    5492                 :            : 
    5493                 :          2 :           if (note != nt
    5494                 :          2 :               && (LABEL_P (note)
    5495                 :          2 :                   || BARRIER_P (note)))
    5496                 :          0 :             note = NEXT_INSN (note);
    5497                 :            : 
    5498                 :          2 :           gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note));
    5499                 :            :         }
    5500                 :            :       else
    5501                 :            :         note = insn;
    5502                 :            : 
    5503                 :    8062540 :       SET_NEXT_INSN (PREV_INSN (insn)) = NEXT_INSN (note);
    5504                 :    8062540 :       SET_PREV_INSN (NEXT_INSN (note)) = PREV_INSN (insn);
    5505                 :            : 
    5506                 :    8062540 :       SET_NEXT_INSN (note) = NEXT_INSN (last);
    5507                 :    8062540 :       SET_PREV_INSN (NEXT_INSN (last)) = note;
    5508                 :            : 
    5509                 :    8062540 :       SET_NEXT_INSN (last) = insn;
    5510                 :    8062540 :       SET_PREV_INSN (insn) = last;
    5511                 :            : 
    5512                 :    8062540 :       bb = BLOCK_FOR_INSN (last);
    5513                 :            : 
    5514                 :    8062540 :       if (jump_p)
    5515                 :            :         {
    5516                 :          2 :           fix_jump_move (insn);
    5517                 :            : 
    5518                 :          2 :           if (BLOCK_FOR_INSN (insn) != bb)
    5519                 :          0 :             move_block_after_check (insn);
    5520                 :            : 
    5521                 :          2 :           gcc_assert (BB_END (bb) == last);
    5522                 :            :         }
    5523                 :            : 
    5524                 :    8062540 :       df_insn_change_bb (insn, bb);
    5525                 :            : 
    5526                 :            :       /* Update BB_END, if needed.  */
    5527                 :    8062540 :       if (BB_END (bb) == last)
    5528                 :          2 :         BB_END (bb) = insn;
    5529                 :            :     }
    5530                 :            : 
    5531                 :   66992900 :   SCHED_GROUP_P (insn) = 0;
    5532                 :   66992900 : }
    5533                 :            : 
    5534                 :            : /* Return true if scheduling INSN will finish current clock cycle.  */
    5535                 :            : static bool
    5536                 :   67010900 : insn_finishes_cycle_p (rtx_insn *insn)
    5537                 :            : {
    5538                 :   67010900 :   if (SCHED_GROUP_P (insn))
    5539                 :            :     /* After issuing INSN, rest of the sched_group will be forced to issue
    5540                 :            :        in order.  Don't make any plans for the rest of cycle.  */
    5541                 :            :     return true;
    5542                 :            : 
    5543                 :            :   /* Finishing the block will, apparently, finish the cycle.  */
    5544                 :   67010800 :   if (current_sched_info->insn_finishes_block_p
    5545                 :   67010800 :       && current_sched_info->insn_finishes_block_p (insn))
    5546                 :    3972540 :     return true;
    5547                 :            : 
    5548                 :            :   return false;
    5549                 :            : }
    5550                 :            : 
    5551                 :            : /* Helper for autopref_multipass_init.  Given a SET in PAT and whether
    5552                 :            :    we're expecting a memory WRITE or not, check that the insn is relevant to
    5553                 :            :    the autoprefetcher modelling code.  Return true iff that is the case.
    5554                 :            :    If it is relevant, record the base register of the memory op in BASE and
    5555                 :            :    the offset in OFFSET.  */
    5556                 :            : 
    5557                 :            : static bool
    5558                 :          0 : analyze_set_insn_for_autopref (rtx pat, bool write, rtx *base, int *offset)
    5559                 :            : {
    5560                 :          0 :   if (GET_CODE (pat) != SET)
    5561                 :            :     return false;
    5562                 :            : 
    5563                 :          0 :   rtx mem = write ? SET_DEST (pat) : SET_SRC (pat);
    5564                 :          0 :   if (!MEM_P (mem))
    5565                 :            :     return false;
    5566                 :            : 
    5567                 :          0 :   struct address_info info;
    5568                 :          0 :   decompose_mem_address (&info, mem);
    5569                 :            : 
    5570                 :            :   /* TODO: Currently only (base+const) addressing is supported.  */
    5571                 :          0 :   if (info.base == NULL || !REG_P (*info.base)
    5572                 :          0 :       || (info.disp != NULL && !CONST_INT_P (*info.disp)))
    5573                 :            :     return false;
    5574                 :            : 
    5575                 :          0 :   *base = *info.base;
    5576                 :          0 :   *offset = info.disp ? INTVAL (*info.disp) : 0;
    5577                 :          0 :   return true;
    5578                 :            : }
    5579                 :            : 
    5580                 :            : /* Functions to model cache auto-prefetcher.
    5581                 :            : 
    5582                 :            :    Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
    5583                 :            :    memory prefetches if it sees instructions with consequitive memory accesses
    5584                 :            :    in the instruction stream.  Details of such hardware units are not published,
    5585                 :            :    so we can only guess what exactly is going on there.
    5586                 :            :    In the scheduler, we model abstract auto-prefetcher.  If there are memory
    5587                 :            :    insns in the ready list (or the queue) that have same memory base, but
    5588                 :            :    different offsets, then we delay the insns with larger offsets until insns
    5589                 :            :    with smaller offsets get scheduled.  If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
    5590                 :            :    is "1", then we look at the ready list; if it is N>1, then we also look
    5591                 :            :    through N-1 queue entries.
    5592                 :            :    If the param is N>=0, then rank_for_schedule will consider auto-prefetching
    5593                 :            :    among its heuristics.
    5594                 :            :    Param value of "-1" disables modelling of the auto-prefetcher.  */
    5595                 :            : 
    5596                 :            : /* Initialize autoprefetcher model data for INSN.  */
    5597                 :            : static void
    5598                 :          0 : autopref_multipass_init (const rtx_insn *insn, int write)
    5599                 :            : {
    5600                 :          0 :   autopref_multipass_data_t data = &INSN_AUTOPREF_MULTIPASS_DATA (insn)[write];
    5601                 :            : 
    5602                 :          0 :   gcc_assert (data->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED);
    5603                 :          0 :   data->base = NULL_RTX;
    5604                 :          0 :   data->offset = 0;
    5605                 :            :   /* Set insn entry initialized, but not relevant for auto-prefetcher.  */
    5606                 :          0 :   data->status = AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
    5607                 :            : 
    5608                 :          0 :   rtx pat = PATTERN (insn);
    5609                 :            : 
    5610                 :            :   /* We have a multi-set insn like a load-multiple or store-multiple.
    5611                 :            :      We care about these as long as all the memory ops inside the PARALLEL
    5612                 :            :      have the same base register.  We care about the minimum and maximum
    5613                 :            :      offsets from that base but don't check for the order of those offsets
    5614                 :            :      within the PARALLEL insn itself.  */
    5615                 :          0 :   if (GET_CODE (pat) == PARALLEL)
    5616                 :            :     {
    5617                 :          0 :       int n_elems = XVECLEN (pat, 0);
    5618                 :            : 
    5619                 :          0 :       int i, offset;
    5620                 :          0 :       rtx base, prev_base = NULL_RTX;
    5621                 :          0 :       int min_offset = INT_MAX;
    5622                 :            : 
    5623                 :          0 :       for (i = 0; i < n_elems; i++)
    5624                 :            :         {
    5625                 :          0 :           rtx set = XVECEXP (pat, 0, i);
    5626                 :          0 :           if (GET_CODE (set) != SET)
    5627                 :            :             return;
    5628                 :            : 
    5629                 :          0 :           if (!analyze_set_insn_for_autopref (set, write, &base, &offset))
    5630                 :            :             return;
    5631                 :            : 
    5632                 :            :           /* Ensure that all memory operations in the PARALLEL use the same
    5633                 :            :              base register.  */
    5634                 :          0 :           if (i > 0 && REGNO (base) != REGNO (prev_base))
    5635                 :            :             return;
    5636                 :          0 :           prev_base = base;
    5637                 :          0 :           min_offset = MIN (min_offset, offset);
    5638                 :            :         }
    5639                 :            : 
    5640                 :            :       /* If we reached here then we have a valid PARALLEL of multiple memory ops
    5641                 :            :          with prev_base as the base and min_offset containing the offset.  */
    5642                 :          0 :       gcc_assert (prev_base);
    5643                 :          0 :       data->base = prev_base;
    5644                 :          0 :       data->offset = min_offset;
    5645                 :          0 :       data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
    5646                 :          0 :       return;
    5647                 :            :     }
    5648                 :            : 
    5649                 :            :   /* Otherwise this is a single set memory operation.  */
    5650                 :          0 :   rtx set = single_set (insn);
    5651                 :          0 :   if (set == NULL_RTX)
    5652                 :            :     return;
    5653                 :            : 
    5654                 :          0 :   if (!analyze_set_insn_for_autopref (set, write, &data->base,
    5655                 :            :                                        &data->offset))
    5656                 :            :     return;
    5657                 :            : 
    5658                 :            :   /* This insn is relevant for the auto-prefetcher.
    5659                 :            :      The base and offset fields will have been filled in the
    5660                 :            :      analyze_set_insn_for_autopref call above.  */
    5661                 :          0 :   data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
    5662                 :            : }
    5663                 :            : 
    5664                 :            : /* Helper function for rank_for_schedule sorting.  */
    5665                 :            : static int
    5666                 :          0 : autopref_rank_for_schedule (const rtx_insn *insn1, const rtx_insn *insn2)
    5667                 :            : {
    5668                 :          0 :   int r = 0;
    5669                 :          0 :   for (int write = 0; write < 2 && !r; ++write)
    5670                 :            :     {
    5671                 :          0 :       autopref_multipass_data_t data1
    5672                 :          0 :         = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
    5673                 :          0 :       autopref_multipass_data_t data2
    5674                 :          0 :         = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
    5675                 :            : 
    5676                 :          0 :       if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
    5677                 :          0 :         autopref_multipass_init (insn1, write);
    5678                 :            : 
    5679                 :          0 :       if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
    5680                 :          0 :         autopref_multipass_init (insn2, write);
    5681                 :            : 
    5682                 :          0 :       int irrel1 = data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
    5683                 :          0 :       int irrel2 = data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
    5684                 :            : 
    5685                 :          0 :       if (!irrel1 && !irrel2)
    5686                 :          0 :         r = data1->offset - data2->offset;
    5687                 :            :       else
    5688                 :          0 :         r = irrel2 - irrel1;
    5689                 :            :     }
    5690                 :            : 
    5691                 :          0 :   return r;
    5692                 :            : }
    5693                 :            : 
    5694                 :            : /* True if header of debug dump was printed.  */
    5695                 :            : static bool autopref_multipass_dfa_lookahead_guard_started_dump_p;
    5696                 :            : 
    5697                 :            : /* Helper for autopref_multipass_dfa_lookahead_guard.
    5698                 :            :    Return "1" if INSN1 should be delayed in favor of INSN2.  */
    5699                 :            : static int
    5700                 :          0 : autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn *insn1,
    5701                 :            :                                           const rtx_insn *insn2, int write)
    5702                 :            : {
    5703                 :          0 :   autopref_multipass_data_t data1
    5704                 :          0 :     = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
    5705                 :          0 :   autopref_multipass_data_t data2
    5706                 :          0 :     = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
    5707                 :            : 
    5708                 :          0 :   if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
    5709                 :          0 :     autopref_multipass_init (insn2, write);
    5710                 :          0 :   if (data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
    5711                 :            :     return 0;
    5712                 :            : 
    5713                 :          0 :   if (rtx_equal_p (data1->base, data2->base)
    5714                 :          0 :       && data1->offset > data2->offset)
    5715                 :            :     {
    5716                 :          0 :       if (sched_verbose >= 2)
    5717                 :            :         {
    5718                 :          0 :           if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
    5719                 :            :             {
    5720                 :          0 :               fprintf (sched_dump,
    5721                 :            :                        ";;\t\tnot trying in max_issue due to autoprefetch "
    5722                 :            :                        "model: ");
    5723                 :          0 :               autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
    5724                 :            :             }
    5725                 :            : 
    5726                 :          0 :           fprintf (sched_dump, " %d(%d)", INSN_UID (insn1), INSN_UID (insn2));
    5727                 :            :         }
    5728                 :            : 
    5729                 :          0 :       return 1;
    5730                 :            :     }
    5731                 :            : 
    5732                 :            :   return 0;
    5733                 :            : }
    5734                 :            : 
    5735                 :            : /* General note:
    5736                 :            : 
    5737                 :            :    We could have also hooked autoprefetcher model into
    5738                 :            :    first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
    5739                 :            :    to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
    5740                 :            :    (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
    5741                 :            :    unblocked).  We don't bother about this yet because target of interest
    5742                 :            :    (ARM Cortex-A15) can issue only 1 memory operation per cycle.  */
    5743                 :            : 
    5744                 :            : /* Implementation of first_cycle_multipass_dfa_lookahead_guard hook.
    5745                 :            :    Return "1" if INSN1 should not be considered in max_issue due to
    5746                 :            :    auto-prefetcher considerations.  */
    5747                 :            : int
    5748                 :          0 : autopref_multipass_dfa_lookahead_guard (rtx_insn *insn1, int ready_index)
    5749                 :            : {
    5750                 :          0 :   int r = 0;
    5751                 :            : 
    5752                 :            :   /* Exit early if the param forbids this or if we're not entering here through
    5753                 :            :      normal haifa scheduling.  This can happen if selective scheduling is
    5754                 :            :      explicitly enabled.  */
    5755                 :          0 :   if (!insn_queue || param_sched_autopref_queue_depth <= 0)
    5756                 :            :     return 0;
    5757                 :            : 
    5758                 :          0 :   if (sched_verbose >= 2 && ready_index == 0)
    5759                 :          0 :     autopref_multipass_dfa_lookahead_guard_started_dump_p = false;
    5760                 :            : 
    5761                 :          0 :   for (int write = 0; write < 2; ++write)
    5762                 :            :     {
    5763                 :          0 :       autopref_multipass_data_t data1
    5764                 :          0 :         = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
    5765                 :            : 
    5766                 :          0 :       if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
    5767                 :          0 :         autopref_multipass_init (insn1, write);
    5768                 :          0 :       if (data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
    5769                 :          0 :         continue;
    5770                 :            : 
    5771                 :          0 :       if (ready_index == 0
    5772                 :          0 :           && data1->status == AUTOPREF_MULTIPASS_DATA_DONT_DELAY)
    5773                 :            :         /* We allow only a single delay on priviledged instructions.
    5774                 :            :            Doing otherwise would cause infinite loop.  */
    5775                 :            :         {
    5776                 :          0 :           if (sched_verbose >= 2)
    5777                 :            :             {
    5778                 :          0 :               if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
    5779                 :            :                 {
    5780                 :          0 :                   fprintf (sched_dump,
    5781                 :            :                            ";;\t\tnot trying in max_issue due to autoprefetch "
    5782                 :            :                            "model: ");
    5783                 :          0 :                   autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
    5784                 :            :                 }
    5785                 :            : 
    5786                 :          0 :               fprintf (sched_dump, " *%d*", INSN_UID (insn1));
    5787                 :            :             }
    5788                 :          0 :           continue;
    5789                 :            :         }
    5790                 :            : 
    5791                 :          0 :       for (int i2 = 0; i2 < ready.n_ready; ++i2)
    5792                 :            :         {
    5793                 :          0 :           rtx_insn *insn2 = get_ready_element (i2);
    5794                 :          0 :           if (insn1 == insn2)
    5795                 :          0 :             continue;
    5796                 :          0 :           r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2, write);
    5797                 :          0 :           if (r)
    5798                 :            :             {
    5799                 :          0 :               if (ready_index == 0)
    5800                 :            :                 {
    5801                 :          0 :                   r = -1;
    5802                 :          0 :                   data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
    5803                 :            :                 }
    5804                 :          0 :               goto finish;
    5805                 :            :             }
    5806                 :            :         }
    5807                 :            : 
    5808                 :          0 :       if (param_sched_autopref_queue_depth == 1)
    5809                 :          0 :         continue;
    5810                 :            : 
    5811                 :            :       /* Everything from the current queue slot should have been moved to
    5812                 :            :          the ready list.  */
    5813                 :          0 :       gcc_assert (insn_queue[NEXT_Q_AFTER (q_ptr, 0)] == NULL_RTX);
    5814                 :            : 
    5815                 :          0 :       int n_stalls = param_sched_autopref_queue_depth - 1;
    5816                 :          0 :       if (n_stalls > max_insn_queue_index)
    5817                 :            :         n_stalls = max_insn_queue_index;
    5818                 :            : 
    5819                 :          0 :       for (int stalls = 1; stalls <= n_stalls; ++stalls)
    5820                 :            :         {
    5821                 :          0 :           for (rtx_insn_list *link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)];
    5822                 :          0 :                link != NULL_RTX;
    5823                 :          0 :                link = link->next ())
    5824                 :            :             {
    5825                 :          0 :               rtx_insn *insn2 = link->insn ();
    5826                 :          0 :               r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2,
    5827                 :            :                                                             write);
    5828                 :          0 :               if (r)
    5829                 :            :                 {
    5830                 :            :                   /* Queue INSN1 until INSN2 can issue.  */
    5831                 :          0 :                   r = -stalls;
    5832                 :          0 :                   if (ready_index == 0)
    5833                 :          0 :                     data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
    5834                 :          0 :                   goto finish;
    5835                 :            :                 }
    5836                 :            :             }
    5837                 :            :         }
    5838                 :            :     }
    5839                 :            : 
    5840                 :          0 :     finish:
    5841                 :          0 :   if (sched_verbose >= 2
    5842                 :          0 :       && autopref_multipass_dfa_lookahead_guard_started_dump_p
    5843                 :          0 :       && (ready_index == ready.n_ready - 1 || r < 0))
    5844                 :            :     /* This does not /always/ trigger.  We don't output EOL if the last
    5845                 :            :        insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
    5846                 :            :        called.  We can live with this.  */
    5847                 :          0 :     fprintf (sched_dump, "\n");
    5848                 :            : 
    5849                 :            :   return r;
    5850                 :            : }
    5851                 :            : 
    5852                 :            : /* Define type for target data used in multipass scheduling.  */
    5853                 :            : #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
    5854                 :            : # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
    5855                 :            : #endif
    5856                 :            : typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t;
    5857                 :            : 
    5858                 :            : /* The following structure describe an entry of the stack of choices.  */
    5859                 :            : struct choice_entry
    5860                 :            : {
    5861                 :            :   /* Ordinal number of the issued insn in the ready queue.  */
    5862                 :            :   int index;
    5863                 :            :   /* The number of the rest insns whose issues we should try.  */
    5864                 :            :   int rest;
    5865                 :            :   /* The number of issued essential insns.  */
    5866                 :            :   int n;
    5867                 :            :   /* State after issuing the insn.  */
    5868                 :            :   state_t state;
    5869                 :            :   /* Target-specific data.  */
    5870                 :            :   first_cycle_multipass_data_t target_data;
    5871                 :            : };
    5872                 :            : 
    5873                 :            : /* The following array is used to implement a stack of choices used in
    5874                 :            :    function max_issue.  */
    5875                 :            : static struct choice_entry *choice_stack;
    5876                 :            : 
    5877                 :            : /* This holds the value of the target dfa_lookahead hook.  */
    5878                 :            : int dfa_lookahead;
    5879                 :            : 
    5880                 :            : /* The following variable value is maximal number of tries of issuing
    5881                 :            :    insns for the first cycle multipass insn scheduling.  We define
    5882                 :            :    this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE).  We would not
    5883                 :            :    need this constraint if all real insns (with non-negative codes)
    5884                 :            :    had reservations because in this case the algorithm complexity is
    5885                 :            :    O(DFA_LOOKAHEAD**ISSUE_RATE).  Unfortunately, the dfa descriptions
    5886                 :            :    might be incomplete and such insn might occur.  For such
    5887                 :            :    descriptions, the complexity of algorithm (without the constraint)
    5888                 :            :    could achieve DFA_LOOKAHEAD ** N , where N is the queue length.  */
    5889                 :            : static int max_lookahead_tries;
    5890                 :            : 
    5891                 :            : /* The following function returns maximal (or close to maximal) number
    5892                 :            :    of insns which can be issued on the same cycle and one of which
    5893                 :            :    insns is insns with the best rank (the first insn in READY).  To
    5894                 :            :    make this function tries different samples of ready insns.  READY
    5895                 :            :    is current queue `ready'.  Global array READY_TRY reflects what
    5896                 :            :    insns are already issued in this try.  The function stops immediately,
    5897                 :            :    if it reached the such a solution, that all instruction can be issued.
    5898                 :            :    INDEX will contain index of the best insn in READY.  The following
    5899                 :            :    function is used only for first cycle multipass scheduling.
    5900                 :            : 
    5901                 :            :    PRIVILEGED_N >= 0
    5902                 :            : 
    5903                 :            :    This function expects recognized insns only.  All USEs,
    5904                 :            :    CLOBBERs, etc must be filtered elsewhere.  */
    5905                 :            : int
    5906                 :   36316100 : max_issue (struct ready_list *ready, int privileged_n, state_t state,
    5907                 :            :            bool first_cycle_insn_p, int *index)
    5908                 :            : {
    5909                 :   36316100 :   int n, i, all, n_ready, best, delay, tries_num;
    5910                 :   36316100 :   int more_issue;
    5911                 :   36316100 :   struct choice_entry *top;
    5912                 :   36316100 :   rtx_insn *insn;
    5913                 :            : 
    5914                 :   36316100 :   if (sched_fusion)
    5915                 :            :     return 0;
    5916                 :            : 
    5917                 :   36316100 :   n_ready = ready->n_ready;
    5918                 :   36316100 :   gcc_assert (dfa_lookahead >= 1 && privileged_n >= 0
    5919                 :            :               && privileged_n <= n_ready);
    5920                 :            : 
    5921                 :            :   /* Init MAX_LOOKAHEAD_TRIES.  */
    5922                 :   36316100 :   if (max_lookahead_tries == 0)
    5923                 :            :     {
    5924                 :     644019 :       max_lookahead_tries = 100;
    5925                 :    3185230 :       for (i = 0; i < issue_rate; i++)
    5926                 :    2541210 :         max_lookahead_tries *= dfa_lookahead;
    5927                 :            :     }
    5928                 :            : 
    5929                 :            :   /* Init max_points.  */
    5930                 :   36316100 :   more_issue = issue_rate - cycle_issued_insns;
    5931                 :   36316100 :   gcc_assert (more_issue >= 0);
    5932                 :            : 
    5933                 :            :   /* The number of the issued insns in the best solution.  */
    5934                 :   36316100 :   best = 0;
    5935                 :            : 
    5936                 :   36316100 :   top = choice_stack;
    5937                 :            : 
    5938                 :            :   /* Set initial state of the search.  */
    5939                 :   36316100 :   memcpy (top->state, state, dfa_state_size);
    5940                 :   36316100 :   top->rest = dfa_lookahead;
    5941                 :   36316100 :   top->n = 0;
    5942                 :   36316100 :   if (targetm.sched.first_cycle_multipass_begin)
    5943                 :   36177300 :     targetm.sched.first_cycle_multipass_begin (&top->target_data,
    5944                 :            :                                                ready_try, n_ready,
    5945                 :            :                                                first_cycle_insn_p);
    5946                 :            : 
    5947                 :            :   /* Count the number of the insns to search among.  */
    5948                 :   97829100 :   for (all = i = 0; i < n_ready; i++)
    5949                 :   61513000 :     if (!ready_try [i])
    5950                 :   56674100 :       all++;
    5951                 :            : 
    5952                 :   36316100 :   if (sched_verbose >= 2)
    5953                 :            :     {
    5954                 :          0 :       fprintf (sched_dump, ";;\t\tmax_issue among %d insns:", all);
    5955                 :          0 :       debug_ready_list_1 (ready, ready_try);
    5956                 :            :     }
    5957                 :            : 
    5958                 :            :   /* I is the index of the insn to try next.  */
    5959                 :            :   i = 0;
    5960                 :            :   tries_num = 0;
    5961                 :  630661000 :   for (;;)
    5962                 :            :     {
    5963                 :  333488000 :       if (/* If we've reached a dead end or searched enough of what we have
    5964                 :            :              been asked...  */
    5965                 :  333488000 :           top->rest == 0
    5966                 :            :           /* or have nothing else to try...  */
    5967                 :  330321000 :           || i >= n_ready
    5968                 :            :           /* or should not issue more.  */
    5969                 :  267485000 :           || top->n >= more_issue)
    5970                 :            :         {
    5971                 :            :           /* ??? (... || i == n_ready).  */
    5972                 :   67284300 :           gcc_assert (i <= n_ready);
    5973                 :            : 
    5974                 :            :           /* We should not issue more than issue_rate instructions.  */
    5975                 :   67284300 :           gcc_assert (top->n <= more_issue);
    5976                 :            : 
    5977                 :   67284300 :           if (top == choice_stack)
    5978                 :            :             break;
    5979                 :            : 
    5980                 :   61944600 :           if (best < top - choice_stack)
    5981                 :            :             {
    5982                 :   35431400 :               if (privileged_n)
    5983                 :            :                 {
    5984                 :            :                   n = privileged_n;
    5985                 :            :                   /* Try to find issued privileged insn.  */
    5986                 :   35428600 :                   while (n && !ready_try[--n])
    5987                 :            :                     ;
    5988                 :            :                 }
    5989                 :            : 
    5990                 :   35431400 :               if (/* If all insns are equally good...  */
    5991                 :            :                   privileged_n == 0
    5992                 :            :                   /* Or a privileged insn will be issued.  */
    5993                 :   35428300 :                   || ready_try[n])
    5994                 :            :                 /* Then we have a solution.  */
    5995                 :            :                 {
    5996                 :   35431200 :                   best = top - choice_stack;
    5997                 :            :                   /* This is the index of the insn issued first in this
    5998                 :            :                      solution.  */
    5999                 :   35431200 :                   *index = choice_stack [1].index;
    6000                 :   35431200 :                   if (top->n == more_issue || best == all)
    6001                 :            :                     break;
    6002                 :            :                 }
    6003                 :            :             }
    6004                 :            : 
    6005                 :            :           /* Set ready-list index to point to the last insn
    6006                 :            :              ('i++' below will advance it to the next insn).  */
    6007                 :   30968300 :           i = top->index;
    6008                 :            : 
    6009                 :            :           /* Backtrack.  */
    6010                 :   30968300 :           ready_try [i] = 0;
    6011                 :            : 
    6012                 :   30968300 :           if (targetm.sched.first_cycle_multipass_backtrack)
    6013                 :   30953800 :             targetm.sched.first_cycle_multipass_backtrack (&top->target_data,
    6014                 :            :                                                            ready_try, n_ready);
    6015                 :            : 
    6016                 :   30968300 :           top--;
    6017                 :   30968300 :           memcpy (state, top->state, dfa_state_size);
    6018                 :            :         }
    6019                 :  266204000 :       else if (!ready_try [i])
    6020                 :            :         {
    6021                 :   85524400 :           tries_num++;
    6022                 :   85524400 :           if (tries_num > max_lookahead_tries)
    6023                 :            :             break;
    6024                 :   85524400 :           insn = ready_element (ready, i);
    6025                 :   85524400 :           delay = state_transition (state, insn);
    6026                 :   85524400 :           if (delay < 0)
    6027                 :            :             {
    6028                 :   67010900 :               if (state_dead_lock_p (state)
    6029                 :   67010900 :                   || insn_finishes_cycle_p (insn))
    6030                 :            :                 /* We won't issue any more instructions in the next
    6031                 :            :                    choice_state.  */
    6032                 :    3972730 :                 top->rest = 0;
    6033                 :            :               else
    6034                 :   63038200 :                 top->rest--;
    6035                 :            : 
    6036                 :   67010900 :               n = top->n;
    6037                 :   67010900 :               if (memcmp (top->state, state, dfa_state_size) != 0)
    6038                 :   66799000 :                 n++;
    6039                 :            : 
    6040                 :            :               /* Advance to the next choice_entry.  */
    6041                 :   67010900 :               top++;
    6042                 :            :               /* Initialize it.  */
    6043                 :   67010900 :               top->rest = dfa_lookahead;
    6044                 :   67010900 :               top->index = i;
    6045                 :   67010900 :               top->n = n;
    6046                 :   67010900 :               memcpy (top->state, state, dfa_state_size);
    6047                 :   67010900 :               ready_try [i] = 1;
    6048                 :            : 
    6049                 :   67010900 :               if (targetm.sched.first_cycle_multipass_issue)
    6050                 :   66853600 :                 targetm.sched.first_cycle_multipass_issue (&top->target_data,
    6051                 :            :                                                            ready_try, n_ready,
    6052                 :            :                                                            insn,
    6053                 :   66853600 :                                                            &((top - 1)
    6054                 :            :                                                              ->target_data));
    6055                 :            : 
    6056                 :            :               i = -1;
    6057                 :            :             }
    6058                 :            :         }
    6059                 :            : 
    6060                 :            :       /* Increase ready-list index.  */
    6061                 :  297172000 :       i++;
    6062                 :            :     }
    6063                 :            : 
    6064                 :   36316100 :   if (targetm.sched.first_cycle_multipass_end)
    6065                 :   71168400 :     targetm.sched.first_cycle_multipass_end (best != 0
    6066                 :   34991100 :                                              ? &choice_stack[1].target_data
    6067                 :            :                                              : NULL);
    6068                 :            : 
    6069                 :            :   /* Restore the original state of the DFA.  */
    6070                 :   36316100 :   memcpy (state, choice_stack->state, dfa_state_size);
    6071                 :            : 
    6072                 :   36316100 :   return best;
    6073                 :            : }
    6074                 :            : 
    6075                 :            : /* The following function chooses insn from READY and modifies
    6076                 :            :    READY.  The following function is used only for first
    6077                 :            :    cycle multipass scheduling.
    6078                 :            :    Return:
    6079                 :            :    -1 if cycle should be advanced,
    6080                 :            :    0 if INSN_PTR is set to point to the desirable insn,
    6081                 :            :    1 if choose_ready () should be restarted without advancing the cycle.  */
    6082                 :            : static int
    6083                 :   39240500 : choose_ready (struct ready_list *ready, bool first_cycle_insn_p,
    6084                 :            :               rtx_insn **insn_ptr)
    6085                 :            : {
    6086                 :   39240500 :   if (dbg_cnt (sched_insn) == false)
    6087                 :            :     {
    6088                 :          0 :       if (nonscheduled_insns_begin == NULL_RTX)
    6089                 :          0 :         nonscheduled_insns_begin = current_sched_info->prev_head;
    6090                 :            : 
    6091                 :          0 :       rtx_insn *insn = first_nonscheduled_insn ();
    6092                 :            : 
    6093                 :          0 :       if (QUEUE_INDEX (insn) == QUEUE_READY)
    6094                 :            :         /* INSN is in the ready_list.  */
    6095                 :            :         {
    6096                 :          0 :           ready_remove_insn (insn);
    6097                 :          0 :           *insn_ptr = insn;
    6098                 :          0 :           return 0;
    6099                 :            :         }
    6100                 :            : 
    6101                 :            :       /* INSN is in the queue.  Advance cycle to move it to the ready list.  */
    6102                 :          0 :       gcc_assert (QUEUE_INDEX (insn) >= 0);
    6103                 :            :       return -1;
    6104                 :            :     }
    6105                 :            : 
    6106                 :   39229000 :   if (dfa_lookahead <= 0 || SCHED_GROUP_P (ready_element (ready, 0))
    6107                 :   76096300 :       || DEBUG_INSN_P (ready_element (ready, 0)))
    6108                 :            :     {
    6109                 :    2384810 :       if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
    6110                 :         19 :         *insn_ptr = ready_remove_first_dispatch (ready);
    6111                 :            :       else
    6112                 :    2384790 :         *insn_ptr = ready_remove_first (ready);
    6113                 :            : 
    6114                 :    2384810 :       return 0;
    6115                 :            :     }
    6116                 :            :   else
    6117                 :            :     {
    6118                 :            :       /* Try to choose the best insn.  */
    6119                 :   36855700 :       int index = 0, i;
    6120                 :   36855700 :       rtx_insn *insn;
    6121                 :            : 
    6122                 :   36855700 :       insn = ready_element (ready, 0);
    6123                 :   36855700 :       if (INSN_CODE (insn) < 0)
    6124                 :            :         {
    6125                 :     542803 :           *insn_ptr = ready_remove_first (ready);
    6126                 :     542803 :           return 0;
    6127                 :            :         }
    6128                 :            : 
    6129                 :            :       /* Filter the search space.  */
    6130                 :   97821000 :       for (i = 0; i < ready->n_ready; i++)
    6131                 :            :         {
    6132                 :   61508100 :           ready_try[i] = 0;
    6133                 :            : 
    6134                 :   61508100 :           insn = ready_element (ready, i);
    6135                 :            : 
    6136                 :            :           /* If this insn is recognizable we should have already
    6137                 :            :              recognized it earlier.
    6138                 :            :              ??? Not very clear where this is supposed to be done.
    6139                 :            :              See dep_cost_1.  */
    6140                 :   61616000 :           gcc_checking_assert (INSN_CODE (insn) >= 0
    6141                 :            :                                || recog_memoized (insn) < 0);
    6142                 :   61508100 :           if (INSN_CODE (insn) < 0)
    6143                 :            :             {
    6144                 :            :               /* Non-recognized insns at position 0 are handled above.  */
    6145                 :     107975 :               gcc_assert (i > 0);
    6146                 :     107975 :               ready_try[i] = 1;
    6147                 :     107975 :               continue;
    6148                 :            :             }
    6149                 :            : 
    6150                 :   61400100 :           if (targetm.sched.first_cycle_multipass_dfa_lookahead_guard)
    6151                 :            :             {
    6152                 :          0 :               ready_try[i]
    6153                 :          0 :                 = (targetm.sched.first_cycle_multipass_dfa_lookahead_guard
    6154                 :          0 :                     (insn, i));
    6155                 :            : 
    6156                 :          0 :               if (ready_try[i] < 0)
    6157                 :            :                 /* Queue instruction for several cycles.
    6158                 :            :                    We need to restart choose_ready as we have changed
    6159                 :            :                    the ready list.  */
    6160                 :            :                 {
    6161                 :          0 :                   change_queue_index (insn, -ready_try[i]);
    6162                 :          0 :                   return 1;
    6163                 :            :                 }
    6164                 :            : 
    6165                 :            :               /* Make sure that we didn't end up with 0'th insn filtered out.
    6166                 :            :                  Don't be tempted to make life easier for backends and just
    6167                 :            :                  requeue 0'th insn if (ready_try[0] == 0) and restart
    6168                 :            :                  choose_ready.  Backends should be very considerate about
    6169                 :            :                  requeueing instructions -- especially the highest priority
    6170                 :            :                  one at position 0.  */
    6171                 :          0 :               gcc_assert (ready_try[i] == 0 || i > 0);
    6172                 :          0 :               if (ready_try[i])
    6173                 :          0 :                 continue;
    6174                 :            :             }
    6175                 :            : 
    6176                 :   61400100 :           gcc_assert (ready_try[i] == 0);
    6177                 :            :           /* INSN made it through the scrutiny of filters!  */
    6178                 :            :         }
    6179                 :            : 
    6180                 :   36312900 :       if (max_issue (ready, 1, curr_state, first_cycle_insn_p, &index) == 0)
    6181                 :            :         {
    6182                 :    1186170 :           *insn_ptr = ready_remove_first (ready);
    6183                 :    1186170 :           if (sched_verbose >= 4)
    6184                 :          0 :             fprintf (sched_dump, ";;\t\tChosen insn (but can't issue) : %s \n",
    6185                 :          0 :                      (*current_sched_info->print_insn) (*insn_ptr, 0));
    6186                 :    1186170 :           return 0;
    6187                 :            :         }
    6188                 :            :       else
    6189                 :            :         {
    6190                 :   35126800 :           if (sched_verbose >= 4)
    6191                 :          0 :             fprintf (sched_dump, ";;\t\tChosen insn : %s\n",
    6192                 :          0 :                      (*current_sched_info->print_insn)
    6193                 :          0 :                      (ready_element (ready, index), 0));
    6194                 :            : 
    6195                 :   35126800 :           *insn_ptr = ready_remove (ready, index);
    6196                 :   35126800 :           return 0;
    6197                 :            :         }
    6198                 :            :     }
    6199                 :            : }
    6200                 :            : 
    6201                 :            : /* This function is called when we have successfully scheduled a
    6202                 :            :    block.  It uses the schedule stored in the scheduled_insns vector
    6203                 :            :    to rearrange the RTL.  PREV_HEAD is used as the anchor to which we
    6204                 :            :    append the scheduled insns; TAIL is the insn after the scheduled
    6205                 :            :    block.  TARGET_BB is the argument passed to schedule_block.  */
    6206                 :            : 
    6207                 :            : static void
    6208                 :    6519130 : commit_schedule (rtx_insn *prev_head, rtx_insn *tail, basic_block *target_bb)
    6209                 :            : {
    6210                 :    6519130 :   unsigned int i;
    6211                 :    6519130 :   rtx_insn *insn;
    6212                 :            : 
    6213                 :    6519130 :   last_scheduled_insn = prev_head;
    6214                 :    6519130 :   for (i = 0;
    6215                 :   73512000 :        scheduled_insns.iterate (i, &insn);
    6216                 :            :        i++)
    6217                 :            :     {
    6218                 :   66992900 :       if (control_flow_insn_p (last_scheduled_insn)
    6219                 :   66992900 :           || current_sched_info->advance_target_bb (*target_bb, insn))
    6220                 :            :         {
    6221                 :         73 :           *target_bb = current_sched_info->advance_target_bb (*target_bb, 0);
    6222                 :            : 
    6223                 :         73 :           if (sched_verbose)
    6224                 :            :             {
    6225                 :          2 :               rtx_insn *x;
    6226                 :            : 
    6227                 :          2 :               x = next_real_insn (last_scheduled_insn);
    6228                 :          2 :               gcc_assert (x);
    6229                 :          2 :               dump_new_block_header (1, *target_bb, x, tail);
    6230                 :            :             }
    6231                 :            : 
    6232                 :         73 :           last_scheduled_insn = bb_note (*target_bb);
    6233                 :            :         }
    6234                 :            : 
    6235                 :   66992900 :       if (current_sched_info->begin_move_insn)
    6236                 :       1315 :         (*current_sched_info->begin_move_insn) (insn, last_scheduled_insn);
    6237                 :   66992900 :       move_insn (insn, last_scheduled_insn,
    6238                 :   66992900 :                  current_sched_info->next_tail);
    6239                 :   66992900 :       if (!DEBUG_INSN_P (insn))
    6240                 :   39238500 :         reemit_notes (insn);
    6241                 :   66992900 :       last_scheduled_insn = insn;
    6242                 :            :     }
    6243                 :            : 
    6244                 :    6519130 :   scheduled_insns.truncate (0);
    6245                 :    6519130 : }
    6246                 :            : 
    6247                 :            : /* Examine all insns on the ready list and queue those which can't be
    6248                 :            :    issued in this cycle.  TEMP_STATE is temporary scheduler state we
    6249                 :            :    can use as scratch space.  If FIRST_CYCLE_INSN_P is true, no insns
    6250                 :            :    have been issued for the current cycle, which means it is valid to
    6251                 :            :    issue an asm statement.
    6252                 :            : 
    6253                 :            :    If SHADOWS_ONLY_P is true, we eliminate all real insns and only
    6254                 :            :    leave those for which SHADOW_P is true.  If MODULO_EPILOGUE is true,
    6255                 :            :    we only leave insns which have an INSN_EXACT_TICK.  */
    6256                 :            : 
    6257                 :            : static void
    6258                 :   45910300 : prune_ready_list (state_t temp_state, bool first_cycle_insn_p,
    6259                 :            :                   bool shadows_only_p, bool modulo_epilogue_p)
    6260                 :            : {
    6261                 :   45910300 :   int i, pass;
    6262                 :   45910300 :   bool sched_group_found = false;
    6263                 :   45910300 :   int min_cost_group = 0;
    6264                 :            : 
    6265                 :   45910300 :   if (sched_fusion)
    6266                 :            :     return;
    6267                 :            : 
    6268                 :  127930000 :   for (i = 0; i < ready.n_ready; i++)
    6269                 :            :     {
    6270                 :   84404900 :       rtx_insn *insn = ready_element (&ready, i);
    6271                 :   84404900 :       if (SCHED_GROUP_P (insn))
    6272                 :            :         {
    6273                 :            :           sched_group_found = true;
    6274                 :            :           break;
    6275                 :            :         }
    6276                 :            :     }
    6277                 :            : 
    6278                 :            :   /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle
    6279                 :            :      such an insn first and note its cost.  If at least one SCHED_GROUP_P insn
    6280                 :            :      gets queued, then all other insns get queued for one cycle later.  */
    6281                 :  150106000 :   for (pass = sched_group_found ? 0 : 1; pass < 2; )
    6282                 :            :     {
    6283                 :   58285800 :       int n = ready.n_ready;
    6284                 :  135802000 :       for (i = 0; i < n; i++)
    6285                 :            :         {
    6286                 :   87506800 :           rtx_insn *insn = ready_element (&ready, i);
    6287                 :   87506800 :           int cost = 0;
    6288                 :   87506800 :           const char *reason = "resource conflict";
    6289                 :            : 
    6290                 :   87506800 :           if (DEBUG_INSN_P (insn))
    6291                 :    3942950 :             continue;
    6292                 :            : 
    6293                 :    4761340 :           if (sched_group_found && !SCHED_GROUP_P (insn)
    6294                 :   83564100 :               && ((pass == 0) || (min_cost_group >= 1)))
    6295                 :            :             {
    6296                 :        174 :               if (pass == 0)
    6297                 :        139 :                 continue;
    6298                 :            :               cost = min_cost_group;
    6299                 :            :               reason = "not in sched group";
    6300                 :            :             }
    6301                 :   83563600 :           else if (modulo_epilogue_p
    6302                 :   83563600 :                    && INSN_EXACT_TICK (insn) == INVALID_TICK)
    6303                 :            :             {
    6304                 :            :               cost = max_insn_queue_index;
    6305                 :            :               reason = "not an epilogue insn";
    6306                 :            :             }
    6307                 :   83563600 :           else if (shadows_only_p && !SHADOW_P (insn))
    6308                 :            :             {
    6309                 :            :               cost = 1;
    6310                 :            :               reason = "not a shadow";
    6311                 :            :             }
    6312                 :   84272100 :           else if (recog_memoized (insn) < 0)
    6313                 :            :             {
    6314                 :     708466 :               if (!first_cycle_insn_p
    6315                 :     708466 :                   && (GET_CODE (PATTERN (insn)) == ASM_INPUT
    6316                 :     555617 :                       || asm_noperands (PATTERN (insn)) >= 0))
    6317                 :            :                 cost = 1;
    6318                 :            :               reason = "asm";
    6319                 :            :             }
    6320                 :   82855200 :           else if (sched_pressure != SCHED_PRESSURE_NONE)
    6321                 :            :             {
    6322                 :      15053 :               if (sched_pressure == SCHED_PRESSURE_MODEL
    6323                 :      15053 :                   && INSN_TICK (insn) <= clock_var)
    6324                 :            :                 {
    6325                 :          0 :                   memcpy (temp_state, curr_state, dfa_state_size);
    6326                 :          0 :                   if (state_transition (temp_state, insn) >= 0)
    6327                 :          0 :                     INSN_TICK (insn) = clock_var + 1;
    6328                 :            :                 }
    6329                 :            :               cost = 0;
    6330                 :            :             }
    6331                 :            :           else
    6332                 :            :             {
    6333                 :   82840100 :               int delay_cost = 0;
    6334                 :            : 
    6335                 :   82840100 :               if (delay_htab)
    6336                 :            :                 {
    6337                 :          0 :                   struct delay_pair *delay_entry;
    6338                 :          0 :                   delay_entry
    6339                 :          0 :                     = delay_htab->find_with_hash (insn,
    6340                 :          0 :                                                   htab_hash_pointer (insn));
    6341                 :          0 :                   while (delay_entry && delay_cost == 0)
    6342                 :            :                     {
    6343                 :          0 :                       delay_cost = estimate_shadow_tick (delay_entry);
    6344                 :          0 :                       if (delay_cost > max_insn_queue_index)
    6345                 :            :                         delay_cost = max_insn_queue_index;
    6346                 :          0 :                       delay_entry = delay_entry->next_same_i1;
    6347                 :            :                     }
    6348                 :            :                 }
    6349                 :            : 
    6350                 :   82840100 :               memcpy (temp_state, curr_state, dfa_state_size);
    6351                 :   82840100 :               cost = state_transition (temp_state, insn);
    6352                 :   82840100 :               if (cost < 0)
    6353                 :            :                 cost = 0;
    6354                 :   15851900 :               else if (cost == 0)
    6355                 :          0 :                 cost = 1;
    6356                 :   82840100 :               if (cost < delay_cost)
    6357                 :            :                 {
    6358                 :          0 :                   cost = delay_cost;
    6359                 :          0 :                   reason = "shadow tick";
    6360                 :            :                 }
    6361                 :            :             }
    6362                 :   82840200 :           if (cost >= 1)
    6363                 :            :             {
    6364                 :   15900100 :               if (SCHED_GROUP_P (insn) && cost > min_cost_group)
    6365                 :            :                 min_cost_group = cost;
    6366                 :   15900100 :               ready_remove (&ready, i);
    6367                 :            :               /* Normally we'd want to queue INSN for COST cycles.  However,
    6368                 :            :                  if SCHED_GROUP_P is set, then we must ensure that nothing
    6369                 :            :                  else comes between INSN and its predecessor.  If there is
    6370                 :            :                  some other insn ready to fire on the next cycle, then that
    6371                 :            :                  invariant would be broken.
    6372                 :            : 
    6373                 :            :                  So when SCHED_GROUP_P is set, just queue this insn for a
    6374                 :            :                  single cycle.  */
    6375                 :   15909200 :               queue_insn (insn, SCHED_GROUP_P (insn) ? 1 : cost, reason);
    6376                 :   15900100 :               if (i + 1 < n)
    6377                 :            :                 break;
    6378                 :            :             }
    6379                 :            :         }
    6380                 :   58285800 :       if (i == n)
    6381                 :   48295400 :         pass++;
    6382                 :            :     }
    6383                 :            : }
    6384                 :            : 
    6385                 :            : /* Called when we detect that the schedule is impossible.  We examine the
    6386                 :            :    backtrack queue to find the earliest insn that caused this condition.  */
    6387                 :            : 
    6388                 :            : static struct haifa_saved_data *
    6389                 :          0 : verify_shadows (void)
    6390                 :            : {
    6391                 :          0 :   struct haifa_saved_data *save, *earliest_fail = NULL;
    6392                 :          0 :   for (save = backtrack_queue; save; save = save->next)
    6393                 :            :     {
    6394                 :          0 :       int t;
    6395                 :          0 :       struct delay_pair *pair = save->delay_pair;
    6396                 :          0 :       rtx_insn *i1 = pair->i1;
    6397                 :            : 
    6398                 :          0 :       for (; pair; pair = pair->next_same_i1)
    6399                 :            :         {
    6400                 :          0 :           rtx_insn *i2 = pair->i2;
    6401                 :            : 
    6402                 :          0 :           if (QUEUE_INDEX (i2) == QUEUE_SCHEDULED)
    6403                 :          0 :             continue;
    6404                 :            : 
    6405                 :          0 :           t = INSN_TICK (i1) + pair_delay (pair);
    6406                 :          0 :           if (t < clock_var)
    6407                 :            :             {
    6408                 :          0 :               if (sched_verbose >= 2)
    6409                 :          0 :                 fprintf (sched_dump,
    6410                 :            :                          ";;\t\tfailed delay requirements for %d/%d (%d->%d)"
    6411                 :            :                          ", not ready\n",
    6412                 :          0 :                          INSN_UID (pair->i1), INSN_UID (pair->i2),
    6413                 :          0 :                          INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2));
    6414                 :            :               earliest_fail = save;
    6415                 :            :               break;
    6416                 :            :             }
    6417                 :          0 :           if (QUEUE_INDEX (i2) >= 0)
    6418                 :            :             {
    6419                 :          0 :               int queued_for = INSN_TICK (i2);
    6420                 :            : 
    6421                 :          0 :               if (t < queued_for)
    6422                 :            :                 {
    6423                 :          0 :                   if (sched_verbose >= 2)
    6424                 :          0 :                     fprintf (sched_dump,
    6425                 :            :                              ";;\t\tfailed delay requirements for %d/%d"
    6426                 :            :                              " (%d->%d), queued too late\n",
    6427                 :          0 :                              INSN_UID (pair->i1), INSN_UID (pair->i2),
    6428                 :          0 :                              INSN_TICK (pair->i1), INSN_EXACT_TICK (pair->i2));
    6429                 :            :                   earliest_fail = save;
    6430                 :            :                   break;
    6431                 :            :                 }
    6432                 :            :             }
    6433                 :            :         }
    6434                 :            :     }
    6435                 :            : 
    6436                 :          0 :   return earliest_fail;
    6437                 :            : }
    6438                 :            : 
    6439                 :            : /* Print instructions together with useful scheduling information between
    6440                 :            :    HEAD and TAIL (inclusive).  */
    6441                 :            : static void
    6442                 :          0 : dump_insn_stream (rtx_insn *head, rtx_insn *tail)
    6443                 :            : {
    6444                 :          0 :   fprintf (sched_dump, ";;\t| insn | prio |\n");
    6445                 :            : 
    6446                 :          0 :   rtx_insn *next_tail = NEXT_INSN (tail);
    6447                 :          0 :   for (rtx_insn *insn = head; insn != next_tail; insn = NEXT_INSN (insn))
    6448                 :            :     {
    6449                 :          0 :       int priority = NOTE_P (insn) ? 0 : INSN_PRIORITY (insn);
    6450                 :          0 :       const char *pattern = (NOTE_P (insn)
    6451                 :          0 :                              ? "note"
    6452                 :          0 :                              : str_pattern_slim (PATTERN (insn)));
    6453                 :            : 
    6454                 :          0 :       fprintf (sched_dump, ";;\t| %4d | %4d | %-30s ",
    6455                 :          0 :                INSN_UID (insn), priority, pattern);
    6456                 :            : 
    6457                 :          0 :       if (sched_verbose >= 4)
    6458                 :            :         {
    6459                 :          0 :           if (NOTE_P (insn) || LABEL_P (insn) || recog_memoized (insn) < 0)
    6460                 :          0 :             fprintf (sched_dump, "nothing");
    6461                 :            :           else
    6462                 :          0 :             print_reservation (sched_dump, insn);
    6463                 :            :         }
    6464                 :          0 :       fprintf (sched_dump, "\n");
    6465                 :            :     }
    6466                 :          0 : }
    6467                 :            : 
    6468                 :            : /* Use forward list scheduling to rearrange insns of block pointed to by
    6469                 :            :    TARGET_BB, possibly bringing insns from subsequent blocks in the same
    6470                 :            :    region.  */
    6471                 :            : 
    6472                 :            : bool
    6473                 :    6519130 : schedule_block (basic_block *target_bb, state_t init_state)
    6474                 :            : {
    6475                 :    6519130 :   int i;
    6476                 :    6519130 :   bool success = modulo_ii == 0;
    6477                 :    6519130 :   struct sched_block_state ls;
    6478                 :    6519130 :   state_t temp_state = NULL;  /* It is used for multipass scheduling.  */
    6479                 :    6519130 :   int sort_p, advance, start_clock_var;
    6480                 :            : 
    6481                 :            :   /* Head/tail info for this block.  */
    6482                 :    6519130 :   rtx_insn *prev_head = current_sched_info->prev_head;
    6483                 :    6519130 :   rtx_insn *next_tail = current_sched_info->next_tail;
    6484                 :    6519130 :   rtx_insn *head = NEXT_INSN (prev_head);
    6485                 :    6519130 :   rtx_insn *tail = PREV_INSN (next_tail);
    6486                 :            : 
    6487                 :    6519130 :   if ((current_sched_info->flags & DONT_BREAK_DEPENDENCIES) == 0
    6488                 :    6519130 :       && sched_pressure != SCHED_PRESSURE_MODEL && !sched_fusion)
    6489                 :    6519130 :     find_modifiable_mems (head, tail);
    6490                 :            : 
    6491                 :            :   /* We used to have code to avoid getting parameters moved from hard
    6492                 :            :      argument registers into pseudos.
    6493                 :            : 
    6494                 :            :      However, it was removed when it proved to be of marginal benefit
    6495                 :            :      and caused problems because schedule_block and compute_forward_dependences
    6496                 :            :      had different notions of what the "head" insn was.  */
    6497                 :            : 
    6498                 :    6519130 :   gcc_assert (head != tail || INSN_P (head));
    6499                 :            : 
    6500                 :    6519130 :   haifa_recovery_bb_recently_added_p = false;
    6501                 :            : 
    6502                 :    6519130 :   backtrack_queue = NULL;
    6503                 :            : 
    6504                 :            :   /* Debug info.  */
    6505                 :    6519130 :   if (sched_verbose)
    6506                 :            :     {
    6507                 :        184 :       dump_new_block_header (0, *target_bb, head, tail);
    6508                 :            : 
    6509                 :        184 :       if (sched_verbose >= 2)
    6510                 :            :         {
    6511                 :          0 :           dump_insn_stream (head, tail);
    6512                 :          0 :           memset (&rank_for_schedule_stats, 0,
    6513                 :            :                   sizeof (rank_for_schedule_stats));
    6514                 :            :         }
    6515                 :            :     }
    6516                 :            : 
    6517                 :    6519130 :   if (init_state == NULL)
    6518                 :        287 :     state_reset (curr_state);
    6519                 :            :   else
    6520                 :    6518840 :     memcpy (curr_state, init_state, dfa_state_size);
    6521                 :            : 
    6522                 :            :   /* Clear the ready list.  */
    6523                 :    6519130 :   ready.first = ready.veclen - 1;
    6524                 :    6519130 :   ready.n_ready = 0;
    6525                 :    6519130 :   ready.n_debug = 0;
    6526                 :            : 
    6527                 :            :   /* It is used for first cycle multipass scheduling.  */
    6528                 :    6519130 :   temp_state = alloca (dfa_state_size);
    6529                 :            : 
    6530                 :    6519130 :   if (targetm.sched.init)
    6531                 :          0 :     targetm.sched.init (sched_dump, sched_verbose, ready.veclen);
    6532                 :            : 
    6533                 :            :   /* We start inserting insns after PREV_HEAD.  */
    6534                 :    6519130 :   last_scheduled_insn = prev_head;
    6535                 :    6519130 :   last_nondebug_scheduled_insn = NULL;
    6536                 :    6519130 :   nonscheduled_insns_begin = NULL;
    6537                 :            : 
    6538                 :    6519130 :   gcc_assert ((NOTE_P (last_scheduled_insn)
    6539                 :            :                || DEBUG_INSN_P (last_scheduled_insn))
    6540                 :            :               && BLOCK_FOR_INSN (last_scheduled_insn) == *target_bb);
    6541                 :            : 
    6542                 :            :   /* Initialize INSN_QUEUE.  Q_SIZE is the total number of insns in the
    6543                 :            :      queue.  */
    6544                 :    6519130 :   q_ptr = 0;
    6545                 :    6519130 :   q_size = 0;
    6546                 :            : 
    6547                 :    6519130 :   insn_queue = XALLOCAVEC (rtx_insn_list *, max_insn_queue_index + 1);
    6548                 :    6519130 :   memset (insn_queue, 0, (max_insn_queue_index + 1) * sizeof (rtx));
    6549                 :            : 
    6550                 :            :   /* Start just before the beginning of time.  */
    6551                 :    6519130 :   clock_var = -1;
    6552                 :            : 
    6553                 :            :   /* We need queue and ready lists and clock_var be initialized
    6554                 :            :      in try_ready () (which is called through init_ready_list ()).  */
    6555                 :    6519130 :   (*current_sched_info->init_ready_list) ();
    6556                 :            : 
    6557                 :    6519130 :   if (sched_pressure)
    6558                 :        987 :     sched_pressure_start_bb (*target_bb);
    6559                 :            : 
    6560                 :            :   /* The algorithm is O(n^2) in the number of ready insns at any given
    6561                 :            :      time in the worst case.  Before reload we are more likely to have
    6562                 :            :      big lists so truncate them to a reasonable size.  */
    6563                 :    6519130 :   if (!reload_completed
    6564                 :       1189 :       && ready.n_ready - ready.n_debug > param_max_sched_ready_insns)
    6565                 :            :     {
    6566                 :          8 :       ready_sort_debug (&ready);
    6567                 :          8 :       ready_sort_real (&ready);
    6568                 :            : 
    6569                 :            :       /* Find first free-standing insn past param_max_sched_ready_insns.
    6570                 :            :          If there are debug insns, we know they're first.  */
    6571                 :          8 :       for (i = param_max_sched_ready_insns + ready.n_debug; i < ready.n_ready;
    6572                 :            :            i++)
    6573                 :          8 :         if (!SCHED_GROUP_P (ready_element (&ready, i)))
    6574                 :            :           break;
    6575                 :            : 
    6576                 :          8 :       if (sched_verbose >= 2)
    6577                 :            :         {
    6578                 :          0 :           fprintf (sched_dump,
    6579                 :            :                    ";;\t\tReady list on entry: %d insns:  ", ready.n_ready);
    6580                 :          0 :           debug_ready_list (&ready);
    6581                 :          0 :           fprintf (sched_dump,
    6582                 :            :                    ";;\t\t before reload => truncated to %d insns\n", i);
    6583                 :            :         }
    6584                 :            : 
    6585                 :            :       /* Delay all insns past it for 1 cycle.  If debug counter is
    6586                 :            :          activated make an exception for the insn right after
    6587                 :            :          nonscheduled_insns_begin.  */
    6588                 :          8 :       {
    6589                 :          8 :         rtx_insn *skip_insn;
    6590                 :            : 
    6591                 :          8 :         if (dbg_cnt (sched_insn) == false)
    6592                 :          0 :           skip_insn = first_nonscheduled_insn ();
    6593                 :            :         else
    6594                 :          8 :           skip_insn = NULL;
    6595                 :            : 
    6596                 :         47 :         while (i < ready.n_ready)
    6597                 :            :           {
    6598                 :         39 :             rtx_insn *insn;
    6599                 :            : 
    6600                 :         39 :             insn = ready_remove (&ready, i);
    6601                 :            : 
    6602                 :         39 :             if (insn != skip_insn)
    6603                 :         39 :               queue_insn (insn, 1, "list truncated");
    6604                 :            :           }
    6605                 :          8 :         if (skip_insn)
    6606                 :          0 :           ready_add (&ready, skip_insn, true);
    6607                 :            :       }
    6608                 :            :     }
    6609                 :            : 
    6610                 :            :   /* Now we can restore basic block notes and maintain precise cfg.  */
    6611                 :    6519130 :   restore_bb_notes (*target_bb);
    6612                 :            : 
    6613                 :    6519130 :   last_clock_var = -1;
    6614                 :            : 
    6615                 :    6519130 :   advance = 0;
    6616                 :            : 
    6617                 :    6519130 :   gcc_assert (scheduled_insns.length () == 0);
    6618                 :    6519130 :   sort_p = TRUE;
    6619                 :    6519130 :   must_backtrack = false;
    6620                 :    6519130 :   modulo_insns_scheduled = 0;
    6621                 :            : 
    6622                 :    6519130 :   ls.modulo_epilogue = false;
    6623                 :    6519130 :   ls.first_cycle_insn_p = true;
    6624                 :            : 
    6625                 :            :   /* Loop until all the insns in BB are scheduled.  */
    6626                 :   30463800 :   while ((*current_sched_info->schedule_more_p) ())
    6627                 :            :     {
    6628                 :   23944700 :       perform_replacements_new_cycle ();
    6629                 :   23944700 :       do
    6630                 :            :         {
    6631                 :   23944700 :           start_clock_var = clock_var;
    6632                 :            : 
    6633                 :   23944700 :           clock_var++;
    6634                 :            : 
    6635                 :   23944700 :           advance_one_cycle ();
    6636                 :            : 
    6637                 :            :           /* Add to the ready list all pending insns that can be issued now.
    6638                 :            :              If there are no ready insns, increment clock until one
    6639                 :            :              is ready and add all pending insns at that point to the ready
    6640                 :            :              list.  */
    6641                 :   23944700 :           queue_to_ready (&ready);
    6642                 :            : 
    6643                 :   23944700 :           gcc_assert (ready.n_ready);
    6644                 :            : 
    6645                 :   23944700 :           if (sched_verbose >= 2)
    6646                 :            :             {
    6647                 :          0 :               fprintf (sched_dump, ";;\t\tReady list after queue_to_ready:");
    6648                 :          0 :               debug_ready_list (&ready);
    6649                 :            :             }
    6650                 :   23944700 :           advance -= clock_var - start_clock_var;
    6651                 :            :         }
    6652                 :   23944700 :       while (advance > 0);
    6653                 :            : 
    6654                 :   23944700 :       if (ls.modulo_epilogue)
    6655                 :            :         {
    6656                 :          0 :           int stage = clock_var / modulo_ii;
    6657                 :          0 :           if (stage > modulo_last_stage * 2 + 2)
    6658                 :            :             {
    6659                 :          0 :               if (sched_verbose >= 2)
    6660                 :          0 :                 fprintf (sched_dump,
    6661                 :            :                          ";;\t\tmodulo scheduled succeeded at II %d\n",
    6662                 :            :                          modulo_ii);
    6663                 :          0 :               success = true;
    6664                 :          0 :               goto end_schedule;
    6665                 :            :             }
    6666                 :            :         }
    6667                 :   23944700 :       else if (modulo_ii > 0)
    6668                 :            :         {
    6669                 :          0 :           int stage = clock_var / modulo_ii;
    6670                 :          0 :           if (stage > modulo_max_stages)
    6671                 :            :             {
    6672                 :          0 :               if (sched_verbose >= 2)
    6673                 :          0 :                 fprintf (sched_dump,
    6674                 :            :                          ";;\t\tfailing schedule due to excessive stages\n");
    6675                 :          0 :               goto end_schedule;
    6676                 :            :             }
    6677                 :          0 :           if (modulo_n_insns == modulo_insns_scheduled
    6678                 :          0 :               && stage > modulo_last_stage)
    6679                 :            :             {
    6680                 :          0 :               if (sched_verbose >= 2)
    6681                 :          0 :                 fprintf (sched_dump,
    6682                 :            :                          ";;\t\tfound kernel after %d stages, II %d\n",
    6683                 :            :                          stage, modulo_ii);
    6684                 :          0 :               ls.modulo_epilogue = true;
    6685                 :            :             }
    6686                 :            :         }
    6687                 :            : 
    6688                 :   23944700 :       prune_ready_list (temp_state, true, false, ls.modulo_epilogue);
    6689                 :   23944700 :       if (ready.n_ready == 0)
    6690                 :       3247 :         continue;
    6691                 :   23941500 :       if (must_backtrack)
    6692                 :          0 :         goto do_backtrack;
    6693                 :            : 
    6694                 :   23941500 :       ls.shadows_only_p = false;
    6695                 :   23941500 :       cycle_issued_insns = 0;
    6696                 :   23941500 :       ls.can_issue_more = issue_rate;
    6697                 :  102311000 :       for (;;)
    6698                 :            :         {
    6699                 :   63126400 :           rtx_insn *insn;
    6700                 :   63126400 :           int cost;
    6701                 :   63126400 :           bool asm_p;
    6702                 :            : 
    6703                 :   63126400 :           if (sort_p && ready.n_ready > 0)
    6704                 :            :             {
    6705                 :            :               /* Sort the ready list based on priority.  This must be
    6706                 :            :                  done every iteration through the loop, as schedule_insn
    6707                 :            :                  may have readied additional insns that will not be
    6708                 :            :                  sorted correctly.  */
    6709                 :   40375500 :               ready_sort (&ready);
    6710                 :            : 
    6711                 :   40375500 :               if (sched_verbose >= 2)
    6712                 :            :                 {
    6713                 :          0 :                   fprintf (sched_dump,
    6714                 :            :                            ";;\t\tReady list after ready_sort:    ");
    6715                 :          0 :                   debug_ready_list (&ready);
    6716                 :            :                 }
    6717                 :            :             }
    6718                 :            : 
    6719                 :            :           /* We don't want md sched reorder to even see debug isns, so put
    6720                 :            :              them out right away.  */
    6721                 :   40375500 :           if (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0))
    6722                 :   67068800 :               && (*current_sched_info->schedule_more_p) ())
    6723                 :            :             {
    6724                 :   31696800 :               while (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0)))
    6725                 :            :                 {
    6726                 :   27754400 :                   rtx_insn *insn = ready_remove_first (&ready);
    6727                 :   27754400 :                   gcc_assert (DEBUG_INSN_P (insn));
    6728                 :   27754400 :                   (*current_sched_info->begin_schedule_ready) (insn);
    6729                 :   27754400 :                   scheduled_insns.safe_push (insn);
    6730                 :   27754400 :                   last_scheduled_insn = insn;
    6731                 :   27754400 :                   advance = schedule_insn (insn);
    6732                 :   27754400 :                   gcc_assert (advance == 0);
    6733                 :   27754400 :                   if (ready.n_ready > 0)
    6734                 :   26646000 :                     ready_sort (&ready);
    6735                 :            :                 }
    6736                 :            :             }
    6737                 :            : 
    6738                 :   63126400 :           if (ls.first_cycle_insn_p && !ready.n_ready)
    6739                 :            :             break;
    6740                 :            : 
    6741                 :   63125700 :         resume_after_backtrack:
    6742                 :            :           /* Allow the target to reorder the list, typically for
    6743                 :            :              better instruction bundling.  */
    6744                 :   63125700 :           if (sort_p
    6745                 :   63125700 :               && (ready.n_ready == 0
    6746                 :  102393000 :                   || !SCHED_GROUP_P (ready_element (&ready, 0))))
    6747                 :            :             {
    6748                 :   60749800 :               if (ls.first_cycle_insn_p && targetm.sched.reorder)
    6749                 :   23929700 :                 ls.can_issue_more
    6750                 :   23929700 :                   = targetm.sched.reorder (sched_dump, sched_verbose,
    6751                 :            :                                            ready_lastpos (&ready),
    6752                 :            :                                            &ready.n_ready, clock_var);
    6753                 :   36820000 :               else if (!ls.first_cycle_insn_p && targetm.sched.reorder2)
    6754                 :          0 :                 ls.can_issue_more
    6755                 :          0 :                   = targetm.sched.reorder2 (sched_dump, sched_verbose,
    6756                 :          0 :                                             ready.n_ready
    6757                 :          0 :                                             ? ready_lastpos (&ready) : NULL,
    6758                 :            :                                             &ready.n_ready, clock_var);
    6759                 :            :             }
    6760                 :            : 
    6761                 :   63125700 :         restart_choose_ready:
    6762                 :   63125700 :           if (sched_verbose >= 2)
    6763                 :            :             {
    6764                 :          0 :               fprintf (sched_dump, ";;\tReady list (t = %3d):  ",
    6765                 :            :                        clock_var);
    6766                 :          0 :               debug_ready_list (&ready);
    6767                 :          0 :               if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
    6768                 :          0 :                 print_curr_reg_pressure ();
    6769                 :            :             }
    6770                 :            : 
    6771                 :   63125700 :           if (ready.n_ready == 0
    6772                 :   23858700 :               && ls.can_issue_more
    6773                 :   22847500 :               && reload_completed)
    6774                 :            :             {
    6775                 :            :               /* Allow scheduling insns directly from the queue in case
    6776                 :            :                  there's nothing better to do (ready list is empty) but
    6777                 :            :                  there are still vacant dispatch slots in the current cycle.  */
    6778                 :   22845300 :               if (sched_verbose >= 6)
    6779                 :          0 :                 fprintf (sched_dump,";;\t\tSecond chance\n");
    6780                 :   22845300 :               memcpy (temp_state, curr_state, dfa_state_size);
    6781                 :   22845300 :               if (early_queue_to_ready (temp_state, &ready))
    6782                 :          4 :                 ready_sort (&ready);
    6783                 :            :             }
    6784                 :            : 
    6785                 :   63125700 :           if (ready.n_ready == 0
    6786                 :   39267000 :               || !ls.can_issue_more
    6787                 :   39240600 :               || state_dead_lock_p (curr_state)
    6788                 :  102366000 :               || !(*current_sched_info->schedule_more_p) ())
    6789                 :            :             break;
    6790                 :            : 
    6791                 :            :           /* Select and remove the insn from the ready list.  */
    6792                 :   39240500 :           if (sort_p)
    6793                 :            :             {
    6794                 :   39240500 :               int res;
    6795                 :            : 
    6796                 :   39240500 :               insn = NULL;
    6797                 :   39240500 :               res = choose_ready (&ready, ls.first_cycle_insn_p, &insn);
    6798                 :            : 
    6799                 :   39240500 :               if (res < 0)
    6800                 :            :                 /* Finish cycle.  */
    6801                 :            :                 break;
    6802                 :   39240500 :               if (res > 0)
    6803                 :          0 :                 goto restart_choose_ready;
    6804                 :            : 
    6805                 :   39240500 :               gcc_assert (insn != NULL_RTX);
    6806                 :            :             }
    6807                 :            :           else
    6808                 :          0 :             insn = ready_remove_first (&ready);
    6809                 :            : 
    6810                 :   39240500 :           if (sched_pressure != SCHED_PRESSURE_NONE
    6811                 :   39240500 :               && INSN_TICK (insn) > clock_var)
    6812                 :            :             {
    6813                 :       2086 :               ready_add (&ready, insn, true);
    6814                 :       2086 :               advance = 1;
    6815                 :       2086 :               break;
    6816                 :            :             }
    6817                 :            : 
    6818                 :   39238500 :           if (targetm.sched.dfa_new_cycle
    6819                 :   39238500 :               && targetm.sched.dfa_new_cycle (sched_dump, sched_verbose,
    6820                 :            :                                               insn, last_clock_var,
    6821                 :            :                                               clock_var, &sort_p))
    6822                 :            :             /* SORT_P is used by the target to override sorting
    6823                 :            :                of the ready list.  This is needed when the target
    6824                 :            :                has modified its internal structures expecting that
    6825                 :            :                the insn will be issued next.  As we need the insn
    6826                 :            :                to have the highest priority (so it will be returned by
    6827                 :            :                the ready_remove_first call above), we invoke
    6828                 :            :                ready_add (&ready, insn, true).
    6829                 :            :                But, still, there is one issue: INSN can be later
    6830                 :            :                discarded by scheduler's front end through
    6831                 :            :                current_sched_info->can_schedule_ready_p, hence, won't
    6832                 :            :                be issued next.  */
    6833                 :            :             {
    6834                 :          0 :               ready_add (&ready, insn, true);
    6835                 :          0 :               break;
    6836                 :            :             }
    6837                 :            : 
    6838                 :   39238500 :           sort_p = TRUE;
    6839                 :            : 
    6840                 :   39238500 :           if (current_sched_info->can_schedule_ready_p
    6841                 :   39238500 :               && ! (*current_sched_info->can_schedule_ready_p) (insn))
    6842                 :            :             /* We normally get here only if we don't want to move
    6843                 :            :                insn from the split block.  */
    6844                 :            :             {
    6845                 :          0 :               TODO_SPEC (insn) = DEP_POSTPONED;
    6846                 :          0 :               goto restart_choose_ready;
    6847                 :            :             }
    6848                 :            : 
    6849                 :   39238500 :           if (delay_htab)
    6850                 :            :             {
    6851                 :            :               /* If this insn is the first part of a delay-slot pair, record a
    6852                 :            :                  backtrack point.  */
    6853                 :          0 :               struct delay_pair *delay_entry;
    6854                 :          0 :               delay_entry
    6855                 :          0 :                 = delay_htab->find_with_hash (insn, htab_hash_pointer (insn));
    6856                 :          0 :               if (delay_entry)
    6857                 :            :                 {
    6858                 :          0 :                   save_backtrack_point (delay_entry, ls);
    6859                 :          0 :                   if (sched_verbose >= 2)
    6860                 :          0 :                     fprintf (sched_dump, ";;\t\tsaving backtrack point\n");
    6861                 :            :                 }
    6862                 :            :             }
    6863                 :            : 
    6864                 :            :           /* DECISION is made.  */
    6865                 :            : 
    6866                 :   39238500 :           if (modulo_ii > 0 && INSN_UID (insn) < modulo_iter0_max_uid)
    6867                 :            :             {
    6868                 :          0 :               modulo_insns_scheduled++;
    6869                 :          0 :               modulo_last_stage = clock_var / modulo_ii;
    6870                 :            :             }
    6871                 :   39238500 :           if (TODO_SPEC (insn) & SPECULATIVE)
    6872                 :          0 :             generate_recovery_code (insn);
    6873                 :            : 
    6874                 :   39238500 :           if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
    6875                 :         15 :             targetm.sched.dispatch_do (insn, ADD_TO_DISPATCH_WINDOW);
    6876                 :            : 
    6877                 :            :           /* Update counters, etc in the scheduler's front end.  */
    6878                 :   39238500 :           (*current_sched_info->begin_schedule_ready) (insn);
    6879                 :   39238500 :           scheduled_insns.safe_push (insn);
    6880                 :   39238500 :           gcc_assert (NONDEBUG_INSN_P (insn));
    6881                 :   39238500 :           last_nondebug_scheduled_insn = last_scheduled_insn = insn;
    6882                 :            : 
    6883                 :   39781400 :           if (recog_memoized (insn) >= 0)
    6884                 :            :             {
    6885                 :   38695500 :               memcpy (temp_state, curr_state, dfa_state_size);
    6886                 :   38695500 :               cost = state_transition (curr_state, insn);
    6887                 :   38695500 :               if (sched_pressure != SCHED_PRESSURE_WEIGHTED && !sched_fusion)
    6888                 :   38689200 :                 gcc_assert (cost < 0);
    6889                 :   38695500 :               if (memcmp (temp_state, curr_state, dfa_state_size) != 0)
    6890                 :   38577500 :                 cycle_issued_insns++;
    6891                 :            :               asm_p = false;
    6892                 :            :             }
    6893                 :            :           else
    6894                 :     542969 :             asm_p = (GET_CODE (PATTERN (insn)) == ASM_INPUT
    6895                 :     542969 :                      || asm_noperands (PATTERN (insn)) >= 0);
    6896                 :            : 
    6897                 :   39238500 :           if (targetm.sched.variable_issue)
    6898                 :          0 :             ls.can_issue_more =
    6899                 :          0 :               targetm.sched.variable_issue (sched_dump, sched_verbose,
    6900                 :            :                                             insn, ls.can_issue_more);
    6901                 :            :           /* A naked CLOBBER or USE generates no instruction, so do
    6902                 :            :              not count them against the issue rate.  */
    6903                 :   39238500 :           else if (GET_CODE (PATTERN (insn)) != USE
    6904                 :   39238500 :                    && GET_CODE (PATTERN (insn)) != CLOBBER)
    6905                 :   38749000 :             ls.can_issue_more--;
    6906                 :   39238500 :           advance = schedule_insn (insn);
    6907                 :            : 
    6908                 :   39238500 :           if (SHADOW_P (insn))
    6909                 :          0 :             ls.shadows_only_p = true;
    6910                 :            : 
    6911                 :            :           /* After issuing an asm insn we should start a new cycle.  */
    6912                 :   39238500 :           if (advance == 0 && asm_p)
    6913                 :      53510 :             advance = 1;
    6914                 :            : 
    6915                 :   39238500 :           if (must_backtrack)
    6916                 :            :             break;
    6917                 :            : 
    6918                 :   39238500 :           if (advance != 0)
    6919                 :            :             break;
    6920                 :            : 
    6921                 :   39185000 :           ls.first_cycle_insn_p = false;
    6922                 :   39185000 :           if (ready.n_ready > 0)
    6923                 :   21965600 :             prune_ready_list (temp_state, false, ls.shadows_only_p,
    6924                 :   21965600 :                               ls.modulo_epilogue);
    6925                 :   39185000 :         }
    6926                 :            : 
    6927                 :   23941500 :     do_backtrack:
    6928                 :   23941500 :       if (!must_backtrack)
    6929                 :   24085200 :         for (i = 0; i < ready.n_ready; i++)
    6930                 :            :           {
    6931                 :     143777 :             rtx_insn *insn = ready_element (&ready, i);
    6932                 :     143777 :             if (INSN_EXACT_TICK (insn) == clock_var)
    6933                 :            :               {
    6934                 :          0 :                 must_backtrack = true;
    6935                 :          0 :                 clock_var++;
    6936                 :          0 :                 break;
    6937                 :            :               }
    6938                 :            :           }
    6939                 :   23941500 :       if (must_backtrack && modulo_ii > 0)
    6940                 :            :         {
    6941                 :          0 :           if (modulo_backtracks_left == 0)
    6942                 :          0 :             goto end_schedule;
    6943                 :          0 :           modulo_backtracks_left--;
    6944                 :            :         }
    6945                 :   23941500 :       while (must_backtrack)
    6946                 :            :         {
    6947                 :          0 :           struct haifa_saved_data *failed;
    6948                 :          0 :           rtx_insn *failed_insn;
    6949                 :            : 
    6950                 :          0 :           must_backtrack = false;
    6951                 :          0 :           failed = verify_shadows ();
    6952                 :          0 :           gcc_assert (failed);
    6953                 :            : 
    6954                 :          0 :           failed_insn = failed->delay_pair->i1;
    6955                 :            :           /* Clear these queues.  */
    6956                 :          0 :           perform_replacements_new_cycle ();
    6957                 :          0 :           toggle_cancelled_flags (false);
    6958                 :          0 :           unschedule_insns_until (failed_insn);
    6959                 :          0 :           while (failed != backtrack_queue)
    6960                 :          0 :             free_topmost_backtrack_point (true);
    6961                 :          0 :           restore_last_backtrack_point (&ls);
    6962                 :          0 :           if (sched_verbose >= 2)
    6963                 :          0 :             fprintf (sched_dump, ";;\t\trewind to cycle %d\n", clock_var);
    6964                 :            :           /* Delay by at least a cycle.  This could cause additional
    6965                 :            :              backtracking.  */
    6966                 :          0 :           queue_insn (failed_insn, 1, "backtracked");
    6967                 :          0 :           advance = 0;
    6968                 :          0 :           if (must_backtrack)
    6969                 :          0 :             continue;
    6970                 :          0 :           if (ready.n_ready > 0)
    6971                 :          0 :             goto resume_after_backtrack;
    6972                 :            :           else
    6973                 :            :             {
    6974                 :          0 :               if (clock_var == 0 && ls.first_cycle_insn_p)
    6975                 :          0 :                 goto end_schedule;
    6976                 :            :               advance = 1;
    6977                 :            :               break;
    6978                 :            :             }
    6979                 :            :         }
    6980                 :   23941500 :       ls.first_cycle_insn_p = true;
    6981                 :            :     }
    6982                 :    6519130 :   if (ls.modulo_epilogue)
    6983                 :          0 :     success = true;
    6984                 :    6519130 :  end_schedule:
    6985                 :    6519130 :   if (!ls.first_cycle_insn_p || advance)
    6986                 :        501 :     advance_one_cycle ();
    6987                 :    6519130 :   perform_replacements_new_cycle ();
    6988                 :    6519130 :   if (modulo_ii > 0)
    6989                 :            :     {
    6990                 :            :       /* Once again, debug insn suckiness: they can be on the ready list
    6991                 :            :          even if they have unresolved dependencies.  To make our view
    6992                 :            :          of the world consistent, remove such "ready" insns.  */
    6993                 :          0 :     restart_debug_insn_loop:
    6994                 :          0 :       for (i = ready.n_ready - 1; i >= 0; i--)
    6995                 :            :         {
    6996                 :          0 :           rtx_insn *x;
    6997                 :            : 
    6998                 :          0 :           x = ready_element (&ready, i);
    6999                 :          0 :           if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x)) != NULL
    7000                 :          0 :               || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x)) != NULL)
    7001                 :            :             {
    7002                 :          0 :               ready_remove (&ready, i);
    7003                 :          0 :               goto restart_debug_insn_loop;
    7004                 :            :             }
    7005                 :            :         }
    7006                 :          0 :       for (i = ready.n_ready - 1; i >= 0; i--)
    7007                 :            :         {
    7008                 :          0 :           rtx_insn *x;
    7009                 :            : 
    7010                 :          0 :           x = ready_element (&ready, i);
    7011                 :          0 :           resolve_dependencies (x);
    7012                 :            :         }
    7013                 :          0 :       for (i = 0; i <= max_insn_queue_index; i++)
    7014                 :            :         {
    7015                 :          0 :           rtx_insn_list *link;
    7016                 :          0 :           while ((link = insn_queue[i]) != NULL)
    7017                 :            :             {
    7018                 :          0 :               rtx_insn *x = link->insn ();
    7019                 :          0 :               insn_queue[i] = link->next ();
    7020                 :          0 :               QUEUE_INDEX (x) = QUEUE_NOWHERE;
    7021                 :          0 :               free_INSN_LIST_node (link);
    7022                 :          0 :               resolve_dependencies (x);
    7023                 :            :             }
    7024                 :            :         }
    7025                 :            :     }
    7026                 :            : 
    7027                 :    6519130 :   if (!success)
    7028                 :          0 :     undo_all_replacements ();
    7029                 :            : 
    7030                 :            :   /* Debug info.  */
    7031                 :    6519130 :   if (sched_verbose)
    7032                 :            :     {
    7033                 :        184 :       fprintf (sched_dump, ";;\tReady list (final):  ");
    7034                 :        184 :       debug_ready_list (&ready);
    7035                 :            :     }
    7036                 :            : 
    7037                 :    6519130 :   if (modulo_ii == 0 && current_sched_info->queue_must_finish_empty)
    7038                 :            :     /* Sanity check -- queue must be empty now.  Meaningless if region has
    7039                 :            :        multiple bbs.  */
    7040                 :    6519010 :     gcc_assert (!q_size && !ready.n_ready && !ready.n_debug);
    7041                 :        120 :   else if (modulo_ii == 0)
    7042                 :            :     {
    7043                 :            :       /* We must maintain QUEUE_INDEX between blocks in region.  */
    7044                 :        156 :       for (i = ready.n_ready - 1; i >= 0; i--)
    7045                 :            :         {
    7046                 :         36 :           rtx_insn *x;
    7047                 :            : 
    7048                 :         36 :           x = ready_element (&ready, i);
    7049                 :         36 :           QUEUE_INDEX (x) = QUEUE_NOWHERE;
    7050                 :         36 :           TODO_SPEC (x) = HARD_DEP;
    7051                 :            :         }
    7052                 :            : 
    7053                 :        120 :       if (q_size)
    7054                 :          0 :         for (i = 0; i <= max_insn_queue_index; i++)
    7055                 :            :           {
    7056                 :          0 :             rtx_insn_list *link;
    7057                 :          0 :             for (link = insn_queue[i]; link; link = link->next ())
    7058                 :            :               {
    7059                 :          0 :                 rtx_insn *x;
    7060                 :            : 
    7061                 :          0 :                 x = link->insn ();
    7062                 :          0 :                 QUEUE_INDEX (x) = QUEUE_NOWHERE;
    7063                 :          0 :                 TODO_SPEC (x) = HARD_DEP;
    7064                 :            :               }
    7065                 :          0 :             free_INSN_LIST_list (&insn_queue[i]);
    7066                 :            :           }
    7067                 :            :     }
    7068                 :            : 
    7069                 :    6519130 :   if (sched_pressure == SCHED_PRESSURE_MODEL)
    7070                 :          0 :     model_end_schedule ();
    7071                 :            : 
    7072                 :    6519130 :   if (success)
    7073                 :            :     {
    7074                 :    6519130 :       commit_schedule (prev_head, tail, target_bb);
    7075                 :    6519130 :       if (sched_verbose)
    7076                 :        184 :         fprintf (sched_dump, ";;   total time = %d\n", clock_var);
    7077                 :            :     }
    7078                 :            :   else
    7079                 :          0 :     last_scheduled_insn = tail;
    7080                 :            : 
    7081                 :    6519130 :   scheduled_insns.truncate (0);
    7082                 :            : 
    7083                 :    6519130 :   if (!current_sched_info->queue_must_finish_empty
    7084                 :    6519010 :       || haifa_recovery_bb_recently_added_p)
    7085                 :            :     {
    7086                 :            :       /* INSN_TICK (minimum clock tick at which the insn becomes
    7087                 :            :          ready) may be not correct for the insn in the subsequent
    7088                 :            :          blocks of the region.  We should use a correct value of
    7089                 :            :          `clock_var' or modify INSN_TICK.  It is better to keep
    7090                 :            :          clock_var value equal to 0 at the start of a basic block.
    7091                 :            :          Therefore we modify INSN_TICK here.  */
    7092                 :        120 :       fix_inter_tick (NEXT_INSN (prev_head), last_scheduled_insn);
    7093                 :            :     }
    7094                 :            : 
    7095                 :    6519130 :   if (targetm.sched.finish)
    7096                 :            :     {
    7097                 :          0 :       targetm.sched.finish (sched_dump, sched_verbose);
    7098                 :            :       /* Target might have added some instructions to the scheduled block
    7099                 :            :          in its md_finish () hook.  These new insns don't have any data
    7100                 :            :          initialized and to identify them we extend h_i_d so that they'll
    7101                 :            :          get zero luids.  */
    7102                 :          0 :       sched_extend_luids ();
    7103                 :            :     }
    7104                 :            : 
    7105                 :            :   /* Update head/tail boundaries.  */
    7106                 :    6519130 :   head = NEXT_INSN (prev_head);
    7107                 :    6519130 :   tail = last_scheduled_insn;
    7108                 :            : 
    7109                 :    6519130 :   if (sched_verbose)
    7110                 :            :     {
    7111                 :        184 :       fprintf (sched_dump, ";;   new head = %d\n;;   new tail = %d\n",
    7112                 :        184 :                INSN_UID (head), INSN_UID (tail));
    7113                 :            : 
    7114                 :        184 :       if (sched_verbose >= 2)
    7115                 :            :         {
    7116                 :          0 :           dump_insn_stream (head, tail);
    7117                 :          0 :           print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats,
    7118                 :            :                                          NULL);
    7119                 :            :         }
    7120                 :            : 
    7121                 :        184 :       fprintf (sched_dump, "\n");
    7122                 :            :     }
    7123                 :            : 
    7124                 :    6519130 :   head = restore_other_notes (head, NULL);
    7125                 :            : 
    7126                 :    6519130 :   current_sched_info->head = head;
    7127                 :    6519130 :   current_sched_info->tail = tail;
    7128                 :            : 
    7129                 :    6519130 :   free_backtrack_queue ();
    7130                 :            : 
    7131                 :    6519130 :   return success;
    7132                 :            : }
    7133                 :            : 
    7134                 :            : /* Set_priorities: compute priority of each insn in the block.  */
    7135                 :            : 
    7136                 :            : int
    7137                 :    6520060 : set_priorities (rtx_insn *head, rtx_insn *tail)
    7138                 :            : {
    7139                 :    6520060 :   rtx_insn *insn;
    7140                 :    6520060 :   int n_insn;
    7141                 :    6520060 :   int sched_max_insns_priority =
    7142                 :    6520060 :         current_sched_info->sched_max_insns_priority;
    7143                 :    6520060 :   rtx_insn *prev_head;
    7144                 :            : 
    7145                 :    6520060 :   if (head == tail && ! INSN_P (head))
    7146                 :          0 :     gcc_unreachable ();
    7147                 :            : 
    7148                 :    6520060 :   n_insn = 0;
    7149                 :            : 
    7150                 :    6520060 :   prev_head = PREV_INSN (head);
    7151                 :  148276000 :   for (insn = tail; insn != prev_head; insn = PREV_INSN (insn))
    7152                 :            :     {
    7153                 :   70877900 :       if (!INSN_P (insn))
    7154                 :    3880770 :         continue;
    7155                 :            : 
    7156                 :   66997100 :       n_insn++;
    7157                 :   66997100 :       (void) priority (insn);
    7158                 :            : 
    7159                 :   66997100 :       gcc_assert (INSN_PRIORITY_KNOWN (insn));
    7160                 :            : 
    7161                 :   70877900 :       sched_max_insns_priority = MAX (sched_max_insns_priority,
    7162                 :            :                                       INSN_PRIORITY (insn));
    7163                 :            :     }
    7164                 :            : 
    7165                 :    6520060 :   current_sched_info->sched_max_insns_priority = sched_max_insns_priority;
    7166                 :            : 
    7167                 :    6520060 :   return n_insn;
    7168                 :            : }
    7169                 :            : 
    7170                 :            : /* Set sched_dump and sched_verbose for the desired debugging output. */
    7171                 :            : void
    7172                 :     644653 : setup_sched_dump (void)
    7173                 :            : {
    7174                 :     644653 :   sched_verbose = sched_verbose_param;
    7175                 :     644653 :   sched_dump = dump_file;
    7176                 :     644653 :   if (!dump_file)
    7177                 :     644616 :     sched_verbose = 0;
    7178                 :     644653 : }
    7179                 :            : 
    7180                 :            : /* Allocate data for register pressure sensitive scheduling.  */
    7181                 :            : static void
    7182                 :     644653 : alloc_global_sched_pressure_data (void)
    7183                 :            : {
    7184                 :     644653 :   if (sched_pressure != SCHED_PRESSURE_NONE)
    7185                 :            :     {
    7186                 :         81 :       int i, max_regno = max_reg_num ();
    7187                 :            : 
    7188                 :         81 :       if (sched_dump != NULL)
    7189                 :            :         /* We need info about pseudos for rtl dumps about pseudo
    7190                 :            :            classes and costs.  */
    7191                 :          0 :         regstat_init_n_sets_and_refs ();
    7192                 :         81 :       ira_set_pseudo_classes (true, sched_verbose ? sched_dump : NULL);
    7193                 :         81 :       sched_regno_pressure_class
    7194                 :         81 :         = (enum reg_class *) xmalloc (max_regno * sizeof (enum reg_class));
    7195                 :      10160 :       for (i = 0; i < max_regno; i++)
    7196                 :      20158 :         sched_regno_pressure_class[i]
    7197                 :      10079 :           = (i < FIRST_PSEUDO_REGISTER
    7198                 :      10079 :              ? ira_pressure_class_translate[REGNO_REG_CLASS (i)]
    7199                 :       3923 :              : ira_pressure_class_translate[reg_allocno_class (i)]);
    7200                 :         81 :       curr_reg_live = BITMAP_ALLOC (NULL);
    7201                 :         81 :       if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
    7202                 :            :         {
    7203                 :         81 :           saved_reg_live = BITMAP_ALLOC (NULL);
    7204                 :         81 :           region_ref_regs = BITMAP_ALLOC (NULL);
    7205                 :            :         }
    7206                 :         81 :       if (sched_pressure == SCHED_PRESSURE_MODEL)
    7207                 :          0 :         tmp_bitmap = BITMAP_ALLOC (NULL);
    7208                 :            : 
    7209                 :            :       /* Calculate number of CALL_SAVED_REGS and FIXED_REGS in register classes
    7210                 :            :          that we calculate register pressure for.  */
    7211                 :        405 :       for (int c = 0; c < ira_pressure_classes_num; ++c)
    7212                 :            :         {
    7213                 :        324 :           enum reg_class cl = ira_pressure_classes[c];
    7214                 :            : 
    7215                 :        324 :           call_saved_regs_num[cl] = 0;
    7216                 :        324 :           fixed_regs_num[cl] = 0;
    7217                 :            : 
    7218                 :       4129 :           for (int i = 0; i < ira_class_hard_regs_num[cl]; ++i)
    7219                 :            :             {
    7220                 :       3805 :               unsigned int regno = ira_class_hard_regs[cl][i];
    7221                 :       3805 :               if (fixed_regs[regno])
    7222                 :          0 :                 ++fixed_regs_num[cl];
    7223                 :       3805 :               else if (!crtl->abi->clobbers_full_reg_p (regno))
    7224                 :        484 :                 ++call_saved_regs_num[cl];
    7225                 :            :             }
    7226                 :            :         }
    7227                 :            :     }
    7228                 :     644653 : }
    7229                 :            : 
    7230                 :            : /*  Free data for register pressure sensitive scheduling.  Also called
    7231                 :            :     from schedule_region when stopping sched-pressure early.  */
    7232                 :            : void
    7233                 :     644653 : free_global_sched_pressure_data (void)
    7234                 :            : {
    7235                 :     644653 :   if (sched_pressure != SCHED_PRESSURE_NONE)
    7236                 :            :     {
    7237                 :         81 :       if (regstat_n_sets_and_refs != NULL)
    7238                 :          0 :         regstat_free_n_sets_and_refs ();
    7239                 :         81 :       if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
    7240                 :            :         {
    7241                 :         81 :           BITMAP_FREE (region_ref_regs);
    7242                 :         81 :           BITMAP_FREE (saved_reg_live);
    7243                 :            :         }
    7244                 :         81 :       if (sched_pressure == SCHED_PRESSURE_MODEL)
    7245                 :          0 :         BITMAP_FREE (tmp_bitmap);
    7246                 :         81 :       BITMAP_FREE (curr_reg_live);
    7247                 :         81 :       free (sched_regno_pressure_class);
    7248                 :            :     }
    7249                 :     644653 : }
    7250                 :            : 
    7251                 :            : /* Initialize some global state for the scheduler.  This function works
    7252                 :            :    with the common data shared between all the schedulers.  It is called
    7253                 :            :    from the scheduler specific initialization routine.  */
    7254                 :            : 
    7255                 :            : void
    7256                 :     644653 : sched_init (void)
    7257                 :            : {
    7258                 :            :   /* Disable speculative loads in their presence if cc0 defined.  */
    7259                 :     644653 :   if (HAVE_cc0)
    7260                 :            :   flag_schedule_speculative_load = 0;
    7261                 :            : 
    7262                 :     644653 :   if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
    7263                 :          3 :     targetm.sched.dispatch_do (NULL, DISPATCH_INIT);
    7264                 :            : 
    7265                 :     644653 :   if (live_range_shrinkage_p)
    7266                 :         20 :     sched_pressure = SCHED_PRESSURE_WEIGHTED;
    7267                 :     644633 :   else if (flag_sched_pressure
    7268                 :        129 :            && !reload_completed
    7269                 :         61 :            && common_sched_info->sched_pass_id == SCHED_RGN_PASS)
    7270                 :         61 :     sched_pressure = ((enum sched_pressure_algorithm)
    7271                 :         61 :                       param_sched_pressure_algorithm);
    7272                 :            :   else
    7273                 :     644572 :     sched_pressure = SCHED_PRESSURE_NONE;
    7274                 :            : 
    7275                 :     644653 :   if (sched_pressure != SCHED_PRESSURE_NONE)
    7276                 :         81 :     ira_setup_eliminable_regset ();
    7277                 :            : 
    7278                 :            :   /* Initialize SPEC_INFO.  */
    7279                 :     644653 :   if (targetm.sched.set_sched_flags)
    7280                 :            :     {
    7281                 :          0 :       spec_info = &spec_info_var;
    7282                 :          0 :       targetm.sched.set_sched_flags (spec_info);
    7283                 :            : 
    7284                 :          0 :       if (spec_info->mask != 0)
    7285                 :            :         {
    7286                 :          0 :           spec_info->data_weakness_cutoff
    7287                 :          0 :             = (param_sched_spec_prob_cutoff * MAX_DEP_WEAK) / 100;
    7288                 :          0 :           spec_info->control_weakness_cutoff
    7289                 :          0 :             = (param_sched_spec_prob_cutoff * REG_BR_PROB_BASE) / 100;
    7290                 :            :         }
    7291                 :            :       else
    7292                 :            :         /* So we won't read anything accidentally.  */
    7293                 :          0 :         spec_info = NULL;
    7294                 :            : 
    7295                 :            :     }
    7296                 :            :   else
    7297                 :            :     /* So we won't read anything accidentally.  */
    7298                 :     644653 :     spec_info = 0;
    7299                 :            : 
    7300                 :            :   /* Initialize issue_rate.  */
    7301                 :     644653 :   if (targetm.sched.issue_rate)
    7302                 :     644653 :     issue_rate = targetm.sched.issue_rate ();
    7303                 :            :   else
    7304                 :          0 :     issue_rate = 1;
    7305                 :            : 
    7306                 :     644653 :   if (targetm.sched.first_cycle_multipass_dfa_lookahead
    7307                 :            :       /* Don't use max_issue with reg_pressure scheduling.  Multipass
    7308                 :            :          scheduling and reg_pressure scheduling undo each other's decisions.  */
    7309                 :     644653 :       && sched_pressure == SCHED_PRESSURE_NONE)
    7310                 :     644572 :     dfa_lookahead = targetm.sched.first_cycle_multipass_dfa_lookahead ();
    7311                 :            :   else
    7312                 :         81 :     dfa_lookahead = 0;
    7313                 :            : 
    7314                 :            :   /* Set to "0" so that we recalculate.  */
    7315                 :     644653 :   max_lookahead_tries = 0;
    7316                 :            : 
    7317                 :     644653 :   if (targetm.sched.init_dfa_pre_cycle_insn)
    7318                 :          0 :     targetm.sched.init_dfa_pre_cycle_insn ();
    7319                 :            : 
    7320                 :     644653 :   if (targetm.sched.init_dfa_post_cycle_insn)
    7321                 :          0 :     targetm.sched.init_dfa_post_cycle_insn ();
    7322                 :            : 
    7323                 :     644653 :   dfa_start ();
    7324                 :     644653 :   dfa_state_size = state_size ();
    7325                 :            : 
    7326                 :     644653 :   init_alias_analysis ();
    7327                 :            : 
    7328                 :     644653 :   if (!sched_no_dce)
    7329                 :     644653 :     df_set_flags (DF_LR_RUN_DCE);
    7330                 :     644653 :   df_note_add_problem ();
    7331                 :            : 
    7332                 :            :   /* More problems needed for interloop dep calculation in SMS.  */
    7333                 :     644653 :   if (common_sched_info->sched_pass_id == SCHED_SMS_PASS)
    7334                 :            :     {
    7335                 :        180 :       df_rd_add_problem ();
    7336                 :        180 :       df_chain_add_problem (DF_DU_CHAIN + DF_UD_CHAIN);
    7337                 :            :     }
    7338                 :            : 
    7339                 :     644653 :   df_analyze ();
    7340                 :            : 
    7341                 :            :   /* Do not run DCE after reload, as this can kill nops inserted
    7342                 :            :      by bundling.  */
    7343                 :     644653 :   if (reload_completed)
    7344                 :     644279 :     df_clear_flags (DF_LR_RUN_DCE);
    7345                 :            : 
    7346                 :     644653 :   regstat_compute_calls_crossed ();
    7347                 :            : 
    7348                 :     644653 :   if (targetm.sched.init_global)
    7349                 :     644653 :     targetm.sched.init_global (sched_dump, sched_verbose, get_max_uid () + 1);
    7350                 :            : 
    7351                 :     644653 :   alloc_global_sched_pressure_data ();
    7352                 :            : 
    7353                 :     644653 :   curr_state = xmalloc (dfa_state_size);
    7354                 :     644653 : }
    7355                 :            : 
    7356                 :            : static void haifa_init_only_bb (basic_block, basic_block);
    7357                 :            : 
    7358                 :            : /* Initialize data structures specific to the Haifa scheduler.  */
    7359                 :            : void
    7360                 :     644525 : haifa_sched_init (void)
    7361                 :            : {
    7362                 :     644525 :   setup_sched_dump ();
    7363                 :     644525 :   sched_init ();
    7364                 :            : 
    7365                 :     644525 :   scheduled_insns.create (0);
    7366                 :            : 
    7367                 :     644525 :   if (spec_info != NULL)
    7368                 :            :     {
    7369                 :          0 :       sched_deps_info->use_deps_list = 1;
    7370                 :          0 :       sched_deps_info->generate_spec_deps = 1;
    7371                 :            :     }
    7372                 :            : 
    7373                 :            :   /* Initialize luids, dependency caches, target and h_i_d for the
    7374                 :            :      whole function.  */
    7375                 :     644525 :   {
    7376                 :     644525 :     sched_init_bbs ();
    7377                 :            : 
    7378                 :     644525 :     auto_vec<basic_block> bbs (n_basic_blocks_for_fn (cfun));
    7379                 :     644525 :     basic_block bb;
    7380                 :    7170510 :     FOR_EACH_BB_FN (bb, cfun)
    7381                 :    6525990 :       bbs.quick_push (bb);
    7382                 :     644525 :     sched_init_luids (bbs);
    7383                 :     644525 :     sched_deps_init (true);
    7384                 :     644525 :     sched_extend_target ();
    7385                 :     644525 :     haifa_init_h_i_d (bbs);
    7386                 :            :   }
    7387                 :            : 
    7388                 :     644525 :   sched_init_only_bb = haifa_init_only_bb;
    7389                 :     644525 :   sched_split_block = sched_split_block_1;
    7390                 :     644525 :   sched_create_empty_bb = sched_create_empty_bb_1;
    7391                 :     644525 :   haifa_recovery_bb_ever_added_p = false;
    7392                 :            : 
    7393                 :     644525 :   nr_begin_data = nr_begin_control = nr_be_in_data = nr_be_in_control = 0;
    7394                 :     644525 :   before_recovery = 0;
    7395                 :     644525 :   after_recovery = 0;
    7396                 :            : 
    7397                 :     644525 :   modulo_ii = 0;
    7398                 :     644525 : }
    7399                 :            : 
    7400                 :            : /* Finish work with the data specific to the Haifa scheduler.  */
    7401                 :            : void
    7402                 :     644525 : haifa_sched_finish (void)
    7403                 :            : {
    7404                 :     644525 :   sched_create_empty_bb = NULL;
    7405                 :     644525 :   sched_split_block = NULL;
    7406                 :     644525 :   sched_init_only_bb = NULL;
    7407                 :            : 
    7408                 :     644525 :   if (spec_info && spec_info->dump)
    7409                 :            :     {
    7410                 :          0 :       char c = reload_completed ? 'a' : 'b';
    7411                 :            : 
    7412                 :          0 :       fprintf (spec_info->dump,
    7413                 :            :                ";; %s:\n", current_function_name ());
    7414                 :            : 
    7415                 :          0 :       fprintf (spec_info->dump,
    7416                 :            :                ";; Procedure %cr-begin-data-spec motions == %d\n",
    7417                 :            :                c, nr_begin_data);
    7418                 :          0 :       fprintf (spec_info->dump,
    7419                 :            :                ";; Procedure %cr-be-in-data-spec motions == %d\n",
    7420                 :            :                c, nr_be_in_data);
    7421                 :          0 :       fprintf (spec_info->dump,
    7422                 :            :                ";; Procedure %cr-begin-control-spec motions == %d\n",
    7423                 :            :                c, nr_begin_control);
    7424                 :          0 :       fprintf (spec_info->dump,
    7425                 :            :                ";; Procedure %cr-be-in-control-spec motions == %d\n",
    7426                 :            :                c, nr_be_in_control);
    7427                 :            :     }
    7428                 :            : 
    7429                 :     644525 :   scheduled_insns.release ();
    7430                 :            : 
    7431                 :            :   /* Finalize h_i_d, dependency caches, and luids for the whole
    7432                 :            :      function.  Target will be finalized in md_global_finish ().  */
    7433                 :     644525 :   sched_deps_finish ();
    7434                 :     644525 :   sched_finish_luids ();
    7435                 :     644525 :   current_sched_info = NULL;
    7436                 :     644525 :   insn_queue = NULL;
    7437                 :     644525 :   sched_finish ();
    7438                 :     644525 : }
    7439                 :            : 
    7440                 :            : /* Free global data used during insn scheduling.  This function works with
    7441                 :            :    the common data shared between the schedulers.  */
    7442                 :            : 
    7443                 :            : void
    7444                 :     644653 : sched_finish (void)
    7445                 :            : {
    7446                 :     644653 :   haifa_finish_h_i_d ();
    7447                 :     644653 :   free_global_sched_pressure_data ();
    7448                 :     644653 :   free (curr_state);
    7449                 :            : 
    7450                 :     644653 :   if (targetm.sched.finish_global)
    7451                 :          0 :     targetm.sched.finish_global (sched_dump, sched_verbose);
    7452                 :            : 
    7453                 :     644653 :   end_alias_analysis ();
    7454                 :            : 
    7455                 :     644653 :   regstat_free_calls_crossed ();
    7456                 :            : 
    7457                 :     644653 :   dfa_finish ();
    7458                 :     644653 : }
    7459                 :            : 
    7460                 :            : /* Free all delay_pair structures that were recorded.  */
    7461                 :            : void
    7462                 :          0 : free_delay_pairs (void)
    7463                 :            : {
    7464                 :          0 :   if (delay_htab)
    7465                 :            :     {
    7466                 :          0 :       delay_htab->empty ();
    7467                 :          0 :       delay_htab_i2->empty ();
    7468                 :            :     }
    7469                 :          0 : }
    7470                 :            : 
    7471                 :            : /* Fix INSN_TICKs of the instructions in the current block as well as
    7472                 :            :    INSN_TICKs of their dependents.
    7473                 :            :    HEAD and TAIL are the begin and the end of the current scheduled block.  */
    7474                 :            : static void
    7475                 :        120 : fix_inter_tick (rtx_insn *head, rtx_insn *tail)
    7476                 :            : {
    7477                 :            :   /* Set of instructions with corrected INSN_TICK.  */
    7478                 :        120 :   auto_bitmap processed;
    7479                 :            :   /* ??? It is doubtful if we should assume that cycle advance happens on
    7480                 :            :      basic block boundaries.  Basically insns that are unconditionally ready
    7481                 :            :      on the start of the block are more preferable then those which have
    7482                 :            :      a one cycle dependency over insn from the previous block.  */
    7483                 :        120 :   int next_clock = clock_var + 1;
    7484                 :            : 
    7485                 :            :   /* Iterates over scheduled instructions and fix their INSN_TICKs and
    7486                 :            :      INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent
    7487                 :            :      across different blocks.  */
    7488                 :       4142 :   for (tail = NEXT_INSN (tail); head != tail; head = NEXT_INSN (head))
    7489                 :            :     {
    7490                 :       2011 :       if (INSN_P (head))
    7491                 :            :         {
    7492                 :       2011 :           int tick;
    7493                 :       2011 :           sd_iterator_def sd_it;
    7494                 :       2011 :           dep_t dep;
    7495                 :            : 
    7496                 :       2011 :           tick = INSN_TICK (head);
    7497                 :       2011 :           gcc_assert (tick >= MIN_TICK);
    7498                 :            : 
    7499                 :            :           /* Fix INSN_TICK of instruction from just scheduled block.  */
    7500                 :       2011 :           if (bitmap_set_bit (processed, INSN_LUID (head)))
    7501                 :            :             {
    7502                 :        132 :               tick -= next_clock;
    7503                 :            : 
    7504                 :        132 :               if (tick < MIN_TICK)
    7505                 :            :                 tick = MIN_TICK;
    7506                 :            : 
    7507                 :        132 :               INSN_TICK (head) = tick;
    7508                 :            :             }
    7509                 :            : 
    7510                 :       2011 :           if (DEBUG_INSN_P (head))
    7511                 :        120 :             continue;
    7512                 :            : 
    7513                 :      10080 :           FOR_EACH_DEP (head, SD_LIST_RES_FORW, sd_it, dep)
    7514                 :            :             {
    7515                 :       8189 :               rtx_insn *next;
    7516                 :            : 
    7517                 :       8189 :               next = DEP_CON (dep);
    7518                 :       8189 :               tick = INSN_TICK (next);
    7519                 :            : 
    7520                 :       8189 :               if (tick != INVALID_TICK
    7521                 :            :                   /* If NEXT has its INSN_TICK calculated, fix it.
    7522                 :            :                      If not - it will be properly calculated from
    7523                 :            :                      scratch later in fix_tick_ready.  */
    7524                 :       8189 :                   && bitmap_set_bit (processed, INSN_LUID (next)))
    7525                 :            :                 {
    7526                 :       1915 :                   tick -= next_clock;
    7527                 :            : 
    7528                 :       1915 :                   if (tick < MIN_TICK)
    7529                 :            :                     tick = MIN_TICK;
    7530                 :            : 
    7531                 :       1915 :                   if (tick > INTER_TICK (next))
    7532                 :       1915 :                     INTER_TICK (next) = tick;
    7533                 :            :                   else
    7534                 :       1915 :                     tick = INTER_TICK (next);
    7535                 :            : 
    7536                 :       1915 :                   INSN_TICK (next) = tick;
    7537                 :            :                 }
    7538                 :            :             }
    7539                 :            :         }
    7540                 :            :     }
    7541                 :        120 : }
    7542                 :            : 
    7543                 :            : /* Check if NEXT is ready to be added to the ready or queue list.
    7544                 :            :    If "yes", add it to the proper list.
    7545                 :            :    Returns:
    7546                 :            :       -1 - is not ready yet,
    7547                 :            :        0 - added to the ready list,
    7548                 :            :    0 < N - queued for N cycles.  */
    7549                 :            : int
    7550                 :  198017000 : try_ready (rtx_insn *next)
    7551                 :            : {
    7552                 :  198017000 :   ds_t old_ts, new_ts;
    7553                 :            : 
    7554                 :  198017000 :   old_ts = TODO_SPEC (next);
    7555                 :            : 
    7556                 :  198017000 :   gcc_assert (!(old_ts & ~(SPECULATIVE | HARD_DEP | DEP_CONTROL | DEP_POSTPONED))
    7557                 :            :               && (old_ts == HARD_DEP
    7558                 :            :                   || old_ts == DEP_POSTPONED
    7559                 :            :                   || (old_ts & SPECULATIVE)
    7560                 :            :                   || old_ts == DEP_CONTROL));
    7561                 :            : 
    7562                 :  198017000 :   new_ts = recompute_todo_spec (next, false);
    7563                 :            : 
    7564                 :  198017000 :   if (new_ts & (HARD_DEP | DEP_POSTPONED))
    7565                 :  262047000 :     gcc_assert (new_ts == old_ts
    7566                 :            :                 && QUEUE_INDEX (next) == QUEUE_NOWHERE);
    7567                 :   66993000 :   else if (current_sched_info->new_ready)
    7568                 :   66991700 :     new_ts = current_sched_info->new_ready (next, new_ts);
    7569                 :            : 
    7570                 :            :   /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might
    7571                 :            :      have its original pattern or changed (speculative) one.  This is due
    7572                 :            :      to changing ebb in region scheduling.
    7573                 :            :      * But if (old_ts & SPECULATIVE), then we are pretty sure that insn
    7574                 :            :      has speculative pattern.
    7575                 :            : 
    7576                 :            :      We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because
    7577                 :            :      control-speculative NEXT could have been discarded by sched-rgn.c
    7578                 :            :      (the same case as when discarded by can_schedule_ready_p ()).  */
    7579                 :            : 
    7580                 :  198017000 :   if ((new_ts & SPECULATIVE)
    7581                 :            :       /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't
    7582                 :            :          need to change anything.  */
    7583                 :          0 :       && new_ts != old_ts)
    7584                 :            :     {
    7585                 :          0 :       int res;
    7586                 :          0 :       rtx new_pat;
    7587                 :            : 
    7588                 :          0 :       gcc_assert ((new_ts & SPECULATIVE) && !(new_ts & ~SPECULATIVE));
    7589                 :            : 
    7590                 :          0 :       res = haifa_speculate_insn (next, new_ts, &new_pat);
    7591                 :            : 
    7592                 :          0 :       switch (res)
    7593                 :            :         {
    7594                 :            :         case -1:
    7595                 :            :           /* It would be nice to change DEP_STATUS of all dependences,
    7596                 :            :              which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP,
    7597                 :            :              so we won't reanalyze anything.  */
    7598                 :            :           new_ts = HARD_DEP;
    7599                 :            :           break;
    7600                 :            : 
    7601                 :          0 :         case 0:
    7602                 :            :           /* We follow the rule, that every speculative insn
    7603                 :            :              has non-null ORIG_PAT.  */
    7604                 :          0 :           if (!ORIG_PAT (next))
    7605                 :          0 :             ORIG_PAT (next) = PATTERN (next);
    7606                 :            :           break;
    7607                 :            : 
    7608                 :          0 :         case 1:
    7609                 :          0 :           if (!ORIG_PAT (next))
    7610                 :            :             /* If we gonna to overwrite the original pattern of insn,
    7611                 :            :                save it.  */
    7612                 :          0 :             ORIG_PAT (next) = PATTERN (next);
    7613                 :            : 
    7614                 :          0 :           res = haifa_change_pattern (next, new_pat);
    7615                 :          0 :           gcc_assert (res);
    7616                 :            :           break;
    7617                 :            : 
    7618                 :          0 :         default:
    7619                 :          0 :           gcc_unreachable ();
    7620                 :            :         }
    7621                 :            :     }
    7622                 :            : 
    7623                 :            :   /* We need to restore pattern only if (new_ts == 0), because otherwise it is
    7624                 :            :      either correct (new_ts & SPECULATIVE),
    7625                 :            :      or we simply don't care (new_ts & HARD_DEP).  */
    7626                 :            : 
    7627                 :  198017000 :   gcc_assert (!ORIG_PAT (next)
    7628                 :            :               || !IS_SPECULATION_BRANCHY_CHECK_P (next));
    7629                 :            : 
    7630                 :  198017000 :   TODO_SPEC (next) = new_ts;
    7631                 :            : 
    7632                 :  198017000 :   if (new_ts & (HARD_DEP | DEP_POSTPONED))
    7633                 :            :     {
    7634                 :            :       /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because
    7635                 :            :          control-speculative NEXT could have been discarded by sched-rgn.c
    7636                 :            :          (the same case as when discarded by can_schedule_ready_p ()).  */
    7637                 :            :       /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/
    7638                 :            : 
    7639                 :  131024000 :       change_queue_index (next, QUEUE_NOWHERE);
    7640                 :            : 
    7641                 :  131024000 :       return -1;
    7642                 :            :     }
    7643                 :   66992900 :   else if (!(new_ts & BEGIN_SPEC)
    7644                 :   66992900 :            && ORIG_PAT (next) && PREDICATED_PAT (next) == NULL_RTX
    7645                 :   66992900 :            && !IS_SPECULATION_CHECK_P (next))
    7646                 :            :     /* We should change pattern of every previously speculative
    7647                 :            :        instruction - and we determine if NEXT was speculative by using
    7648                 :            :        ORIG_PAT field.  Except one case - speculation checks have ORIG_PAT
    7649                 :            :        pat too, so skip them.  */
    7650                 :            :     {
    7651                 :          0 :       bool success = haifa_change_pattern (next, ORIG_PAT (next));
    7652                 :          0 :       gcc_assert (success);
    7653                 :          0 :       ORIG_PAT (next) = 0;
    7654                 :            :     }
    7655                 :            : 
    7656                 :   66992900 :   if (sched_verbose >= 2)
    7657                 :            :     {
    7658                 :          0 :       fprintf (sched_dump, ";;\t\tdependencies resolved: insn %s",
    7659                 :          0 :                (*current_sched_info->print_insn) (next, 0));
    7660                 :            : 
    7661                 :          0 :       if (spec_info && spec_info->dump)
    7662                 :            :         {
    7663                 :          0 :           if (new_ts & BEGIN_DATA)
    7664                 :          0 :             fprintf (spec_info->dump, "; data-spec;");
    7665                 :          0 :           if (new_ts & BEGIN_CONTROL)
    7666                 :          0 :             fprintf (spec_info->dump, "; control-spec;");
    7667                 :          0 :           if (new_ts & BE_IN_CONTROL)
    7668                 :          0 :             fprintf (spec_info->dump, "; in-control-spec;");
    7669                 :            :         }
    7670                 :          0 :       if (TODO_SPEC (next) & DEP_CONTROL)
    7671                 :          0 :         fprintf (sched_dump, " predicated");
    7672                 :          0 :       fprintf (sched_dump, "\n");
    7673                 :            :     }
    7674                 :            : 
    7675                 :   66992900 :   adjust_priority (next);
    7676                 :            : 
    7677                 :   66992900 :   return fix_tick_ready (next);
    7678                 :            : }
    7679                 :            : 
    7680                 :            : /* Calculate INSN_TICK of NEXT and add it to either ready or queue list.  */
    7681                 :            : static int
    7682                 :   67192400 : fix_tick_ready (rtx_insn *next)
    7683                 :            : {
    7684                 :   67192400 :   int tick, delay;
    7685                 :            : 
    7686                 :   67192400 :   if (!DEBUG_INSN_P (next) && !sd_lists_empty_p (next, SD_LIST_RES_BACK))
    7687                 :            :     {
    7688                 :   28342900 :       int full_p;
    7689                 :   28342900 :       sd_iterator_def sd_it;
    7690                 :   28342900 :       dep_t dep;
    7691                 :            : 
    7692                 :   28342900 :       tick = INSN_TICK (next);
    7693                 :            :       /* if tick is not equal to INVALID_TICK, then update
    7694                 :            :          INSN_TICK of NEXT with the most recent resolved dependence
    7695                 :            :          cost.  Otherwise, recalculate from scratch.  */
    7696                 :   28342900 :       full_p = (tick == INVALID_TICK);
    7697                 :            : 
    7698                 :  127612000 :       FOR_EACH_DEP (next, SD_LIST_RES_BACK, sd_it, dep)
    7699                 :            :         {
    7700                 :   99269200 :           rtx_insn *pro = DEP_PRO (dep);
    7701                 :   99269200 :           int tick1;
    7702                 :            : 
    7703                 :   99269200 :           gcc_assert (INSN_TICK (pro) >= MIN_TICK);
    7704                 :            : 
    7705                 :   99269200 :           tick1 = INSN_TICK (pro) + dep_cost (dep);
    7706                 :   99269200 :           if (tick1 > tick)
    7707                 :            :             tick = tick1;
    7708                 :            : 
    7709                 :   99269200 :           if (!full_p)
    7710                 :            :             break;
    7711                 :            :         }
    7712                 :            :     }
    7713                 :            :   else
    7714                 :            :     tick = -1;
    7715                 :            : 
    7716                 :   67192400 :   INSN_TICK (next) = tick;
    7717                 :            : 
    7718                 :   67192400 :   delay = tick - clock_var;
    7719                 :   67192400 :   if (delay <= 0 || sched_pressure != SCHED_PRESSURE_NONE || sched_fusion)
    7720                 :   54246900 :     delay = QUEUE_READY;
    7721                 :            : 
    7722                 :   67192400 :   change_queue_index (next, delay);
    7723                 :            : 
    7724                 :   67192400 :   return delay;
    7725                 :            : }
    7726                 :            : 
    7727                 :            : /* Move NEXT to the proper queue list with (DELAY >= 1),
    7728                 :            :    or add it to the ready list (DELAY == QUEUE_READY),
    7729                 :            :    or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE).  */
    7730                 :            : static void
    7731                 :  198216000 : change_queue_index (rtx_insn *next, int delay)
    7732                 :            : {
    7733                 :  198216000 :   int i = QUEUE_INDEX (next);
    7734                 :            : 
    7735                 :  198216000 :   gcc_assert (QUEUE_NOWHERE <= delay && delay <= max_insn_queue_index
    7736                 :            :               && delay != 0);
    7737                 :  198216000 :   gcc_assert (i != QUEUE_SCHEDULED);
    7738                 :            : 
    7739                 :  198216000 :   if ((delay > 0 && NEXT_Q_AFTER (q_ptr, delay) == i)
    7740                 :  198168000 :       || (delay < 0 && delay == i))
    7741                 :            :     /* We have nothing to do.  */
    7742                 :            :     return;
    7743                 :            : 
    7744                 :            :   /* Remove NEXT from wherever it is now.  */
    7745                 :   67108700 :   if (i == QUEUE_READY)
    7746                 :          0 :     ready_remove_insn (next);
    7747                 :   67108700 :   else if (i >= 0)
    7748                 :     115734 :     queue_remove (next);
    7749                 :            : 
    7750                 :            :   /* Add it to the proper place.  */
    7751                 :   67108700 :   if (delay == QUEUE_READY)
    7752                 :   54211600 :     ready_add (readyp, next, false);
    7753                 :   12897100 :   else if (delay >= 1)
    7754                 :   12897100 :     queue_insn (next, delay, "change queue index");
    7755                 :            : 
    7756                 :   67108700 :   if (sched_verbose >= 2)
    7757                 :            :     {
    7758                 :          0 :       fprintf (sched_dump, ";;\t\ttick updated: insn %s",
    7759                 :          0 :                (*current_sched_info->print_insn) (next, 0));
    7760                 :            : 
    7761                 :          0 :       if (delay == QUEUE_READY)
    7762                 :          0 :         fprintf (sched_dump, " into ready\n");
    7763                 :          0 :       else if (delay >= 1)
    7764                 :          0 :         fprintf (sched_dump, " into queue with cost=%d\n", delay);
    7765                 :            :       else
    7766                 :          0 :         fprintf (sched_dump, " removed from ready or queue lists\n");
    7767                 :            :     }
    7768                 :            : }
    7769                 :            : 
    7770                 :            : static int sched_ready_n_insns = -1;
    7771                 :            : 
    7772                 :            : /* Initialize per region data structures.  */
    7773                 :            : void
    7774                 :    6525460 : sched_extend_ready_list (int new_sched_ready_n_insns)
    7775                 :            : {
    7776                 :    6525460 :   int i;
    7777                 :            : 
    7778                 :    6525460 :   if (sched_ready_n_insns == -1)
    7779                 :            :     /* At the first call we need to initialize one more choice_stack
    7780                 :            :        entry.  */
    7781                 :            :     {
    7782                 :    6525370 :       i = 0;
    7783                 :    6525370 :       sched_ready_n_insns = 0;
    7784                 :    6525370 :       scheduled_insns.reserve (new_sched_ready_n_insns);
    7785                 :            :     }
    7786                 :            :   else
    7787                 :         89 :     i = sched_ready_n_insns + 1;
    7788                 :            : 
    7789                 :    6525460 :   ready.veclen = new_sched_ready_n_insns + issue_rate;
    7790                 :    6525460 :   ready.vec = XRESIZEVEC (rtx_insn *, ready.vec, ready.veclen);
    7791                 :            : 
    7792                 :    6525460 :   gcc_assert (new_sched_ready_n_insns >= sched_ready_n_insns);
    7793                 :            : 
    7794                 :    6525460 :   ready_try = (signed char *) xrecalloc (ready_try, new_sched_ready_n_insns,
    7795                 :            :                                          sched_ready_n_insns,
    7796                 :            :                                          sizeof (*ready_try));
    7797                 :            : 
    7798                 :            :   /* We allocate +1 element to save initial state in the choice_stack[0]
    7799                 :            :      entry.  */
    7800                 :    6525460 :   choice_stack = XRESIZEVEC (struct choice_entry, choice_stack,
    7801                 :            :                              new_sched_ready_n_insns + 1);
    7802                 :            : 
    7803                 :   80044900 :   for (; i <= new_sched_ready_n_insns; i++)
    7804                 :            :     {
    7805                 :   73519500 :       choice_stack[i].state = xmalloc (dfa_state_size);
    7806                 :            : 
    7807                 :   73519500 :       if (targetm.sched.first_cycle_multipass_init)
    7808                 :   73293600 :         targetm.sched.first_cycle_multipass_init (&(choice_stack[i]
    7809                 :            :                                                     .target_data));
    7810                 :            :     }
    7811                 :            : 
    7812                 :    6525460 :   sched_ready_n_insns = new_sched_ready_n_insns;
    7813                 :    6525460 : }
    7814                 :            : 
    7815                 :            : /* Free per region data structures.  */
    7816                 :            : void
    7817                 :    6525370 : sched_finish_ready_list (void)
    7818                 :            : {
    7819                 :    6525370 :   int i;
    7820                 :            : 
    7821                 :    6525370 :   free (ready.vec);
    7822                 :    6525370 :   ready.vec = NULL;<