Query running long despite having "good" plan - oracle

Question

I have a query which is a UNION query which joins multiple tables. All the stats are updated. The query used to run fine and completed within 30-40 seconds but in 12c it's taking insane amount of time(~10 mins to 5 mins)
I have captured the actual plan.
lan hash value: 1038754298
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------
| Id | Operation | Name | Starts | E-Rows | A-Rows | A-Time | Buffers | Reads | OMem | 1Mem | O/1/M |
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------
| 0 | SELECT STATEMENT | | 1 | | 50 |00:00:49.93 | 960K| 7 | | | |
| 1 | SORT ORDER BY | | 1 | 1791 | 50 |00:00:49.93 | 960K| 7 | 267K| 267K| 1/0/0|
| 2 | UNION-ALL | | 1 | | 2648 |00:00:49.93 | 960K| 7 | | | |
| 3 | NESTED LOOPS SEMI | | 1 | 1 | 2648 |00:00:42.36 | 498K| 0 | | | |
|* 4 | HASH JOIN | | 1 | 1589 | 9111 |00:00:42.22 | 469K| 0 | 7900K| 2299K| 1/0/0|
|* 5 | HASH JOIN | | 1 | 1140K| 50569 |00:00:19.50 | 342K| 0 | 8321K| 2379K| 1/0/0|
|* 6 | HASH JOIN | | 1 | 44335 | 59660 |00:00:16.01 | 252K| 0 | 1817K| 1817K| 1/0/0|
|* 7 | TABLE ACCESS FULL | PROMHEAD | 1 | 870 | 870 |00:00:00.01 | 602 | 0 | | | |
|* 8 | HASH JOIN | | 1 | 1168K| 172K|00:00:15.94 | 251K| 0 | 1148K| 1148K| 1/0/0|
|* 9 | TABLE ACCESS FULL | PRICE_BATCH_TRAN | 1 | 857 | 857 |00:00:00.01 | 315 | 0 | | | |
| 10 | TABLE ACCESS FULL | PROMSTORE | 1 | 36M| 36M|00:00:04.64 | 251K| 0 | | | |
|* 11 | TABLE ACCESS FULL | PROMSKU | 1 | 6665K| 6665K|00:00:01.49 | 90224 | 0 | | | |
| 12 | PARTITION RANGE ALL | | 1 | 37M| 37M|00:00:10.72 | 127K| 0 | | | |
| 13 | INDEX FAST FULL SCAN | PK_ITEM_ZONE_PRICE | 24 | 37M| 37M|00:00:04.75 | 127K| 0 | | | |
| 14 | PARTITION RANGE ITERATOR | | 9111 | 32376 | 2648 |00:00:00.13 | 28752 | 0 | | | |
| 15 | PARTITION HASH ITERATOR | | 9111 | 32376 | 2648 |00:00:00.11 | 28752 | 0 | | | |
|* 16 | TABLE ACCESS BY LOCAL INDEX ROWID BATCHED | FDT_PRICE_FUTURE | 9111 | 32376 | 2648 |00:00:00.09 | 28752 | 0 | | | |
|* 17 | INDEX RANGE SCAN | FDT_PRICE_FUTURE_PK | 9111 | 1 | 2668 |00:00:00.07 | 26089 | 0 | | | |
|* 18 | HASH JOIN | | 1 | 1790 | 0 |00:00:07.31 | 450K| 7 | 1599K| 1599K| 1/0/0|
| 19 | TABLE ACCESS FULL | PRICE_SUSP_HEAD | 1 | 2056 | 2056 |00:00:00.01 | 965 | 0 | | | |
|* 20 | HASH JOIN | | 1 | 1912 | 0 |00:00:07.30 | 449K| 7 | 39M| 8299K| 1/0/0|
| 21 | PART JOIN FILTER CREATE | :BF0000 | 1 | 745K| 745K|00:00:00.12 | 1682 | 0 | | | |
| 22 | TABLE ACCESS FULL | PRICE_ZONE_GROUP_STORE | 1 | 745K| 745K|00:00:00.03 | 1682 | 0 | | | |
|* 23 | HASH JOIN | | 1 | 496K| 0 |00:00:07.03 | 448K| 7 | 73M| 6638K| 1/0/0|
| 24 | PARTITION RANGE ALL | | 1 | 1138K| 1138K|00:00:01.70 | 154K| 7 | | | |
| 25 | PARTITION HASH ALL | | 26 | 1138K| 1138K|00:00:01.53 | 154K| 7 | | | |
| 26 | TABLE ACCESS BY LOCAL INDEX ROWID BATCHED| FDT_PRICE_FUTURE | 208 | 1138K| 1138K|00:00:01.35 | 154K| 7 | | | |
|* 27 | INDEX SKIP SCAN | FDT_PRICE_FUTURE_I1 | 208 | 1138K| 1138K|00:00:00.89 | 125K| 7 | | | |
|* 28 | HASH JOIN | | 1 | 25M| 0 |00:00:04.84 | 293K| 0 | 977K| 977K| 1/0/0|
|* 29 | HASH JOIN | | 1 | 252K| 1123 |00:00:03.33 | 90826 | 0 | 1344K| 1344K| 1/0/0|
|* 30 | TABLE ACCESS FULL | PROMHEAD | 1 | 870 | 870 |00:00:00.01 | 602 | 0 | | | |
|* 31 | TABLE ACCESS FULL | PROMSKU | 1 | 6665K| 6665K|00:00:01.47 | 90224 | 0 | | | |
| 32 | PARTITION RANGE JOIN-FILTER | | 1 | 6719K| 32778 |00:00:01.51 | 202K| 0 | | | |
|* 33 | TABLE ACCESS FULL | PRICE_SUSP_DETAIL | 24 | 6719K| 32778 |00:00:01.51 | 202K| 0 | | | |
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------
Predicate Information (identified by operation id):
---------------------------------------------------
4 - access("IZP"."ITEM"="PSK"."SKU" AND "IZP"."ZONE_GROUP_ID"="PBT"."ZONE_GROUP_ID" AND "IZP"."ZONE_ID"="PBT"."NEW_ZONE_ID")
5 - access("PS"."PROMOTION"="PSK"."PROMOTION")
6 - access("PH"."PROMOTION"="PS"."PROMOTION")
7 - filter("PH"."STATUS"='A')
8 - access("PS"."STORE"="PBT"."STORE")
filter("PS"."START_DATE">"PBT"."EFFECTIVE_DATE")
9 - filter("PBT"."FDC_STATUS"='A')
11 - filter(("PSK"."CHANGE_TYPE"='MA' OR "PSK"."CHANGE_TYPE"='PC'))
16 - filter("PRICE_MOD_TYPE"='ZC')
17 - access("STORE"="PS"."STORE" AND "SKU"="PSK"."SKU" AND "START_DATE"<"PS"."START_DATE")
18 - access("PSH"."PRICE_CHANGE"="PSD"."PRICE_CHANGE")
filter("FDT"."START_DATE">"PSH"."ACTIVE_DATE")
20 - access("PSD"."ZONE_ID"="PZGS"."ZONE_ID" AND "PSD"."ZONE_GROUP_ID"="PZGS"."ZONE_GROUP_ID" AND "FDT"."STORE"="PZGS"."STORE")
23 - access("PSK"."PROMOTION"="FDT"."PRICE_MOD_NO" AND "PSD"."SKU"="FDT"."SKU")
27 - access("FDT"."PRICE_MOD_TYPE"='PR')
filter("FDT"."PRICE_MOD_TYPE"='PR')
28 - access("PSK"."SKU"="PSD"."SKU")
29 - access("PH"."PROMOTION"="PSK"."PROMOTION")
30 - filter("PH"."STATUS"='A')
31 - filter(("PSK"."CHANGE_TYPE"='MA' OR "PSK"."CHANGE_TYPE"='PC'))
33 - filter("PSD"."STATUS"='A')
Note
-----
- this is an adaptive plan
As you can see Oracle predicts the cardinality correctly in most cases. Can you suggest where should I start looking from.
SELECT
PBT.STORE ,
PSK.SKU ,
0 MULTI_UNITS ,
0 MULTI_UNIT_RETAIL ,
PS.START_DATE PS_START_DATE ,
PBT.EFFECTIVE_DATE PC_ACTIVE_DATE ,
PBT.ZONE_CHANGE PRICE_CHANGE ,
PH.PROMOTION PRICE_MOD_NO ,
'PR' PRICE_TYPE ,
'U' REC_TYPE
FROM PROMHEAD PH ,
PROMSTORE PS ,
PROMSKU PSK ,
PRICE_BATCH_TRAN PBT,
ITEM_ZONE_PRICE IZP
WHERE PH.PROMOTION =PS.PROMOTION
AND PS.PROMOTION =PSK.PROMOTION
AND PBT.FDC_STATUS ='A'
AND PH.STATUS ='A'
AND PS.STORE =PBT.STORE
AND PSK.CHANGE_TYPE IN('MA','PC')
AND IZP.ITEM =PSK.SKU
AND IZP.ZONE_GROUP_ID=PBT.ZONE_GROUP_ID
AND IZP.ZONE_ID =PBT.NEW_ZONE_ID
AND PS.START_DATE > PBT.EFFECTIVE_DATE
AND EXISTS
(
SELECT 1
FROM FDT_PRICE_FUTURE
WHERE SKU =PSK.SKU
AND STORE =PS.STORE
AND START_DATE < PS.START_DATE
AND PRICE_MOD_TYPE = 'ZC'
)
UNION ALL
SELECT FDT.STORE ,
FDT.SKU ,
0 MULTI_UNITS ,
0 MULTI_UNIT_RETAIL ,
FDT.START_DATE PS_START_DATE ,
PSH.ACTIVE_DATE PC_ACTIVE_DATE ,
PSD.PRICE_CHANGE PRICE_CHANGE ,
PH.PROMOTION PRICE_MOD_NO ,
'PR' PRICE_TYPE ,
'U' REC_TYPE
FROM FDT_PRICE_FUTURE FDT ,
PROMHEAD PH ,
PROMSKU PSK ,
PRICE_SUSP_DETAIL PSD,
PRICE_SUSP_HEAD PSH ,
PRICE_ZONE_GROUP_STORE PZGS
WHERE PH.PROMOTION =PSK.PROMOTION
AND PSK.PROMOTION =FDT.PRICE_MOD_NO
AND FDT.PRICE_MOD_TYPE='PR'
AND PH.STATUS ='A'
AND PSH.PRICE_CHANGE =PSD.PRICE_CHANGE
AND PSD.STATUS ='A'
AND PSD.ZONE_ID =PZGS.ZONE_ID
AND PSD.ZONE_GROUP_ID =PZGS.ZONE_GROUP_ID
AND FDT.STORE =PZGS.STORE
AND PSK.SKU =PSD.SKU
AND PSD.SKU =FDT.SKU
AND PSK.CHANGE_TYPE IN('MA','PC')
AND FDT.START_DATE > PSH.ACTIVE_DATE
ORDER BY 1,2,7;

Answer 1

Also, if Diagnostics and Tuning Pack licenses available, you can run the SQL Tuning Advisor on the SQL.
-- SQLTUNE.sql
-- Updated 04-Mar-2015 RDCornejo for spool error
-- takes sql_id,plan_hash_value, start and ent snap_id's as parameters
-- created sql tuning report on c:\temp
set echo off
set feedback off
set term off
set verify off
set head on
-- uncomment for debugging if neeed
-- set echo on term on
set termout on
accept SQL_ID char prompt 'Enter the sql_id to Tune? '
accept hash_value char prompt 'Enter the hash_value to Tune? '
accept BEGIN_SNAP char prompt 'Enter the begin snap_id to Tune? '
accept END_SNAP char prompt 'Enter the end snap_id to Tune? '
-- using bind variables:
variable sql_id_var varchar2(25);
variable plan_hash_value varchar2(25);
variable begin_snap_var varchar2(25);
variable end_snap_var varchar2(25);
variable task_name varchar2(100);
exec :sql_id_var := '&SQL_ID';
exec :plan_hash_value := '&hash_value';
exec :begin_snap_var := '&BEGIN_SNAP';
exec :end_snap_var := '&END_SNAP';
exec :task_name := :sql_id_var || '_' || :plan_hash_value || '_AWR';
column sql_id format a25
column plan_hash_value format 999999999999
column begin_snap format a10
column end_snap format a10
select :sql_id_var as sql_id
, :plan_hash_value as plan_hash_value
, :begin_snap_var as begin_snap
, :end_snap_var as end_snap
from dual;
-- get my prefered text in file name
column MY_TEXT new_val FILE_NAME noprint
Select instance_name||'_'|| :SQL_ID_VAR || '_' || :plan_hash_value|| '_' || :BEGIN_SNAP_VAR || '_' || :END_SNAP_VAR || '_' || to_char(sysdate, 'yyyy_mm_dd_hh24_mi') MY_TEXT
from
(select upper(substr(global_name, 1, instr(global_name,'.' )-1)) instance_name from global_name);
-- turn spooling on to capture any error and the report.
set serveroutput on
spool c:\temp\AWR_Tuning_Task_&FILE_NAME..txt
set linesize 210
set pagesize 9999
set long 10000000
set time on
set timing on
SET SERVEROUTPUT ON
prompt Drop existing task if exists. Ignore error if not
-- Drop existig tuning task; assuming I'm running new tasks and if one exists, it it no longer needed; can ignore error if no task exists yet
execute DBMS_SQLTUNE.drop_tuning_task (task_name => :task_name);
prompt Create Tuning Task
-- Tuning task created for specific a statement from the AWR. [SEE BELOW FOR EXAPLES USING OTHER METHODS OF IDENTIFYING THE SQL]
--If the TASK_NAME parameter is specified it's value is returned as the SQL tune task identifier.
--If ommitted a system generated name like "TASK_1478" is returned.
--If the SCOPE parameter is set to scope_limited the SQL profiling analysis is omitted.
--The TIME_LIMIT parameter simply restricts the time the optimizer can spend compiling the recommendations.
/**/
DECLARE
l_sql_tune_task_id VARCHAR2(100);
BEGIN
l_sql_tune_task_id := DBMS_SQLTUNE.create_tuning_task (
begin_snap => to_number(:BEGIN_SNAP_VAR) ,
end_snap => to_number(:END_SNAP_VAR) ,
sql_id => :sql_id_var,
plan_hash_value => to_number(:plan_hash_value),
scope => DBMS_SQLTUNE.scope_comprehensive,
time_limit => 1200, -- in seconds
task_name => :task_name,
description => 'Tuning task for statement '|| :sql_id_var || ' hash: ' || :plan_hash_value|| ' in AWR ' || :BEGIN_SNAP_VAR || ' - ' || :END_SNAP_VAR || ' .');
DBMS_OUTPUT.put_line('l_sql_tune_task_id: ' || l_sql_tune_task_id);
END;
/
prompt Execute Tuning Task
-- With the tuning task defined the next step is to execute it using the EXECUTE_TUNING_TASK procedure.
EXEC DBMS_SQLTUNE.execute_tuning_task(task_name => :task_name);
/**/
-- Once the tuning task has executed successfully the recommendations can be displayed using the REPORT_TUNING_TASK function.
set linesize 210
set pagesize 9999
set long 10000000
set time on
set timing on
SET SERVEROUTPUT ON
-- width of output should not need more than 200
column recommendations format a200
prompt Report Tuning Task
SELECT DBMS_SQLTUNE.report_tuning_task(:task_name) AS recommendations FROM dual;
spool off