Optimize bloat queries (#45)

* Fix for tables bloat

* Fix for indexes bloat

Author: mfvanek

sql/bloated_indexes.sql

@@ -54,6 +54,7 @@ nested_indexes_attributes as (
 named_indexes_attributes as (
     select
         ic.table_oid,
+        ct.relnamespace,
         ic.inner_index_name,
         ic.attpos,
         ic.indkey,
@@ -92,10 +93,12 @@ rows_data_stats as (
             else 2 + ((32 + 8 - 1) / 8) /* indextupledata size + indexattributebitmapdata size (max num filed per index + 8 - 1 /8) */
             end as index_tuple_header_size,
         /* remove null values and save space using it fractional part from stats */
-        sum((1 - coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 0)) as null_data_width
+        sum((1 - coalesce(s.null_frac, 0)) * coalesce(s.avg_width, 0)) as null_data_width,
+        max(case when i.atttypid = 'pg_catalog.name'::regtype then 1 else 0 end) > 0 as stats_not_available
     from
         named_indexes_attributes i
-        join pg_catalog.pg_stats s on s.schemaname = i.nspname and s.tablename = i.attrelname and s.attname = i.attname
+        join pg_catalog.pg_namespace n on n.oid = i.relnamespace
+        join pg_catalog.pg_stats s on s.schemaname = n.nspname and s.tablename = i.attrelname and s.attname = i.attname
     group by 1, 2, 3, 4, 5, 6, 7, 8, 9
 ),
 rows_header_stats as (
@@ -118,20 +121,22 @@ rows_header_stats as (
                 else null_data_width::integer % max_align end
             )::numeric as null_data_header_width,
         page_header_size,
-        page_opaque_data_size
+        page_opaque_data_size,
+        stats_not_available
     from rows_data_stats
 ),
 relation_stats as (
     select
-        /* itemiddata size + computed avg size of a tuple (nulldatahdrwidth) */
+        /* itemiddata size + computed avg size of a tuple (null_data_header_width) */
         coalesce(1 +
                  ceil(reltuples / floor((block_size - page_opaque_data_size - page_header_size) * fill_factor / (100 * (4 + null_data_header_width)::float))),
             0)::bigint as estimated_pages_count,
         block_size,
         table_oid::regclass::text as table_name,
         index_oid::regclass::text as index_name,
         pg_relation_size(index_oid) as index_size,
-        relpages
+        relpages,
+        stats_not_available
     from rows_header_stats
 ),
 corrected_relation_stats as (
@@ -141,9 +146,8 @@ corrected_relation_stats as (
         index_size,
         block_size,
         relpages,
-        (case
-             when relpages - estimated_pages_count > 0 then relpages - estimated_pages_count
-             else 0 end)::bigint as pages_ff_diff
+        (case when relpages - estimated_pages_count > 0 then relpages - estimated_pages_count else 0 end)::bigint as pages_ff_diff,
+        stats_not_available
     from relation_stats
 ),
  bloat_stats as (
@@ -152,11 +156,13 @@ corrected_relation_stats as (
         index_name,
         index_size,
         block_size * pages_ff_diff as bloat_size,
-        round(100 * block_size * pages_ff_diff / index_size::float)::integer as bloat_percentage
-     from
-        corrected_relation_stats
+        round(100 * block_size * pages_ff_diff / index_size::numeric, 2)::numeric(5, 2) as bloat_percentage,
+        stats_not_available
+     from corrected_relation_stats
  )
 select *
 from bloat_stats
-where bloat_percentage >= :bloat_percentage_threshold::integer
+where
+    not stats_not_available and
+    bloat_percentage >= :bloat_percentage_threshold::numeric(5, 2)
 order by table_name, index_name;

sql/bloated_tables.sql

@@ -29,12 +29,13 @@ with tables_stats as (
         24 as page_header_size,
         23 + case when max(coalesce(ps.null_frac, 0)) > 0 then (7 + count(ps.attname)) / 8 else 0::int end +
             case when bool_or(pa.attname = 'oid' and pa.attnum < 0) then 4 else 0 end as table_tuple_header_size,
-        sum((1 - coalesce(ps.null_frac, 0)) * coalesce(ps.avg_width, 0)) as null_data_width
+        sum((1 - coalesce(ps.null_frac, 0)) * coalesce(ps.avg_width, 0)) as null_data_width,
+        bool_or(pa.atttypid = 'pg_catalog.name'::regtype) or sum(case when pa.attnum > 0 then 1 else 0 end) <> count(ps.attname) as stats_not_available
     from
         pg_attribute as pa
         join pg_class as pc on pa.attrelid = pc.oid
         join pg_namespace as pn on pn.oid = pc.relnamespace
-        join pg_stats as ps on ps.schemaname = pn.nspname and ps.tablename = pc.relname and ps.inherited = false and ps.attname = pa.attname
+        left join pg_stats as ps on ps.schemaname = pn.nspname and ps.tablename = pc.relname and ps.inherited = false and ps.attname = pa.attname
         left join pg_class as toast on pc.reltoastrelid = toast.oid
     where
         not pa.attisdropped and
@@ -58,7 +59,8 @@ tables_pages_size as (
         block_size,
         page_header_size,
         table_oid,
-        fill_factor
+        fill_factor,
+        stats_not_available
     from tables_stats
 ),
 relation_stats as (
@@ -68,7 +70,8 @@ relation_stats as (
         table_pages_count,
         block_size,
         table_oid::regclass::text as table_name,
-        pg_table_size(table_oid) as table_size
+        pg_table_size(table_oid) as table_size,
+        stats_not_available
     from tables_pages_size
 ),
 corrected_relation_stats as (
@@ -77,18 +80,22 @@ corrected_relation_stats as (
         table_size,
         (case when table_pages_count - estimated_pages_count > 0 then table_pages_count - estimated_pages_count
             else 0 end)::bigint as pages_ff_diff,
-        block_size
+        block_size,
+        stats_not_available
     from relation_stats
 ),
 bloat_stats as (
     select
         table_name,
         table_size,
         block_size * pages_ff_diff as bloat_size,
-        case when table_size > 0 then round(100 * block_size * pages_ff_diff / table_size::float)::integer else 0 end as bloat_percentage
+        case when table_size > 0 then round(100 * block_size * pages_ff_diff / table_size::numeric, 2) else 0 end ::numeric(5, 2) as bloat_percentage,
+        stats_not_available
     from corrected_relation_stats
 )
 select *
 from bloat_stats
-where bloat_percentage >= :bloat_percentage_threshold::integer
+where
+    not stats_not_available and
+    bloat_percentage >= :bloat_percentage_threshold::numeric(5, 2)
 order by table_name;