User-agent: *
Allow: /social_instagram/
Sitemap: https://pomeranzbooks.co.il/sitemap.xml


##############
#   custom   #
##############

##############################################################################
# Faceted /shop URLs, added 2026-08-22 (Scale Up Right).
#
# 368 public categories x 8,364 authors x 2,950 publishers, multiplied again by
# the sort and price parameters, is a crawl space of several million URLs. Each
# such page costs 1-2 seconds to render on an idle server and about 5 seconds
# under the load below. Crawlers found the space and saturated production:
# concurrent requests pinned at 60, PostgreSQL CPU at 40-90%, and a single-row
# RPC read queueing 14 seconds behind the backlog.
#
# Product pages, category pages, author pages and publisher pages stay fully
# indexable. Only the PARAMETERISED variants are closed, so nothing that earns
# search traffic is lost.
##############################################################################
User-agent: *
Crawl-delay: 10
Disallow: /*author=
Disallow: /*publisher=
Disallow: /*tag=
Disallow: /*tags=
Disallow: /*attrib=
Disallow: /*attribute_value=
Disallow: /*order=
Disallow: /*min_price=
Disallow: /*max_price=
Disallow: /*ppg=
Disallow: /*ppr=
Disallow: /*search=